From 7851c6c447a6fb8edcff0930693ad790c620e017 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Sun, 24 Dec 2023 18:22:27 -0500 Subject: [PATCH] proto: add `timestamp` field to `SubscribeUpdate` message --- .github/workflows/release.yml | 11 ++++--- .github/workflows/test.yml | 12 ++++--- CHANGELOG.md | 33 +++++++++++++++++++ Cargo.lock | 11 ++++--- Cargo.toml | 15 +++++---- examples/rust/Cargo.toml | 2 +- yellowstone-grpc-client/Cargo.toml | 2 +- yellowstone-grpc-geyser/Cargo.toml | 2 +- yellowstone-grpc-geyser/src/filters.rs | 23 ++++++++----- yellowstone-grpc-geyser/src/grpc.rs | 3 ++ yellowstone-grpc-proto/Cargo.toml | 3 +- yellowstone-grpc-proto/proto/geyser.proto | 2 ++ yellowstone-grpc-proto/src/lib.rs | 1 + yellowstone-grpc-tools/Cargo.toml | 2 +- .../src/bin/grpc-google-pubsub.rs | 8 ++--- yellowstone-grpc-tools/src/kafka/grpc.rs | 19 +++++++---- 16 files changed, 105 insertions(+), 44 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 021eb2e..706dc02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get install -y libsasl2-dev + - name: Set rust version run: | source ci/rust-version.sh @@ -54,10 +57,10 @@ jobs: target key: ${{ matrix.os }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('**/Cargo.lock') }}-0001 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libsasl2-dev + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y libsasl2-dev - name: Check Solana version run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48406a6..0b088e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,7 @@ on: - 'master' - 'v1.16' - 'v1.17' + - 'v1.17-gamma' workflow_dispatch: env: @@ -25,6 +26,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get install -y libsasl2-dev + - name: Set rust version run: | source ci/rust-version.sh @@ -53,10 +57,10 @@ jobs: target key: ${{ matrix.os }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('**/Cargo.lock') }}-0001 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libsasl2-dev + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y libsasl2-dev - name: cargo tree run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 24aecca..05a3197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,17 @@ The minor version will be incremented upon a breaking change and the patch versi ## 2024-02-06 +- yellowstone-grpc-client-1.13.0+solana.1.17.20.gamma +- yellowstone-grpc-geyser-1.12.0+solana.1.17.20.gamma +- yellowstone-grpc-proto-1.12.0+solana.1.17.20.gamma +- yellowstone-grpc-tools-1.0.0-rc.9+solana.1.17.20.gamma + +### Features + +- proto: add `timestamp` field to `SubscribeUpdate` message + +## 2024-02-06 + - yellowstone-grpc-client-1.13.0+solana.1.17.20 - yellowstone-grpc-geyser-1.12.0+solana.1.17.20 - yellowstone-grpc-proto-1.12.0+solana.1.17.20 @@ -117,6 +128,17 @@ The minor version will be incremented upon a breaking change and the patch versi - solana: update to 1.17.16 ([#274](https://github.com/rpcpool/yellowstone-grpc/pull/274)) +## 2024-01-09 + +- yellowstone-grpc-client-1.13.0+solana.1.17.15.gamma +- yellowstone-grpc-geyser-1.12.0+solana.1.17.15.gamma +- yellowstone-grpc-proto-1.12.0+solana.1.17.15.gamma +- yellowstone-grpc-tools-1.0.0-rc.9+solana.1.17.15.gamma + +### Features + +- proto: add `timestamp` field to `SubscribeUpdate` message + ## 2024-01-08 - yellowstone-grpc-client-1.13.0+solana.1.17.15 @@ -129,6 +151,17 @@ The minor version will be incremented upon a breaking change and the patch versi - proto: add more convert functions ([#264](https://github.com/rpcpool/yellowstone-grpc/pull/264)) - geyser: set plugin name to `{name}-{version}` ([#270](https://github.com/rpcpool/yellowstone-grpc/pull/270)) +## 2023-12-24 + +- yellowstone-grpc-client-1.13.0+solana.1.16.21 +- yellowstone-grpc-geyser-1.12.0+solana.1.16.21 +- yellowstone-grpc-proto-1.12.0+solana.1.16.21 +- yellowstone-grpc-tools-1.0.0-rc.10+solana.1.16.21 + +### Features + +- proto: add `timestamp` field to `SubscribeUpdate` message + ## 2023-12-22 - yellowstone-grpc-client-1.12.0+solana.1.17.12 diff --git a/Cargo.lock b/Cargo.lock index 5a54aa8..d15cad4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5174,7 +5174,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "1.14.0+solana.1.17.28" +version = "1.14.0+solana.1.17.28.gamma" dependencies = [ "bytes", "futures", @@ -5188,7 +5188,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client-simple" -version = "1.12.0+solana.1.17.28" +version = "1.12.0+solana.1.17.28.gamma" dependencies = [ "anyhow", "backoff", @@ -5211,7 +5211,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-geyser" -version = "1.13.0+solana.1.17.28" +version = "1.13.0+solana.1.17.28.gamma" dependencies = [ "anyhow", "base64 0.21.7", @@ -5244,11 +5244,12 @@ dependencies = [ [[package]] name = "yellowstone-grpc-proto" -version = "1.13.0+solana.1.17.28" +version = "1.13.0+solana.1.17.28.gamma" dependencies = [ "anyhow", "bincode", "prost 0.12.3", + "prost-types 0.12.3", "protobuf-src", "solana-account-decoder", "solana-sdk", @@ -5259,7 +5260,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-tools" -version = "1.0.0-rc.10+solana.1.17.28" +version = "1.0.0-rc.10+solana.1.17.28.gamma" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b853f01..e0b3171 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [workspace] resolver = "2" members = [ - "examples/rust", # 1.12.0+solana.1.17.28 - "yellowstone-grpc-client", # 1.14.0+solana.1.17.28 - "yellowstone-grpc-geyser", # 1.13.0+solana.1.17.28 - "yellowstone-grpc-proto", # 1.13.0+solana.1.17.28 - "yellowstone-grpc-tools", # 1.0.0-rc.10+solana.1.17.28 + "examples/rust", # 1.12.0+solana.1.17.28.gamma + "yellowstone-grpc-client", # 1.14.0+solana.1.17.28.gamma + "yellowstone-grpc-geyser", # 1.13.0+solana.1.17.28.gamma + "yellowstone-grpc-proto", # 1.13.0+solana.1.17.28.gamma + "yellowstone-grpc-tools", # 1.0.0-rc.10+solana.1.17.28.gamma ] [workspace.package] @@ -45,6 +45,7 @@ log = "0.4.17" maplit = "1.0.2" prometheus = "0.13.2" prost = "0.12.1" +prost-types = "0.12.1" protobuf-src = "1.1.0" rdkafka = "0.34.0" serde = "1.0.145" @@ -66,8 +67,8 @@ tonic-health = "0.10.2" tracing = "0.1.37" tracing-subscriber = "0.3.17" vergen = "8.2.1" -yellowstone-grpc-client = { path = "yellowstone-grpc-client", version = "=1.14.0+solana.1.17.28" } -yellowstone-grpc-proto = { path = "yellowstone-grpc-proto", version = "=1.13.0+solana.1.17.28" } +yellowstone-grpc-client = { path = "yellowstone-grpc-client", version = "=1.14.0+solana.1.17.28.gamma" } +yellowstone-grpc-proto = { path = "yellowstone-grpc-proto", version = "=1.13.0+solana.1.17.28.gamma" } [profile.release] debug = true diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml index acce7b5..e2d4c0e 100644 --- a/examples/rust/Cargo.toml +++ b/examples/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-client-simple" -version = "1.12.0+solana.1.17.28" +version = "1.12.0+solana.1.17.28.gamma" authors = { workspace = true } edition = { workspace = true } homepage = { workspace = true } diff --git a/yellowstone-grpc-client/Cargo.toml b/yellowstone-grpc-client/Cargo.toml index 42279ee..0399817 100644 --- a/yellowstone-grpc-client/Cargo.toml +++ b/yellowstone-grpc-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-client" -version = "1.14.0+solana.1.17.28" +version = "1.14.0+solana.1.17.28.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Geyser Simple Client" diff --git a/yellowstone-grpc-geyser/Cargo.toml b/yellowstone-grpc-geyser/Cargo.toml index 8cac576..a715bdd 100644 --- a/yellowstone-grpc-geyser/Cargo.toml +++ b/yellowstone-grpc-geyser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-geyser" -version = "1.13.0+solana.1.17.28" +version = "1.13.0+solana.1.17.28.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Geyser Plugin" diff --git a/yellowstone-grpc-geyser/src/filters.rs b/yellowstone-grpc-geyser/src/filters.rs index d0a41fc..b00c12d 100644 --- a/yellowstone-grpc-geyser/src/filters.rs +++ b/yellowstone-grpc-geyser/src/filters.rs @@ -16,15 +16,20 @@ use { std::{ collections::{HashMap, HashSet}, str::FromStr, + time::SystemTime, }, - yellowstone_grpc_proto::prelude::{ - subscribe_request_filter_accounts_filter::Filter as AccountsFilterDataOneof, - subscribe_request_filter_accounts_filter_memcmp::Data as AccountsFilterMemcmpOneof, - subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequest, - SubscribeRequestAccountsDataSlice, SubscribeRequestFilterAccounts, - SubscribeRequestFilterAccountsFilter, SubscribeRequestFilterBlocks, - SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterEntry, SubscribeRequestFilterSlots, - SubscribeRequestFilterTransactions, SubscribeUpdate, SubscribeUpdatePong, + yellowstone_grpc_proto::{ + prelude::{ + subscribe_request_filter_accounts_filter::Filter as AccountsFilterDataOneof, + subscribe_request_filter_accounts_filter_memcmp::Data as AccountsFilterMemcmpOneof, + subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequest, + SubscribeRequestAccountsDataSlice, SubscribeRequestFilterAccounts, + SubscribeRequestFilterAccountsFilter, SubscribeRequestFilterBlocks, + SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterEntry, + SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, SubscribeUpdate, + SubscribeUpdatePong, + }, + prost_types::Timestamp, }, }; @@ -119,6 +124,7 @@ impl Filter { Some(SubscribeUpdate { filters, update_oneof: Some(message.to_proto(&self.accounts_data_slice)), + timestamp: Some(Timestamp::from(SystemTime::now())), }) } }) @@ -129,6 +135,7 @@ impl Filter { self.ping.map(|id| SubscribeUpdate { filters: vec![], update_oneof: Some(UpdateOneof::Pong(SubscribeUpdatePong { id })), + timestamp: Some(Timestamp::from(SystemTime::now())), }) } } diff --git a/yellowstone-grpc-geyser/src/grpc.rs b/yellowstone-grpc-geyser/src/grpc.rs index 288451b..2ec9208 100644 --- a/yellowstone-grpc-geyser/src/grpc.rs +++ b/yellowstone-grpc-geyser/src/grpc.rs @@ -23,6 +23,7 @@ use { atomic::{AtomicUsize, Ordering}, Arc, }, + time::SystemTime, }, tokio::{ fs, @@ -52,6 +53,7 @@ use { SubscribeUpdateBlockMeta, SubscribeUpdateEntry, SubscribeUpdatePing, SubscribeUpdateSlot, SubscribeUpdateTransaction, SubscribeUpdateTransactionInfo, }, + prost_types::Timestamp, }, }; @@ -1234,6 +1236,7 @@ impl Geyser for GrpcService { let ping_msg = SubscribeUpdate { filters: vec![], update_oneof: Some(UpdateOneof::Ping(SubscribeUpdatePing {})), + timestamp: Some(Timestamp::from(SystemTime::now())), }; loop { diff --git a/yellowstone-grpc-proto/Cargo.toml b/yellowstone-grpc-proto/Cargo.toml index 6d3dd73..e3bb385 100644 --- a/yellowstone-grpc-proto/Cargo.toml +++ b/yellowstone-grpc-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-proto" -version = "1.13.0+solana.1.17.28" +version = "1.13.0+solana.1.17.28.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Geyser Protobuf Definitions" @@ -13,6 +13,7 @@ publish = true [dependencies] bincode = { workspace = true } prost = { workspace = true } +prost-types = { workspace = true } solana-account-decoder = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } diff --git a/yellowstone-grpc-proto/proto/geyser.proto b/yellowstone-grpc-proto/proto/geyser.proto index 2250a13..5f17c9c 100644 --- a/yellowstone-grpc-proto/proto/geyser.proto +++ b/yellowstone-grpc-proto/proto/geyser.proto @@ -1,5 +1,6 @@ syntax = "proto3"; +import "google/protobuf/timestamp.proto"; import public "solana-storage.proto"; option go_package = "github.com/rpcpool/solana-geyser-grpc/golang/proto"; @@ -102,6 +103,7 @@ message SubscribeUpdate { SubscribeUpdateBlockMeta block_meta = 7; SubscribeUpdateEntry entry = 8; } + google.protobuf.Timestamp timestamp = 10; } message SubscribeUpdateAccount { diff --git a/yellowstone-grpc-proto/src/lib.rs b/yellowstone-grpc-proto/src/lib.rs index f4d5bb6..425aee7 100644 --- a/yellowstone-grpc-proto/src/lib.rs +++ b/yellowstone-grpc-proto/src/lib.rs @@ -18,6 +18,7 @@ pub mod prelude { } pub use prost; +pub use prost_types; pub use tonic; pub mod convert_to { diff --git a/yellowstone-grpc-tools/Cargo.toml b/yellowstone-grpc-tools/Cargo.toml index 4e053df..05bc975 100644 --- a/yellowstone-grpc-tools/Cargo.toml +++ b/yellowstone-grpc-tools/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-tools" -version = "1.0.0-rc.10+solana.1.17.28" +version = "1.0.0-rc.10+solana.1.17.28.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Tools" diff --git a/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs b/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs index 7ffa809..f3050d7 100644 --- a/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs +++ b/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs @@ -185,9 +185,9 @@ impl ArgsAction { .context("failed to get message from gRPC")?; match &message { - SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Ping(_)) } => prom::recv_inc(GprcMessageKind::Ping), - SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Pong(_)) } => prom::recv_inc(GprcMessageKind::Pong), - SubscribeUpdate { filters: _, update_oneof: Some(value) } => { + SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Ping(_)), timestamp: _ } => prom::recv_inc(GprcMessageKind::Ping), + SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Pong(_)), timestamp: _ } => prom::recv_inc(GprcMessageKind::Pong), + SubscribeUpdate { filters: _, update_oneof: Some(value), timestamp: _ } => { if let UpdateOneof::Slot(slot) = value { prom::set_slot_tip( CommitmentLevel::try_from(slot.status).expect("valid commitment"), @@ -204,7 +204,7 @@ impl ArgsAction { prom::recv_inc(prom_kind); }, - SubscribeUpdate { filters: _, update_oneof: None } => anyhow::bail!("received empty updat emessage"), + SubscribeUpdate { filters: _, update_oneof: None, timestamp: _ } => anyhow::bail!("received empty updat emessage"), }; } }; diff --git a/yellowstone-grpc-tools/src/kafka/grpc.rs b/yellowstone-grpc-tools/src/kafka/grpc.rs index bb9997c..f2c703a 100644 --- a/yellowstone-grpc-tools/src/kafka/grpc.rs +++ b/yellowstone-grpc-tools/src/kafka/grpc.rs @@ -7,6 +7,7 @@ use { atomic::{AtomicUsize, Ordering}, Arc, }, + time::SystemTime, }, tokio::{ sync::{broadcast, mpsc, Notify}, @@ -24,13 +25,16 @@ use { }, tonic_health::server::health_reporter, tracing::{error, info}, - yellowstone_grpc_proto::prelude::{ - geyser_server::{Geyser, GeyserServer}, - subscribe_update::UpdateOneof, - GetBlockHeightRequest, GetBlockHeightResponse, GetLatestBlockhashRequest, - GetLatestBlockhashResponse, GetSlotRequest, GetSlotResponse, GetVersionRequest, - GetVersionResponse, IsBlockhashValidRequest, IsBlockhashValidResponse, PingRequest, - PongResponse, SubscribeRequest, SubscribeUpdate, SubscribeUpdatePing, + yellowstone_grpc_proto::{ + prelude::{ + geyser_server::{Geyser, GeyserServer}, + subscribe_update::UpdateOneof, + GetBlockHeightRequest, GetBlockHeightResponse, GetLatestBlockhashRequest, + GetLatestBlockhashResponse, GetSlotRequest, GetSlotResponse, GetVersionRequest, + GetVersionResponse, IsBlockhashValidRequest, IsBlockhashValidResponse, PingRequest, + PongResponse, SubscribeRequest, SubscribeUpdate, SubscribeUpdatePing, + }, + prost_types::Timestamp, }, }; @@ -119,6 +123,7 @@ impl Geyser for GrpcService { let ping_msg = SubscribeUpdate { filters: vec![], update_oneof: Some(UpdateOneof::Ping(SubscribeUpdatePing {})), + timestamp: Some(Timestamp::from(SystemTime::now())), }; loop {