diff --git a/fog/api/proto/ledger.proto b/fog/api/proto/ledger.proto index 6112345c72..4dd5fb211d 100644 --- a/fog/api/proto/ledger.proto +++ b/fog/api/proto/ledger.proto @@ -36,6 +36,27 @@ message GetOutputsResponse { uint64 num_blocks = 2; /// The total number of Txos in the ledger at the time the request is evaluated uint64 global_txo_count = 3; + /// The latest block_version of a block in the block chain + /// + /// This may be needed when building transactions, so that use of new transaction + /// features can be gated on the block version being increased. + /// + /// Clients may also choose to prompt users to update their software if + /// the block version increases beyond what was "known" when the software + /// was built. + uint32 latest_block_version = 4; + /// The max of latest_block_version and the MAX_BLOCK_VERSION value + /// in mc-transaction-core (in this deploy of fog ledger). + /// + /// Usually when we redeploy consensus, we also redeploy fog. So this should + /// usually be equal to the MAX_BLOCK_VERSION value in the consensus enclave. + /// (In case it isn't, it won't be less than latest_block_version.) + /// + /// This is possibly an additional signal that clients can use to discover + /// that there is a new version of transaction-core that may be available + /// for an update (by comparing to their local value of max_block_version). + uint32 max_block_version = 5; + } message OutputResult { @@ -114,6 +135,26 @@ message CheckKeyImagesResponse { uint64 global_txo_count = 2; /// The results for each key image query repeated KeyImageResult results = 3; + /// The latest block_version of a block in the block chain + /// + /// This may be needed when building transactions, so that use of new transaction + /// features can be gated on the block version being increased. + /// + /// Clients may also choose to prompt users to update their software if + /// the block version increases beyond what was "known" when the software + /// was built. + uint32 latest_block_version = 4; + /// The max of latest_block_version and the MAX_BLOCK_VERSION value + /// in mc-transaction-core (in this deploy of fog ledger). + /// + /// Usually when we redeploy consensus, we also redeploy fog. So this should + /// usually be equal to the MAX_BLOCK_VERSION value in the consensus enclave. + /// (In case it isn't, it won't be less than latest_block_version.) + /// + /// This is possibly an additional signal that clients can use to discover + /// that there is a new version of transaction-core that may be available + /// for an update (by comparing to their local value of max_block_version). + uint32 max_block_version = 5; } message KeyImageResult { diff --git a/fog/ledger/enclave/api/src/lib.rs b/fog/ledger/enclave/api/src/lib.rs index 4650d3b108..5d6a189ad1 100644 --- a/fog/ledger/enclave/api/src/lib.rs +++ b/fog/ledger/enclave/api/src/lib.rs @@ -51,6 +51,12 @@ pub struct UntrustedKeyImageQueryResponse { /// The cumulative txo count of the last known block. pub last_known_block_cumulative_txo_count: u64, + + /// The latest value of block version in the blockchain + pub latest_block_version: u32, + + /// The (max of) latest_block_version and mc_transaction_core::BLOCK_VERSION + pub max_block_version: u32, } /// The API for interacting with a ledger node's enclave. diff --git a/fog/ledger/enclave/impl/src/lib.rs b/fog/ledger/enclave/impl/src/lib.rs index 11c34bbe63..df2f41827a 100644 --- a/fog/ledger/enclave/impl/src/lib.rs +++ b/fog/ledger/enclave/impl/src/lib.rs @@ -140,7 +140,7 @@ where fn check_key_images( &self, msg: EnclaveMessage, - untrusted_keyimagequery_response: UntrustedKeyImageQueryResponse, + untrusted_key_image_query_response: UntrustedKeyImageQueryResponse, ) -> Result> { let channel_id = msg.channel_id.clone(); //client session does not implement copy trait so clone let user_plaintext = self.ake.client_decrypt(msg)?; @@ -151,10 +151,12 @@ where })?; let mut resp = CheckKeyImagesResponse { - num_blocks: untrusted_keyimagequery_response.highest_processed_block_count, + num_blocks: untrusted_key_image_query_response.highest_processed_block_count, results: Default::default(), - global_txo_count: untrusted_keyimagequery_response + global_txo_count: untrusted_key_image_query_response .last_known_block_cumulative_txo_count, + latest_block_version: untrusted_key_image_query_response.latest_block_version, + max_block_version: untrusted_key_image_query_response.max_block_version, }; // Do the scope lock of keyimagetore diff --git a/fog/ledger/server/src/db_fetcher.rs b/fog/ledger/server/src/db_fetcher.rs index 673b179f61..7ff419f156 100644 --- a/fog/ledger/server/src/db_fetcher.rs +++ b/fog/ledger/server/src/db_fetcher.rs @@ -247,6 +247,19 @@ impl DbFetche shared_state.last_known_block_cumulative_txo_count = global_txo_count; } } + match self.db.get_latest_block() { + Err(e) => { + log::error!( + self.logger, + "Unexpected error when checking for ledger latest block version {}: {:?}", + self.next_block_index, + e + ); + } + Ok(block) => { + shared_state.latest_block_version = block.version; + } + } }); self.next_block_index += 1; diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index 7ba9c66f9e..825a654d8a 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -72,17 +72,27 @@ impl KeyImageService { ) -> Result { log::trace!(self.logger, "Getting encrypted request"); - let (highest_processed_block_count, last_known_block_cumulative_txo_count) = { + let ( + highest_processed_block_count, + last_known_block_cumulative_txo_count, + latest_block_version, + ) = { let shared_state = self.db_poll_shared_state.lock().expect("mutex poisoned"); ( shared_state.highest_processed_block_count, shared_state.last_known_block_cumulative_txo_count, + shared_state.latest_block_version, ) }; let untrusted_query_response = UntrustedKeyImageQueryResponse { highest_processed_block_count, last_known_block_cumulative_txo_count, + latest_block_version, + max_block_version: core::cmp::max( + latest_block_version, + mc_transaction_core::BLOCK_VERSION, + ), }; let result_blob = self diff --git a/fog/ledger/server/src/merkle_proof_service.rs b/fog/ledger/server/src/merkle_proof_service.rs index 34014a841b..4d2393e065 100644 --- a/fog/ledger/server/src/merkle_proof_service.rs +++ b/fog/ledger/server/src/merkle_proof_service.rs @@ -104,6 +104,12 @@ impl MerkleProofService { )); } + let latest_block_version = self + .ledger + .get_latest_block() + .map_err(|err| rpc_database_err(err, &self.logger))? + .version; + Ok(GetOutputsResponse { num_blocks: self .ledger @@ -134,6 +140,11 @@ impl MerkleProofService { }) .collect::, DbError>>() .map_err(|err| rpc_database_err(err, &self.logger))?, + latest_block_version, + max_block_version: core::cmp::max( + latest_block_version, + mc_transaction_core::BLOCK_VERSION, + ), }) } @@ -267,6 +278,7 @@ mod test { let highest_index: u32 = num_tx_outs - 1; mock_ledger.num_tx_outs = num_tx_outs as u64; + mock_ledger.num_blocks = 1; for (index, tx_out) in get_tx_outs(num_tx_outs).into_iter().enumerate() { mock_ledger.tx_out_by_index.insert(index as u64, tx_out); @@ -319,6 +331,7 @@ mod test { let mut mock_ledger = MockLedger::default(); let num_tx_outs: u32 = 100; mock_ledger.num_tx_outs = num_tx_outs as u64; + mock_ledger.num_blocks = 1; // Populate the mock ledger with TxOuts and membership proofs. for (index, tx_out) in get_tx_outs(num_tx_outs).into_iter().enumerate() { diff --git a/fog/ledger/server/src/server.rs b/fog/ledger/server/src/server.rs index d77b3b763e..c5df20a558 100644 --- a/fog/ledger/server/src/server.rs +++ b/fog/ledger/server/src/server.rs @@ -247,4 +247,7 @@ pub struct DbPollSharedState { /// The cumulative txo count of the last known block. pub last_known_block_cumulative_txo_count: u64, + + /// The latest value of `block_version` in the blockchain + pub latest_block_version: u32, } diff --git a/fog/ledger/test_infra/src/lib.rs b/fog/ledger/test_infra/src/lib.rs index e2ca053b30..a97f1f9f80 100644 --- a/fog/ledger/test_infra/src/lib.rs +++ b/fog/ledger/test_infra/src/lib.rs @@ -105,8 +105,12 @@ impl Ledger for MockLedger { Ok(self.num_blocks) } - fn get_block(&self, _block_number: u64) -> Result { - unimplemented!() + fn get_block(&self, block_number: u64) -> Result { + if block_number < self.num_blocks { + Ok(Block::default()) + } else { + Err(Error::NotFound) + } } fn get_block_signature(&self, _block_number: u64) -> Result { diff --git a/fog/types/src/common.rs b/fog/types/src/common.rs index 65e3589215..f596c31968 100644 --- a/fog/types/src/common.rs +++ b/fog/types/src/common.rs @@ -6,13 +6,16 @@ use serde::{Deserialize, Serialize}; /// A half-open [a, b) range of blocks #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Message, Serialize, Deserialize)] pub struct BlockRange { + /// The first block in the range #[prost(uint64, tag = "1")] pub start_block: u64, + /// The end block, which is one past the end of the range. #[prost(uint64, tag = "2")] pub end_block: u64, } impl BlockRange { + /// Create a new block range pub fn new(start_block: u64, end_block: u64) -> Self { Self { start_block, @@ -20,14 +23,17 @@ impl BlockRange { } } + /// Test if a block index is in the range pub fn contains(&self, block: u64) -> bool { block >= self.start_block && block < self.end_block } + /// Test if a block range is well-formed pub fn is_valid(&self) -> bool { self.end_block > self.start_block } + /// Test if two block ranges overlap pub fn overlaps(&self, other: &BlockRange) -> bool { self.start_block < other.end_block && other.start_block < self.end_block } diff --git a/fog/types/src/ledger.rs b/fog/types/src/ledger.rs index a0c2949a29..a63b1d8666 100644 --- a/fog/types/src/ledger.rs +++ b/fog/types/src/ledger.rs @@ -42,8 +42,35 @@ pub struct GetOutputsResponse { /// Number of txos in the ledger #[prost(uint64, tag = "3")] pub global_txo_count: u64, + + /// The latest block_version of a block in the block chain + /// + /// This may be needed when building transactions, so that use of new + /// transaction features can be gated on the block version being + /// increased. + /// + /// Clients may also choose to prompt users to update their software if + /// the block version increases beyond what was "known" when the software + /// was built. + #[prost(uint32, tag = "4")] + pub latest_block_version: u32, + + /// The max of latest_block_version and the MAX_BLOCK_VERSION value + /// in mc-transaction-core (in this deploy of fog ledger). + /// + /// Usually when we redeploy consensus, we also redeploy fog. So this should + /// usually be equal to the MAX_BLOCK_VERSION value in the consensus + /// enclave. (In case it isn't, it won't be less than + /// latest_block_version.) + /// + /// This is possibly an additional signal that clients can use to discover + /// that there is a new version of transaction-core that may be available + /// for an update (by comparing to their local value of max_block_version). + #[prost(uint32, tag = "5")] + pub max_block_version: u32, } +/// The result of an individual query for an output and membership proof #[derive(Clone, Message, Eq, PartialEq, Serialize, Deserialize)] pub struct OutputResult { /// Index that was queried (global index of a txo) @@ -77,9 +104,11 @@ pub struct CheckKeyImagesRequest { /// Query about a particular key image #[derive(Message, Eq, PartialEq)] pub struct KeyImageQuery { + /// The key image to query about #[prost(message, required, tag = "1")] pub key_image: KeyImage, + /// A lower bound on the range to search. This is an optimization. #[prost(fixed64, tag = "2")] pub start_block: u64, } @@ -102,21 +131,47 @@ pub struct CheckKeyImagesResponse { /// Results of key image checks #[prost(message, repeated, tag = "3")] pub results: Vec, + + /// The latest block_version of a block in the block chain + /// + /// This may be needed when building transactions, so that use of new + /// transaction features can be gated on the block version being + /// increased. + /// + /// Clients may also choose to prompt users to update their software if + /// the block version increases beyond what was "known" when the software + /// was built. + #[prost(uint32, tag = "4")] + pub latest_block_version: u32, + + /// The max of latest_block_version and the MAX_BLOCK_VERSION value + /// in mc-transaction-core (in this deploy of fog ledger). + /// + /// Usually when we redeploy consensus, we also redeploy fog. So this should + /// usually be equal to the MAX_BLOCK_VERSION value in the consensus + /// enclave. (In case it isn't, it won't be less than + /// latest_block_version.) + /// + /// This is possibly an additional signal that clients can use to discover + /// that there is a new version of transaction-core that may be available + /// for an update (by comparing to their local value of max_block_version). + #[prost(uint32, tag = "5")] + pub max_block_version: u32, } /// A result which tells for a given key image, whether it was spent or not /// and at what height. #[derive(Clone, Message, Eq, PartialEq, Serialize, Deserialize)] pub struct KeyImageResult { + /// The key image which was queried #[prost(message, required, tag = "1")] pub key_image: KeyImage, - // Note: We should perhaps add - // [prost(... , default = "!064")] - // here to force prost to emit this field even if spent_at = 0, because - // spent_at = 0 indicates a miss and we don't want to leak that. - // But it's kind of a hack... - // proto3 does not support defaults, so this cannot be in the .proto AFAIU - // There's probably a simpler fix... + + /// The block index of the block in which this key image appeared + // + // Note: prost will omit this field if spent_at = 0, but that never + // happens in real life, because a Tx cannot be spent in the origin block, + // and in the case that a Tx is not spent, we set this field to a nonzero value. #[prost(fixed64, tag = "2")] pub spent_at: u64, @@ -138,6 +193,7 @@ pub struct KeyImageResult { pub key_image_result_code: u32, } +/// An enum corresponding to the KeyImageResultCode proto enum #[derive(PartialEq, Eq, Debug, Display)] #[repr(u32)] pub enum KeyImageResultCode { diff --git a/fog/types/src/lib.rs b/fog/types/src/lib.rs index ed50cfd748..aa1a7edd21 100644 --- a/fog/types/src/lib.rs +++ b/fog/types/src/lib.rs @@ -2,15 +2,25 @@ #![no_std] +//! Enclave-compatible types used by fog. +//! Particularly prosty versions of protobuf types, but also some enclave api +//! types. + +#![deny(missing_docs)] + extern crate alloc; use alloc::vec::Vec; use prost::Message; use serde::{Deserialize, Serialize}; +/// Types from or related to fog_common.proto pub mod common; +/// Types related to fog ingest pub mod ingest; +/// Types related to fog ledger pub mod ledger; +/// Types related to fog view pub mod view; /// An Encrypted Tx Out Record, consisting of a fog search_key (rng output), @@ -59,5 +69,6 @@ impl core::fmt::Display for BlockCount { } impl BlockCount { + /// The largest possible BlockCount pub const MAX: BlockCount = BlockCount(u64::MAX); } diff --git a/fog/types/src/view.rs b/fog/types/src/view.rs index 50b7dbf0e1..c596879f1c 100644 --- a/fog/types/src/view.rs +++ b/fog/types/src/view.rs @@ -20,8 +20,13 @@ pub use mc_fog_kex_rng::KexRngPubkey; // These are synced with types in fog_api view.proto, and tests enforce that // they round trip These are NOT expected to be synced with Db schema types +/// The QueryRequestAAD structure, which should be passed as AAD when making +/// an attested fog view request. #[derive(Clone, Eq, PartialEq, Message)] pub struct QueryRequestAAD { + /// The first id of a user event to return. If you set this larger than 0, + /// it means that you already saw some of the events and you don't want + /// to be sent them again. #[prost(int64, tag = "1")] pub start_from_user_event_id: i64, @@ -31,38 +36,62 @@ pub struct QueryRequestAAD { pub start_from_block_index: u64, } +/// The QueryRequest structure, which should be passed as the encrypted data +/// when making an attested fog view request #[derive(Clone, Eq, PartialEq, Message)] pub struct QueryRequest { + /// The search keys to query for TxOut's + /// These should all be values that came from KexRng's #[prost(bytes, repeated, tag = "1")] pub get_txos: Vec>, } +/// The QueryResponse structure, returned by the enclave in response to an +/// attested request #[derive(Clone, Eq, PartialEq, Message)] pub struct QueryResponse { + /// The number of blocks processed by this view enclave at the time that the + /// request was evaluated. #[prost(uint64, tag = "1")] pub highest_processed_block_count: u64, + /// The timestamp of the highest processed block #[prost(uint64, tag = "2")] pub highest_processed_block_signature_timestamp: u64, + /// The index of the next user id event that the user should query + /// (all events from request.start_from_user_event_id, up to this number, + /// were returned) #[prost(int64, tag = "3")] pub next_start_from_user_event_id: i64, + /// Any block ranges corresponding to missed block events. Any block in the + /// range should be downloaded and scanned to ensure correct balance + /// computation. #[prost(message, repeated, tag = "4")] pub missed_block_ranges: Vec, + /// Any RNG records that the users can now use to build RNGs and then search + /// for transactions. #[prost(message, repeated, tag = "5")] pub rng_records: Vec, + /// Any records of decommissioned ingest invocations, which implies that an + /// RNG will no longer be used. #[prost(message, repeated, tag = "6")] pub decommissioned_ingest_invocations: Vec, + /// The results of each tx out search query #[prost(message, repeated, tag = "7")] pub tx_out_search_results: Vec, + /// The last known block count. This may be larger than the highest known + /// block count, if fog is stuck or this view server is behind. #[prost(uint64, tag = "8")] pub last_known_block_count: u64, + /// The last known block cumulative txo count. This is provided to help + /// clients sample for mixins. #[prost(uint64, tag = "9")] pub last_known_block_cumulative_txo_count: u64, } @@ -88,9 +117,11 @@ pub struct RngRecord { /// Information about a decommissioned ingest invocation. #[derive(Clone, Eq, PartialEq, Hash, Message, Serialize, Deserialize)] pub struct DecommissionedIngestInvocation { + /// The ingest invocation id which is decommissioned #[prost(int64, tag = "1")] pub ingest_invocation_id: i64, + /// The last block which it ingested #[prost(uint64, tag = "2")] pub last_ingested_block: u64, } @@ -152,13 +183,13 @@ pub struct TxOutSearchResult { pub ciphertext: Vec, } -// TxOutRecord is what information the fog service preserves for a user about -// their TxOut. These are created by the ingest server and then encrypted. The -// encrypted blobs are eventually returned to the user, who must deserialize -// them. -// -// Note: There are conformance tests in fog_api that check that this matches -// proto +/// TxOutRecord is what information the fog service preserves for a user about +/// their TxOut. These are created by the ingest server and then encrypted. The +/// encrypted blobs are eventually returned to the user, who must deserialize +/// them. +/// +/// Note: There are conformance tests in fog-api that check that this matches +/// the proto #[derive(Clone, Eq, Hash, PartialEq, Message)] pub struct TxOutRecord { /// The (compressed ristretto) bytes of commitment associated to amount @@ -279,9 +310,9 @@ impl TxOutRecord { } } -// FogTxOut is a redacted version of the TxOut, removing the fog hint, and with -// reduced data about Amount. The hint is only used during ingest, so we don't -// need to persist it. +/// FogTxOut is a redacted version of the TxOut, removing the fog hint, and with +/// reduced data about Amount. The hint is only used during ingest, so we don't +/// need to persist it. #[derive(Clone, Eq, Hash, PartialEq, Debug, Default)] pub struct FogTxOut { /// The one-time public address of this output. diff --git a/ledger/db/src/ledger_trait.rs b/ledger/db/src/ledger_trait.rs index b24de49d26..43ebe40c05 100644 --- a/ledger/db/src/ledger_trait.rs +++ b/ledger/db/src/ledger_trait.rs @@ -81,4 +81,13 @@ pub trait Ledger: Send { /// Get the tx out root membership element from the tx out Merkle Tree. fn get_root_tx_out_membership_element(&self) -> Result; + + /// Get the latest block header, if any. NotFound if the ledger-db is empty. + fn get_latest_block(&self) -> Result { + let num_blocks = self.num_blocks()?; + if num_blocks == 0 { + return Err(Error::NotFound); + } + self.get_block(num_blocks - 1) + } }