Fog passes latest_block_version to clients in its responses (#1461)

* Fog passes `latest_block_version` to clients in its responses

This makes fog-ledger pass `latest_block_version` to clients in
its responses when they do `KeyImage` checks or get merkle proofs.

This will mean that they will get an update on this value when
they do a balance check or before they submit a transaction,
giving them a chance to potentially:

* Pass the `latest_block_version` to the transaction builder, as
  envisioned in MCIP #26.
* Maybe signal the need for an update if in the course of a balance
  check we learn that the block version has advanced beyond what
  our software was built to support.

This is not a breaking change, this is just the part where fog
communicates the `block_version` which is backwards compatible,
so it is safe to merge at any time.

The non-fog clients can already ask the ledger for the latest
block version directly.

We could add an API to consensus as well to get the latest block
header, but it doesn't seem that it's a requirement right now
so we didn't.

* Update ledger/db/src/ledger_trait.rs

Co-authored-by: Remoun Metyas <[email protected]>

* try to get more fog ledger tests to pass

* also pass `max_block_version` to the clients when we pass `block_version`

This addresses a review suggestion from Eran

Co-authored-by: Remoun Metyas <[email protected]>

fog/api/proto/ledger.proto

@@ -36,6 +36,27 @@ message GetOutputsResponse {
     uint64 num_blocks = 2;
     /// The total number of Txos in the ledger at the time the request is evaluated
     uint64 global_txo_count = 3;
+    /// The latest block_version of a block in the block chain
+    ///
+    /// This may be needed when building transactions, so that use of new transaction
+    /// features can be gated on the block version being increased.
+    ///
+    /// Clients may also choose to prompt users to update their software if
+    /// the block version increases beyond what was "known" when the software
+    /// was built.
+    uint32 latest_block_version = 4;
+    /// The max of latest_block_version and the MAX_BLOCK_VERSION value
+    /// in mc-transaction-core (in this deploy of fog ledger).
+    ///
+    /// Usually when we redeploy consensus, we also redeploy fog. So this should
+    /// usually be equal to the MAX_BLOCK_VERSION value in the consensus enclave.
+    /// (In case it isn't, it won't be less than latest_block_version.)
+    ///
+    /// This is possibly an additional signal that clients can use to discover
+    /// that there is a new version of transaction-core that may be available
+    /// for an update (by comparing to their local value of max_block_version).
+    uint32 max_block_version = 5;
+
 }
 
 message OutputResult {
@@ -114,6 +135,26 @@ message CheckKeyImagesResponse {
     uint64 global_txo_count = 2;
     /// The results for each key image query
     repeated KeyImageResult results = 3;
+    /// The latest block_version of a block in the block chain
+    ///
+    /// This may be needed when building transactions, so that use of new transaction
+    /// features can be gated on the block version being increased.
+    ///
+    /// Clients may also choose to prompt users to update their software if
+    /// the block version increases beyond what was "known" when the software
+    /// was built.
+    uint32 latest_block_version = 4;
+    /// The max of latest_block_version and the MAX_BLOCK_VERSION value
+    /// in mc-transaction-core (in this deploy of fog ledger).
+    ///
+    /// Usually when we redeploy consensus, we also redeploy fog. So this should
+    /// usually be equal to the MAX_BLOCK_VERSION value in the consensus enclave.
+    /// (In case it isn't, it won't be less than latest_block_version.)
+    ///
+    /// This is possibly an additional signal that clients can use to discover
+    /// that there is a new version of transaction-core that may be available
+    /// for an update (by comparing to their local value of max_block_version).
+    uint32 max_block_version = 5;
 }
 
 message KeyImageResult {

fog/ledger/enclave/api/src/lib.rs

@@ -51,6 +51,12 @@ pub struct UntrustedKeyImageQueryResponse {
 
     /// The cumulative txo count of the last known block.
     pub last_known_block_cumulative_txo_count: u64,
+
+    /// The latest value of block version in the blockchain
+    pub latest_block_version: u32,
+
+    /// The (max of) latest_block_version and mc_transaction_core::BLOCK_VERSION
+    pub max_block_version: u32,
 }
 
 /// The API for interacting with a ledger node's enclave.

fog/ledger/enclave/impl/src/lib.rs

@@ -140,7 +140,7 @@ where
     fn check_key_images(
         &self,
         msg: EnclaveMessage<ClientSession>,
-        untrusted_keyimagequery_response: UntrustedKeyImageQueryResponse,
+        untrusted_key_image_query_response: UntrustedKeyImageQueryResponse,
     ) -> Result<Vec<u8>> {
         let channel_id = msg.channel_id.clone(); //client session does not implement copy trait so clone
         let user_plaintext = self.ake.client_decrypt(msg)?;
@@ -151,10 +151,12 @@ where
         })?;
 
         let mut resp = CheckKeyImagesResponse {
-            num_blocks: untrusted_keyimagequery_response.highest_processed_block_count,
+            num_blocks: untrusted_key_image_query_response.highest_processed_block_count,
             results: Default::default(),
-            global_txo_count: untrusted_keyimagequery_response
+            global_txo_count: untrusted_key_image_query_response
                 .last_known_block_cumulative_txo_count,
+            latest_block_version: untrusted_key_image_query_response.latest_block_version,
+            max_block_version: untrusted_key_image_query_response.max_block_version,
         };
 
         // Do the scope lock of keyimagetore

fog/ledger/server/src/db_fetcher.rs

@@ -247,6 +247,19 @@ impl<DB: Ledger, E: LedgerEnclaveProxy + Clone + Send + Sync + 'static> DbFetche
                             shared_state.last_known_block_cumulative_txo_count = global_txo_count;
                         }
                     }
+                    match self.db.get_latest_block() {
+                        Err(e) => {
+                            log::error!(
+                                self.logger,
+                                "Unexpected error when checking for ledger latest block version {}: {:?}",
+                                self.next_block_index,
+                                e
+                            );
+                        }
+                        Ok(block) => {
+                            shared_state.latest_block_version = block.version;
+                        }
+                    }
                 });
 
                 self.next_block_index += 1;

fog/ledger/server/src/key_image_service.rs

@@ -72,17 +72,27 @@ impl<L: Ledger + Clone, E: LedgerEnclaveProxy> KeyImageService<L, E> {
     ) -> Result<attest::Message, RpcStatus> {
         log::trace!(self.logger, "Getting encrypted request");
 
-        let (highest_processed_block_count, last_known_block_cumulative_txo_count) = {
+        let (
+            highest_processed_block_count,
+            last_known_block_cumulative_txo_count,
+            latest_block_version,
+        ) = {
             let shared_state = self.db_poll_shared_state.lock().expect("mutex poisoned");
             (
                 shared_state.highest_processed_block_count,
                 shared_state.last_known_block_cumulative_txo_count,
+                shared_state.latest_block_version,
             )
         };
 
         let untrusted_query_response = UntrustedKeyImageQueryResponse {
             highest_processed_block_count,
             last_known_block_cumulative_txo_count,
+            latest_block_version,
+            max_block_version: core::cmp::max(
+                latest_block_version,
+                mc_transaction_core::BLOCK_VERSION,
+            ),
         };
 
         let result_blob = self

fog/ledger/server/src/merkle_proof_service.rs

@@ -104,6 +104,12 @@ impl<L: Ledger + Clone, E: LedgerEnclaveProxy> MerkleProofService<L, E> {
             ));
         }
 
+        let latest_block_version = self
+            .ledger
+            .get_latest_block()
+            .map_err(|err| rpc_database_err(err, &self.logger))?
+            .version;
+
         Ok(GetOutputsResponse {
             num_blocks: self
                 .ledger
@@ -134,6 +140,11 @@ impl<L: Ledger + Clone, E: LedgerEnclaveProxy> MerkleProofService<L, E> {
                 })
                 .collect::<Result<Vec<_>, DbError>>()
                 .map_err(|err| rpc_database_err(err, &self.logger))?,
+            latest_block_version,
+            max_block_version: core::cmp::max(
+                latest_block_version,
+                mc_transaction_core::BLOCK_VERSION,
+            ),
         })
     }
 
@@ -267,6 +278,7 @@ mod test {
         let highest_index: u32 = num_tx_outs - 1;
 
         mock_ledger.num_tx_outs = num_tx_outs as u64;
+        mock_ledger.num_blocks = 1;
 
         for (index, tx_out) in get_tx_outs(num_tx_outs).into_iter().enumerate() {
             mock_ledger.tx_out_by_index.insert(index as u64, tx_out);
@@ -319,6 +331,7 @@ mod test {
         let mut mock_ledger = MockLedger::default();
         let num_tx_outs: u32 = 100;
         mock_ledger.num_tx_outs = num_tx_outs as u64;
+        mock_ledger.num_blocks = 1;
 
         // Populate the mock ledger with TxOuts and membership proofs.
         for (index, tx_out) in get_tx_outs(num_tx_outs).into_iter().enumerate() {

fog/ledger/server/src/server.rs

@@ -247,4 +247,7 @@ pub struct DbPollSharedState {
 
     /// The cumulative txo count of the last known block.
     pub last_known_block_cumulative_txo_count: u64,
+
+    /// The latest value of `block_version` in the blockchain
+    pub latest_block_version: u32,
 }

fog/ledger/test_infra/src/lib.rs

@@ -105,8 +105,12 @@ impl Ledger for MockLedger {
         Ok(self.num_blocks)
     }
 
-    fn get_block(&self, _block_number: u64) -> Result<Block, Error> {
-        unimplemented!()
+    fn get_block(&self, block_number: u64) -> Result<Block, Error> {
+        if block_number < self.num_blocks {
+            Ok(Block::default())
+        } else {
+            Err(Error::NotFound)
+        }
     }
 
     fn get_block_signature(&self, _block_number: u64) -> Result<BlockSignature, Error> {

fog/types/src/common.rs

@@ -6,28 +6,34 @@ use serde::{Deserialize, Serialize};
 /// A half-open [a, b) range of blocks
 #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Message, Serialize, Deserialize)]
 pub struct BlockRange {
+    /// The first block in the range
     #[prost(uint64, tag = "1")]
     pub start_block: u64,
+    /// The end block, which is one past the end of the range.
     #[prost(uint64, tag = "2")]
     pub end_block: u64,
 }
 
 impl BlockRange {
+    /// Create a new block range
     pub fn new(start_block: u64, end_block: u64) -> Self {
         Self {
             start_block,
             end_block,
         }
     }
 
+    /// Test if a block index is in the range
     pub fn contains(&self, block: u64) -> bool {
         block >= self.start_block && block < self.end_block
     }
 
+    /// Test if a block range is well-formed
     pub fn is_valid(&self) -> bool {
         self.end_block > self.start_block
     }
 
+    /// Test if two block ranges overlap
     pub fn overlaps(&self, other: &BlockRange) -> bool {
         self.start_block < other.end_block && other.start_block < self.end_block
     }

fog/types/src/ledger.rs

@@ -42,8 +42,35 @@ pub struct GetOutputsResponse {
     /// Number of txos in the ledger
     #[prost(uint64, tag = "3")]
     pub global_txo_count: u64,
+
+    /// The latest block_version of a block in the block chain
+    ///
+    /// This may be needed when building transactions, so that use of new
+    /// transaction features can be gated on the block version being
+    /// increased.
+    ///
+    /// Clients may also choose to prompt users to update their software if
+    /// the block version increases beyond what was "known" when the software
+    /// was built.
+    #[prost(uint32, tag = "4")]
+    pub latest_block_version: u32,
+
+    /// The max of latest_block_version and the MAX_BLOCK_VERSION value
+    /// in mc-transaction-core (in this deploy of fog ledger).
+    ///
+    /// Usually when we redeploy consensus, we also redeploy fog. So this should
+    /// usually be equal to the MAX_BLOCK_VERSION value in the consensus
+    /// enclave. (In case it isn't, it won't be less than
+    /// latest_block_version.)
+    ///
+    /// This is possibly an additional signal that clients can use to discover
+    /// that there is a new version of transaction-core that may be available
+    /// for an update (by comparing to their local value of max_block_version).
+    #[prost(uint32, tag = "5")]
+    pub max_block_version: u32,
 }
 
+/// The result of an individual query for an output and membership proof
 #[derive(Clone, Message, Eq, PartialEq, Serialize, Deserialize)]
 pub struct OutputResult {
     /// Index that was queried (global index of a txo)
@@ -77,9 +104,11 @@ pub struct CheckKeyImagesRequest {
 /// Query about a particular key image
 #[derive(Message, Eq, PartialEq)]
 pub struct KeyImageQuery {
+    /// The key image to query about
     #[prost(message, required, tag = "1")]
     pub key_image: KeyImage,
 
+    /// A lower bound on the range to search. This is an optimization.
     #[prost(fixed64, tag = "2")]
     pub start_block: u64,
 }
@@ -102,21 +131,47 @@ pub struct CheckKeyImagesResponse {
     /// Results of key image checks
     #[prost(message, repeated, tag = "3")]
     pub results: Vec<KeyImageResult>,
+
+    /// The latest block_version of a block in the block chain
+    ///
+    /// This may be needed when building transactions, so that use of new
+    /// transaction features can be gated on the block version being
+    /// increased.
+    ///
+    /// Clients may also choose to prompt users to update their software if
+    /// the block version increases beyond what was "known" when the software
+    /// was built.
+    #[prost(uint32, tag = "4")]
+    pub latest_block_version: u32,
+
+    /// The max of latest_block_version and the MAX_BLOCK_VERSION value
+    /// in mc-transaction-core (in this deploy of fog ledger).
+    ///
+    /// Usually when we redeploy consensus, we also redeploy fog. So this should
+    /// usually be equal to the MAX_BLOCK_VERSION value in the consensus
+    /// enclave. (In case it isn't, it won't be less than
+    /// latest_block_version.)
+    ///
+    /// This is possibly an additional signal that clients can use to discover
+    /// that there is a new version of transaction-core that may be available
+    /// for an update (by comparing to their local value of max_block_version).
+    #[prost(uint32, tag = "5")]
+    pub max_block_version: u32,
 }
 
 /// A result which tells for a given key image, whether it was spent or not
 /// and at what height.
 #[derive(Clone, Message, Eq, PartialEq, Serialize, Deserialize)]
 pub struct KeyImageResult {
+    /// The key image which was queried
     #[prost(message, required, tag = "1")]
     pub key_image: KeyImage,
-    // Note: We should perhaps add
-    // [prost(... , default = "!064")]
-    // here to force prost to emit this field even if spent_at = 0, because
-    // spent_at = 0 indicates a miss and we don't want to leak that.
-    // But it's kind of a hack...
-    // proto3 does not support defaults, so this cannot be in the .proto AFAIU
-    // There's probably a simpler fix...
+
+    /// The block index of the block in which this key image appeared
+    //
+    // Note: prost will omit this field if spent_at = 0, but that never
+    // happens in real life, because a Tx cannot be spent in the origin block,
+    // and in the case that a Tx is not spent, we set this field to a nonzero value.
     #[prost(fixed64, tag = "2")]
     pub spent_at: u64,
 
@@ -138,6 +193,7 @@ pub struct KeyImageResult {
     pub key_image_result_code: u32,
 }
 
+/// An enum corresponding to the KeyImageResultCode proto enum
 #[derive(PartialEq, Eq, Debug, Display)]
 #[repr(u32)]
 pub enum KeyImageResultCode {

fog/types/src/lib.rs

@@ -2,15 +2,25 @@
 
 #![no_std]
 
+//! Enclave-compatible types used by fog.
+//! Particularly prosty versions of protobuf types, but also some enclave api
+//! types.
+
+#![deny(missing_docs)]
+
 extern crate alloc;
 
 use alloc::vec::Vec;
 use prost::Message;
 use serde::{Deserialize, Serialize};
 
+/// Types from or related to fog_common.proto
 pub mod common;
+/// Types related to fog ingest
 pub mod ingest;
+/// Types related to fog ledger
 pub mod ledger;
+/// Types related to fog view
 pub mod view;
 
 /// An Encrypted Tx Out Record, consisting of a fog search_key (rng output),
@@ -59,5 +69,6 @@ impl core::fmt::Display for BlockCount {
 }
 
 impl BlockCount {
+    /// The largest possible BlockCount
     pub const MAX: BlockCount = BlockCount(u64::MAX);
 }