Merge pull request #166 from tonlabs/tmp-rc

March 2023 release

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file.
 
+## Version: 0.52.2
+
+### New
+
+- Made Jaeger tracing optional
+
+## Version: 0.52.1
+
+### New
+
+- Bumped block version to 37
+- Supported CapFeeInGasUnits capability
+
 ## Version: 0.50.23
 
 ### New

Cargo.toml

@@ -2,7 +2,7 @@
 build = 'common/build/build.rs'
 edition = '2021'
 name = 'ton_node'
-version = '0.51.55'
+version = '0.52.2'
 
 [workspace]
 members = [ 'storage' ]
@@ -39,37 +39,35 @@ sha2 = '0.10'
 spin = '0.7.1'
 stream-cancel = '0.8.0'
 string-builder = '^0.2.0'
-adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.7.131' }
+adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.7.143' }
 catchain = { path = 'catchain' }
 ctrlc = { features = [ 'termination' ], version = '3.2.1' }
-dht = { git = 'https://github.com/tonlabs/ever-dht.git', tag = '0.5.141' }
-ever-crypto = { git = 'https://github.com/tonlabs/ever-crypto.git', tag = '0.1.91' }
+dht = { git = 'https://github.com/tonlabs/ever-dht.git', tag = '0.5.154' }
+ever-crypto = { git = 'https://github.com/tonlabs/ever-crypto.git', tag = '0.1.101' }
 lockfree = { git = 'https://github.com/tonlabs/lockfree.git' }
-overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.6.124' }
-rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.7.125' }
+overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.6.138' }
+rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.7.138' }
 rustracing = { optional = true, version = '0.4.0' }
 rustracing_jaeger = { optional = true, version = '0.4.1' }
 statsd = { optional = true, version = '0.15' }
 storage = { path = 'storage' }
 tokio = { features = [ 'rt-multi-thread' ], version = '1.5' }
-ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.3.65' }
-ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.2.178' }
-ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.28' }
-ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.95' }
-ton_executor = { git = 'https://github.com/tonlabs/ever-executor.git', tag = '1.15.175' }
+ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.3.75' }
+ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.2.188' }
+ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.39' }
+ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.108' }
+ton_executor = { git = 'https://github.com/tonlabs/ever-executor.git', tag = '1.15.189' }
 ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '1.12.7' }
-ton_vm = { git = 'https://github.com/tonlabs/ever-vm.git', tag = '1.8.116' }
+ton_vm = { git = 'https://github.com/tonlabs/ever-vm.git', tag = '1.8.127' }
 validator_session = { path = 'validator-session' }
 
 [features]
-async_ss_storage = [ 'storage/ref_count_gc' ]
 ci_run = [ 'storage/ci_run' ]
-default = [ 'telemetry', 'async_ss_storage', 'remp_emergency' ]
+default = [ 'telemetry' ]
 export_key = [ 'catchain/export_key', 'ever-crypto/export_key' ]
 external_db = [ 'rdkafka' ]
 gosh = [ 'ton_block/gosh', 'ton_vm/gosh' ]
 metrics = [ 'statsd' ]
-remp_emergency = [  ]
 signature_with_id = [ 'ton_block/signature_with_id', 'ton_vm/signature_with_id', 'ton_executor/signature_with_id' ]
 slashing = [ 'ton_abi', 'validator_session/slashing' ]
 telemetry = [ 'adnl/telemetry', 'dht/telemetry', 'rldp/telemetry', 'overlay/telemetry', 'storage/telemetry' ]

catchain/Cargo.toml

@@ -19,13 +19,13 @@ metrics-runtime = '0.13.0'
 rand = '0.8'
 regex = '1.3.1'
 sha2 = '0.10'
-adnl = { features = [ 'node' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.7.131' }
-ever-crypto = { git = 'https://github.com/tonlabs/ever-crypto.git', tag = '0.1.91' }
-overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.6.124' }
-rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.7.125' }
+adnl = { features = [ 'node' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.7.143' }
+ever-crypto = { git = 'https://github.com/tonlabs/ever-crypto.git', tag = '0.1.101' }
+overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.6.138' }
+rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.7.138' }
 storage = { path = '../storage' }
 tokio = { features = [ 'rt-multi-thread' ], version = '1.5' }
-ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.2.178' }
+ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.2.188' }
 ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '1.12.7' }
 
 [features]

src/collator_test_bundle.rs

@@ -176,16 +176,16 @@ pub fn create_block_handle_storage() -> BlockHandleStorage {
         Arc::new(NodeStateDb::in_memory()),
         Arc::new(NodeStateDb::in_memory()),
         #[cfg(feature = "telemetry")]
-        create_storage_telemetry(),
-        create_storage_allocated()
+        Arc::new(StorageTelemetry::default()),
+        Arc::new(StorageAlloc::default()),
     )
 }
 
 #[cfg(feature = "telemetry")]
 pub fn create_engine_telemetry() -> Arc<EngineTelemetry> {
     Arc::new(
         EngineTelemetry {
-            storage: create_storage_telemetry(),
+            storage: Arc::new(StorageTelemetry::default()),
             awaiters: Metric::without_totals("", 1),
             catchain_clients: Metric::without_totals("", 1),
             cells: Metric::without_totals("", 1),
@@ -202,7 +202,7 @@ pub fn create_engine_telemetry() -> Arc<EngineTelemetry> {
 pub fn create_engine_allocated() -> Arc<EngineAlloc> {
     Arc::new(
         EngineAlloc {
-            storage: create_storage_allocated(),
+            storage: Arc::new(StorageAlloc::default()),
             awaiters: Arc::new(AtomicU64::new(0)),
             catchain_clients: Arc::new(AtomicU64::new(0)),
             overlay_clients: Arc::new(AtomicU64::new(0)),
@@ -215,29 +215,6 @@ pub fn create_engine_allocated() -> Arc<EngineAlloc> {
     )
 }
 
-#[cfg(feature = "telemetry")]
-fn create_storage_telemetry() -> Arc<StorageTelemetry> {
-    Arc::new(
-        StorageTelemetry {
-            file_entries: Metric::without_totals("", 1),
-            handles: Metric::without_totals("", 1),
-            packages: Metric::without_totals("", 1),
-            storage_cells: Metric::without_totals("", 1)
-        }
-    )
-}
-
-fn create_storage_allocated() -> Arc<StorageAlloc> {
-    Arc::new(
-        StorageAlloc {
-            file_entries: Arc::new(AtomicU64::new(0)),
-            handles: Arc::new(AtomicU64::new(0)),
-            packages: Arc::new(AtomicU64::new(0)),
-            storage_cells: Arc::new(AtomicU64::new(0))
-        }
-    )
-}
-
 pub struct CollatorTestBundle {
     index: CollatorTestBundleIndex,
     top_shard_blocks: Vec<Arc<TopBlockDescrStuff>>,

src/config.rs

@@ -73,6 +73,23 @@ impl Default for CellsGcConfig {
     }
 }
 
+#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)]
+pub struct CellsDbConfig {
+    pub states_db_queue_len: u32,
+    pub max_pss_slowdown_mcs: u32,
+    pub prefill_cells_cunters: bool,
+}
+
+impl Default for CellsDbConfig {
+    fn default() -> Self {
+        Self {
+            states_db_queue_len: 1000,
+            max_pss_slowdown_mcs: 750,
+            prefill_cells_cunters: false,
+        }
+    }
+}
+
 #[derive(serde::Deserialize, serde::Serialize)]
 pub struct TonNodeConfig {
     log_config_name: Option<String>,
@@ -113,6 +130,8 @@ pub struct TonNodeConfig {
     restore_db: bool,
     #[serde(default)]
     low_memory_mode: bool,
+    #[serde(default)]
+    cells_db_config: CellsDbConfig,
 }
 
 pub struct TonNodeGlobalConfig(TonNodeGlobalConfigJson);
@@ -207,9 +226,6 @@ pub struct RempConfig {
     client_enabled: bool,
     remp_client_pool: Option<u8>,
     service_enabled: bool,
-    #[cfg(feature="remp_emergency")]
-    #[serde(default)]
-    forcedly_disable_remp_cap: bool,
 }
 
 impl RempConfig {
@@ -238,10 +254,6 @@ impl RempConfig {
         self.remp_client_pool
     }
 
-    #[cfg(feature="remp_emergency")]
-    pub fn forcedly_disable_remp_cap(&self) -> bool {
-        self.forcedly_disable_remp_cap
-    }
 }
 
 #[derive(Debug, Default, serde::Deserialize, serde::Serialize, Clone)]
@@ -406,11 +418,6 @@ impl TonNodeConfig {
 
         config_json.connectivity_check_config.check()?;
 
-        #[cfg(not(feature = "async_ss_storage"))]
-        if config_json.low_memory_mode {
-            fail!("'low_memory_mode' is applied only with 'async_ss_storage' feature");
-        }
-
         config_json.configs_dir = configs_dir.to_string();
         config_json.file_name = json_file_name.to_string();
 
@@ -525,6 +532,9 @@ impl TonNodeConfig {
     pub fn low_memory_mode(&self) -> bool {
         self.low_memory_mode
     }
+    pub fn cells_db_config(&self) -> &CellsDbConfig {
+        &self.cells_db_config
+    }
 
 
     pub fn load_global_config(&self) -> Result<TonNodeGlobalConfig> {

src/engine.rs

@@ -66,7 +66,7 @@ use crate::{
     validator::telemetry::{CollatorValidatorTelemetry, RempCoreTelemetry},
 };
 #[cfg(feature = "telemetry")]
-use adnl::telemetry::{Metric, TelemetryItem, TelemetryPrinter};
+use adnl::telemetry::{Metric, MetricBuilder, TelemetryItem, TelemetryPrinter};
 use overlay::QueriesConsumer;
 #[cfg(feature = "metrics")]
 use statsd::client;
@@ -125,8 +125,6 @@ pub struct Engine {
     sync_status: AtomicU32,
     low_memory_mode: bool,
     remp_capability: AtomicBool,
-    #[cfg(feature="remp_emergency")]
-    forcedly_disable_remp_cap: bool,
 
     test_bundles_config: CollatorTestBundlesGeneralConfig,
 
@@ -518,14 +516,7 @@ impl Engine {
 
         #[cfg(feature = "telemetry")] 
         let (metrics, engine_telemetry) = Self::create_telemetry();
-        let storage_allocated = Arc::new(
-            StorageAlloc {
-                file_entries: Arc::new(AtomicU64::new(0)),
-                handles: Arc::new(AtomicU64::new(0)),
-                packages: Arc::new(AtomicU64::new(0)),
-                storage_cells: Arc::new(AtomicU64::new(0))
-            }
-        );
+        let storage_allocated = Arc::new(StorageAlloc::default());
         let engine_allocated = Arc::new(
             EngineAlloc {
                 storage: storage_allocated,
@@ -557,9 +548,11 @@ impl Engine {
             }
         };
 
+        let cells_db_config = general_config.cells_db_config().clone();
         let db_config = InternalDbConfig { 
             db_directory: general_config.internal_db_path().to_string(), 
-            cells_gc_interval_sec: general_config.cells_gc_config().gc_interval_sec
+            cells_gc_interval_sec: general_config.cells_gc_config().gc_interval_sec,
+            cells_db_config: cells_db_config.clone(),
         };
         let control_config = general_config.control_server()?;
         let global_config = general_config.load_global_config()?;
@@ -651,6 +644,7 @@ impl Engine {
             enable_shard_state_persistent_gc,
             cells_lifetime_sec,
             stopper.clone(),
+            cells_db_config.states_db_queue_len + 10,
             #[cfg(feature = "telemetry")]
             engine_telemetry.clone(),
             engine_allocated.clone()
@@ -727,8 +721,6 @@ impl Engine {
             sync_status: AtomicU32::new(0),
             low_memory_mode,
             remp_capability: AtomicBool::new(false),
-            #[cfg(feature="remp_emergency")]
-            forcedly_disable_remp_cap: remp_config.forcedly_disable_remp_cap(),
             test_bundles_config,
             shard_states_keeper: shard_states_keeper.clone(),
             #[cfg(feature="workchains")]
@@ -971,10 +963,6 @@ impl Engine {
         self.remp_capability.store(value, Ordering::Relaxed);
     }
 
-    #[cfg(feature="remp_emergency")]
-    pub fn forcedly_disable_remp_cap(&self) -> bool {
-        self.forcedly_disable_remp_cap
-    }
 
     pub async fn download_and_apply_block_worker(
         self: Arc<Self>, 
@@ -1237,13 +1225,27 @@ impl Engine {
         fn create_metric(name: &str) -> Arc<Metric> {
             Metric::without_totals(name, Engine::TIMEOUT_TELEMETRY_SEC)
         }
+        fn create_metric_ex(name: &str) -> Arc<MetricBuilder> {
+            MetricBuilder::with_metric_and_period(
+                Metric::with_total_amount(name, Engine::TIMEOUT_TELEMETRY_SEC),
+                1_000_000_000 // 1 sec in nanos
+            )
+        }
 
         let storage_telemetry = Arc::new(
             StorageTelemetry {
                 file_entries: create_metric("Alloc NODE file entries"),
                 handles: create_metric("Alloc NODE block handles"),
                 packages: create_metric("Alloc NODE packages"),
-                storage_cells: create_metric("Alloc NODE storage cells")
+                storage_cells: create_metric("Alloc NODE storage cells"),
+                shardstates_queue: create_metric("Alloc NODE shardstates queue"),
+                cells_counters: create_metric("Alloc NODE cells counters"),
+                cell_counter_from_cache: create_metric_ex("NODE read cache cell_counters/sec"),
+                cell_counter_from_db: create_metric_ex("NODE read db cell_counters/sec"),
+                updated_old_cells: create_metric_ex("NODE old format update cells/sec"),
+                updated_cells: create_metric_ex("NODE update cell_counters/sec"),
+                new_cells: create_metric_ex("NODE create new cells/sec"),
+                deleted_cells: create_metric_ex("NODE delete cells/sec"),
             }
         );
         let engine_telemetry = Arc::new(
@@ -1265,6 +1267,14 @@ impl Engine {
             TelemetryItem::Metric(engine_telemetry.storage.handles.clone()),
             TelemetryItem::Metric(engine_telemetry.storage.packages.clone()),
             TelemetryItem::Metric(engine_telemetry.storage.storage_cells.clone()),
+            TelemetryItem::Metric(engine_telemetry.storage.shardstates_queue.clone()),
+            TelemetryItem::Metric(engine_telemetry.storage.cells_counters.clone()),
+            TelemetryItem::MetricBuilder(engine_telemetry.storage.cell_counter_from_cache.clone()),
+            TelemetryItem::MetricBuilder(engine_telemetry.storage.cell_counter_from_db.clone()),
+            TelemetryItem::MetricBuilder(engine_telemetry.storage.updated_old_cells.clone()),
+            TelemetryItem::MetricBuilder(engine_telemetry.storage.updated_cells.clone()),
+            TelemetryItem::MetricBuilder(engine_telemetry.storage.new_cells.clone()),
+            TelemetryItem::MetricBuilder(engine_telemetry.storage.deleted_cells.clone()),
             TelemetryItem::Metric(engine_telemetry.awaiters.clone()),
             TelemetryItem::Metric(engine_telemetry.catchain_clients.clone()),
             TelemetryItem::Metric(engine_telemetry.cells.clone()),
@@ -1363,9 +1373,6 @@ impl Engine {
     }
 
     fn process_ext_msg_broadcast(&self, broadcast: ExternalMessageBroadcast, src: Arc<KeyId>) {
-        #[cfg(feature="remp_emergency")]
-        let remp = !self.forcedly_disable_remp_cap() && self.remp_capability();
-        #[cfg(not(feature="remp_emergency"))]
         let remp = self.remp_capability();
         // just add to list
         if !self.is_validator() {
@@ -1655,7 +1662,7 @@ impl Engine {
         let join_handle = tokio::spawn(async move {
             engine.acquire_stop(Engine::MASK_SERVICE_ARCHIVES_GC);
             if let Err(e) = Self::archives_gc_worker(&engine, archives_gc_block).await {
-                log::error!("FATAL!!! Unexpected error in archives gc: {:?}", e);
+                log::error!("CRITICAL!!! Unexpected error in archives gc: {:?}", e);
             }
             engine.release_stop(Engine::MASK_SERVICE_ARCHIVES_GC);
         });

src/engine_operations.rs

@@ -709,9 +709,6 @@ impl EngineOperations for Engine {
             fail!("Can't process external message because node is out of sync");
         }
 
-        #[cfg(feature="remp_emergency")]
-        let remp_way = !self.forcedly_disable_remp_cap() && self.remp_capability();
-        #[cfg(not(feature="remp_emergency"))]
         let remp_way = self.remp_capability();
         if remp_way {
             self.remp_client()
@@ -870,10 +867,6 @@ impl EngineOperations for Engine {
         )
     }
 
-    #[cfg(feature="remp_emergency")]
-    fn forcedly_disable_remp_cap(&self) -> bool {
-        self.forcedly_disable_remp_cap()
-    }
 
     // Get current list of new shard blocks with respect to last mc block.
     // If given mc_seq_no is not equal to last mc seq_no - function fails.