Update cardano-node after addition of the new BulkSync implementation

bench/tx-generator/tx-generator.cabal

@@ -141,7 +141,7 @@ library
                       , ouroboros-network-framework
                       , ouroboros-network-protocols
                       , plutus-ledger-api
-                      , plutus-tx
+                      , plutus-tx ^>= 1.30
                       , random
                       , serialise
                       , streaming

cabal.project

@@ -14,7 +14,7 @@ repository cardano-haskell-packages
 -- you need to run if you change them
 index-state:
   , hackage.haskell.org 2024-06-23T23:01:13Z
-  , cardano-haskell-packages 2024-07-03T01:26:49Z
+  , cardano-haskell-packages 2024-09-01T03:50:08Z
 
 packages:
   cardano-node
@@ -65,3 +65,27 @@ allow-newer: katip:Win32
 -- IMPORTANT
 -- Do NOT add more source-repository-package stanzas here unless they are strictly
 -- temporary! Please read the section in CONTRIBUTING about updating dependencies.
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-network
+  tag: 5c304e5adbd27907c675bd60a2282264a65f117c
+  --sha256: Jn3Qqlsirlyu23bsFN2SO054LIe2rcsHUW1tN8Pm1Ks=
+  subdir:
+    ouroboros-network
+    ouroboros-network-api
+    ouroboros-network-protocols
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  -- back-port of BulkSync for genesis to ouroboros-consensus-0.20.0.0
+  tag: 0874ad4a61854f50bf9303d9f662ae6b129fac6d
+  --sha256: iw9IWAnUz4SX6OIBfe4HsRukZr8adyLbCS36CJo+8OE=
+  subdir:
+    ouroboros-consensus
+    ouroboros-consensus-cardano
+    ouroboros-consensus-diffusion
+    ouroboros-consensus-protocol
+    sop-extras
+    strict-sop-core

cardano-node/ChangeLog.md

@@ -9,6 +9,8 @@
   - `--mempool-capacity-override` and `--no-mempool-capacity-override` can be set in the configuration file via the key `MempoolCapacityBytesOverride`.
   - `--snapshot-interval` can be set in the configuration file via the key `SnapshotInterval`.
   - `--num-of-disk-snapshots` can be set in the configuration file via the key `NumOfDiskSnapshots`.
+- Update tracing with Genesis messages
+- Update configuration with Genesis parameters
 
 ## 8.2.1 -- August 2023
 

cardano-node/cardano-node.cabal

@@ -157,7 +157,7 @@ library
                       , cardano-ledger-byron
                       , cardano-ledger-conway
                       , cardano-ledger-core
-                      , cardano-ledger-shelley
+                      , cardano-ledger-shelley >= 1.12.3.0
                       , cardano-prelude
                       , cardano-protocol-tpraos >= 1.0.2
                       , cardano-slotting >= 0.2
@@ -175,10 +175,10 @@ library
                       , generic-data
                       , hostname
                       , io-classes >= 1.4
-                      , iohk-monitoring
+                      , iohk-monitoring ^>= 0.1
                       , iproute
                       , lobemo-backend-aggregation
-                      , lobemo-backend-ekg
+                      , lobemo-backend-ekg ^>= 0.1
                       , lobemo-backend-monitoring
                       , lobemo-backend-trace-forwarder
                       , mtl

cardano-node/src/Cardano/Node/Configuration/POM.hs

@@ -34,6 +34,8 @@ import           Cardano.Tracing.OrphanInstances.Network ()
 import           Ouroboros.Consensus.Mempool (MempoolCapacityBytes (..),
                    MempoolCapacityBytesOverride (..))
 import qualified Ouroboros.Consensus.Node as Consensus (NetworkP2PMode (..))
+import           Ouroboros.Consensus.Node.Genesis (GenesisConfig,
+                   GenesisConfigFlags (..), defaultGenesisConfigFlags, mkGenesisConfig)
 import           Ouroboros.Consensus.Storage.LedgerDB.DiskPolicy (NumOfDiskSnapshots (..),
                    SnapshotInterval (..))
 import           Ouroboros.Network.NodeToNode (AcceptedConnectionsLimit (..), DiffusionMode (..))
@@ -162,6 +164,9 @@ data NodeConfiguration
 
          -- Enable Peer Sharing
        , ncPeerSharing :: PeerSharing
+
+       -- Genesis syncing protocol configuration
+       , ncGenesisConfig :: GenesisConfig
        } deriving (Eq, Show)
 
 
@@ -225,6 +230,10 @@ data PartialNodeConfiguration
 
          -- Peer Sharing
        , pncPeerSharing :: !(Last PeerSharing)
+
+         -- Genesis syncing protocol
+       , pncEnableGenesis      :: !(Last Bool)
+       , pncGenesisConfigFlags :: !(Last GenesisConfigFlags)
        } deriving (Eq, Generic, Show)
 
 instance AdjustFilePaths PartialNodeConfiguration where
@@ -321,6 +330,11 @@ instance FromJSON PartialNodeConfiguration where
       -- DISABLED BY DEFAULT
       pncPeerSharing <- Last <$> v .:? "PeerSharing" .!= Just PeerSharingDisabled
 
+      -- Genesis syncing protocol
+      -- DISABLED BY DEFAULT
+      pncEnableGenesis      <- Last <$> v .:? "EnableGenesis" .!= Just False
+      pncGenesisConfigFlags <- Last <$> v .:? "LowLevelGenesisOptions"
+
       pure PartialNodeConfiguration {
              pncProtocolConfig
            , pncSocketConfig = Last . Just $ SocketConfig mempty mempty mempty pncSocketPath
@@ -355,6 +369,8 @@ instance FromJSON PartialNodeConfiguration where
            , pncTargetNumberOfActiveBigLedgerPeers
            , pncEnableP2P
            , pncPeerSharing
+           , pncEnableGenesis
+           , pncGenesisConfigFlags
            }
     where
       parseMempoolCapacityBytesOverride v = parseNoOverride <|> parseOverride
@@ -531,6 +547,8 @@ defaultPartialNodeConfiguration =
     , pncTargetNumberOfActiveBigLedgerPeers      = Last (Just 5)
     , pncEnableP2P                      = Last (Just EnabledP2PMode)
     , pncPeerSharing                    = Last (Just PeerSharingDisabled)
+    , pncEnableGenesis                  = Last (Just False)
+    , pncGenesisConfigFlags             = Last (Just defaultGenesisConfigFlags)
     }
 
 lastOption :: Parser a -> Parser (Last a)
@@ -596,6 +614,16 @@ makeNodeConfiguration pnc = do
     lastToEither "Missing PeerSharing"
     $ pncPeerSharing pnc
 
+  enableGenesis <-
+    lastToEither "Missing EnableGenesis"
+    $ pncEnableGenesis pnc
+
+  mGenesisConfigFlags <- if enableGenesis
+    then fmap Just <$>
+      lastToEither "Missing GenesisConfigFlags"
+      $ pncGenesisConfigFlags pnc
+    else pure Nothing
+
   -- TODO: This is not mandatory
   experimentalProtocols <-
     lastToEither "Missing ExperimentalProtocolsEnabled" $
@@ -643,6 +671,7 @@ makeNodeConfiguration pnc = do
                  EnabledP2PMode  -> SomeNetworkP2PMode Consensus.EnabledP2PMode
                  DisabledP2PMode -> SomeNetworkP2PMode Consensus.DisabledP2PMode
              , ncPeerSharing
+             , ncGenesisConfig = mkGenesisConfig mGenesisConfigFlags
              }
 
 ncProtocol :: NodeConfiguration -> Protocol

cardano-node/src/Cardano/Node/Orphans.hs

@@ -7,6 +7,7 @@ module Cardano.Node.Orphans () where
 
 import           Cardano.Api ()
 
+import           Ouroboros.Consensus.Node.Genesis (GenesisConfigFlags (..))
 import           Ouroboros.Network.NodeToNode (AcceptedConnectionsLimit (..))
 import           Ouroboros.Network.SizeInBytes (SizeInBytes (..))
 
@@ -38,3 +39,15 @@ instance FromJSON AcceptedConnectionsLimit where
       <$> v .: "hardLimit"
       <*> v .: "softLimit"
       <*> v .: "delay"
+
+instance FromJSON GenesisConfigFlags where
+  parseJSON = withObject "GenesisConfigFlags" $ \v ->
+    GenesisConfigFlags
+      <$> v .:? "EnableCSJ"       .!= True
+      <*> v .:? "EnableLoEAndGDD" .!= True
+      <*> v .:? "EnableLoP"       .!= True
+      <*> v .:? "BlockFetchGracePeriod"
+      <*> v .:? "BucketCapacity"
+      <*> v .:? "BucketRate"
+      <*> v .:? "CSJJumpSize"
+      <*> v .:? "GDDRateLimit"

cardano-node/src/Cardano/Node/Parsers.hs

@@ -126,6 +126,8 @@ nodeRunParser = do
            , pncTargetNumberOfActiveBigLedgerPeers = mempty
            , pncEnableP2P = mempty
            , pncPeerSharing = mempty
+           , pncEnableGenesis = mempty
+           , pncGenesisConfigFlags = mempty
            }
 
 parseSocketPath :: Text -> Parser SocketPath

cardano-node/src/Cardano/Node/Run.hs

@@ -6,9 +6,11 @@
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE PackageImports #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TupleSections #-}
+{-# LANGUAGE TypeApplications #-}
 
 {-# LANGUAGE TypeApplications #-}
 
@@ -45,7 +47,7 @@ import           "contra-tracer" Control.Tracer
 import           Data.Either (partitionEithers)
 import           Data.Map.Strict (Map)
 import qualified Data.Map.Strict as Map
-import           Data.Maybe (catMaybes, fromMaybe, mapMaybe)
+import           Data.Maybe (catMaybes, fromMaybe, mapMaybe, isJust)
 import           Data.Monoid (Last (..))
 import           Data.Proxy (Proxy (..))
 import           Data.Text (Text, breakOn, pack)
@@ -93,9 +95,11 @@ import           Cardano.Tracing.Config (TraceOptions (..), TraceSelection (..))
 
 import qualified Ouroboros.Consensus.Config as Consensus
 import           Ouroboros.Consensus.Config.SupportsNode (ConfigSupportsNode (..))
+import qualified Ouroboros.Consensus.MiniProtocol.ChainSync.Client as ChainSync.Client
 import           Ouroboros.Consensus.Node (DiskPolicyArgs (..), NetworkP2PMode (..),
                    RunNodeArgs (..), StdRunNodeArgs (..))
 import qualified Ouroboros.Consensus.Node as Node (getChainDB, run)
+import qualified Ouroboros.Consensus.Node.Genesis as Genesis
 import           Ouroboros.Consensus.Node.NetworkProtocolVersion
 import           Ouroboros.Consensus.Node.ProtocolInfo
 import           Ouroboros.Consensus.Util.Orphans ()
@@ -476,6 +480,7 @@ handleSimpleNode blockType runP p2pMode tracers nc onKernel = do
               , rnEnableP2P      = p2pMode
               , rnPeerSharing    = ncPeerSharing nc
               , rnGetUseBootstrapPeers = readTVar useBootstrapVar
+              , rnGenesisConfig = ncGenesisConfig nc
               }
 #ifdef UNIX
         -- initial `SIGHUP` handler, which only rereads the topology file but
@@ -559,6 +564,7 @@ handleSimpleNode blockType runP p2pMode tracers nc onKernel = do
                 , rnEnableP2P      = p2pMode
                 , rnPeerSharing    = ncPeerSharing nc
                 , rnGetUseBootstrapPeers = pure DontUseBootstrapPeers
+                , rnGenesisConfig = ncGenesisConfig nc
                 }
 #ifdef UNIX
         -- initial `SIGHUP` handler; it only warns that neither updating of

cardano-node/src/Cardano/Node/Tracing/Consistency.hs

@@ -38,6 +38,7 @@ import           Ouroboros.Consensus.BlockchainTime.WallClock.Types (RelativeTim
 import           Ouroboros.Consensus.BlockchainTime.WallClock.Util (TraceBlockchainTimeEvent (..))
 import           Ouroboros.Consensus.Cardano.Block
 import           Ouroboros.Consensus.Ledger.Query (Query)
+import           Ouroboros.Consensus.Genesis.Governor (TraceGDDEvent)
 import           Ouroboros.Consensus.Ledger.SupportsMempool (ApplyTxErr, GenTxId)
 import           Ouroboros.Consensus.Mempool (TraceEventMempool (..))
 import           Ouroboros.Consensus.MiniProtocol.BlockFetch.Server
@@ -46,11 +47,13 @@ import           Ouroboros.Consensus.MiniProtocol.ChainSync.Client (TraceChainSy
 import           Ouroboros.Consensus.MiniProtocol.ChainSync.Server (TraceChainSyncServerEvent)
 import           Ouroboros.Consensus.MiniProtocol.LocalTxSubmission.Server
                    (TraceLocalTxSubmissionServerEvent (..))
+import           Ouroboros.Consensus.Node.GSM (TraceGsmEvent)
 import qualified Ouroboros.Consensus.Protocol.Ledger.HotKey as HotKey
 import qualified Ouroboros.Consensus.Storage.ChainDB as ChainDB
+import qualified Ouroboros.Consensus.MiniProtocol.ChainSync.Client.Jumping as CSJumping
 import           Ouroboros.Network.Block (Point (..), SlotNo, Tip)
 import qualified Ouroboros.Network.BlockFetch.ClientState as BlockFetch
-import           Ouroboros.Network.BlockFetch.Decision
+import           Ouroboros.Network.BlockFetch.Decision.Trace as BlockFetch
 import           Ouroboros.Network.ConnectionHandler (ConnectionHandlerTrace (..))
 import           Ouroboros.Network.ConnectionId (ConnectionId)
 import           Ouroboros.Network.ConnectionManager.Types (ConnectionManagerTrace (..))
@@ -151,9 +154,7 @@ getAllNamespaces =
         chainSyncServerBlockNS = map (nsGetTuple . nsReplacePrefix ["ChainSync", "ServerBlock"])
                         (allNamespaces :: [Namespace (TraceChainSyncServerEvent blk)])
         blockFetchDecisionNS = map (nsGetTuple . nsReplacePrefix ["BlockFetch", "Decision"])
-                        (allNamespaces :: [Namespace [BlockFetch.TraceLabelPeer
-                                                      remotePeer
-                                                      (FetchDecision [Point (Header blk)])]])
+                        (allNamespaces :: [Namespace (TraceDecisionEvent remotePeer (Header blk))])
         blockFetchClientNS = map (nsGetTuple . nsReplacePrefix ["BlockFetch", "Client"])
                         (allNamespaces :: [Namespace (BlockFetch.TraceLabelPeer
                                                       remotePeer
@@ -185,6 +186,15 @@ getAllNamespaces =
         blockchainTimeNS = map (nsGetTuple . nsReplacePrefix  ["BlockchainTime"])
                         (allNamespaces :: [Namespace (TraceBlockchainTimeEvent RelativeTime)])
 
+        gsmEventNS = map (nsGetTuple . nsReplacePrefix  ["GsmEvent"])
+                         (allNamespaces :: [Namespace (TraceGsmEvent selection)])
+
+        csjEventNS = map nsGetTuple
+                         (allNamespaces :: [Namespace (CSJumping.TraceEvent peer)])
+
+        gddEventNS = map nsGetTuple
+                         (allNamespaces :: [Namespace (TraceGDDEvent peer blk)])
+
 -- Node to client
         keepAliveClientNS = map (nsGetTuple . nsReplacePrefix ["Net"])
                                 (allNamespaces :: [Namespace  (TraceKeepAliveClient peer)])
@@ -390,6 +400,9 @@ getAllNamespaces =
             <> mempoolNS
             <> forgeNS
             <> blockchainTimeNS
+            <> gsmEventNS
+            <> csjEventNS
+            <> gddEventNS
 -- NodeToClient
             <> keepAliveClientNS
             <> chainSyncNS

cardano-node/src/Cardano/Node/Tracing/Documentation.hs

@@ -59,7 +59,7 @@ import qualified Ouroboros.Consensus.Protocol.Ledger.HotKey as HotKey
 import qualified Ouroboros.Consensus.Storage.ChainDB as ChainDB
 import           Ouroboros.Network.Block (Point (..), Serialised, SlotNo, Tip)
 import qualified Ouroboros.Network.BlockFetch.ClientState as BlockFetch
-import           Ouroboros.Network.BlockFetch.Decision
+import qualified Ouroboros.Network.BlockFetch.Decision.Trace as BlockFetch
 import           Ouroboros.Network.ConnectionHandler (ConnectionHandlerTrace (..))
 import           Ouroboros.Network.ConnectionId (ConnectionId)
 import           Ouroboros.Network.ConnectionManager.Types (ConnectionManagerTrace (..))
@@ -278,9 +278,7 @@ docTracersFirstPhase condConfigFileName = do
                 ["BlockFetch", "Decision"]
     configureTracers configReflection trConfig [blockFetchDecisionTr]
     blockFetchDecisionTrDoc <- documentTracer (blockFetchDecisionTr ::
-       Trace IO [BlockFetch.TraceLabelPeer
-                                      remotePeer
-                                      (FetchDecision [Point (Header blk)])])
+       Trace IO (BlockFetch.TraceDecisionEvent remotePeer (Header blk)))
 
     blockFetchClientTr  <- mkCardanoTracer
                 trBase trForward mbTrEKG

cardano-node/src/Cardano/Node/Tracing/Formatting.hs

@@ -1,3 +1,5 @@
+{-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 
@@ -11,6 +13,7 @@ import           Cardano.Logging (LogFormatting (..))
 import           Cardano.Node.Tracing.Render (renderHeaderHashForDetails)
 import           Ouroboros.Consensus.Block (ConvertRawHash (..), RealPoint, realPointHash,
                    realPointSlot)
+import qualified Ouroboros.Network.AnchoredFragment as AF
 import           Ouroboros.Network.Block
 
 import           Data.Aeson (Value (String), toJSON, (.=))
@@ -51,3 +54,22 @@ instance ConvertRawHash blk
         , "slot" .= unSlotNo (realPointSlot p)
         , "hash" .= renderHeaderHashForDetails (Proxy @blk) dtal (realPointHash p)
         ]
+
+instance (ConvertRawHash blk) => LogFormatting (AF.Anchor blk) where
+  forMachine dtal = \case
+    AF.AnchorGenesis -> mconcat
+      [ "kind" .= String "AnchorGenesis" ]
+    AF.Anchor slot hash bno -> mconcat
+      [ "kind" .= String "Anchor"
+      , "slot" .= toJSON (unSlotNo slot)
+      , "headerHash" .= renderHeaderHashForDetails (Proxy @blk) dtal hash
+      , "blockNo" .= toJSON (unBlockNo bno)
+      ]
+
+instance (ConvertRawHash blk, HasHeader blk) => LogFormatting (AF.AnchoredFragment blk) where
+  forMachine dtal frag = mconcat
+    [ "kind" .= String "AnchoredFragment"
+    , "anchor" .= forMachine dtal (AF.anchor frag)
+    , "headPoint" .= forMachine dtal (AF.headPoint frag)
+    , "length" .= toJSON (AF.length frag)
+    ]