Merge #3705

3705: Prepare for membench benchmarks r=deepfire a=jutaro Add configurations to old and new tracing to make it comparable. Co-authored-by: Yupanqui <jnf@arcor.de> Co-authored-by: Kosyrev Serge <serge.kosyrev@iohk.io>
IntersectMBO · Mar 15, 2022 · a3fc02b · a3fc02b
2 parents d534505 + a796a90
commit a3fc02b
Show file tree

Hide file tree

Showing 8 changed files with 432 additions and 152 deletions.
diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs
@@ -823,6 +823,19 @@ docChainDBInitChainSel = [
       []
       "Candidate contains headers from the future which exceed the\
       \ clock skew, making them invalid."
+    , DocMsg
+      (ChainDB.InitChainSelValidation
+          (ChainDB.UpdateLedgerDbTraceEvent anyProto))
+      []
+      "UpdateLedgerDb"
+   ,  DocMsg
+        ChainDB.StartedInitChainSelection
+      []
+      "StartedInitChainSelection"
+   ,  DocMsg
+        ChainDB.InitalChainSelected
+      []
+      "InitalChainSelected"
   ]
 
 --------------------------------------------------------------------------------

diff --git a/cardano-tracer/configuration/membench-example.yaml b/cardano-tracer/configuration/membench-example.yaml
@@ -0,0 +1,10 @@
+---
+networkMagic: 764824073
+network:
+  tag: AcceptAt
+  contents: "/tmp/forwarder.sock"
+logging:
+- logRoot: "/home/yupanqui/IOG/docs"
+  logMode: FileMode
+  logFormat: ForMachine
+verbosity: Minimum
diff --git a/configuration/cardano/membench-config-new.yaml b/configuration/cardano/membench-config-new.yaml
@@ -0,0 +1,171 @@
+################################################################################
+# Mainnet Cardano Node Configuration
+
+##### Locations #####
+
+AlonzoGenesisFile: mainnet-alonzo-genesis.json
+AlonzoGenesisHash: 7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874
+ByronGenesisFile: mainnet-byron-genesis.json
+ByronGenesisHash: 5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb
+ShelleyGenesisFile: mainnet-shelley-genesis.json
+ShelleyGenesisHash: 1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81
+
+##### Core protocol parameters #####
+
+Protocol: Cardano
+
+# The mainnet does not include the network magic into addresses. Testnets do.
+RequiresNetworkMagic: RequiresNoMagic
+
+##### Update system parameters #####
+
+# This protocol version number gets used by block producing nodes as part
+# of the system for agreeing on and synchronising protocol updates.
+#
+# See https://github.com/input-output-hk/cardano-node/blob/master/cardano-node/src/Cardano/Node/Protocol/Cardano.hs#L199
+LastKnownBlockVersion-Major: 3
+LastKnownBlockVersion-Minor: 0
+LastKnownBlockVersion-Alt: 0
+MaxKnownMajorProtocolVersion: 2
+
+# In the Byron era some software versions are also published on the chain.
+# We do this only for Byron compatibility now.
+ApplicationName: cardano-sl
+ApplicationVersion: 1
+
+##### Logging configuration #####
+
+# Enable or disable logging overall
+TurnOnLogging: True
+
+##### New logging model #####
+# Options for new tracing
+
+# Use old tracing as standard for now
+UseTraceDispatcher: True
+
+
+TraceOptionSeverity:
+  # Show messages of Severity Notice or higher as default
+  - ns: ""
+    severity: Silence
+
+  # But modify severity for the following namespaces
+  - ns: Node.ChainDB
+    severity: Info
+
+  - ns: Node.ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate
+    severity: Silence
+
+  - ns: Node.ChainDB.InitChainSelEvent.UpdateLedgerDb
+    severity: Silence
+
+  - ns: Node.ConnectionManager
+    severity: Info
+
+  - ns: Node.AcceptPolicy
+    severity: Info
+
+  - ns: Node.DNSResolver
+    severity: Info
+
+  - ns: Node.DNSSubscription
+    severity: Info
+
+  - ns: Node.DiffusionInit
+    severity: Info
+
+  - ns: Node.ErrorPolicy
+    severity: Info
+
+  - ns: Node.Forge
+    severity: Info
+
+  - ns: Node.InboundGovernor
+    severity: Info
+
+  - ns: Node.IpSubscription
+    severity: Info
+
+  - ns: Node.IpSubscription
+    severity: Info
+
+  - ns: Node.LedgerPeers
+    severity: Info
+
+  - ns: Node.LocalErrorPolicy
+    severity: Info
+
+  - ns: Node.LocalRootPeers
+    severity: Info
+
+  - ns: Node.Mempool
+    severity: Info
+
+  - ns: Node.PeerSelection
+    severity: Info
+
+  - ns: Node.PeerSelectionActions
+    severity: Info
+
+  - ns: Node.PublicRootPeers
+    severity: Info
+
+  - ns: Node.Resources
+    severity: Silence
+
+TraceOptionDetail:
+  # All messages are shown with normal detail level
+  - ns: ""
+    detail: DNormal
+
+TraceOptionBackend:
+  # Use these backends
+  - ns: ""
+    backends:
+      - Stdout MachineFormat
+      - EKGBackend
+      - Forwarder
+
+# Limit the frequency of the following messages
+TraceOptionLimiter:
+  - ns: Node.ChainDB.AddBlockEvent.AddedBlockToQueue
+    limiterName: AddedBlockToQueueLimiter
+    limiterFrequency: 2.0
+
+  - ns: Node.ChainDB.AddBlockEvent.AddedBlockToVolatileDB
+    limiterName: AddedBlockToVolatileDBLimiter
+    limiterFrequency: 2.0
+
+  - ns: Node.ChainDB.CopyToImmutableDBEvent.CopiedBlockToImmutableDB
+    limiterName: CopiedBlockToImmutableDBLimiter
+    limiterFrequency: 2.0
+
+  - ns: Node.ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate
+    limiterName: ValidCandidateLimiter
+    limiterFrequency: 2.0
+
+  - ns: Node.BlockFetchClient.CompletedBlockFetch
+    limiterName: CompletedBlockFetchLimiter
+    limiterFrequency: 2.0
+
+# Options for the trace forwarder
+TraceOptionForwarder:
+    address:
+      filePath: /tmp/forwarder.sock
+    mode: Initiator
+
+TraceOptionPeerFreqency: 2000
+
+TraceOptionResourceFreqency: 5000
+
+
+##### Stubs for legacy logging config #####
+
+TurnOnLogMetrics: False
+minSeverity: Critical
+setupScribes: []
+setupBackends: []
+defaultScribes: []
+defaultBackends: []
+options: {}
diff --git a/configuration/cardano/membench-config-old.json b/configuration/cardano/membench-config-old.json
@@ -0,0 +1,115 @@
+{
+  "AlonzoGenesisFile": "mainnet-alonzo-genesis.json",
+  "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874",
+  "ApplicationName": "cardano-sl",
+  "ApplicationVersion": 1,
+  "ByronGenesisFile": "mainnet-byron-genesis.json",
+  "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb",
+  "LastKnownBlockVersion-Alt": 0,
+  "LastKnownBlockVersion-Major": 3,
+  "LastKnownBlockVersion-Minor": 0,
+  "MaxKnownMajorProtocolVersion": 2,
+  "Protocol": "Cardano",
+  "RequiresNetworkMagic": "RequiresNoMagic",
+  "ShelleyGenesisFile": "mainnet-shelley-genesis.json",
+  "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81",
+  "TraceAcceptPolicy": true,
+  "TraceBlockFetchClient": false,
+  "TraceBlockFetchDecisions": false,
+  "TraceBlockFetchProtocol": false,
+  "TraceBlockFetchProtocolSerialised": false,
+  "TraceBlockFetchServer": false,
+  "TraceChainDb": true,
+  "TraceChainSyncBlockServer": false,
+  "TraceChainSyncClient": false,
+  "TraceChainSyncHeaderServer": false,
+  "TraceChainSyncProtocol": false,
+  "TraceConnectionManager": true,
+  "TraceDNSResolver": true,
+  "TraceDNSSubscription": true,
+  "TraceDiffusionInitialization": true,
+  "TraceErrorPolicy": true,
+  "TraceForge": true,
+  "TraceHandshake": false,
+  "TraceInboundGovernor": true,
+  "TraceIpSubscription": true,
+  "TraceLedgerPeers": true,
+  "TraceLocalChainSyncProtocol": false,
+  "TraceLocalErrorPolicy": true,
+  "TraceLocalHandshake": false,
+  "TraceLocalRootPeers": true,
+  "TraceLocalTxSubmissionProtocol": false,
+  "TraceLocalTxSubmissionServer": false,
+  "TraceMempool": true,
+  "TraceMux": false,
+  "TracePeerSelection": true,
+  "TracePeerSelectionActions": true,
+  "TracePublicRootPeers": true,
+  "TraceServer": true,
+  "TraceTxInbound": false,
+  "TraceTxOutbound": false,
+  "TraceTxSubmissionProtocol": false,
+  "TracingVerbosity": "NormalVerbosity",
+  "TurnOnLogMetrics": true,
+  "TurnOnLogging": true,
+  "defaultBackends": [
+    "KatipBK"
+  ],
+  "defaultScribes": [
+    [
+      "StdoutSK",
+      "stdout"
+    ],
+    [
+      "FileSK",
+      "log.json"
+    ]
+  ],
+  "hasEKG": 12788,
+  "hasPrometheus": [
+    "127.0.0.1",
+    12798
+  ],
+  "minSeverity": "Info",
+  "options": {
+    "mapBackends": {
+      "cardano.node.metrics": [
+        "EKGViewBK"
+      ],
+      "cardano.node.resources": [
+        "EKGViewBK"
+      ]
+    },
+    "mapSubtrace": {
+      "cardano.node.metrics": {
+        "subtrace": "Neutral"
+      }
+    }
+  },
+  "rotation": {
+    "rpKeepFilesNum": 10,
+    "rpLogLimitBytes": 5000000,
+    "rpMaxAgeHours": 24
+  },
+  "setupBackends": [
+    "KatipBK"
+  ],
+  "setupScribes": [
+    {
+      "scFormat": "ScJson",
+      "scKind": "StdoutSK",
+      "scName": "stdout",
+      "scRotation": null
+    },
+    {
+      "scFormat": "ScJson",
+      "scKind": "FileSK",
+      "scName": "log.json",
+      "scRotation": {
+        "rpLogLimitBytes": 300000000,
+        "rpMaxAgeHours": 24,
+        "rpKeepFilesNum": 20
+      }
+    }
+  ]
+}
diff --git a/nix/workbench/profiles/derived.jq b/nix/workbench/profiles/derived.jq
@@ -128,7 +128,7 @@ def add_derived_params:
          , cluster_startup_overhead_s:
                 (($gsis.utxo + $gsis.delegators) as $dataset_size
                 | if $dataset_size < 10000 then 5
-                  else $dataset_size / 25000
+                  else $dataset_size / 20000
                   end)
          }
      }

diff --git a/scripts/lite/membench-new-tracing.sh b/scripts/lite/membench-new-tracing.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# This script connects a node to mainnet
+
+ROOT="$(realpath "$(dirname "$0")/../..")"
+configuration="${ROOT}/configuration/cardano"
+
+data_dir=mainnetsingle
+mkdir -p "${data_dir}"
+db_dir="${data_dir}/db/node"
+mkdir -p "${db_dir}"
+socket_dir="${data_dir}/socket"
+mkdir -p "${socket_dir}"
+
+# Launch a node
+cabal run exe:cardano-node -- run \
+  --config "${configuration}/membench-config-new.yaml" \
+  --topology "${configuration}/mainnet-topology.json" \
+  --database-path "${db_dir}" \
+  --socket-path "${socket_dir}/node-1-socket" \
+  --host-addr "127.0.0.1" \
+  --port "3001"
+
+
+
+function cleanup()
+{
+  for child in $(jobs -p); do
+    echo kill "$child" && kill "$child"
+  done
+}
+
+trap cleanup EXIT
diff --git a/scripts/lite/membench-old-tracing.sh b/scripts/lite/membench-old-tracing.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# This script connects a node to mainnet
+
+ROOT="$(realpath "$(dirname "$0")/../..")"
+configuration="${ROOT}/configuration/cardano"
+
+data_dir=mainnetsingle
+mkdir -p "${data_dir}"
+db_dir="${data_dir}/db/node"
+mkdir -p "${db_dir}"
+socket_dir="${data_dir}/socket"
+mkdir -p "${socket_dir}"
+
+# Launch a node
+cabal run exe:cardano-node -- run \
+  --config "${configuration}/membench-config-old.json" \
+  --topology "${configuration}/mainnet-topology.json" \
+  --database-path "${db_dir}" \
+  --socket-path "${socket_dir}/node-1-socket" \
+  --host-addr "127.0.0.1" \
+  --port "3001"
+
+
+
+function cleanup()
+{
+  for child in $(jobs -p); do
+    echo kill "$child" && kill "$child"
+  done
+}
+
+trap cleanup EXIT