merge master branch

Author: jangko

Makefile

@@ -66,13 +66,13 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 
 # Fluffy debugging tools + testing tools
 FLUFFY_TOOLS := \
-	beacon_chain_bridge \
+	beacon_lc_bridge \
 	eth_data_exporter \
 	content_verifier \
 	blockwalk \
 	portalcli
 FLUFFY_TOOLS_DIRS := \
-	fluffy/tools/bridge \
+	fluffy/tools/beacon_lc_bridge \
 	fluffy/tools
 # comma-separated values for the "clean" target
 FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS))

fluffy/conf.nim

@@ -10,10 +10,11 @@
 import
   std/os,
   uri, confutils, confutils/std/net, chronicles,
-  eth/keys, eth/net/nat, eth/p2p/discoveryv5/[enr, node, routing_table],
+  eth/keys, eth/p2p/discoveryv5/[enr, node, routing_table],
   json_rpc/rpcproxy,
   nimcrypto/hash,
   stew/byteutils,
+  eth/net/nat, # must be late (compilation annoyance)
   ./logging,
   ./network/wire/portal_protocol_config
 
@@ -239,51 +240,51 @@ func completeCmdArg*(T: type TrustedDigest, input: string): seq[string] =
   return @[]
 
 proc parseCmdArg*(T: type enr.Record, p: string): T
-    {.raises: [ConfigurationError].} =
+    {.raises: [ValueError].} =
   if not fromURI(result, p):
-    raise newException(ConfigurationError, "Invalid ENR")
+    raise newException(ValueError, "Invalid ENR")
 
 proc completeCmdArg*(T: type enr.Record, val: string): seq[string] =
   return @[]
 
 proc parseCmdArg*(T: type Node, p: string): T
-    {.raises: [ConfigurationError].} =
+    {.raises: [ValueError].} =
   var record: enr.Record
   if not fromURI(record, p):
-    raise newException(ConfigurationError, "Invalid ENR")
+    raise newException(ValueError, "Invalid ENR")
 
   let n = newNode(record)
   if n.isErr:
-    raise newException(ConfigurationError, $n.error)
+    raise newException(ValueError, $n.error)
 
   if n[].address.isNone():
-    raise newException(ConfigurationError, "ENR without address")
+    raise newException(ValueError, "ENR without address")
 
   n[]
 
 proc completeCmdArg*(T: type Node, val: string): seq[string] =
   return @[]
 
 proc parseCmdArg*(T: type PrivateKey, p: string): T
-    {.raises: [ConfigurationError].} =
+    {.raises: [ValueError].} =
   try:
     result = PrivateKey.fromHex(p).tryGet()
   except CatchableError:
-    raise newException(ConfigurationError, "Invalid private key")
+    raise newException(ValueError, "Invalid private key")
 
 proc completeCmdArg*(T: type PrivateKey, val: string): seq[string] =
   return @[]
 
 proc parseCmdArg*(T: type ClientConfig, p: string): T
-      {.raises: [ConfigurationError].} =
+      {.raises: [ValueError].} =
   let uri = parseUri(p)
   if (uri.scheme == "http" or uri.scheme == "https"):
     getHttpClientConfig(p)
   elif (uri.scheme == "ws" or uri.scheme == "wss"):
     getWebSocketClientConfig(p)
   else:
     raise newException(
-      ConfigurationError, "Proxy uri should have defined scheme (http/https/ws/wss)"
+      ValueError, "Proxy uri should have defined scheme (http/https/ws/wss)"
     )
 
 proc completeCmdArg*(T: type ClientConfig, val: string): seq[string] =

fluffy/docs/content_seeding.md

@@ -89,15 +89,15 @@ Run a Fluffy node with the JSON-RPC API enabled.
 ./build/fluffy --rpc --table-ip-limit:1024 --bucket-ip-limit:24
 ```
 
-Build & run the `beacon_chain_bridge`:
+Build & run the `beacon_lc_bridge`:
 ```bash
-make fluffy-tools
+make beacon_lc_bridge
 
 TRUSTED_BLOCK_ROOT=0x1234567890123456789012345678901234567890123456789012345678901234 # Replace this
-./build/beacon_chain_bridge --trusted-block-root=${TRUSTED_BLOCK_ROOT}
+./build/beacon_lc_bridge --trusted-block-root=${TRUSTED_BLOCK_ROOT}
 ```
 
-The `beacon_chain_bridge` will start with the consensus light client sync follow
+The `beacon_lc_bridge` will start with the consensus light client sync follow
 beacon block gossip. Once it is synced, the execution payload of new beacon
 blocks will be extracted and injected in the Portal network as execution headers
 and blocks.

fluffy/network/history/history_network.nim

@@ -9,12 +9,12 @@
 
 import
   stew/results, chronos, chronicles,
-  eth/[common/eth_types_rlp, rlp, trie, trie/db],
+  eth/[common/eth_types_rlp, rlp],
   eth/p2p/discoveryv5/[protocol, enr],
   ../../common/common_types,
   ../../content_db,
   ../../network_metadata,
-  ../../../nimbus/constants,
+  ../../../nimbus/[constants, db/core_db],
   ../wire/[portal_protocol, portal_stream, portal_protocol_config],
   "."/[history_content, accumulator]
 
@@ -184,7 +184,7 @@ func encode*(receipts: seq[Receipt]): seq[byte] =
 # for if/when peer scoring/banning is added.
 
 proc calcRootHash(items: Transactions | PortalReceipts | Withdrawals): Hash256 =
-  var tr = initHexaryTrie(newMemoryDB())
+  var tr = newCoreDbRef(LegacyDbMemory).mptPrune
   for i, item in items:
     try:
       tr.put(rlp.encode(i), item.asSeq())
@@ -209,7 +209,7 @@ func validateBlockHeaderBytes*(
 
   let header = ? decodeRlp(bytes, BlockHeader)
 
-  if header.excessDataGas.isSome:
+  if header.excessBlobGas.isSome:
     return err("EIP-4844 not yet implemented")
 
   # TODO: Verify timestamp with Shanghai timestamp to if isSome()

fluffy/network/wire/portal_protocol_config.nim

@@ -54,7 +54,7 @@ proc init*(
   )
 
 proc parseCmdArg*(T: type RadiusConfig, p: string): T
-    {.raises: [ConfigurationError].} =
+    {.raises: [ValueError].} =
   if p.startsWith("dynamic") and len(p) == 7:
     RadiusConfig(kind: Dynamic)
   elif p.startsWith("static:"):
@@ -64,11 +64,11 @@ proc parseCmdArg*(T: type RadiusConfig, p: string): T
         uint16.parseCmdArg(num)
       except ValueError:
         let msg = "Provided logRadius: " & num & " is not a valid number"
-        raise newException(ConfigurationError, msg)
+        raise newException(ValueError, msg)
 
     if parsed > 256:
       raise newException(
-        ConfigurationError, "Provided logRadius should be <= 256"
+        ValueError, "Provided logRadius should be <= 256"
       )
 
     RadiusConfig(kind: Static, logRadius: parsed)
@@ -80,11 +80,11 @@ proc parseCmdArg*(T: type RadiusConfig, p: string): T
         let msg =
           "Not supported radius config option: " & p & " . " &
           "Supported options: dynamic and static:logRadius"
-        raise newException(ConfigurationError, msg)
+        raise newException(ValueError, msg)
 
     if parsed > 256:
       raise newException(
-        ConfigurationError, "Provided logRadius should be <= 256")
+        ValueError, "Provided logRadius should be <= 256")
 
     RadiusConfig(kind: Static, logRadius: parsed)
 

fluffy/scripts/launch_local_testnet.sh

@@ -34,7 +34,7 @@ if [ ${PIPESTATUS[0]} != 4 ]; then
 fi
 
 OPTS="h:n:d"
-LONGOPTS="help,nodes:,data-dir:,enable-htop,log-level:,base-port:,base-rpc-port:,base-metrics-port:,reuse-existing-data-dir,timeout:,kill-old-processes"
+LONGOPTS="help,nodes:,data-dir:,enable-htop,log-level:,base-port:,base-rpc-port:,lc-bridge,trusted-block-root:,base-metrics-port:,reuse-existing-data-dir,timeout:,kill-old-processes"
 
 # default values
 NUM_NODES="64"
@@ -48,6 +48,8 @@ REUSE_EXISTING_DATA_DIR="0"
 TIMEOUT_DURATION="0"
 KILL_OLD_PROCESSES="0"
 SCRIPTS_DIR="fluffy/scripts/"
+LC_BRIDGE="0"
+TRUSTED_BLOCK_ROOT=""
 
 print_help() {
   cat <<EOF
@@ -61,6 +63,8 @@ E.g.: $(basename "$0") --nodes ${NUM_NODES} --data-dir "${DATA_DIR}" # defaults
   --base-port                 bootstrap node's discv5 port (default: ${BASE_PORT})
   --base-rpc-port             bootstrap node's RPC port (default: ${BASE_RPC_PORT})
   --base-metrics-port         bootstrap node's metrics server port (default: ${BASE_METRICS_PORT})
+  --lc-bridge                 run an beacon lc bridge attached to the bootstrap node
+  --trusted-block-root        recent trusted finalized block root to initialize the consensus light client from
   --enable-htop               use "htop" to see the fluffy processes without doing any tests
   --log-level                 set the log level (default: ${LOG_LEVEL})
   --reuse-existing-data-dir   instead of deleting and recreating the data dir, keep it and reuse everything we can from it
@@ -107,6 +111,14 @@ while true; do
       BASE_RPC_PORT="$2"
       shift 2
       ;;
+    --lc-bridge)
+      LC_BRIDGE="1"
+      shift
+      ;;
+    --trusted-block-root)
+      TRUSTED_BLOCK_ROOT="$2"
+      shift 2
+      ;;
     --base-metrics-port)
       BASE_METRICS_PORT="$2"
       shift 2
@@ -183,6 +195,9 @@ fi
 
 # Build the binaries
 BINARIES="fluffy"
+if [[ "${LC_BRIDGE}" == "1" ]]; then
+  BINARIES="${BINARIES} beacon_lc_bridge"
+fi
 TEST_BINARIES="test_portal_testnet"
 $MAKE -j ${NPROC} LOG_LEVEL=TRACE ${BINARIES}
 $MAKE -j ${NPROC} LOG_LEVEL=INFO ${TEST_BINARIES}
@@ -191,9 +206,13 @@ $MAKE -j ${NPROC} LOG_LEVEL=INFO ${TEST_BINARIES}
 # instance as the parent and the target process name as a pattern to the
 # "pkill" command.
 cleanup() {
-  pkill -f -P $$ fluffy &>/dev/null || true
+  for BINARY in ${BINARIES}; do
+    pkill -f -P $$ ${BINARY} &>/dev/null || true
+  done
   sleep 2
-  pkill -f -9 -P $$ fluffy &>/dev/null || true
+  for BINARY in ${BINARIES}; do
+    pkill -f -9 -P $$ ${BINARY} &>/dev/null || true
+  done
 
   # Delete the binaries we just built, because these are with none default logs.
   # TODO: When fluffy gets run time log options a la nimbus-eth2 we can keep
@@ -219,7 +238,7 @@ if [[ "${TIMEOUT_DURATION}" != "0" ]]; then
 fi
 
 PIDS=""
-NUM_JOBS=${NUM_NODES}
+NUM_JOBS=$(( NUM_NODES + LC_BRIDGE ))
 
 dump_logs() {
   LOG_LINES=20
@@ -234,6 +253,11 @@ BOOTSTRAP_NODE=0
 BOOTSTRAP_TIMEOUT=5 # in seconds
 BOOTSTRAP_ENR_FILE="${DATA_DIR}/node${BOOTSTRAP_NODE}/fluffy_node.enr"
 
+TRUSTED_BLOCK_ROOT_ARG=""
+if [[ -z ${TRUSTED_BLOCK_ROOT} ]]; then
+  TRUSTED_BLOCK_ROOT_ARG="--trusted-block-root=${TRUSTED_BLOCK_ROOT}"
+fi
+
 for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   NODE_DATA_DIR="${DATA_DIR}/node${NUM_NODE}"
   rm -rf "${NODE_DATA_DIR}"
@@ -275,6 +299,7 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
     --log-level="${LOG_LEVEL}" \
     --udp-port=$(( BASE_PORT + NUM_NODE )) \
     --data-dir="${NODE_DATA_DIR}" \
+    --network="none" \
     ${BOOTSTRAP_ARG} \
     --rpc \
     --rpc-address="127.0.0.1" \
@@ -285,6 +310,7 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
     --table-ip-limit=1024 \
     --bucket-ip-limit=24 \
     --bits-per-hop=1 \
+    ${TRUSTED_BLOCK_ROOT_ARG} \
     ${RADIUS_ARG} \
     ${EXTRA_ARGS} \
     > "${DATA_DIR}/log${NUM_NODE}.txt" 2>&1 &
@@ -296,6 +322,21 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   fi
 done
 
+if [[ "$LC_BRIDGE" == "1" ]]; then
+  echo "Starting bridge node."
+  LC_BRIDGE_DATA_DIR="${DATA_DIR}/lc_bridge"
+  ./build/beacon_lc_bridge \
+    --data-dir="${LC_BRIDGE_DATA_DIR}" \
+    --udp-port=$(( BASE_PORT + NUM_NODES )) \
+    --rpc-address="127.0.0.1" \
+    --rpc-port="${BASE_RPC_PORT}" \
+    --beacon-light-client \
+    ${TRUSTED_BLOCK_ROOT_ARG} \
+    > "${DATA_DIR}/log_lc_bridge.txt" 2>&1 &
+
+  PIDS="${PIDS},$!"
+fi
+
 # give the regular nodes time to crash
 sleep 5
 BG_JOBS="$(jobs | wc -l | tr -d ' ')"

fluffy/tests/test_state_network.nim

@@ -8,22 +8,22 @@
 import
   std/os,
   testutils/unittests, chronos,
-  eth/[common/eth_hash, keys, trie/db, trie/hexary],
+  eth/[common/eth_hash, keys],
   eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
-  ../../nimbus/[config, db/db_chain, db/state_db],
+  ../../nimbus/[config, db/core_db, db/state_db],
   ../../nimbus/common/[chain_config, genesis],
   ../network/wire/[portal_protocol, portal_stream],
   ../network/state/[state_content, state_network],
   ../content_db,
   ./test_helpers
 
-proc genesisToTrie(filePath: string): HexaryTrie =
+proc genesisToTrie(filePath: string): CoreDbMptRef =
   # TODO: Doing our best here with API that exists, to be improved.
   var cn: NetworkParams
   if not loadNetworkParams(filePath, cn):
     quit(1)
 
-  let sdb  = newStateDB(newMemoryDB(), false)
+  let sdb  = newStateDB(newCoreDbRef LegacyDbMemory, false)
   let map  = toForkTransitionTable(cn.config)
   let fork = map.toHardFork(forkDeterminationInfo(0.toBlockNumber, cn.genesis.timestamp))
   discard toGenesisHeader(cn.genesis, sdb, fork)