Run all EEST engine tests, not just ones in static folder (#3846)

* Run all EEST engine tests, not just ones in static folder.

* Try deleting unneeded tests to save disk space.

* Updates to support running all tests, including tests using older new payload/fcu versions.

Author: bhartnett

hive_integration/engine_client.nim

@@ -59,14 +59,14 @@ proc forkchoiceUpdatedV1*(client: RpcClient,
 
 proc forkchoiceUpdatedV2*(client: RpcClient,
       update: ForkchoiceStateV1,
-      payloadAttributes = Opt.none(PayloadAttributes)):
+      payloadAttributes = Opt.none(PayloadAttributesV2)):
         Result[ForkchoiceUpdatedResponse, string] =
   wrapTrySimpleRes:
     client.engine_forkchoiceUpdatedV2(update, payloadAttributes)
 
 proc forkchoiceUpdatedV3*(client: RpcClient,
       update: ForkchoiceStateV1,
-      payloadAttributes = Opt.none(PayloadAttributes)):
+      payloadAttributes = Opt.none(PayloadAttributesV3)):
         Result[ForkchoiceUpdatedResponse, string] =
   wrapTrySimpleRes:
     client.engine_forkchoiceUpdatedV3(update, payloadAttributes)
@@ -78,8 +78,8 @@ proc forkchoiceUpdated*(client: RpcClient,
                           Result[ForkchoiceUpdatedResponse, string] =
   case version
   of Version.V1: return client.forkchoiceUpdatedV1(update, attr.V1)
-  of Version.V2: return client.forkchoiceUpdatedV2(update, attr)
-  of Version.V3: return client.forkchoiceUpdatedV3(update, attr)
+  of Version.V2: return client.forkchoiceUpdatedV2(update, attr.V2)
+  of Version.V3: return client.forkchoiceUpdatedV3(update, attr.V3)
   of Version.V4, Version.V5, Version.V6: discard
 
 proc getPayloadV1*(client: RpcClient, payloadId: Bytes8): Result[ExecutionPayloadV1, string] =
@@ -98,10 +98,38 @@ proc getPayloadV4*(client: RpcClient, payloadId: Bytes8): Result[GetPayloadV4Res
   wrapTrySimpleRes:
     client.engine_getPayloadV4(payloadId)
 
+proc getPayloadV5*(client: RpcClient, payloadId: Bytes8): Result[GetPayloadV5Response, string] =
+  wrapTrySimpleRes:
+    client.engine_getPayloadV5(payloadId)
+
+proc getPayloadV6*(client: RpcClient, payloadId: Bytes8): Result[GetPayloadV6Response, string] =
+  wrapTrySimpleRes:
+    client.engine_getPayloadV6(payloadId)
+
 proc getPayload*(client: RpcClient,
                  version: Version,
                  payloadId: Bytes8): Result[GetPayloadResponse, string] =
-  if version == Version.V4:
+  if version == Version.V6:
+    let x = client.getPayloadV6(payloadId).valueOr:
+      return err(error)
+    ok(GetPayloadResponse(
+      executionPayload: executionPayload(x.executionPayload),
+      blockValue: Opt.some(x.blockValue),
+      blobsBundleV2: Opt.some(x.blobsBundle),
+      shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder),
+      executionRequests: Opt.some(x.executionRequests),
+    ))
+  elif version == Version.V5:
+    let x = client.getPayloadV5(payloadId).valueOr:
+      return err(error)
+    ok(GetPayloadResponse(
+      executionPayload: executionPayload(x.executionPayload),
+      blockValue: Opt.some(x.blockValue),
+      blobsBundleV2: Opt.some(x.blobsBundle),
+      shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder),
+      executionRequests: Opt.some(x.executionRequests),
+    ))
+  elif version == Version.V4:
     let x = client.getPayloadV4(payloadId).valueOr:
       return err(error)
     ok(GetPayloadResponse(
@@ -171,6 +199,16 @@ proc newPayloadV4*(client: RpcClient,
     client.engine_newPayloadV4(payload, versionedHashes,
       parentBeaconBlockRoot, executionRequests)
 
+proc newPayloadV5*(client: RpcClient,
+      payload: ExecutionPayloadV4,
+      versionedHashes: seq[VersionedHash],
+      parentBeaconBlockRoot: Hash32,
+      executionRequests: seq[seq[byte]]):
+        Result[PayloadStatusV1, string] =
+  wrapTrySimpleRes:
+    client.engine_newPayloadV5(payload, versionedHashes,
+      parentBeaconBlockRoot, executionRequests)
+
 proc newPayloadV1*(client: RpcClient,
       payload: ExecutionPayload):
         Result[PayloadStatusV1, string] =
@@ -202,21 +240,38 @@ proc newPayloadV4*(client: RpcClient,
     client.engine_newPayloadV4(payload, versionedHashes,
       parentBeaconBlockRoot, executionRequests)
 
+proc newPayloadV5*(client: RpcClient,
+      payload: ExecutionPayload,
+      versionedHashes: Opt[seq[VersionedHash]],
+      parentBeaconBlockRoot: Opt[Hash32],
+      executionRequests: Opt[seq[seq[byte]]]):
+        Result[PayloadStatusV1, string] =
+  wrapTrySimpleRes:
+    client.engine_newPayloadV5(payload, versionedHashes,
+      parentBeaconBlockRoot, executionRequests)
+
 proc newPayload*(client: RpcClient,
                  version: Version,
                  payload: ExecutableData): Result[PayloadStatusV1, string] =
   case version
-  of Version.V1: return client.newPayloadV1(payload.basePayload)
-  of Version.V2: return client.newPayloadV2(payload.basePayload)
+  of Version.V1:
+    return client.newPayloadV1(payload.basePayload)
+  of Version.V2:
+    return client.newPayloadV2(payload.basePayload)
   of Version.V3:
     return client.newPayloadV3(payload.basePayload,
       payload.versionedHashes,
       payload.beaconRoot)
-  of Version.V4, Version.V5:   # Osaka doesn't define any new newPayloadV5
+  of Version.V4:
     return client.newPayloadV4(payload.basePayload,
       payload.versionedHashes,
       payload.beaconRoot,
       payload.executionRequests)
+  of Version.V5:
+    return client.newPayloadV5(payload.basePayload,
+      payload.versionedHashes,
+      payload.beaconRoot,
+      payload.executionRequests)
   of Version.V6: discard
 
 proc exchangeCapabilities*(client: RpcClient,

scripts/eest_ci_cache.sh

@@ -63,16 +63,21 @@ download_and_extract() {
   tar -xzf "${archive}" -C "${dest_dir}" --strip-components=1
 
   rm -rf "${dest_dir}/.meta"
+
   mv "${dest_dir}/blockchain_tests/static/state_tests/"* "${dest_dir}/blockchain_tests" 2>/dev/null || true
   rm -rf "${dest_dir}/blockchain_tests/static"
 
-  mkdir -p "${dest_dir}/engine_tests"
-  mv "${dest_dir}/blockchain_tests_engine/static/state_tests/"* "${dest_dir}/engine_tests" 2>/dev/null || true
-  rm -rf "${dest_dir}/blockchain_tests_engine"
+  mv "${dest_dir}/blockchain_tests_engine/static/state_tests/"* "${dest_dir}/blockchain_tests_engine" 2>/dev/null || true
+  rm -rf "${dest_dir}/blockchain_tests_engine/static"
 
   mv "${dest_dir}/state_tests/static/state_tests/"* "${dest_dir}/state_tests" 2>/dev/null || true
   rm -rf "${dest_dir}/state_tests/static"
 
+  # Remove unused tests
+  rm -rf "${dest_dir}/blockchain_tests_engine_x"
+  rm -rf "${dest_dir}/blockchain_tests_sync"
+  rm -rf "${dest_dir}/transaction_tests"
+
   rm -f "${archive}"
 
   echo "${version}" > "${dest_dir}/version.txt"

tests/eest/eest_engine.nim

@@ -30,7 +30,13 @@ proc sendNewPayload(env: TestEnv, version: uint64, param: PayloadParam): Result[
   if not env.client.isSome:
     return err("Client is not initialized")
 
-  if version == 3:
+  if version == 1:
+    env.client.get().newPayloadV1(
+      param.payload)
+  elif version == 2:
+    env.client.get().newPayloadV2(
+      param.payload)
+  elif version == 3:
     env.client.get().newPayloadV3(
       param.payload,
       param.versionedHashes,
@@ -40,17 +46,30 @@ proc sendNewPayload(env: TestEnv, version: uint64, param: PayloadParam): Result[
       param.payload,
       param.versionedHashes,
       param.parentBeaconBlockRoot,
-      param.excutionRequests)
+      param.executionRequests)
+  elif version == 5:
+    env.client.get().newPayloadV5(
+      param.payload,
+      param.versionedHashes,
+      param.parentBeaconBlockRoot,
+      param.executionRequests)
   else:
     err("Unsupported NewPayload version: " & $version)
 
 proc sendFCU(env: TestEnv, version: uint64, param: PayloadParam): Result[ForkchoiceUpdatedResponse, string] =
+  if not env.client.isSome:
+    return err("Client is not initialized")
+
   let update = ForkchoiceStateV1(
     headblockHash:      param.payload.blockHash,
     finalizedblockHash: param.payload.blockHash
   )
 
-  if version == 3 and env.client.isSome:
+  if version == 1:
+    env.client.get().forkchoiceUpdatedV1(update)
+  elif version == 2:
+    env.client.get().forkchoiceUpdatedV2(update)
+  elif version == 3:
     env.client.get().forkchoiceUpdatedV3(update)
   else:
     err("Unsupported FCU version: " & $version)
@@ -60,8 +79,12 @@ proc runTest(env: TestEnv, unit: EngineUnitEnv): Result[void, string] =
     return err("Client is not initialized")
 
   for enp in unit.engineNewPayloads:
+
     var status = env.sendNewPayload(enp.newPayloadVersion.uint64, enp.params).valueOr:
-      return err(error)
+      if enp.validationError.isSome():
+        continue
+      else:
+        return err(error)
 
     discard status
     when false:
@@ -80,8 +103,7 @@ proc runTest(env: TestEnv, unit: EngineUnitEnv): Result[void, string] =
       if status.validationError.isSome:
         return err(status.validationError.value)
 
-  let header = env.client.get().latestHeader().valueOr:
-    return err(error)
+  let header = env.chain.latestHeader()
 
   if unit.lastblockhash != header.computeRlpHash:
     return err("last block hash mismatch")

tests/eest/eest_engine_test.nim

@@ -17,7 +17,7 @@ import
 
 const
   baseFolder = "tests/fixtures"
-  eestType = "engine_tests"
+  eestType = "blockchain_tests_engine"
   eestReleases = [
     "eest_develop",
     "eest_devnet"

tests/eest/eest_helpers.nim

@@ -69,12 +69,13 @@ type
     payload*: ExecutionPayload
     versionedHashes*: Opt[seq[Hash32]]
     parentBeaconBlockRoot*: Opt[Hash32]
-    excutionRequests*: Opt[seq[seq[byte]]]
+    executionRequests*: Opt[seq[seq[byte]]]
 
   PayloadItem* = object
     params*: PayloadParam
     newPayloadVersion*: Numero
     forkchoiceUpdatedVersion*: Numero
+    validationError*: Opt[string]
 
   EnvConfig* = object
     network*: string
@@ -156,7 +157,7 @@ proc readValue*(
       of 2:
         r.readValue(val.parentBeaconBlockRoot)
       of 3:
-        r.readValue(val.excutionRequests)
+        r.readValue(val.executionRequests)
       else:
         r.raiseUnexpectedValue("Unexpected element")