Remove backtest checksum

Instead of checking whether firedancer-dev backtest randomly
modified inputs, just ban backtest from modifying inputs.

Author: riptl

.github/workflows/benchmark.yml

@@ -60,7 +60,7 @@ jobs:
         DUMP_DIR=../dump ./src/flamenco/runtime/tests/run_ledger_backtest.sh \
           -l mainnet-368528500-perf \
           -y 5 -m 2000000 -e 368529500 -c 3.0.0 \
-          --log baseline.log --skip-checksum
+          --log baseline.log
         SPS_BASELINE=$(grep "Backtest playback done." baseline.log | awk -F 'sec/slot: ' '{print $2}')
         TOTAL_ELAPSED_BASELINE=$(grep "Backtest playback done." baseline.log | awk -F 'elapsed: ' '{print $2}' | awk '{print $1}')
         SNAPSHOT_LOAD_BASELINE=$(grep "accounts from snapshot in" baseline.log | awk -F 'accounts from snapshot in ' '{print $2}' | awk '{print $1}')

contrib/test/ledger_common.sh

@@ -7,56 +7,3 @@ echo_notice() {
 echo_error() {
   echo -e "\033[31m$1$2\033[0m"
 }
-
-create_checksum() {
-  # generate checksum of ledger
-  find $DUMP/$LEDGER -type f \( -name "snapshot*" -o -name "genesis*" -o -wholename "*rocksdb*" \) -print0 | xargs -0 md5sum | sort > $DUMP/$LEDGER/checksum.txt
-}
-
-redownload_ledger_or_fail() {
-  if [[ $REDOWNLOAD -eq 1 ]]; then
-    echo_notice "Redownloading ledger $LEDGER"
-    rm -rf $DUMP/$LEDGER
-    download_and_extract_ledger
-    create_checksum
-  else
-    echo_error "If you want to redownload the ledger, please remove the -nr flag and try again."
-    exit 1
-  fi
-}
-
-check_ledger_checksum() {
-  if [ ! -f "$DUMP/$LEDGER/checksum.txt" ]; then
-    echo_error "original checksum file does not exist."
-    exit 1
-  fi
-
-  # check existing checksum
-  find $DUMP/$LEDGER -type f \( -name "snapshot*" -o -name "genesis*" -o -wholename "*rocksdb*" \) -print0 | xargs -0 md5sum | sort > $DUMP/$LEDGER/checksum_temp.txt
-
-  if ! diff -u $DUMP/$LEDGER/checksum.txt $DUMP/$LEDGER/checksum_temp.txt > /dev/null; then
-    echo_error "Checksum mismatch for existing ledger $LEDGER. This likely means the ledger is corrupted. \
-vimdiff $DUMP/$LEDGER/checksum.txt $DUMP/$LEDGER/checksum_temp.txt to see the differences."
-  else
-    rm $DUMP/$LEDGER/checksum_temp.txt
-  fi
-}
-
-check_ledger_checksum_and_redownload() {
-  if [ ! -f "$DUMP/$LEDGER/checksum.txt" ]; then
-    echo_error "original checksum file does not exist."
-    redownload_ledger_or_fail
-  fi
-
-  # check existing checksum
-  find $DUMP/$LEDGER -type f \( -name "snapshot*" -o -name "genesis*" -o -wholename "*rocksdb*" \) -print0 | xargs -0 md5sum | sort > $DUMP/$LEDGER/checksum_temp.txt
-
-  if ! diff -u $DUMP/$LEDGER/checksum.txt $DUMP/$LEDGER/checksum_temp.txt > /dev/null; then
-    echo_error "Checksum mismatch for existing ledger $LEDGER. This likely means the ledger is corrupted. \
-vimdiff $DUMP/$LEDGER/checksum.txt $DUMP/$LEDGER/checksum_temp.txt to see the differences. \
-If you are running locally, please disable redownload with -nr and inspect the differences. "
-    redownload_ledger_or_fail
-  else
-    rm $DUMP/$LEDGER/checksum_temp.txt
-  fi
-}

contrib/test/run_solcap_tests.sh

@@ -44,9 +44,6 @@ download_and_extract_ledger() {
 
 if [[ ! -e $DUMP/$LEDGER && SKIP_INGEST -eq 0 ]]; then
   download_and_extract_ledger
-  create_checksum
-else
-  check_ledger_checksum_and_redownload
 fi
 
 rm -rf $DUMP/$LEDGER/devnet-398736132_current.toml
@@ -68,6 +65,3 @@ $OBJDIR/bin/firedancer-dev configure fini all --config $DUMP/$LEDGER/devnet-3987
 
 $OBJDIR/bin/fd_solcap_import $DUMP/$LEDGER/bank_hash_details/ $DUMP/$LEDGER/solana.solcap
 $OBJDIR/bin/fd_solcap_diff $DUMP/$LEDGER/solana.solcap $DUMP/$LEDGER/fd.solcap -v 4
-
-# check that the ledger is not corrupted after a run
-check_ledger_checksum

src/flamenco/runtime/tests/run_ledger_backtest.sh

@@ -22,7 +22,6 @@ HUGE_TLBFS_MOUNT_PATH=${HUGE_TLBFS_MOUNT_PATH:="/mnt/.fd"}
 HUGE_TLBFS_ALLOW_HUGEPAGE_INCREASE=${HUGE_TLBFS_ALLOW_HUGEPAGE_INCREASE:="true"}
 HAS_INCREMENTAL="false"
 REDOWNLOAD=1
-SKIP_CHECKSUM=0
 DEBUG=( )
 WATCH=( )
 
@@ -111,10 +110,6 @@ while [[ $# -gt 0 ]]; do
         DEBUG=( gdb -q -x contrib/debug.gdb --args )
         shift
         ;;
-    --skip-checksum)
-        SKIP_CHECKSUM=1
-        shift
-        ;;
     --log)
         LOG="$2"
         shift
@@ -166,13 +161,6 @@ download_and_extract_ledger() {
 
 if [[ ! -e $DUMP/$LEDGER && SKIP_INGEST -eq 0 ]]; then
   download_and_extract_ledger
-  if [[ $SKIP_CHECKSUM -eq 0 ]]; then
-    create_checksum
-  fi
-else
-  if [[ $SKIP_CHECKSUM -eq 0 ]]; then
-    check_ledger_checksum_and_redownload
-  fi
 fi
 
 chmod -R 0700 $DUMP/$LEDGER
@@ -244,11 +232,6 @@ sudo rm -rf $DUMP/$LEDGER/backtest.blockstore $DUMP/$LEDGER/backtest.funk &> /de
 
 echo "Log for ledger $LEDGER at $LOG"
 
-# check that the ledger is not corrupted after a run
-if [[ $SKIP_CHECKSUM -eq 0 ]]; then
-  check_ledger_checksum
-fi
-
 if [ "$status" -eq 0 ]; then
   echo_notice "Finished on-demand ingest and replay\n"
   exit 0