Postgresql-setup script updates

docker.nix: Bump postgresql to 17 (as restore snapshot fails if snapshots are made using postgres 17), postgres 14 is ancient anyways and there are various improvements (most prominent one being non-locking index maintenance)
docker-compose*.yml: Bump postgresql version to 17, node version to 10.1.4 and dbsync version to 13.6.0.4
postgresql-setup.sh:
- Parse psql output using csv instead of manipulating white chars or table seperators
- Use strict checks for ${PGDATABASE} as there can be another database with a prefix/suffix attached
- When creating/restoring a snapshot, existence of ledger_file should be optional, as there are dbsync config options that don't require lstate files, accordingly - make presence of ledger-state-file optional
- When restoring a snapshot (eg: across postgres versions), there will be cases where an informational error may be expected. The presence of exit-on-error prevents use of these snapshots, especially when run from docker created by nix-ops (where it isnt as straightforward to override scripts)

Author: rdlrt

docker-compose.example.yml

@@ -4,7 +4,7 @@ name: cardano-db-sync
 
 services:
   postgres:
-    image: postgres:14.10-alpine
+    image: postgres:17.2-alpine
     environment:
       - POSTGRES_LOGGING=true
       - POSTGRES_DB=cexplorer
@@ -36,7 +36,7 @@ services:
         max-file: "10"
 
   cardano-node:
-    image: ghcr.io/intersectmbo/cardano-node:9.0.0
+    image: ghcr.io/intersectmbo/cardano-node:10.1.4
     environment:
       - NETWORK=${NETWORK:-mainnet}
     networks:
@@ -59,7 +59,7 @@ services:
         max-file: "10"
 
   cardano-db-sync:
-    image: ghcr.io/intersectmbo/cardano-db-sync:13.3.0.0
+    image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4
     environment:
       - NETWORK=${NETWORK:-mainnet}
       - POSTGRES_HOST=postgres

docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.9"
 
 services:
   postgres:
-    image: postgres:14.10-alpine
+    image: postgres:17.2-alpine
     environment:
       - POSTGRES_LOGGING=true
       - POSTGRES_DB_FILE=/run/secrets/postgres_db
@@ -33,7 +33,7 @@ services:
         max-file: "10"
 
   cardano-node:
-    image: ghcr.io/intersectmbo/cardano-node:10.1.3
+    image: ghcr.io/intersectmbo/cardano-node:10.1.4
     environment:
       - NETWORK=${NETWORK:-mainnet}
     volumes:

docker-test.yml

@@ -2,7 +2,7 @@ version: "3.5"
 
 services:
   postgres:
-    image: postgres:14.10-alpine
+    image: postgres:17.2-alpine
     environment:
       - POSTGRES_LOGGING=true
       - POSTGRES_DB_FILE=/run/secrets/postgres_db

nix/docker.nix

@@ -10,7 +10,7 @@
 , bashInteractive, cacert, cardano-cli, cardano-db-sync, cardano-db-tool
 , cardano-smash-server, coreutils, curl, findutils, getconf, glibcLocales
 , gnused, gnutar, gzip, jq, iana-etc, iproute, iputils, lib, libidn, libpqxx
-, postgresql_14, socat, utillinux
+, postgresql_17, socat, utillinux
 }:
 
 let
@@ -46,7 +46,7 @@ let
         iputils # Useful utilities for Linux networking
         libidn # Library for internationalized domain names
         libpqxx # A C++ library to access PostgreSQL databases
-        postgresql_14 # A powerful, open source object-relational database system
+        postgresql_17 # A powerful, open source object-relational database system
         socat # Utility for bidirectional data transfer
         utillinux # System utilities for Linux
         cardano-cli # tool for interacting with cardano-node

scripts/postgresql-setup.sh

@@ -98,7 +98,7 @@ function check_connect_as_user {
 
 function check_db_exists {
 	set +e
-	count=$(psql -l "${PGDATABASE}" | grep -c "${PGDATABASE} ")
+	count=$(psql -t -l --csv "${PGDATABASE}" | grep -c "^${PGDATABASE},")
 	if test "${count}" -lt 1 ; then
 		echo
 		echo "Error : No '${PGDATABASE}' database."
@@ -108,7 +108,7 @@ function check_db_exists {
 		echo
 		exit 1
 		fi
-	count=$(psql -l "${PGDATABASE}" | grep "${PGDATABASE} " | cut -d \| -f 3 | grep -c UTF8)
+	count=$(psql -t -l --csv "${PGDATABASE}" | grep "^${PGDATABASE}," | grep -c UTF8)
 	if test "${count}" -ne 1 ; then
 		echo
 		echo "Error : '${PGDATABASE}' database exists, but is not UTF8."
@@ -164,19 +164,20 @@ function create_snapshot {
 	tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
 	echo $"Working directory: ${tmp_dir}"
 	pg_dump --no-owner --schema=public --jobs="${numcores}" "${PGDATABASE}" --format=directory --file="${tmp_dir}/db/"
-	lstate_gz_file=$(basename "${ledger_file}").gz
-	gzip --to-stdout "${ledger_file}" > "${tmp_dir}/$(basename "${ledger_file}").gz"
-	tree "${tmp_dir}"
+	if [ -n "${ledger_file}" ]; then
+	  lstate_gz_file=$(basename "${ledger_file}").gz
+	  gzip --to-stdout "${ledger_file}" > "${tmp_dir}/${lstate_gz_file}"
+	fi
 	# Use plain tar here because the database dump files and the ledger state file are already gzipped.
-	tar cvf - --directory "${tmp_dir}" "db" "${lstate_gz_file}" | tee "${tgz_file}.tmp" \
-		| sha256sum | head -c 64 | sed -e "s/$/  ${tgz_file}\n/" > "${tgz_file}.sha256sum"
+	tar cvf - --directory "${tmp_dir}" "db" $( [ -f "/${tmp_dir}/${lstate_gz_file}" ] && echo "${lstate_gz_file}" ) | tee "${tgz_file}.tmp" | \
+	  sha256sum | head -c 64 | sed -e "s/$/  ${tgz_file}\n/" > "${tgz_file}.sha256sum"
 	mv "${tgz_file}.tmp" "${tgz_file}"
 	rm "${recursive}" "${force}" "${tmp_dir}"
 	if test "$(tar "${test}" --file "${tgz_file}")" ; then
 	  echo "Tar reports the snapshot file as being corrupt."
 	  echo "It is not safe to drop the database and restore using this file."
 	  exit 1
-	  fi
+	fi
 	echo "Created ${tgz_file} + .sha256sum"
 }
 
@@ -185,24 +186,26 @@ function restore_snapshot {
 	if test "${file_count}" -gt 0 ; then
 	  echo "Ledger state directory ($2) is not empty. Please empty it and then retry."
 	  exit 1
-	  fi
+	fi
 	tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
 	tar xvf "$1" --directory "$tmp_dir"
 	if test -d "${tmp_dir}/db/" ; then
 	  # New pg_dump format
 	  lstate_gz_file=$(find "${tmp_dir}/" -iname "*.lstate.gz")
-	  lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
-	  gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
+	  if [ -n "${lstate_gz_file}" ] ; then
+	    lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
+	    gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
+	  fi
 
-          # Important: specify --schema=public below to skip over `create schema public`
-          # statement generated by pg_dump
+	  # Important: specify --schema=public below to skip over `create schema public`
+	  # statement generated by pg_dump
 	  pg_restore \
 	    --schema=public \
 	    --jobs="${numcores}" \
 	    --format=directory \
 	    --dbname="${PGDATABASE}" \
-            --no-owner \
-	    --exit-on-error \
+	    --no-owner \
+	    $( [ -z "${SKIP_RESTORE_ERROR}" ] && echo " --exit-on-error" ) \
 	    "${tmp_dir}/db/"
 	else
 	  # Old snapshot format produced by this script
@@ -228,7 +231,7 @@ function usage_exit {
 	echo "    $progname --dump-schema       - Dump the schema of the database."
 	echo
 	echo "    - Create a db-sync state snapshot"
-	echo "      $progname --create-snapshot <snapshot-file> <ledger-state-file>"
+	echo "      $progname --create-snapshot <snapshot-file> [<ledger-state-file>]"
 	echo
 	echo "    - Restore a db-sync state snapshot."
 	echo "      $progname --restore-snapshot <snapshot-file> <ledger-state-dir>"
@@ -300,22 +303,18 @@ case "${1:-""}" in
 	--create-snapshot)
 		check_pgpass_file
 		check_db_exists
-		if test $# -ne 3 ; then
-		  echo "Expecting exactly 2 more arguments, the snapshot file name template and the ledger state file."
+		if test $# -lt 2 ; then
+		  echo "Expecting the snapshot file name (without extension) and optionally the ledger state file as arguments."
 		  exit 1
-		  fi
+		fi
 		if test -z "$2" ; then
-		  echo "Second argument should be the snapshot file name template."
-		  exit 1
-		  fi
-		if test -z "$3" ; then
-		  echo "Third argument should be the ledger state file."
+		  echo "Second argument should be the snapshot file name (without extension)."
 		  exit 1
-		  fi
-		if test -d "$3" ; then
-		  echo "Third argument is a directory and expecting a file."
+		fi
+		if test -n "$3" && test -d "$3" ; then
+		  echo "Third argument provided is a directory but expecting a file."
 		  exit 1
-		  fi
+		fi
 		create_snapshot "$2" "$3"
 		;;
 	--restore-snapshot)