Skip to content

Commit

Permalink
Postgresql-setup script updates
Browse files Browse the repository at this point in the history 
docker-compose*.yml: Bump node version to 10.1.4 and dbsync version to 13.6.0.4
postgresql-setup.sh:
- Parse psql output using csv instead of manipulating white chars or table seperators
- Use strict checks for ${PGDATABASE} as there can be another database with a prefix/suffix attached
- When creating/restoring a snapshot, existence of ledger_file should be optional, as there are dbsync config options that don't require lstate files, accordingly - make presence of ledger-state-file optional
- When restoring a snapshot (eg: across postgres versions), there will be cases where an informational error may be expected. The presence of exit-on-error prevents use of these snapshots, especially when run from docker created by nix-ops (where it isnt as straightforward to override scripts)
  • Loading branch information
@rdlrt @sgillespie
rdlrt authored and sgillespie committed Feb 5, 2025
1 parent cea19a3 commit 16b4cea
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 40 deletions.
4 changes: 2 additions & 2 deletions docker-compose.example.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ services:
max-file: "10"

cardano-node:
image: ghcr.io/intersectmbo/cardano-node:9.0.0
image: ghcr.io/intersectmbo/cardano-node:10.1.4
environment:
- NETWORK=${NETWORK:-mainnet}
networks:
Expand All @@ -59,7 +59,7 @@ services:
max-file: "10"

cardano-db-sync:
image: ghcr.io/intersectmbo/cardano-db-sync:13.3.0.0
image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4
environment:
- NETWORK=${NETWORK:-mainnet}
- POSTGRES_HOST=postgres
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ services:
max-file: "10"

cardano-node:
image: ghcr.io/intersectmbo/cardano-node:10.1.3
image: ghcr.io/intersectmbo/cardano-node:10.1.4
environment:
- NETWORK=${NETWORK:-mainnet}
volumes:
Expand Down
70 changes: 33 additions & 37 deletions scripts/postgresql-setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ function check_connect_as_user {

function check_db_exists {
set +e
count=$(psql -l "${PGDATABASE}" | grep -c "${PGDATABASE} ")
count=$(psql -t -l --csv "${PGDATABASE}" | grep -c "^${PGDATABASE},")
if test "${count}" -lt 1 ; then
echo
echo "Error : No '${PGDATABASE}' database."
Expand All @@ -108,7 +108,7 @@ function check_db_exists {
echo
exit 1
fi
count=$(psql -l "${PGDATABASE}" | grep "${PGDATABASE} " | cut -d \| -f 3 | grep -c UTF8)
count=$(psql -t -l --csv "${PGDATABASE}" | grep "^${PGDATABASE}," | grep -c UTF8)
if test "${count}" -ne 1 ; then
echo
echo "Error : '${PGDATABASE}' database exists, but is not UTF8."
Expand Down Expand Up @@ -164,19 +164,21 @@ function create_snapshot {
tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
echo $"Working directory: ${tmp_dir}"
pg_dump --no-owner --schema=public --jobs="${numcores}" "${PGDATABASE}" --format=directory --file="${tmp_dir}/db/"
lstate_gz_file=$(basename "${ledger_file}").gz
gzip --to-stdout "${ledger_file}" > "${tmp_dir}/$(basename "${ledger_file}").gz"
tree "${tmp_dir}"
# Use plain tar here because the database dump files and the ledger state file are already gzipped.
tar cvf - --directory "${tmp_dir}" "db" "${lstate_gz_file}" | tee "${tgz_file}.tmp" \
| sha256sum | head -c 64 | sed -e "s/$/ ${tgz_file}\n/" > "${tgz_file}.sha256sum"
if [ -n "${ledger_file}" ]; then
lstate_gz_file=$(basename "${ledger_file}").gz
gzip --to-stdout "${ledger_file}" > "${tmp_dir}/${lstate_gz_file}"
fi
# Use plain tar here because the database dump files and the ledger state file are already gzipped. Disable Shellcheck SC2046 to avoid empty '' getting added while quoting
# shellcheck disable=SC2046
tar cvf - --directory "${tmp_dir}" "db" $( [ -n "${lstate_gz_file:-}" ] && [ -f "/${tmp_dir}/${lstate_gz_file}" ] && echo "${lstate_gz_file}" ) | tee "${tgz_file}.tmp" | \
sha256sum | head -c 64 | sed -e "s/$/ ${tgz_file}\n/" > "${tgz_file}.sha256sum"
mv "${tgz_file}.tmp" "${tgz_file}"
rm "${recursive}" "${force}" "${tmp_dir}"
if test "$(tar "${test}" --file "${tgz_file}")" ; then
echo "Tar reports the snapshot file as being corrupt."
echo "It is not safe to drop the database and restore using this file."
exit 1
fi
fi
echo "Created ${tgz_file} + .sha256sum"
}

Expand All @@ -185,24 +187,26 @@ function restore_snapshot {
if test "${file_count}" -gt 0 ; then
echo "Ledger state directory ($2) is not empty. Please empty it and then retry."
exit 1
fi
fi
tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
tar xvf "$1" --directory "$tmp_dir"
if test -d "${tmp_dir}/db/" ; then
# New pg_dump format
lstate_gz_file=$(find "${tmp_dir}/" -iname "*.lstate.gz")
lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
if [ -n "${lstate_gz_file:-}" ] ; then
lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
fi

# Important: specify --schema=public below to skip over `create schema public`
# statement generated by pg_dump
# Important: specify --schema=public below to skip over `create schema public`
# statement generated by pg_dump
# shellcheck disable=SC2046
pg_restore \
--schema=public \
--jobs="${numcores}" \
--format=directory \
--dbname="${PGDATABASE}" \
--no-owner \
--exit-on-error \
--no-owner \
$( [ -z "${SKIP_RESTORE_ERROR:-}" ] && echo "--exit-on-error" ) \
"${tmp_dir}/db/"
else
# Old snapshot format produced by this script
Expand All @@ -228,7 +232,7 @@ function usage_exit {
echo " $progname --dump-schema - Dump the schema of the database."
echo
echo " - Create a db-sync state snapshot"
echo " $progname --create-snapshot <snapshot-file> <ledger-state-file>"
echo " $progname --create-snapshot <snapshot-file> [<ledger-state-file>]"
echo
echo " - Restore a db-sync state snapshot."
echo " $progname --restore-snapshot <snapshot-file> <ledger-state-dir>"
Expand Down Expand Up @@ -300,23 +304,19 @@ case "${1:-""}" in
--create-snapshot)
check_pgpass_file
check_db_exists
if test $# -ne 3 ; then
echo "Expecting exactly 2 more arguments, the snapshot file name template and the ledger state file."
if test $# -lt 2 ; then
echo "Expecting the snapshot file name (without extension) and optionally the ledger state file as arguments."
exit 1
fi
fi
if test -z "$2" ; then
echo "Second argument should be the snapshot file name template."
echo "Second argument should be the snapshot file name (without extension)."
exit 1
fi
if test -z "$3" ; then
echo "Third argument should be the ledger state file."
exit 1
fi
if test -d "$3" ; then
echo "Third argument is a directory and expecting a file."
fi
if test -n "${3:-}" && test -d "${3}" ; then
echo "Third argument provided is a directory but expecting a file."
exit 1
fi
create_snapshot "$2" "$3"
fi
create_snapshot "$2" "${3:-}"
;;
--restore-snapshot)
check_pgpass_file
Expand All @@ -334,15 +334,11 @@ case "${1:-""}" in
echo "Second argument should be the snapshot file."
exit 1
fi
if test -z "$3" ; then
echo "Third argument should be the ledger state directory."
exit 1
fi
if test -f "$3" ; then
if test -n "${3:-}" && test -f "${3}"; then
echo "Third argument is a file and expecting a directory."
exit 1
fi
restore_snapshot "$2" "$3"
restore_snapshot "$2" "${3:-}"
;;
*)
usage_exit
Expand Down

0 comments on commit 16b4cea

Please sign in to comment.