From 2b4d4f7b399c2892acbe8b1953fd5f0aaeb80f41 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 1 Dec 2025 14:58:31 +0400 Subject: [PATCH 1/3] Upgrade PostgreSQL from 13 to 17 in Docker compose file Signed-off-by: tdruez --- CHANGELOG.rst | 7 ++++ Dockerfile | 2 +- docker-compose.yml | 50 ++++++++++++++++++++++++++++- docs/installation.rst | 4 +-- migrate-pg13-to-17.sh | 74 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 4 deletions(-) create mode 100755 migrate-pg13-to-17.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0f49ed140..f4ce709cc4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Changelog ========= +v36.0.0 (unrelease) +-------------------- + +WARNING: This version now requires PostgreSQL v17 or later. +If you are running ScanCode.io using the provided docker-compose.yml, you need to +run the migration script before starting the stack: `./migrate-pg13-to-17.sh` + v35.5.0 (2025-12-01) -------------------- diff --git a/Dockerfile b/Dockerfile index 937fb31d3b..0b552343bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ ENV PYTHONUNBUFFERED=1 # Do not write Python .pyc files ENV PYTHONDONTWRITEBYTECODE=1 # Add the app dir in the Python path for entry points availability -ENV PYTHONPATH=$PYTHONPATH:$APP_DIR +ENV PYTHONPATH=$APP_DIR # OS requirements as per # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html diff --git a/docker-compose.yml b/docker-compose.yml index 2098369fcc..f078e88cf4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,55 @@ name: scancodeio services: + # This service checks the PostgreSQL data version before starting the database. + # PostgreSQL major versions store data in incompatible formats, so starting + # PostgreSQL 17 with data from PostgreSQL 13 would fail or cause corruption. + # + # The check reads the PG_VERSION file from the data volume (mounted read-only) + # and blocks startup if the data is from an older PostgreSQL version. + # + # If this check fails, run the migration script before starting the stack: + # ./migrate-pg13-to-17.sh + # + # For fresh installations (no existing data), this check passes automatically. + db-check: + image: docker.io/library/postgres:17 + volumes: + - db_data:/var/lib/postgresql/data/:ro + entrypoint: [ "/bin/bash", "-c" ] + command: + - | + DATA_DIR="/var/lib/postgresql/data" + if [ ! -f "$$DATA_DIR/PG_VERSION" ]; then + echo "Fresh install detected, no upgrade needed." + exit 0 + fi + + OLD_VERSION=$$(cat "$$DATA_DIR/PG_VERSION") + echo "Found PostgreSQL data version: $$OLD_VERSION" + + if [ "$$OLD_VERSION" -lt 17 ]; then + echo "" + echo "╔════════════════════════════════════════════════════════════════════╗" + echo "║ ERROR: PostgreSQL $$OLD_VERSION data detected, version 17 required ║" + echo "╠════════════════════════════════════════════════════════════════════╣" + echo "║ Your database volume contains data from an older PostgreSQL. ║" + echo "║ ║" + echo "║ To migrate, run: ║" + echo "║ ./migrate-pg13-to-17.sh ║" + echo "║ ║" + echo "╚════════════════════════════════════════════════════════════════════╝" + echo "" + exit 1 + fi + + echo "PostgreSQL version OK." + restart: "no" + db: - image: docker.io/library/postgres:13 + image: docker.io/library/postgres:17 + depends_on: + db-check: + condition: service_completed_successfully env_file: - docker.env volumes: diff --git a/docs/installation.rst b/docs/installation.rst index 2f38728beb..b29de683a2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -285,7 +285,7 @@ Before you install ScanCode.io, make sure you have the following prerequisites: * **Python: versions 3.10 to 3.13** found at https://www.python.org/downloads/ * **Git**: most recent release available at https://git-scm.com/ - * **PostgreSQL**: release 13 or later found at https://www.postgresql.org/ or + * **PostgreSQL**: release 17 or later found at https://www.postgresql.org/ or https://postgresapp.com/ on macOS .. _system_dependencies: @@ -519,7 +519,7 @@ Once Helm is properly set up, add the ``scancode-kube`` repo as follows:: # sample output # NAME VERSION REPOSITORY STATUS # nginx 9.x.x https://charts.bitnami.com/bitnami ok - # postgresql 11.x.x https://charts.bitnami.com/bitnami ok + # postgresql 17.x.x https://charts.bitnami.com/bitnami ok # redis 16.x.x https://charts.bitnami.com/bitnami ok # install scancode helm charts diff --git a/migrate-pg13-to-17.sh b/migrate-pg13-to-17.sh new file mode 100755 index 0000000000..949908bfc8 --- /dev/null +++ b/migrate-pg13-to-17.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -e +echo "=== PostgreSQL 13 to 17 Migration ===" + +POSTGRES_DB="scancodeio" +POSTGRES_USER="scancodeio" +BACKUP_FILE="backup_pg13_$(date +%Y%m%d_%H%M%S).dump" +VOLUME_NAME="scancodeio_db_data" +VOLUME_BACKUP="${VOLUME_NAME}_pg13_backup" + +# Stop all compose services first +echo "Stopping all services..." +docker compose down + +# Cleanup any leftover container from previous run +docker rm -f pg13_backup 2>/dev/null || true + +# Check volume exists +if ! docker volume inspect "$VOLUME_NAME" &>/dev/null; then + echo "ERROR: Volume $VOLUME_NAME not found" + exit 1 +fi + +echo "Step 1/5: Starting temporary PG13 container for backup..." +docker run -d --name pg13_backup \ + -v "$VOLUME_NAME":/var/lib/postgresql/data \ + postgres:13 + +echo " Waiting for PG13 to be ready..." +until docker exec pg13_backup pg_isready 2>/dev/null; do + sleep 2 +done + +echo "Step 2/5: Creating backup of $POSTGRES_DB (this may take a while)..." +docker exec pg13_backup pg_dump -U "$POSTGRES_USER" -Fc "$POSTGRES_DB" > "$BACKUP_FILE" +BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1) +echo " Backup saved to: $BACKUP_FILE ($BACKUP_SIZE)" + +if [ ! -s "$BACKUP_FILE" ]; then + echo "ERROR: Backup file is empty" + docker stop pg13_backup && docker rm pg13_backup + exit 1 +fi + +echo "Step 3/5: Stopping temporary container and renaming old volume..." +docker stop pg13_backup && docker rm pg13_backup + +docker volume create "$VOLUME_BACKUP" +docker run --rm \ + -v "$VOLUME_NAME":/from:ro \ + -v "$VOLUME_BACKUP":/to \ + alpine sh -c "cp -a /from/. /to/" +docker volume rm "$VOLUME_NAME" +echo " Old volume preserved as: $VOLUME_BACKUP" + +echo "Step 4/5: Starting fresh PG17..." +docker compose up -d db +echo " Waiting for PG17 to be ready..." +until docker compose exec -T db pg_isready 2>/dev/null; do + sleep 2 +done + +echo "Step 5/5: Restoring data (this may take a while)..." +docker compose exec -T db pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --jobs=4 --no-owner --no-acl < "$BACKUP_FILE" + +echo "" +echo "=== Migration complete! ===" +echo "Backup retained at: $BACKUP_FILE" +echo "Old volume preserved as: $VOLUME_BACKUP" +echo "" +echo "Once verified, you can delete the old volume with:" +echo " docker volume rm $VOLUME_BACKUP" +echo "" +echo "Verify with: docker compose exec db psql -U scancodeio -c 'SELECT version();'" From 88220d6dc6c10419bac46fea403f93015293cb2d Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 2 Dec 2025 15:52:25 +0400 Subject: [PATCH 2/3] Refine the PostgreSQL 13 to 17 migration script Signed-off-by: tdruez --- migrate-pg13-to-17.sh | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/migrate-pg13-to-17.sh b/migrate-pg13-to-17.sh index 949908bfc8..4c458a942a 100755 --- a/migrate-pg13-to-17.sh +++ b/migrate-pg13-to-17.sh @@ -1,13 +1,39 @@ #!/bin/bash +# ============================================================================= +# PostgreSQL 13 to 17 Migration Script for ScanCode.io +# ============================================================================= +# +# This script migrates the PostgreSQL database from version 13 to 17. +# +# Usage: +# ./migrate-pg13-to-17.sh [backup_directory] +# +# Arguments: +# backup_directory Optional. Directory to store the backup file. +# Defaults to current directory. +# +# Examples: +# ./migrate-pg13-to-17.sh +# ./migrate-pg13-to-17.sh /path/to/backups +# +# ============================================================================= + set -e echo "=== PostgreSQL 13 to 17 Migration ===" POSTGRES_DB="scancodeio" POSTGRES_USER="scancodeio" -BACKUP_FILE="backup_pg13_$(date +%Y%m%d_%H%M%S).dump" +BACKUP_DIR="${1:-.}" +BACKUP_FILE="$BACKUP_DIR/backup_pg13_$(date +%Y%m%d_%H%M%S).dump" VOLUME_NAME="scancodeio_db_data" VOLUME_BACKUP="${VOLUME_NAME}_pg13_backup" +# Check backup directory exists +if [ ! -d "$BACKUP_DIR" ]; then + echo "ERROR: Backup directory $BACKUP_DIR does not exist" + exit 1 +fi + # Stop all compose services first echo "Stopping all services..." docker compose down @@ -61,7 +87,9 @@ until docker compose exec -T db pg_isready 2>/dev/null; do done echo "Step 5/5: Restoring data (this may take a while)..." -docker compose exec -T db pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --jobs=4 --no-owner --no-acl < "$BACKUP_FILE" +docker cp "$BACKUP_FILE" scancodeio-db-1:/tmp/backup.dump +docker compose exec -T db pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --no-acl /tmp/backup.dump +docker compose exec -T db rm /tmp/backup.dump echo "" echo "=== Migration complete! ===" From f1ff580a2be002414290e401e6b2306d5938a614 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 3 Dec 2025 09:14:22 +0400 Subject: [PATCH 3/3] Improve the "Breaking Change" warning Signed-off-by: tdruez --- CHANGELOG.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f4ce709cc4..e7a0871f31 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,12 +1,14 @@ Changelog ========= -v36.0.0 (unrelease) +v36.0.0 (unreleased) -------------------- -WARNING: This version now requires PostgreSQL v17 or later. -If you are running ScanCode.io using the provided docker-compose.yml, you need to -run the migration script before starting the stack: `./migrate-pg13-to-17.sh` +**Breaking Change:** PostgreSQL 17 is now required (previously 13). + +Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before starting +the stack. +Fresh installations require no action. v35.5.0 (2025-12-01) --------------------