diff --git a/infra/README.md b/infra/README.md index dd487d5..6ff4e0b 100644 --- a/infra/README.md +++ b/infra/README.md @@ -278,6 +278,24 @@ cutover) is documented in [`infra/runbooks/production-migration.md`](runbooks/pr --- +## Backups (production) + +`infra/scripts/backup.sh` runs daily at 03:45 on the production server via +`/etc/cron.d/church-website-backup` (log: `/var/log/church-website-backup.log`): + +- **Local** (`/opt/backups` on production): `pg_dump -Fc` of the database + + tarball of both upload volumes, 7 days retention (~5 GB). +- **Off-site** (staging server `178.104.35.59`, `/opt/backups/production`): + DB dumps with 14 days retention + an rsync mirror of the uploads (single + current copy — the staging server is short on disk, so no tarball history + there). + +Production's root SSH key (`/root/.ssh/id_ed25519`) is authorized on the +staging server for the transfer. To restore, use the drop/restore + volume +steps from the migration runbook with the backup files as source. + +--- + ## Troubleshooting ### Build fails with OOM diff --git a/infra/ansible/inventory/production.yml b/infra/ansible/inventory/production.yml index abad405..ed2809e 100644 --- a/infra/ansible/inventory/production.yml +++ b/infra/ansible/inventory/production.yml @@ -3,7 +3,7 @@ all: production-vps: ansible_host: 217.154.211.139 ansible_user: root - ansible_ssh_private_key_file: ~/.ssh/id_ed25519 + ansible_ssh_private_key_file: ~/.ssh/id_rsa vars: # No Forgejo on production — code is pulled from git.skick.app @@ -19,13 +19,13 @@ all: postgres_container_name: postgres postgres_image: postgis/postgis:16-3.4 postgres_volume: pgdata - # Tuned for 4 vCPU / 8 GB shared with the Next.js app and on-server builds + # Tuned for 4 vCPU / 4 GB shared with the Next.js app and on-server builds postgres_server_args: >- -c max_connections=40 - -c shared_buffers=512MB - -c effective_cache_size=2GB - -c work_mem=16MB - -c maintenance_work_mem=128MB + -c shared_buffers=256MB + -c effective_cache_size=1GB + -c work_mem=8MB + -c maintenance_work_mem=64MB -c checkpoint_completion_target=0.9 -c max_wal_size=1GB -c random_page_cost=1.1 diff --git a/infra/ansible/inventory/test.yml b/infra/ansible/inventory/test.yml index 42877c8..ed8a6cd 100644 --- a/infra/ansible/inventory/test.yml +++ b/infra/ansible/inventory/test.yml @@ -26,12 +26,15 @@ all: # Caddy caddy_domains: - - domain: mutter-teresa.skick.app - proxy_port: 3001 - domain: mutter-teresa-test.skick.app proxy_port: 3002 - domain: git.skick.app proxy_port: 3003 + # Old staging domain redirects to production (app-staging container is + # stopped; the site now lives on the production VPS) + caddy_redirects: + - from: mutter-teresa.skick.app + to: hl-mutter-teresa-chemnitz.de # Forgejo forgejo_domain: git.skick.app diff --git a/infra/runbooks/production-migration.md b/infra/runbooks/production-migration.md index 8148837..ae5526b 100644 --- a/infra/runbooks/production-migration.md +++ b/infra/runbooks/production-migration.md @@ -45,7 +45,7 @@ Sanity checks: ```bash ssh $NEW "curl -sI http://127.0.0.1:3001 | head -1" # HTTP 200/30x -ssh $NEW "docker exec postgres psql -U postgres -c 'SHOW shared_buffers;'" # 512MB +ssh $NEW "docker exec postgres psql -U postgres -c 'SHOW shared_buffers;'" # 256MB ssh $NEW "swapon --show" # 4G /swapfile ``` @@ -112,8 +112,12 @@ ssh $OLD "docker run --rm -v uploads-staging-documents:/src:ro alpine tar -C /sr ### 2c. Restart + verify +Recreate the container rather than just starting it — Next.js caches rendered +pages in the container's filesystem, and a plain `docker start` serves stale +pages (e.g. a cached 404 for `/`) from before the restore: + ```bash -ssh $NEW "docker start app-production" +ssh $NEW "docker rm -f app-production && docker run -d --name app-production --restart unless-stopped --network church-website-net --env-file /opt/church-website/envs/production/.env -v uploads-production-media:/app/media -v uploads-production-documents:/app/documents -p 127.0.0.1:3001:3000 church-website:production" ssh $NEW "docker exec -u 0 app-production chown -R 1001:1001 /app/media /app/documents" ssh $NEW "curl -sI http://127.0.0.1:3001 | head -1" # 200 ssh $NEW "curl -sI http://127.0.0.1:3001/admin | head -1" # 200/30x diff --git a/infra/scripts/backup.sh b/infra/scripts/backup.sh new file mode 100644 index 0000000..77d575e --- /dev/null +++ b/infra/scripts/backup.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Daily production backup. +# - Local point-in-time archives (DB dump + uploads tarball), 7 days retention. +# - Off-site copy on the staging server: DB dumps (14 days) + incremental +# uploads mirror (rsync, single current copy — the staging server is short +# on disk, so no remote tarball history). +# Installed at /opt/church-website/scripts/backup.sh, run by +# /etc/cron.d/church-website-backup. +set -euo pipefail + +DATE=$(date +%F) +BACKUP_DIR=/opt/backups +REMOTE=root@178.104.35.59 +REMOTE_DIR=/opt/backups/production +VOLUMES_DIR=/var/lib/docker/volumes + +mkdir -p "$BACKUP_DIR" + +# Local point-in-time backups +docker exec postgres pg_dump -U postgres -Fc church_website > "$BACKUP_DIR/db-$DATE.dump" +tar -czf "$BACKUP_DIR/uploads-$DATE.tar.gz" -C "$VOLUMES_DIR" \ + uploads-production-media/_data uploads-production-documents/_data + +# Off-site copy to the staging server +ssh -o StrictHostKeyChecking=accept-new "$REMOTE" "mkdir -p $REMOTE_DIR/db $REMOTE_DIR/uploads/media $REMOTE_DIR/uploads/documents" +scp -q "$BACKUP_DIR/db-$DATE.dump" "$REMOTE:$REMOTE_DIR/db/" +rsync -a --delete "$VOLUMES_DIR/uploads-production-media/_data/" "$REMOTE:$REMOTE_DIR/uploads/media/" +rsync -a --delete "$VOLUMES_DIR/uploads-production-documents/_data/" "$REMOTE:$REMOTE_DIR/uploads/documents/" + +# Retention +find "$BACKUP_DIR" -type f -mtime +7 -delete +ssh "$REMOTE" "find $REMOTE_DIR/db -type f -mtime +14 -delete" + +echo "$(date -Is) backup OK: db-$DATE.dump + uploads-$DATE.tar.gz (local), synced to $REMOTE"