feature: infra

infra/README.md

@@ -278,6 +278,24 @@ cutover) is documented in [`infra/runbooks/production-migration.md`](runbooks/pr
 
 ---
 
+## Backups (production)
+
+`infra/scripts/backup.sh` runs daily at 03:45 on the production server via
+`/etc/cron.d/church-website-backup` (log: `/var/log/church-website-backup.log`):
+
+- **Local** (`/opt/backups` on production): `pg_dump -Fc` of the database +
+  tarball of both upload volumes, 7 days retention (~5 GB).
+- **Off-site** (staging server `178.104.35.59`, `/opt/backups/production`):
+  DB dumps with 14 days retention + an rsync mirror of the uploads (single
+  current copy — the staging server is short on disk, so no tarball history
+  there).
+
+Production's root SSH key (`/root/.ssh/id_ed25519`) is authorized on the
+staging server for the transfer. To restore, use the drop/restore + volume
+steps from the migration runbook with the backup files as source.
+
+---
+
 ## Troubleshooting
 
 ### Build fails with OOM

infra/ansible/inventory/production.yml

@@ -3,7 +3,7 @@ all:
     production-vps:
       ansible_host: 217.154.211.139
       ansible_user: root
-      ansible_ssh_private_key_file: ~/.ssh/id_ed25519
+      ansible_ssh_private_key_file: ~/.ssh/id_rsa
 
   vars:
     # No Forgejo on production — code is pulled from git.skick.app
@@ -19,13 +19,13 @@ all:
     postgres_container_name: postgres
     postgres_image: postgis/postgis:16-3.4
     postgres_volume: pgdata
-    # Tuned for 4 vCPU / 8 GB shared with the Next.js app and on-server builds
+    # Tuned for 4 vCPU / 4 GB shared with the Next.js app and on-server builds
     postgres_server_args: >-
       -c max_connections=40
-      -c shared_buffers=512MB
+      -c shared_buffers=256MB
-      -c effective_cache_size=2GB
+      -c effective_cache_size=1GB
-      -c work_mem=16MB
+      -c work_mem=8MB
-      -c maintenance_work_mem=128MB
+      -c maintenance_work_mem=64MB
       -c checkpoint_completion_target=0.9
       -c max_wal_size=1GB
       -c random_page_cost=1.1

infra/ansible/inventory/test.yml

@@ -26,12 +26,15 @@ all:
 
     # Caddy
     caddy_domains:
-      - domain: mutter-teresa.skick.app
-        proxy_port: 3001
       - domain: mutter-teresa-test.skick.app
         proxy_port: 3002
       - domain: git.skick.app
         proxy_port: 3003
+    # Old staging domain redirects to production (app-staging container is
+    # stopped; the site now lives on the production VPS)
+    caddy_redirects:
+      - from: mutter-teresa.skick.app
+        to: hl-mutter-teresa-chemnitz.de
 
     # Forgejo
     forgejo_domain: git.skick.app

infra/runbooks/production-migration.md

@@ -45,7 +45,7 @@ Sanity checks:
 
 ```bash
 ssh $NEW "curl -sI http://127.0.0.1:3001 | head -1"          # HTTP 200/30x
-ssh $NEW "docker exec postgres psql -U postgres -c 'SHOW shared_buffers;'"   # 512MB
+ssh $NEW "docker exec postgres psql -U postgres -c 'SHOW shared_buffers;'"   # 256MB
 ssh $NEW "swapon --show"                                     # 4G /swapfile
 ```
 
@@ -112,8 +112,12 @@ ssh $OLD "docker run --rm -v uploads-staging-documents:/src:ro alpine tar -C /sr
 
 ### 2c. Restart + verify
 
+Recreate the container rather than just starting it — Next.js caches rendered
+pages in the container's filesystem, and a plain `docker start` serves stale
+pages (e.g. a cached 404 for `/`) from before the restore:
+
 ```bash
-ssh $NEW "docker start app-production"
+ssh $NEW "docker rm -f app-production && docker run -d --name app-production --restart unless-stopped --network church-website-net --env-file /opt/church-website/envs/production/.env -v uploads-production-media:/app/media -v uploads-production-documents:/app/documents -p 127.0.0.1:3001:3000 church-website:production"
 ssh $NEW "docker exec -u 0 app-production chown -R 1001:1001 /app/media /app/documents"
 ssh $NEW "curl -sI http://127.0.0.1:3001 | head -1"          # 200
 ssh $NEW "curl -sI http://127.0.0.1:3001/admin | head -1"    # 200/30x

infra/scripts/backup.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Daily production backup.
+# - Local point-in-time archives (DB dump + uploads tarball), 7 days retention.
+# - Off-site copy on the staging server: DB dumps (14 days) + incremental
+#   uploads mirror (rsync, single current copy — the staging server is short
+#   on disk, so no remote tarball history).
+# Installed at /opt/church-website/scripts/backup.sh, run by
+# /etc/cron.d/church-website-backup.
+set -euo pipefail
+
+DATE=$(date +%F)
+BACKUP_DIR=/opt/backups
+REMOTE=root@178.104.35.59
+REMOTE_DIR=/opt/backups/production
+VOLUMES_DIR=/var/lib/docker/volumes
+
+mkdir -p "$BACKUP_DIR"
+
+# Local point-in-time backups
+docker exec postgres pg_dump -U postgres -Fc church_website > "$BACKUP_DIR/db-$DATE.dump"
+tar -czf "$BACKUP_DIR/uploads-$DATE.tar.gz" -C "$VOLUMES_DIR" \
+  uploads-production-media/_data uploads-production-documents/_data
+
+# Off-site copy to the staging server
+ssh -o StrictHostKeyChecking=accept-new "$REMOTE" "mkdir -p $REMOTE_DIR/db $REMOTE_DIR/uploads/media $REMOTE_DIR/uploads/documents"
+scp -q "$BACKUP_DIR/db-$DATE.dump" "$REMOTE:$REMOTE_DIR/db/"
+rsync -a --delete "$VOLUMES_DIR/uploads-production-media/_data/" "$REMOTE:$REMOTE_DIR/uploads/media/"
+rsync -a --delete "$VOLUMES_DIR/uploads-production-documents/_data/" "$REMOTE:$REMOTE_DIR/uploads/documents/"
+
+# Retention
+find "$BACKUP_DIR" -type f -mtime +7 -delete
+ssh "$REMOTE" "find $REMOTE_DIR/db -type f -mtime +14 -delete"
+
+echo "$(date -Is) backup OK: db-$DATE.dump + uploads-$DATE.tar.gz (local), synced to $REMOTE"