backup before migrating: once per set of pending changes, not on every restart of a crash-looping api

apps/api/src/db/client.ts

@@ -24,19 +24,24 @@ export async function runMigrations(db: Db): Promise<void> {
 }
 
 /**
- * This image has migrations the database hasn't run yet. False for an empty database (a fresh
- * install: nothing to lose). Reads drizzle's own bookkeeping, `drizzle.__drizzle_migrations`.
+ * Migrations this image has and the database hasn't run yet: how many are applied, of how many.
+ * Null when none are, or for an empty database (a fresh install: nothing to lose). Reads drizzle's
+ * own bookkeeping, `drizzle.__drizzle_migrations`.
  */
-export async function migrationsPending(db: Db, folder = MIGRATIONS_DIR): Promise<boolean> {
+export async function migrationsPending(
+  db: Db,
+  folder = MIGRATIONS_DIR,
+): Promise<{ applied: number; total: number } | null> {
   const journal = JSON.parse(await readFile(join(folder, 'meta', '_journal.json'), 'utf8')) as {
     entries: unknown[];
   };
   const t = await db.execute<{ t: string | null }>(
     sql`select to_regclass('drizzle.__drizzle_migrations')::text as t`,
   );
-  if (!t.rows[0]?.t) return false;
+  if (!t.rows[0]?.t) return null;
   const r = await db.execute<{ n: number }>(
     sql`select count(*)::int as n from drizzle.__drizzle_migrations`,
   );
-  return r.rows[0]!.n < journal.entries.length;
+  const applied = r.rows[0]!.n;
+  return applied < journal.entries.length ? { applied, total: journal.entries.length } : null;
 }

apps/api/src/index.ts

@@ -84,12 +84,13 @@ watchWebhooks(db, live, app.log);
 watchEvents(db, live, ami, app.log);
 
 // A new version with database changes: back up first, so a bad update can be undone (08 §3.1).
-if (await migrationsPending(db)) {
+const pending = await migrationsPending(db);
+if (pending) {
   if (config.SKIP_BACKUP_BEFORE_MIGRATE)
     app.log.warn('database changes pending; SKIP_BACKUP_BEFORE_MIGRATE is set, so no backup first');
   else {
-    app.log.info('database changes pending: backing up first');
-    const backup = await backupBeforeMigrate(config, db).catch((err: unknown) => {
+    app.log.info(pending, 'database changes pending: backing up first');
+    const backup = await backupBeforeMigrate(config, db, pending).catch((err: unknown) => {
       app.log.fatal(
         { err, dir: config.BACKUPS_DIR },
         'backup before migrating failed, so the database was left as it is. Fix the cause ' +
@@ -98,7 +99,11 @@ if (await migrationsPending(db)) {
       );
       process.exit(1);
     });
-    app.log.info({ backup: backup.name }, 'backup before migrating made');
+    // update.sh finds this line to restore from (the name, then the message).
+    app.log.info(
+      { backup: backup.name },
+      backup.made ? 'backup before migrating made' : 'backup before migrating made earlier, kept',
+    );
   }
 }
 await runMigrations(db);

apps/api/src/services/backup.ts

@@ -5,7 +5,7 @@
 // Restore is scripts/restore.sh only.
 import { execFile } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { mkdtemp, readdir, rename, rm, stat, writeFile, chmod } from 'node:fs/promises';
+import { chmod, mkdtemp, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { basename, dirname, join, resolve } from 'node:path';
 import { promisify } from 'node:util';
@@ -160,10 +160,29 @@ export function runBackup(config: Config, db: Db): Promise<BackupDto> {
  * backup has, without recordings (big, and no migration touches them). Prunes nothing, so it
  * doesn't push a daily backup out; the daily job's pruning removes it in time. Reads as little of
  * the old schema as it can: the time zone only, falling back to TZ.
+ *
+ * With `step` (the migrations about to run), once per step: a crash-looping api would otherwise
+ * back up on every restart until the disk fills. `.before-migrate.json` in the backups folder
+ * remembers the step and its backup; `made: false` means that one is reused.
  */
-export async function backupBeforeMigrate(config: Config, db: Db): Promise<BackupDto> {
+export async function backupBeforeMigrate(
+  config: Config,
+  db: Db,
+  step?: { applied: number; total: number },
+): Promise<BackupDto & { made: boolean }> {
+  const dir = resolve(config.BACKUPS_DIR);
+  const marker = join(dir, '.before-migrate.json');
+  const key = step && `${step.applied}-${step.total}`;
+  if (key) {
+    const last = await readFile(marker, 'utf8').then(
+      (s) => JSON.parse(s) as { key: string; name: string },
+      () => null,
+    );
+    const kept = last?.key === key && (await listBackups(dir)).find((b) => b.name === last.name);
+    if (kept) return { ...kept, made: false };
+  }
   const timezone = await getSetting(db, 'general.timezone').catch(() => config.TZ);
-  return createBackup(
+  const made = await createBackup(
     {
       databaseUrl: config.DATABASE_URL,
       appSecret: config.APP_SECRET,
@@ -177,6 +196,8 @@ export async function backupBeforeMigrate(config: Config, db: Db): Promise<Backu
     },
     { keep: Infinity },
   );
+  if (key) await writeFile(marker, JSON.stringify({ key, name: made.name }));
+  return { ...made, made: true };
 }
 
 /** The daily backup is due: enabled, past `backup.time` here, and none made today yet. */

apps/api/test/migrations.test.ts

@@ -29,10 +29,11 @@ describe.skipIf(!hasTestDb)('migrationsPending', () => {
     await rm(newer, { recursive: true, force: true });
   });
 
-  it('false when up to date, true when the image has more, false for an empty database', async () => {
-    expect(await migrationsPending(t.db)).toBe(false);
-    expect(await migrationsPending(t.db, newer)).toBe(true);
+  it('none when up to date, counts when the image has more, none for an empty database', async () => {
+    expect(await migrationsPending(t.db)).toBeNull();
+    const pending = await migrationsPending(t.db, newer);
+    expect(pending).toEqual({ applied: pending!.total - 1, total: pending!.total });
     await t.db.execute(sql`drop schema drizzle cascade`);
-    expect(await migrationsPending(t.db, newer)).toBe(false);
+    expect(await migrationsPending(t.db, newer)).toBeNull();
   });
 });