mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
fix(series): repair legacy series index upgrades
Normalize legacy float values before migration 0076 so 2.7 databases can upgrade without constraint failures or precision loss. Serialize the compatibility conversion across concurrent startup processes. Fixes #1189
This commit is contained in:
neon
@@ -304,6 +304,24 @@ export const E2E_SUITES = Object.freeze({
|
||||
...SHARED_DB_AND_HELPER_PATHS,
|
||||
],
|
||||
},
|
||||
"series-index-migration": {
|
||||
id: "series-index-migration",
|
||||
name: "Series Index Migration",
|
||||
timeout: 60,
|
||||
lane: "full",
|
||||
description: "Legacy series index upgrade compatibility suite",
|
||||
vitestTarget: "test/series-index-migration.e2e-spec.ts",
|
||||
junitOutput: `${TEST_RESULTS_DIR}/series-index-migration-e2e-junit.xml`,
|
||||
prepareDedicatedDatabase: true,
|
||||
useDedicatedDatabase: true,
|
||||
changedPaths: [
|
||||
"server/src/db/migrations/**",
|
||||
"server/src/scripts/migrate.ts",
|
||||
"server/src/scripts/series-index-migration-compatibility.ts",
|
||||
"server/test/series-index-migration.e2e-spec.ts",
|
||||
...SHARED_DB_AND_HELPER_PATHS,
|
||||
],
|
||||
},
|
||||
"reading-attempts": {
|
||||
id: "reading-attempts",
|
||||
name: "Reading Attempts",
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
"start:debug": "nest start --debug --watch --config nest-cli.dev.json",
|
||||
"start:prod": "node dist/main",
|
||||
"db:generate": "drizzle-kit generate --name",
|
||||
"db:migrate": "tsx --env-file-if-exists=.env src/scripts/install-postgres-extensions.ts && drizzle-kit migrate",
|
||||
"db:migrate": "tsx --env-file-if-exists=.env src/scripts/migrate.ts",
|
||||
"e2e:db:migrate": "DATABASE_URL=postgres://bookorbit:bookorbit@localhost:5432/bookorbit_e2e pnpm db:migrate",
|
||||
"db:migrate:runtime": "node dist/scripts/migrate.js",
|
||||
"e2e:db:prepare": "tsx src/scripts/db-prepare-e2e.ts",
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
import { Pool } from 'pg';
|
||||
|
||||
import { createPostgresClientConfig } from '../db/postgres-connection-config';
|
||||
import { installPostgresExtensions } from './postgres-extensions';
|
||||
|
||||
async function run(): Promise<void> {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
if (!connectionString) throw new Error('DATABASE_URL is required');
|
||||
|
||||
const pool = new Pool(createPostgresClientConfig(connectionString));
|
||||
try {
|
||||
await installPostgresExtensions(pool);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
void run();
|
||||
@@ -6,6 +6,7 @@ import { Pool } from 'pg';
|
||||
|
||||
import { createPostgresClientConfig } from '../db/postgres-connection-config';
|
||||
import { installPostgresExtensions } from './postgres-extensions';
|
||||
import { prepareLegacySeriesIndexColumns } from './series-index-migration-compatibility';
|
||||
|
||||
function resolveMigrationsFolder(): string {
|
||||
const candidates = [
|
||||
@@ -38,6 +39,7 @@ async function runMigrations() {
|
||||
|
||||
try {
|
||||
await installPostgresExtensions(pool);
|
||||
await prepareLegacySeriesIndexColumns(pool);
|
||||
|
||||
const migrationsFolder = resolveMigrationsFolder();
|
||||
await migrate(drizzle(pool), { migrationsFolder });
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { Pool } from 'pg';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { prepareLegacySeriesIndexColumns } from './series-index-migration-compatibility';
|
||||
|
||||
function createPool(rows: Array<{ tableName: string }>, counts: Record<string, string> = {}) {
|
||||
const query = vi.fn((statement: string) => {
|
||||
if (statement.includes('information_schema.columns')) return Promise.resolve({ rows });
|
||||
const table = Object.keys(counts).find((name) => statement.includes(`FROM public.${name}`));
|
||||
if (table) return Promise.resolve({ rows: [{ count: counts[table] }] });
|
||||
return Promise.resolve({ rows: [] });
|
||||
});
|
||||
const release = vi.fn();
|
||||
const pool = { connect: vi.fn(() => Promise.resolve({ query, release })) } as unknown as Pick<Pool, 'connect'>;
|
||||
return { pool, query, release };
|
||||
}
|
||||
|
||||
describe('prepareLegacySeriesIndexColumns', () => {
|
||||
it('does nothing when the legacy real columns are absent', async () => {
|
||||
const { pool, query, release } = createPool([]);
|
||||
|
||||
await expect(prepareLegacySeriesIndexColumns(pool)).resolves.toEqual({ convertedColumns: 0, clearedValues: 0 });
|
||||
expect(query).toHaveBeenCalledTimes(4);
|
||||
expect(query).toHaveBeenNthCalledWith(1, 'BEGIN');
|
||||
expect(query).toHaveBeenNthCalledWith(2, `SELECT pg_advisory_xact_lock(hashtext('bookorbit'), hashtext('series-index-migration-compatibility'))`);
|
||||
expect(query).toHaveBeenLastCalledWith('COMMIT');
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('normalizes both legacy columns before migration 0076 runs', async () => {
|
||||
const { pool, query } = createPool([{ tableName: 'book_metadata' }, { tableName: 'book_series_memberships' }], {
|
||||
book_metadata: '2',
|
||||
book_series_memberships: '1',
|
||||
});
|
||||
|
||||
await expect(prepareLegacySeriesIndexColumns(pool)).resolves.toEqual({ convertedColumns: 2, clearedValues: 3 });
|
||||
|
||||
const statements = query.mock.calls.map(([statement]) => statement);
|
||||
const counts = statements.filter((statement) => statement.includes('SELECT count(*)::text AS count'));
|
||||
const alters = statements.filter((statement) => statement.includes('ALTER TABLE'));
|
||||
expect(counts).toHaveLength(2);
|
||||
for (const statement of counts) {
|
||||
expect(statement).toContain('length(trim_scale(series_index::text::numeric)::text) > 20');
|
||||
}
|
||||
expect(alters).toHaveLength(2);
|
||||
expect(alters).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining('ALTER TABLE public.book_metadata'),
|
||||
expect.stringContaining('ALTER TABLE public.book_series_memberships'),
|
||||
]),
|
||||
);
|
||||
for (const statement of alters) {
|
||||
expect(statement).toContain('trim_scale(series_index::text::numeric)::text');
|
||||
expect(statement).toContain("series_index::text IN ('NaN', 'Infinity', '-Infinity')");
|
||||
expect(statement).toContain('WHEN series_index < 0 THEN NULL');
|
||||
expect(statement).toContain('WHEN length(trim_scale(series_index::text::numeric)::text) > 20 THEN NULL');
|
||||
}
|
||||
});
|
||||
|
||||
it('rolls back and releases the client when conversion fails', async () => {
|
||||
const query = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ rows: [] })
|
||||
.mockResolvedValueOnce({ rows: [] })
|
||||
.mockResolvedValueOnce({ rows: [{ tableName: 'book_metadata' }] })
|
||||
.mockResolvedValueOnce({ rows: [{ count: '0' }] })
|
||||
.mockRejectedValueOnce(new Error('conversion failed'))
|
||||
.mockResolvedValueOnce({ rows: [] });
|
||||
const release = vi.fn();
|
||||
const pool = { connect: vi.fn(() => Promise.resolve({ query, release })) } as unknown as Pick<Pool, 'connect'>;
|
||||
|
||||
await expect(prepareLegacySeriesIndexColumns(pool)).rejects.toThrow('conversion failed');
|
||||
expect(query).toHaveBeenLastCalledWith('ROLLBACK');
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { Pool } from 'pg';
|
||||
|
||||
import { sanitizeLogValue } from '../common/utils/log-sanitize.utils';
|
||||
|
||||
const LEGACY_SERIES_INDEX_TABLES = ['book_series_memberships', 'book_metadata'] as const;
|
||||
|
||||
type LegacySeriesIndexTable = (typeof LEGACY_SERIES_INDEX_TABLES)[number];
|
||||
|
||||
type CompatibilityResult = {
|
||||
convertedColumns: number;
|
||||
clearedValues: number;
|
||||
};
|
||||
|
||||
function isLegacySeriesIndexTable(value: string): value is LegacySeriesIndexTable {
|
||||
return LEGACY_SERIES_INDEX_TABLES.includes(value as LegacySeriesIndexTable);
|
||||
}
|
||||
|
||||
export async function prepareLegacySeriesIndexColumns(pool: Pick<Pool, 'connect'>): Promise<CompatibilityResult> {
|
||||
const client = await pool.connect();
|
||||
const startedAt = Date.now();
|
||||
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
await client.query(`SELECT pg_advisory_xact_lock(hashtext('bookorbit'), hashtext('series-index-migration-compatibility'))`);
|
||||
const columns = await client.query<{ tableName: string }>(`
|
||||
SELECT table_name AS "tableName"
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND column_name = 'series_index'
|
||||
AND data_type = 'real'
|
||||
AND table_name IN ('book_series_memberships', 'book_metadata')
|
||||
ORDER BY table_name
|
||||
`);
|
||||
const tables = columns.rows.map((row) => row.tableName).filter(isLegacySeriesIndexTable);
|
||||
|
||||
if (tables.length === 0) {
|
||||
await client.query('COMMIT');
|
||||
return { convertedColumns: 0, clearedValues: 0 };
|
||||
}
|
||||
|
||||
console.log(`[db.series-index-compatibility] [start] columns=${tables.length} - legacy series index conversion started`);
|
||||
|
||||
let clearedValues = 0;
|
||||
for (const table of tables) {
|
||||
const countResult = await client.query<{ count: string }>(`
|
||||
SELECT count(*)::text AS count
|
||||
FROM public.${table}
|
||||
WHERE series_index IS NOT NULL
|
||||
AND (
|
||||
series_index::text IN ('NaN', 'Infinity', '-Infinity')
|
||||
OR series_index < 0
|
||||
OR length(trim_scale(series_index::text::numeric)::text) > 20
|
||||
)
|
||||
`);
|
||||
clearedValues += Number.parseInt(countResult.rows[0]?.count ?? '0', 10);
|
||||
|
||||
await client.query(`
|
||||
ALTER TABLE public.${table}
|
||||
ALTER COLUMN series_index TYPE varchar(20)
|
||||
USING CASE
|
||||
WHEN series_index IS NULL THEN NULL
|
||||
WHEN series_index::text IN ('NaN', 'Infinity', '-Infinity') THEN NULL
|
||||
WHEN series_index < 0 THEN NULL
|
||||
WHEN length(trim_scale(series_index::text::numeric)::text) > 20 THEN NULL
|
||||
ELSE trim_scale(series_index::text::numeric)::text
|
||||
END
|
||||
`);
|
||||
}
|
||||
|
||||
await client.query('COMMIT');
|
||||
const endMessage = `[db.series-index-compatibility] [end] durationMs=${Date.now() - startedAt} columns=${tables.length} cleared=${clearedValues} - legacy series index conversion completed`;
|
||||
if (clearedValues > 0) console.warn(endMessage);
|
||||
else console.log(endMessage);
|
||||
return { convertedColumns: tables.length, clearedValues };
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
const errorClass = error instanceof Error ? error.name : typeof error;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(
|
||||
`[db.series-index-compatibility] [fail] durationMs=${Date.now() - startedAt} errorClass=${errorClass} error="${sanitizeLogValue(message)}" - legacy series index conversion failed`,
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { readFile, readdir } from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { Client, Pool } from 'pg';
|
||||
|
||||
import { prepareLegacySeriesIndexColumns } from '../src/scripts/series-index-migration-compatibility';
|
||||
|
||||
const MIGRATIONS_DIRECTORY = join(dirname(fileURLToPath(import.meta.url)), '../src/db/migrations');
|
||||
|
||||
function escapeIdentifier(identifier: string): string {
|
||||
return `"${identifier.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
async function applyMigrations(client: Client, filenames: string[]): Promise<void> {
|
||||
for (const filename of filenames) {
|
||||
await client.query(await readFile(join(MIGRATIONS_DIRECTORY, filename), 'utf8'));
|
||||
}
|
||||
}
|
||||
|
||||
describe('Series index migration compatibility (e2e)', { timeout: 60_000 }, () => {
|
||||
it('repairs legacy values before applying the 2.8 series index migration', async () => {
|
||||
const baseUrl = new URL(process.env.DATABASE_URL!);
|
||||
const databaseName = `series_index_${randomUUID().replace(/-/g, '').slice(0, 12)}_e2e`;
|
||||
const targetUrl = new URL(baseUrl);
|
||||
targetUrl.pathname = `/${databaseName}`;
|
||||
const adminUrl = new URL(baseUrl);
|
||||
adminUrl.pathname = '/postgres';
|
||||
|
||||
const admin = new Client({ connectionString: adminUrl.toString() });
|
||||
let target: Client | undefined;
|
||||
let pool: Pool | undefined;
|
||||
await admin.connect();
|
||||
|
||||
try {
|
||||
await admin.query(`CREATE DATABASE ${escapeIdentifier(databaseName)}`);
|
||||
target = new Client({ connectionString: targetUrl.toString() });
|
||||
await target.connect();
|
||||
await target.query(`
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm";
|
||||
CREATE EXTENSION IF NOT EXISTS "unaccent";
|
||||
CREATE EXTENSION IF NOT EXISTS "vector";
|
||||
CREATE OR REPLACE FUNCTION public.bookorbit_unaccent(value text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE
|
||||
PARALLEL SAFE
|
||||
STRICT
|
||||
AS $function$ SELECT public.unaccent('public.unaccent', value) $function$;
|
||||
`);
|
||||
|
||||
const migrationFiles = (await readdir(MIGRATIONS_DIRECTORY)).filter((filename) => filename.endsWith('.sql')).sort();
|
||||
await applyMigrations(
|
||||
target,
|
||||
migrationFiles.filter((filename) => filename < '0076_add_series_index_labels.sql'),
|
||||
);
|
||||
await target.query(`
|
||||
WITH new_library AS (
|
||||
INSERT INTO libraries (name) VALUES ('Series Index Migration') RETURNING id
|
||||
), new_folder AS (
|
||||
INSERT INTO library_folders (library_id, path)
|
||||
SELECT id, '/series-index-migration' FROM new_library
|
||||
RETURNING id, library_id
|
||||
), new_book AS (
|
||||
INSERT INTO books (library_id, library_folder_id, folder_path)
|
||||
SELECT library_id, id, '/series-index-migration/book' FROM new_folder
|
||||
RETURNING id
|
||||
), new_series AS (
|
||||
INSERT INTO book_series (name, normalized_name)
|
||||
VALUES ('Legacy Series', 'legacy series')
|
||||
RETURNING id
|
||||
), new_membership AS (
|
||||
INSERT INTO book_series_memberships (book_id, series_id, series_index)
|
||||
SELECT new_book.id, new_series.id, -1::real
|
||||
FROM new_book, new_series
|
||||
)
|
||||
INSERT INTO book_metadata (book_id, series_index)
|
||||
SELECT id, 1e-5::real FROM new_book;
|
||||
`);
|
||||
await target.query(`
|
||||
WITH samples(folder_path, series_index) AS (
|
||||
VALUES
|
||||
('/series-index-migration/precision-1234567', 123456.7::real),
|
||||
('/series-index-migration/precision-16777215', 16777215::real),
|
||||
('/series-index-migration/precision-10485765', 1048576.5::real)
|
||||
), folder AS (
|
||||
SELECT id, library_id
|
||||
FROM library_folders
|
||||
WHERE path = '/series-index-migration'
|
||||
), new_books AS (
|
||||
INSERT INTO books (library_id, library_folder_id, folder_path)
|
||||
SELECT folder.library_id, folder.id, samples.folder_path
|
||||
FROM folder, samples
|
||||
RETURNING id, folder_path
|
||||
)
|
||||
INSERT INTO book_metadata (book_id, series_index)
|
||||
SELECT new_books.id, samples.series_index
|
||||
FROM new_books
|
||||
JOIN samples USING (folder_path);
|
||||
`);
|
||||
|
||||
const seriesIndexMigration = await readFile(join(MIGRATIONS_DIRECTORY, '0076_add_series_index_labels.sql'), 'utf8');
|
||||
await target.query('BEGIN');
|
||||
await expect(target.query(seriesIndexMigration)).rejects.toThrow('book_series_memberships_series_index_format_chk');
|
||||
await target.query('ROLLBACK');
|
||||
|
||||
pool = new Pool({ connectionString: targetUrl.toString() });
|
||||
const compatibilityResults = await Promise.all([prepareLegacySeriesIndexColumns(pool), prepareLegacySeriesIndexColumns(pool)]);
|
||||
expect(compatibilityResults).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ convertedColumns: 2, clearedValues: 1 },
|
||||
{ convertedColumns: 0, clearedValues: 0 },
|
||||
]),
|
||||
);
|
||||
await target.query(seriesIndexMigration);
|
||||
|
||||
const values = await target.query<{ source: string; seriesIndex: string | null }>(`
|
||||
SELECT source, "seriesIndex"
|
||||
FROM (
|
||||
SELECT 0 AS sort_order, 'membership' AS source, series_index AS "seriesIndex" FROM book_series_memberships
|
||||
UNION ALL
|
||||
SELECT 1, books.folder_path, book_metadata.series_index
|
||||
FROM book_metadata
|
||||
INNER JOIN books ON books.id = book_metadata.book_id
|
||||
) AS migrated_values
|
||||
ORDER BY sort_order, source
|
||||
`);
|
||||
expect(values.rows).toEqual([
|
||||
{ source: 'membership', seriesIndex: null },
|
||||
{ source: '/series-index-migration/book', seriesIndex: '0.00001' },
|
||||
{ source: '/series-index-migration/precision-10485765', seriesIndex: '1048576.5' },
|
||||
{ source: '/series-index-migration/precision-1234567', seriesIndex: '123456.7' },
|
||||
{ source: '/series-index-migration/precision-16777215', seriesIndex: '16777215' },
|
||||
]);
|
||||
|
||||
await applyMigrations(
|
||||
target,
|
||||
migrationFiles.filter((filename) => filename > '0076_add_series_index_labels.sql'),
|
||||
);
|
||||
} finally {
|
||||
await pool?.end();
|
||||
await target?.end();
|
||||
await admin.query(`DROP DATABASE IF EXISTS ${escapeIdentifier(databaseName)} WITH (FORCE)`);
|
||||
await admin.end();
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user