fix(scanner): repair incomplete audiobook durations

Repair legacy null durations during ordinary and incremental scans. Derive completion only after every asset has a measured duration.

Fixes #1498
This commit is contained in:
neon
2026-09-23 00:05:32 +00:00
parent c25a756ed7
commit 759ec730d9
5 changed files with 198 additions and 20 deletions
@@ -150,6 +150,47 @@ describe('AudiobookService', () => {
); );
}); });
it('uses complete per-file durations instead of stale aggregate metadata', async () => {
const { service, bookService } = makeFixture();
const detail = await bookService.getDetail();
bookService.getDetail.mockResolvedValue({
...detail,
audioMetadata: { ...detail.audioMetadata, durationSeconds: 60 },
});
const manifest = await service.getManifest(30, makeUser());
expect(manifest.totalDurationMs).toBe(30_125);
});
it('does not derive false completion while any asset duration is missing', async () => {
const { service, repo, bookService } = makeFixture();
const files = await repo.findAudioFiles();
repo.findAudioFiles.mockResolvedValue(files.map((file, index) => (index === 0 ? { ...file, durationSeconds: null } : file)));
const manifest = await service.getManifest(30, makeUser());
repo.createPlaybackState.mockResolvedValue({
capturedAt: new Date('2026-02-01T00:00:00.000Z'),
revision: 1,
});
const saved = await service.putPlaybackState(
30,
{
assetId: manifest.assets[1]!.assetId,
positionMs: 20_000,
capturedAt: '2026-02-01T00:00:00.000Z',
operationId: '99999999-9999-4999-8999-999999999999',
baseRevision: 0,
manifestRevision: manifest.revision,
},
makeUser(),
);
expect(saved.percentage).toBe(0);
expect(saved.completed).toBe(false);
expect(bookService.autoUpdateReadStatusForProgress).toHaveBeenCalledWith(7, { bookId: 30, libraryId: 4 }, 0, {});
});
it('syncs EPUB progress only after the revision-controlled write is accepted', async () => { it('syncs EPUB progress only after the revision-controlled write is accepted', async () => {
const { service, repo, bookService } = makeFixture(); const { service, repo, bookService } = makeFixture();
const manifest = await service.getManifest(30, makeUser()); const manifest = await service.getManifest(30, makeUser());
@@ -28,6 +28,7 @@ interface ManifestContext {
manifest: AudiobookManifest; manifest: AudiobookManifest;
files: AudioFileRow[]; files: AudioFileRow[];
libraryId: number; libraryId: number;
hasCompleteDurations: boolean;
} }
@Injectable() @Injectable()
@@ -99,7 +100,9 @@ export class AudiobookService {
const elapsedBefore = context.manifest.assets.slice(0, fileIndex).reduce((sum, item) => sum + (item.durationMs ?? 0), 0); const elapsedBefore = context.manifest.assets.slice(0, fileIndex).reduce((sum, item) => sum + (item.durationMs ?? 0), 0);
const absolutePositionMs = elapsedBefore + dto.positionMs; const absolutePositionMs = elapsedBefore + dto.positionMs;
const percentage = const percentage =
context.manifest.totalDurationMs > 0 ? Math.max(0, Math.min(100, (absolutePositionMs / context.manifest.totalDurationMs) * 100)) : 0; context.hasCompleteDurations && context.manifest.totalDurationMs > 0
? Math.max(0, Math.min(100, (absolutePositionMs / context.manifest.totalDurationMs) * 100))
: (previous?.percentage ?? 0);
const values = { const values = {
currentFileId: context.files[fileIndex]!.id, currentFileId: context.files[fileIndex]!.id,
positionSeconds: dto.positionMs / 1000, positionSeconds: dto.positionMs / 1000,
@@ -219,14 +222,18 @@ export class AudiobookService {
.update(`${file.publicId}:${file.sizeBytes ?? ''}:${file.mtime?.getTime() ?? ''}`) .update(`${file.publicId}:${file.sizeBytes ?? ''}:${file.mtime?.getTime() ?? ''}`)
.digest('hex'), .digest('hex'),
})); }));
const totalDurationMs = const hasCompleteDurations = assets.every((asset) => asset.durationMs !== null && asset.durationMs > 0);
detail.audioMetadata?.durationSeconds !== null && detail.audioMetadata?.durationSeconds !== undefined const measuredDurationMs = assets.reduce((sum, asset) => sum + (asset.durationMs ?? 0), 0);
const totalDurationMs = hasCompleteDurations
? measuredDurationMs
: detail.audioMetadata?.durationSeconds !== null && detail.audioMetadata?.durationSeconds !== undefined
? Math.round(detail.audioMetadata.durationSeconds * 1000) ? Math.round(detail.audioMetadata.durationSeconds * 1000)
: assets.reduce((sum, asset) => sum + (asset.durationMs ?? 0), 0); : measuredDurationMs;
const chapters = this.buildChapters(bookId, revision, detail.audioMetadata?.chapters ?? [], assets, totalDurationMs); const chapters = this.buildChapters(bookId, revision, detail.audioMetadata?.chapters ?? [], assets, totalDurationMs);
return { return {
libraryId: detail.libraryId, libraryId: detail.libraryId,
files, files,
hasCompleteDurations,
manifest: { manifest: {
schema: AUDIOBOOK_MANIFEST_SCHEMA, schema: AUDIOBOOK_MANIFEST_SCHEMA,
schemaVersion: AUDIOBOOK_MANIFEST_VERSION, schemaVersion: AUDIOBOOK_MANIFEST_VERSION,
@@ -244,7 +244,9 @@ export class ScannerRepository {
sizeBytes: bookFiles.sizeBytes, sizeBytes: bookFiles.sizeBytes,
mtime: bookFiles.mtime, mtime: bookFiles.mtime,
fileHash: bookFiles.fileHash, fileHash: bookFiles.fileHash,
format: bookFiles.format,
sortOrder: bookFiles.sortOrder, sortOrder: bookFiles.sortOrder,
durationSeconds: bookFiles.durationSeconds,
mediaOverlayAvailable: bookFiles.mediaOverlayAvailable, mediaOverlayAvailable: bookFiles.mediaOverlayAvailable,
mediaOverlayCheckedAt: bookFiles.mediaOverlayCheckedAt, mediaOverlayCheckedAt: bookFiles.mediaOverlayCheckedAt,
}) })
@@ -67,6 +67,7 @@ function makeBookFile(overrides: Record<string, unknown> = {}) {
format: 'epub', format: 'epub',
role: 'content', role: 'content',
sortOrder: 0, sortOrder: 0,
durationSeconds: 600,
createdAt: new Date(), createdAt: new Date(),
updatedAt: new Date(), updatedAt: new Date(),
...overrides, ...overrides,
@@ -2207,6 +2208,71 @@ describe('incremental scan — no re-extraction on unchanged winner', () => {
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled(); expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
expect(mockMetadata.aggregateAudioDuration).not.toHaveBeenCalled(); expect(mockMetadata.aggregateAudioDuration).not.toHaveBeenCalled();
}); });
it('repairs a missing duration when an otherwise unchanged audio candidate is processed', async () => {
const m4b = makeFileStat({ absolutePath: '/library/Book/book.m4b', relPath: 'Book/book.m4b', ino: 10001n });
const repo = makeRepo({
findBooksByLibraryFolder: vi
.fn()
.mockResolvedValue([{ id: 1, libraryId: 1, libraryFolderId: 1, folderPath: '/library/Book', status: 'present' }]),
findBookFilesByLibraryFolder: vi
.fn()
.mockResolvedValue([makeBookFile({ id: 10, bookId: 1, absolutePath: m4b.absolutePath, ino: m4b.ino, format: 'm4b', durationSeconds: null })]),
});
mockFindCandidates.mockResolvedValue({
candidates: [makeCandidate('/library/Book', [m4b])],
skippedDirs: new Set(),
unchangedDirs: new Set(),
dirMtimes: new Map(),
});
const done = awaitScan(repo);
const { service } = makeService(repo);
await service.startScan(1, 'manual');
await done;
expect(mockMetadata.extractAudioFileDuration).toHaveBeenCalledOnce();
expect(mockMetadata.extractAudioFileDuration).toHaveBeenCalledWith(1, m4b.absolutePath);
expect(mockMetadata.aggregateAudioDuration).toHaveBeenCalledWith(1);
});
it('repairs every missing track duration when an incremental scan skips an unchanged audiobook directory', async () => {
const files = Array.from({ length: 16 }, (_, index) =>
makeBookFile({
id: index + 10,
bookId: 1,
absolutePath: `/library/Book/chapter-${String(index + 1).padStart(2, '0')}.mp3`,
relPath: `Book/chapter-${String(index + 1).padStart(2, '0')}.mp3`,
ino: BigInt(10001 + index),
format: 'mp3',
durationSeconds: null,
}),
);
const repo = makeRepo({
findBooksByLibraryFolder: vi
.fn()
.mockResolvedValue([{ id: 1, libraryId: 1, libraryFolderId: 1, folderPath: '/library/Book', status: 'present' }]),
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue(files),
});
mockFindCandidates.mockResolvedValue({
candidates: [],
skippedDirs: new Set(),
unchangedDirs: new Set(['/library/Book']),
dirMtimes: new Map(),
});
const done = awaitScan(repo);
const { service } = makeService(repo);
await service.startScan(1, 'manual');
await done;
expect(mockMetadata.extractAudioFileDuration).toHaveBeenCalledTimes(16);
for (const file of files) {
expect(mockMetadata.extractAudioFileDuration).toHaveBeenCalledWith(1, file.absolutePath);
}
expect(mockMetadata.aggregateAudioDuration).toHaveBeenCalledOnce();
expect(mockMetadata.aggregateAudioDuration).toHaveBeenCalledWith(1);
});
}); });
describe('missing book restoration', () => { describe('missing book restoration', () => {
+78 -16
View File
@@ -1,5 +1,6 @@
import { ConflictException, Injectable, Logger, NotFoundException, OnApplicationBootstrap, Optional } from '@nestjs/common'; import { ConflictException, Injectable, Logger, NotFoundException, OnApplicationBootstrap, Optional } from '@nestjs/common';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
import { mapWithConcurrency } from '../../common/utils/batch.utils';
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils'; import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
import { naturalCompare } from '../../common/utils/natural-sort.utils'; import { naturalCompare } from '../../common/utils/natural-sort.utils';
import { pathsReferToSameEntry } from '../../common/utils/path-identity.utils'; import { pathsReferToSameEntry } from '../../common/utils/path-identity.utils';
@@ -61,6 +62,7 @@ interface FileByPathEntry {
sizeBytes: number | null; sizeBytes: number | null;
mtime: Date | null; mtime: Date | null;
fileHash: string | null; fileHash: string | null;
durationSeconds?: number | null;
sortOrder: number | null; sortOrder: number | null;
mediaOverlayAvailable: boolean; mediaOverlayAvailable: boolean;
mediaOverlayCheckedAt: Date | null; mediaOverlayCheckedAt: Date | null;
@@ -121,6 +123,7 @@ const BOOK_MISSING_NOTIFY_DEBOUNCE_MS = 5000;
const WATCHER_NOTIFY_DEBOUNCE_MS = 30_000; const WATCHER_NOTIFY_DEBOUNCE_MS = 30_000;
const TARGETED_BOOK_SCAN_MAX_CONCURRENCY = 8; const TARGETED_BOOK_SCAN_MAX_CONCURRENCY = 8;
const MISSING_FILE_STAT_BATCH_SIZE = 50; const MISSING_FILE_STAT_BATCH_SIZE = 50;
const AUDIO_DURATION_PROBE_CONCURRENCY = 4;
type OrganizationMode = 'book_per_file' | 'book_per_folder'; type OrganizationMode = 'book_per_file' | 'book_per_folder';
interface ScanCounts { interface ScanCounts {
@@ -137,6 +140,7 @@ interface RegisteredFile {
role: FileRole; role: FileRole;
absolutePath: string; absolutePath: string;
sizeBytes: number; sizeBytes: number;
durationSeconds: number | null;
isNew: boolean; isNew: boolean;
wasReassigned: boolean; wasReassigned: boolean;
wasChanged: boolean; wasChanged: boolean;
@@ -322,6 +326,7 @@ export class ScannerService implements OnApplicationBootstrap {
sizeBytes: number | null; sizeBytes: number | null;
mtime: Date | null; mtime: Date | null;
fileHash: string | null; fileHash: string | null;
durationSeconds?: number | null;
sortOrder?: number | null; sortOrder?: number | null;
mediaOverlayAvailable?: boolean | null; mediaOverlayAvailable?: boolean | null;
mediaOverlayCheckedAt?: Date | null; mediaOverlayCheckedAt?: Date | null;
@@ -366,6 +371,7 @@ export class ScannerService implements OnApplicationBootstrap {
sizeBytes: f.sizeBytes, sizeBytes: f.sizeBytes,
mtime: f.mtime, mtime: f.mtime,
fileHash: f.fileHash, fileHash: f.fileHash,
durationSeconds: f.durationSeconds ?? null,
sortOrder: f.sortOrder ?? null, sortOrder: f.sortOrder ?? null,
mediaOverlayAvailable: f.mediaOverlayAvailable === true, mediaOverlayAvailable: f.mediaOverlayAvailable === true,
mediaOverlayCheckedAt: f.mediaOverlayCheckedAt ?? null, mediaOverlayCheckedAt: f.mediaOverlayCheckedAt ?? null,
@@ -1033,6 +1039,7 @@ export class ScannerService implements OnApplicationBootstrap {
sizeBytes: f.sizeBytes, sizeBytes: f.sizeBytes,
mtime: f.mtime, mtime: f.mtime,
fileHash: f.fileHash, fileHash: f.fileHash,
durationSeconds: f.durationSeconds,
sortOrder: f.sortOrder, sortOrder: f.sortOrder,
mediaOverlayAvailable: f.mediaOverlayAvailable, mediaOverlayAvailable: f.mediaOverlayAvailable,
mediaOverlayCheckedAt: f.mediaOverlayCheckedAt, mediaOverlayCheckedAt: f.mediaOverlayCheckedAt,
@@ -1514,6 +1521,8 @@ export class ScannerService implements OnApplicationBootstrap {
} }
} }
await this.repairMissingAudioDurations(knownFiles, unchangedDirs, seenBookIds);
// Deferred prune: delete book files not retained by any candidate. // Deferred prune: delete book files not retained by any candidate.
// Must happen after ALL batches so cross-book file moves are visible. // Must happen after ALL batches so cross-book file moves are visible.
const pruneCounts = { added: 0, updated: 0 }; const pruneCounts = { added: 0, updated: 0 };
@@ -1566,6 +1575,57 @@ export class ScannerService implements OnApplicationBootstrap {
} }
} }
private async repairMissingAudioDurations(
knownFiles: Array<{
bookId: number;
absolutePath: string;
format: string | null;
durationSeconds: number | null;
}>,
unchangedDirs: Set<string>,
processedBookIds: Set<number>,
): Promise<void> {
if (unchangedDirs.size === 0) return;
const isUnderUnchangedDir = (absolutePath: string) => {
for (const dir of unchangedDirs) {
if (absolutePath === dir || absolutePath.startsWith(dir + sep)) return true;
}
return false;
};
const filesToRepair = knownFiles.filter(
(file) =>
!processedBookIds.has(file.bookId) &&
file.format !== null &&
isAudioFormat(file.format) &&
(file.durationSeconds === null || file.durationSeconds <= 0) &&
isUnderUnchangedDir(file.absolutePath) &&
!this.selfWriteRegistry.isSuppressed(file.absolutePath),
);
const repairedBookIds = new Set<number>();
await mapWithConcurrency(filesToRepair, AUDIO_DURATION_PROBE_CONCURRENCY, async (file) => {
try {
await this.metadataService.extractAudioFileDuration(file.bookId, file.absolutePath);
repairedBookIds.add(file.bookId);
} catch (err) {
this.logger.warn(
`[scanner.extract_audio_duration] [fail] bookId=${file.bookId} path="${sanitizeLogValue(file.absolutePath)}" errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - missing audio duration repair failed`,
);
}
});
await mapWithConcurrency([...repairedBookIds], AUDIO_DURATION_PROBE_CONCURRENCY, async (bookId) => {
try {
await this.metadataService.aggregateAudioDuration(bookId);
} catch (err) {
this.logger.warn(
`[scanner.aggregate_audio_duration] [fail] bookId=${bookId} errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - repaired audio duration aggregation failed`,
);
}
});
}
private async processCandidate( private async processCandidate(
candidate: BookCandidate, candidate: BookCandidate,
libraryId: number, libraryId: number,
@@ -1617,6 +1677,7 @@ export class ScannerService implements OnApplicationBootstrap {
const fileCount: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 }; const fileCount: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
let processResult: ProcessedFileResult; let processResult: ProcessedFileResult;
const knownDurationSeconds = fileByPath.get(fileStat.absolutePath)?.durationSeconds ?? null;
try { try {
processResult = await this.processFile( processResult = await this.processFile(
@@ -1647,6 +1708,7 @@ export class ScannerService implements OnApplicationBootstrap {
role, role,
absolutePath: fileStat.absolutePath, absolutePath: fileStat.absolutePath,
sizeBytes: fileStat.sizeBytes, sizeBytes: fileStat.sizeBytes,
durationSeconds: knownDurationSeconds,
isNew: processResult.isNew, isNew: processResult.isNew,
wasReassigned: processResult.reassigned, wasReassigned: processResult.reassigned,
wasChanged: processResult.changed, wasChanged: processResult.changed,
@@ -1684,6 +1746,9 @@ export class ScannerService implements OnApplicationBootstrap {
metadataSources.some((source) => hasMetadataSourceChanged(source.file)) || (book.primaryFileId === null && winner !== null); metadataSources.some((source) => hasMetadataSourceChanged(source.file)) || (book.primaryFileId === null && winner !== null);
const audioContentFiles = contentFiles.filter((f) => f.format !== null && isAudioFormat(f.format!)); const audioContentFiles = contentFiles.filter((f) => f.format !== null && isAudioFormat(f.format!));
const changedAudioFiles = audioContentFiles.filter(hasMetadataSourceChanged); const changedAudioFiles = audioContentFiles.filter(hasMetadataSourceChanged);
const audioFilesNeedingDuration = audioContentFiles.filter(
(file) => hasMetadataSourceChanged(file) || file.durationSeconds === null || file.durationSeconds <= 0,
);
const winnerIsAudio = winner !== null && winner.format !== null && isAudioFormat(winner.format); const winnerIsAudio = winner !== null && winner.format !== null && isAudioFormat(winner.format);
// 3a: Extract audio-specific fields (chapters, narrators) from the first audio file if any audio // 3a: Extract audio-specific fields (chapters, narrators) from the first audio file if any audio
@@ -1718,25 +1783,22 @@ export class ScannerService implements OnApplicationBootstrap {
await this.extractFirstAvailableMetadataSource(book.id, metadataSources); await this.extractFirstAvailableMetadataSource(book.id, metadataSources);
} }
// 3c: Write per-file duration to bookFiles for every new/reassigned/changed audio file. // 3c: Write per-file duration for new, changed, or historically unprobed audio files.
// Running this for all new audio files (including the winner) ensures // Including the winner ensures aggregateAudioDuration has accurate data for the total.
// aggregateAudioDuration has accurate per-file data for the total. if (audioFilesNeedingDuration.length > 0) {
if (changedAudioFiles.length > 0) { await mapWithConcurrency(audioFilesNeedingDuration, AUDIO_DURATION_PROBE_CONCURRENCY, async (audioFile) => {
await Promise.all( try {
changedAudioFiles.map(async (audioFile) => { await this.metadataService.extractAudioFileDuration(book.id, audioFile.absolutePath);
try { } catch (err) {
await this.metadataService.extractAudioFileDuration(book.id, audioFile.absolutePath); this.logger.warn(
} catch (err) { `[scanner.extract_audio_duration] [fail] bookId=${book.id} path="${sanitizeLogValue(audioFile.absolutePath)}" errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - audio duration extraction failed`,
this.logger.warn( );
`[scanner.extract_audio_duration] [fail] bookId=${book.id} path="${sanitizeLogValue(audioFile.absolutePath)}" errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - audio duration extraction failed`, }
); });
}
}),
);
} }
// 3d: Re-aggregate total duration whenever audio files exist and anything changed. // 3d: Re-aggregate total duration whenever audio files exist and anything changed.
if (audioContentFiles.length > 0 && (shouldExtractMetadata || changedAudioFiles.length > 0)) { if (audioContentFiles.length > 0 && (shouldExtractMetadata || audioFilesNeedingDuration.length > 0)) {
try { try {
await this.metadataService.aggregateAudioDuration(book.id); await this.metadataService.aggregateAudioDuration(book.id);
} catch (err) { } catch (err) {