mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
perf(kobo): stop re-hashing a delivery's source on every Kobo sync
Every position conversion resolved the delivered artifact by hashing the whole source file, so a multi-gigabyte read-along outlasted the Kobo's 30 s request timeout. Remember each source's hash by its file signature and hash again only after the file changes. Reusing the stamped Kobo location on device syncs (defect 3) is left out: that stamp is not per device. Ref #1786
This commit is contained in:
@@ -139,6 +139,37 @@ describe('DeliveryArtifactService', () => {
|
||||
await expect(service.prepare(source, 'original', original.sourceRevision)).rejects.toThrow();
|
||||
await original.release();
|
||||
});
|
||||
it('reuses a verified source hash while the file is unchanged and rehashes it after a write', async () => {
|
||||
const path = join(directory, 'source.epub');
|
||||
await fixture(path, false);
|
||||
const source = { id: 9, absolutePath: path, format: 'epub' };
|
||||
const first = await service.prepare(source, 'original');
|
||||
await first.release();
|
||||
const actual = await vi.importActual<typeof import('fs/promises')>('fs/promises');
|
||||
let fullReads = 0;
|
||||
vi.mocked(open).mockImplementation(async (...args: Parameters<typeof actual.open>) => {
|
||||
const handle = await actual.open(...args);
|
||||
const createReadStream = handle.createReadStream.bind(handle);
|
||||
vi.spyOn(handle, 'createReadStream').mockImplementation((options) => {
|
||||
fullReads++;
|
||||
return createReadStream(options);
|
||||
});
|
||||
return handle;
|
||||
});
|
||||
try {
|
||||
const again = await service.prepare(source, 'original', first.sourceRevision);
|
||||
expect(again.sourceRevision).toBe(first.sourceRevision);
|
||||
await again.release();
|
||||
expect(fullReads).toBe(0);
|
||||
|
||||
await writeFile(path, 'rewritten in place');
|
||||
await expect(service.prepare(source, 'original', first.sourceRevision)).rejects.toThrow();
|
||||
expect(fullReads).toBe(1);
|
||||
} finally {
|
||||
vi.mocked(open).mockImplementation(actual.open);
|
||||
}
|
||||
});
|
||||
|
||||
it('detects external replacement from the recorded source stat', async () => {
|
||||
const path = join(directory, 'source.epub');
|
||||
await fixture(path, false);
|
||||
|
||||
@@ -16,6 +16,7 @@ import { inspectDeliveryEpub } from './epub-delivery-validation';
|
||||
|
||||
const DELIVERY_DETECTOR_VERSION = 'overlay-inspection-v2';
|
||||
export const DELIVERY_TRANSFORM_VERSION = 'text-only-v2';
|
||||
const MAX_REMEMBERED_REVISIONS = 4096;
|
||||
export interface DeliverySource {
|
||||
id: number;
|
||||
absolutePath: string;
|
||||
@@ -44,6 +45,10 @@ export class DeliveryArtifactService {
|
||||
private readonly root: string;
|
||||
private readonly appRoot: string;
|
||||
private readonly builds = new Map<string, Promise<Omit<DeliveryArtifact, 'release'>>>();
|
||||
// A source's SHA-256 keyed by its file signature. Every Kobo sync that converts a position resolves
|
||||
// the delivered artifact again, and re-reading a multi-gigabyte read-along each time outlasted the
|
||||
// device's request timeout. Any write moves the signature's change time, so a stale entry never matches.
|
||||
private readonly revisions = new Map<string, string>();
|
||||
private readonly leased = new Map<string, number>();
|
||||
private publication: Promise<void> = Promise.resolve();
|
||||
private active = 0;
|
||||
@@ -109,13 +114,16 @@ export class DeliveryArtifactService {
|
||||
(source.ino !== undefined && source.ino !== (await handle.stat({ bigint: true })).ino))
|
||||
)
|
||||
throw new ConflictException({ code: 'DELIVERY_SOURCE_CHANGED' });
|
||||
const sourceRevision = await digestHandle(handle);
|
||||
const revisionKey = `${source.id}:${resolvedSource}:${deliveryFileSignature(before)}`;
|
||||
const knownRevision = this.revisions.get(revisionKey);
|
||||
const sourceRevision = knownRevision ?? (await digestHandle(handle));
|
||||
if (
|
||||
deliveryFileSignature(before) !== deliveryFileSignature(await handle.stat()) ||
|
||||
deliveryFileSignature(before) !== deliveryFileSignature(await stat(source.absolutePath))
|
||||
) {
|
||||
throw new ConflictException({ code: 'DELIVERY_SOURCE_CHANGED' });
|
||||
}
|
||||
this.rememberRevision(revisionKey, sourceRevision);
|
||||
if (expectedRevision && sourceRevision !== expectedRevision) throw new ConflictException({ code: 'DELIVERY_SOURCE_CHANGED' });
|
||||
const format = source.format?.toLowerCase() ?? 'unknown';
|
||||
const transform =
|
||||
@@ -176,6 +184,12 @@ export class DeliveryArtifactService {
|
||||
}
|
||||
}
|
||||
|
||||
private rememberRevision(key: string, revision: string): void {
|
||||
this.revisions.delete(key);
|
||||
this.revisions.set(key, revision);
|
||||
if (this.revisions.size > MAX_REMEMBERED_REVISIONS) this.revisions.delete(this.revisions.keys().next().value!);
|
||||
}
|
||||
|
||||
private async build(
|
||||
source: DeliverySource,
|
||||
sourceRevision: string,
|
||||
|
||||
Reference in New Issue
Block a user