fix(annotations): preserve source highlight timestamps

Use KOReader highlight times for annotation history while retaining ingestion times for diagnostics. Export all stored location representations and backfill existing rows during sync.

Fixes #1300
This commit is contained in:
neon
2026-09-08 16:52:36 +00:00
parent cf40058cb6
commit b9609b198e
38 changed files with 18700 additions and 42 deletions
@@ -137,7 +137,7 @@ const metadataItems = computed(() => {
if (props.annotation.chapterTitle) items.push(props.annotation.chapterTitle)
if (props.annotation.pageno != null) items.push(t('annotations.listItem.pageNumber', { page: props.annotation.pageno }))
if (props.annotation.chapterIndex != null) items.push(t('annotations.listItem.chapterNumber', { chapter: props.annotation.chapterIndex + 1 }))
const date = formatDate(props.annotation.createdAt)
const date = formatDate(props.annotation.highlightedAt)
if (date) items.push(date)
return items
})
@@ -31,6 +31,7 @@ function makeAnnotation(overrides: Partial<AnnotationItem> = {}): AnnotationItem
origin: 'web',
positionStatus: 'exact',
chapterIndex: 0,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
...overrides,
}
@@ -64,7 +64,7 @@ const { t } = useI18n()
const STYLE_ICONS = { highlight: Highlighter, underline: Underline, strikethrough: Strikethrough, squiggly: Waves, invert: Contrast }
const created = computed(() => new Date(props.annotation.createdAt))
const created = computed(() => new Date(props.annotation.highlightedAt))
const dayLabel = computed(() => formatDate(created.value, { day: 'numeric', month: 'short' }))
const yearLabel = computed(() => formatDate(created.value, { year: 'numeric' }))
const hub = computed(() => ('bookTitle' in props.annotation ? props.annotation : null))
@@ -86,7 +86,7 @@ function isFirstOfDay(groupIndex: number, itemIndex: number): boolean {
if (!group) return true
const previous = group.items[itemIndex - 1]
if (!previous) return true
return dayKey(previous.createdAt) !== dayKey(group.items[itemIndex]!.createdAt)
return dayKey(previous.highlightedAt) !== dayKey(group.items[itemIndex]!.highlightedAt)
}
function isFirstOfChapter(groupIndex: number, itemIndex: number): boolean {
@@ -18,6 +18,7 @@ function annotation(overrides: Partial<AnnotationHubItem> = {}): AnnotationHubIt
origin: 'web',
positionStatus: 'exact',
chapterIndex: 0,
highlightedAt: '2026-07-12T00:00:00.000Z',
createdAt: '2026-07-12T00:00:00.000Z',
bookTitle: 'A Book',
author: 'An Author',
@@ -17,6 +17,7 @@ function item(overrides: Partial<AnnotationHubItem> = {}): AnnotationHubItem {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-08-24T10:00:00.000Z',
createdAt: '2026-08-24T10:00:00.000Z',
bookTitle: 'Tartufo',
author: 'Kira Jane Buxton',
@@ -46,11 +47,11 @@ describe('HUB_VIEWS', () => {
describe('hubGroupKey', () => {
it('keys each mode off the field the server sorted by', () => {
const row = item({ bookId: 42, color: '#38BDF8', origin: 'kobo', createdAt: '2026-03-04T00:00:00.000Z' })
const row = item({ bookId: 42, color: '#38BDF8', origin: 'kobo', highlightedAt: '2026-03-04T00:00:00.000Z' })
expect(hubGroupKey(row, 'book')).toBe('42')
expect(hubGroupKey(row, 'color')).toBe('#38BDF8')
expect(hubGroupKey(row, 'source')).toBe('kobo')
expect(hubGroupKey(row, 'month')).toBe(monthKey(row.createdAt))
expect(hubGroupKey(row, 'month')).toBe(monthKey(row.highlightedAt))
})
})
@@ -58,7 +58,7 @@ export function hubGroupKey(item: AnnotationHubItem, mode: AnnotationHubGroupMod
case 'source':
return item.origin
default:
return monthKey(item.createdAt)
return monthKey(item.highlightedAt)
}
}
@@ -89,7 +89,7 @@ const groups = computed<StreamGroup[]>(() =>
}
return {
key: group.key,
label: formatDate(new Date(lead.createdAt), { month: 'long', year: 'numeric' }),
label: formatDate(new Date(lead.highlightedAt), { month: 'long', year: 'numeric' }),
count: group.items.length,
items: group.items,
}
@@ -91,6 +91,7 @@ function exportJson() {
color: h.color,
style: h.style,
chapter: h.chapterTitle,
highlightedAt: h.highlightedAt,
createdAt: h.createdAt,
})),
}
@@ -73,7 +73,7 @@ const groups = computed<StreamGroup[]>(() =>
const first = group.items[0]
return {
key: group.key,
label: first ? formatDate(new Date(first.createdAt), { day: 'numeric', month: 'long', year: 'numeric' }) : group.key,
label: first ? formatDate(new Date(first.highlightedAt), { day: 'numeric', month: 'long', year: 'numeric' }) : group.key,
count: group.total,
items: group.items,
}
@@ -18,6 +18,7 @@ function makeHighlight(overrides: Partial<AnnotationItem> = {}): AnnotationItem
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
...overrides,
}
@@ -17,6 +17,7 @@ function item(overrides: Partial<AnnotationItem> = {}): AnnotationItem {
origin: 'kobo',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-04-02T19:30:00.000Z',
createdAt: '2026-04-02T19:30:00.000Z',
...overrides,
}
@@ -157,21 +158,21 @@ describe('buildHighlightGroups by colour', () => {
describe('buildHighlightGroups by day', () => {
it('groups by the viewer time zone and puts the newest day first', () => {
const groups = buildHighlightGroups(
[item({ id: 1, createdAt: '2026-04-02T12:00:00.000Z' }), item({ id: 2, createdAt: '2026-04-05T12:00:00.000Z' })],
[item({ id: 1, highlightedAt: '2026-04-02T12:00:00.000Z' }), item({ id: 2, highlightedAt: '2026-04-05T12:00:00.000Z' })],
stats(),
'day',
)
expect(groups.map((g) => g.label)).toEqual([
highlightDay(item({ createdAt: '2026-04-05T12:00:00.000Z' })),
highlightDay(item({ createdAt: '2026-04-02T12:00:00.000Z' })),
highlightDay(item({ highlightedAt: '2026-04-05T12:00:00.000Z' })),
highlightDay(item({ highlightedAt: '2026-04-02T12:00:00.000Z' })),
])
expect(groups[0].total).toBe(1)
})
it('puts highlights made on the same day in one group', () => {
const groups = buildHighlightGroups(
[item({ id: 1, createdAt: '2026-04-02T09:00:00.000Z' }), item({ id: 2, createdAt: '2026-04-02T11:00:00.000Z' })],
[item({ id: 1, highlightedAt: '2026-04-02T09:00:00.000Z' }), item({ id: 2, highlightedAt: '2026-04-02T11:00:00.000Z' })],
stats(),
'day',
)
@@ -183,6 +184,6 @@ describe('buildHighlightGroups by day', () => {
describe('highlightDay', () => {
it('returns an empty key for an unparseable timestamp', () => {
expect(highlightDay(item({ createdAt: 'not a date' }))).toBe('')
expect(highlightDay(item({ highlightedAt: 'not a date' }))).toBe('')
})
})
@@ -24,7 +24,7 @@ export interface HighlightGroup {
/** The day an annotation was made, in the viewer's own time zone. */
export function highlightDay(annotation: AnnotationItem): string {
const date = new Date(annotation.createdAt)
const date = new Date(annotation.highlightedAt)
if (Number.isNaN(date.getTime())) return ''
const month = `${date.getMonth() + 1}`.padStart(2, '0')
const day = `${date.getDate()}`.padStart(2, '0')
@@ -161,6 +161,7 @@ describe('ReaderSidebar', () => {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-02-14T12:00:00.000Z',
createdAt: '2026-02-14T12:00:00.000Z',
},
{
@@ -177,6 +178,7 @@ describe('ReaderSidebar', () => {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-02-15T12:00:00.000Z',
createdAt: '2026-02-15T12:00:00.000Z',
},
],
@@ -237,6 +239,7 @@ describe('ReaderSidebar', () => {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-02-13T12:00:00.000Z',
createdAt: '2026-02-13T12:00:00.000Z',
},
{
@@ -253,6 +256,7 @@ describe('ReaderSidebar', () => {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-02-16T12:00:00.000Z',
createdAt: '2026-02-16T12:00:00.000Z',
},
],
@@ -348,6 +352,7 @@ describe('ReaderSidebar', () => {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-02-11T12:00:00.000Z',
createdAt: '2026-02-11T12:00:00.000Z',
},
],
@@ -27,6 +27,7 @@ function makeAnnotation(id: number, cfi = `epubcfi(/6/${id})`): Annotation {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
}
}
@@ -18,6 +18,7 @@ function makeAnnotation(overrides: Partial<Annotation> = {}): Annotation {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
...overrides,
}
@@ -167,6 +167,7 @@ function makeAnnotation(id: number, overrides: Partial<AnnotationItem> = {}): An
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
pdf: { page: 2, rect: { x: 10, y: 20, width: 30, height: 8 }, rects: [{ x: 10, y: 20, width: 30, height: 8 }] },
...overrides,
@@ -25,6 +25,7 @@ function makeAnnotation(overrides: Partial<AnnotationItem> = {}): AnnotationItem
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
pdf: { page: 1, rect: { x: 10, y: 20, width: 30, height: 8 }, rects: [{ x: 10, y: 20, width: 30, height: 8 }] },
...overrides,
@@ -37,6 +37,7 @@ function makeAnnotation(id: number): AnnotationItem {
origin: 'web',
positionStatus: 'exact',
chapterIndex: null,
highlightedAt: '2026-01-01T00:00:00.000Z',
createdAt: '2026-01-01T00:00:00.000Z',
pdf: PDF_POSITION,
}
+1
View File
@@ -96,6 +96,7 @@ export interface AnnotationItem {
origin: "web" | "koreader" | "kobo";
positionStatus: AnnotationPositionStatus | null;
chapterIndex: number | null;
highlightedAt: string;
createdAt: string;
/** Present for PDF highlights; null/absent for EPUB and device-synced formats. */
pdf?: AnnotationPdfPosition | null;
@@ -0,0 +1,4 @@
DROP INDEX "annotations_user_created_active_idx";--> statement-breakpoint
ALTER TABLE "annotations" ADD COLUMN "source_created_at" timestamp with time zone;--> statement-breakpoint
CREATE INDEX "annotations_user_book_created_active_idx" ON "annotations" USING btree ("user_id","book_id",coalesce("source_created_at", "created_at") desc,"id" DESC NULLS LAST) WHERE "annotations"."deleted_at" is null;--> statement-breakpoint
CREATE INDEX "annotations_user_created_active_idx" ON "annotations" USING btree ("user_id",coalesce("source_created_at", "created_at") desc,"id" DESC NULLS LAST) WHERE "annotations"."deleted_at" is null;
File diff suppressed because it is too large Load Diff
@@ -603,6 +603,13 @@
"when": 1788321273908,
"tag": "0085_add_authentication_method",
"breakpoints": true
},
{
"idx": 86,
"version": "7",
"when": 1788884825211,
"tag": "0086_add_annotation_source_created_at",
"breakpoints": true
}
]
}
+5 -1
View File
@@ -408,6 +408,7 @@ export const annotations = pgTable(
// deviceCreatedAt doubles as the KOReader-side identity datetime for synced annotations.
deviceCreatedAt: varchar('device_created_at', { length: 19 }),
deviceUpdatedAt: varchar('device_updated_at', { length: 19 }),
sourceCreatedAt: timestamp('source_created_at', { withTimezone: true }),
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
updatedAt: timestamp('updated_at', { withTimezone: true })
.notNull()
@@ -425,7 +426,10 @@ export const annotations = pgTable(
// The annotations hub is an infinite stream ordered newest-first over everything a
// user owns, so its default query sorts the whole set without this.
index('annotations_user_created_active_idx')
.on(t.userId, t.createdAt.desc(), t.id.desc())
.on(t.userId, sql`coalesce(${t.sourceCreatedAt}, ${t.createdAt}) desc`, t.id.desc())
.where(sql`${t.deletedAt} is null`),
index('annotations_user_book_created_active_idx')
.on(t.userId, t.bookId, sql`coalesce(${t.sourceCreatedAt}, ${t.createdAt}) desc`, t.id.desc())
.where(sql`${t.deletedAt} is null`),
check('annotations_style_chk', sql`${t.style} in ('highlight', 'underline', 'strikethrough', 'squiggly', 'invert')`),
check('annotations_origin_chk', sql`${t.origin} in ('web', 'koreader', 'kobo')`),
@@ -18,6 +18,7 @@ function makeRow(overrides: Partial<HubAnnotationRow> = {}): HubAnnotationRow {
deletedAt: null,
deviceCreatedAt: null,
deviceUpdatedAt: null,
sourceCreatedAt: null,
createdAt: new Date('2026-06-01T10:00:00Z'),
updatedAt: new Date('2026-06-01T10:00:00Z'),
cfi: 'epubcfi(/6/2!/4/2,/1:0,/1:5)',
@@ -26,6 +27,7 @@ function makeRow(overrides: Partial<HubAnnotationRow> = {}): HubAnnotationRow {
bookTitle: 'Test Book',
jumpFileId: 10,
pageno: null,
xpointer: null,
...overrides,
} as HubAnnotationRow;
}
@@ -59,7 +61,7 @@ describe('AnnotationExportService', () => {
expect(result.contentType).toContain('csv');
expect(result.content).toContain('"He said ""hi"", twice\nand left"');
expect(result.content.split('\r\n')[0]).toBe('book,chapter,text,note,color,style,origin,createdAt');
expect(result.content.split('\r\n')[0]).toBe('book,chapter,text,note,color,style,origin,page,cfi,xpointer,highlightedAt,createdAt');
});
it('renders JSON with stable fields', () => {
@@ -75,4 +77,31 @@ describe('AnnotationExportService', () => {
cfi: 'epubcfi(/6/2!/4/2,/1:0,/1:5)',
});
});
it('exports source time and KOReader position while preserving ingestion time', () => {
const result = service.export(
[
makeRow({
origin: 'koreader',
sourceCreatedAt: new Date('2026-01-15T16:30:00.000Z'),
createdAt: new Date('2026-09-06T17:10:19.332Z'),
cfi: null,
pageno: 123,
xpointer: '/body/DocFragment[5]/body/p[2]/text().0',
}),
],
'json',
'library',
);
expect(JSON.parse(result.content)).toEqual([
expect.objectContaining({
page: 123,
cfi: null,
xpointer: '/body/DocFragment[5]/body/p[2]/text().0',
highlightedAt: '2026-01-15T16:30:00.000Z',
createdAt: '2026-09-06T17:10:19.332Z',
}),
]);
});
});
@@ -62,13 +62,13 @@ export class AnnotationExportService {
.join('\n');
lines.push(quote);
if (row.note) lines.push('', `Note: ${row.note}`);
lines.push('', `<sub>${row.style}, ${row.color}, ${row.origin}, ${row.createdAt.toISOString().slice(0, 10)}</sub>`, '');
lines.push('', `<sub>${row.style}, ${row.color}, ${row.origin}, ${this.highlightedAt(row).toISOString().slice(0, 10)}</sub>`, '');
}
return lines.join('\n');
}
private toCsv(rows: HubAnnotationRow[]): string {
const header = ['book', 'chapter', 'text', 'note', 'color', 'style', 'origin', 'createdAt'];
const header = ['book', 'chapter', 'text', 'note', 'color', 'style', 'origin', 'page', 'cfi', 'xpointer', 'highlightedAt', 'createdAt'];
const escape = (value: string | null): string => {
const raw = value ?? '';
return /[",\n\r]/.test(raw) ? `"${raw.replace(/"/g, '""')}"` : raw;
@@ -84,6 +84,10 @@ export class AnnotationExportService {
escape(row.color),
escape(row.style),
escape(row.origin),
escape(row.pageno == null ? null : String(row.pageno)),
escape(row.cfi),
escape(row.xpointer),
escape(this.highlightedAt(row).toISOString()),
escape(row.createdAt.toISOString()),
].join(','),
);
@@ -103,11 +107,18 @@ export class AnnotationExportService {
color: row.color,
style: row.style,
origin: row.origin,
page: row.pageno,
cfi: row.cfi,
xpointer: row.xpointer,
highlightedAt: this.highlightedAt(row).toISOString(),
createdAt: row.createdAt.toISOString(),
})),
null,
2,
);
}
private highlightedAt(row: HubAnnotationRow): Date {
return row.sourceCreatedAt ?? row.createdAt;
}
}
@@ -20,6 +20,7 @@ function makeHubRow(overrides: Record<string, unknown> = {}) {
deletedAt: null,
deviceCreatedAt: null,
deviceUpdatedAt: null,
sourceCreatedAt: null,
createdAt: new Date('2026-01-10T00:00:00Z'),
updatedAt: new Date('2026-01-10T00:00:00Z'),
bookTitle: 'A Book',
@@ -27,6 +28,7 @@ function makeHubRow(overrides: Record<string, unknown> = {}) {
jumpFileId: 9,
jumpFileFormat: 'fb2',
pageno: null,
xpointer: null,
...overrides,
};
}
@@ -204,6 +206,27 @@ describe('AnnotationHubService', () => {
expect(result.items[0]).toMatchObject({ cfi: null, jumpFileId: 9, jumpFileFormat: 'pdf', pageno: 4 });
});
it('returns source highlight time separately from ingestion time', async () => {
const { service, annotationRepo } = makeService();
annotationRepo.findHubPaginated.mockResolvedValueOnce({
items: [
makeHubRow({
origin: 'koreader',
sourceCreatedAt: new Date('2026-01-15T16:30:00.000Z'),
createdAt: new Date('2026-09-06T17:10:19.332Z'),
}),
],
total: 1,
});
const result = await service.list(10, {});
expect(result.items[0]).toMatchObject({
highlightedAt: '2026-01-15T16:30:00.000Z',
createdAt: '2026-09-06T17:10:19.332Z',
});
});
});
describe('listBooks', () => {
@@ -171,6 +171,7 @@ export class AnnotationHubService {
jumpFileId: null,
jumpFileFormat: null,
pageno: null,
xpointer: null,
},
);
}
@@ -266,6 +267,7 @@ export class AnnotationHubService {
origin: row.origin,
positionStatus: (row.cfi != null || row.cfiStatus != null ? (row.cfiStatus ?? 'exact') : null) as AnnotationHubItem['positionStatus'],
chapterIndex: typeof chapterIndex === 'number' ? chapterIndex : null,
highlightedAt: (row.sourceCreatedAt ?? row.createdAt).toISOString(),
createdAt: row.createdAt.toISOString(),
bookTitle: row.bookTitle,
author: row.author,
@@ -350,6 +350,27 @@ export class AnnotationSyncRepository {
await ex.update(annotations).set({ deviceUpdatedAt }).where(eq(annotations.id, annotationId));
}
async setSourceCreatedAtSilent(annotationId: number, sourceCreatedAt: Date, ex: Executor = this.db): Promise<void> {
await ex
.update(annotations)
.set({ sourceCreatedAt })
.where(and(eq(annotations.id, annotationId), isNull(annotations.sourceCreatedAt)));
}
async setSourceCreatedAtsSilent(entries: { annotationId: number; sourceCreatedAt: Date }[], ex: Executor = this.db): Promise<void> {
if (entries.length === 0) return;
const values = sql.join(
entries.map((entry) => sql`(${entry.annotationId}::int, ${entry.sourceCreatedAt}::timestamptz)`),
sql`, `,
);
await ex.execute(sql`
update ${annotations} set source_created_at = v.source_created_at
from (values ${values}) as v(annotation_id, source_created_at)
where ${annotations.id} = v.annotation_id
and ${annotations.sourceCreatedAt} is null
`);
}
async bumpVersion(annotationId: number, ex: Executor = this.db): Promise<number> {
const [row] = await ex
.update(annotations)
@@ -14,6 +14,7 @@ const TX = Symbol('tx');
function makeIncoming(overrides: Partial<IncomingDeviceAnnotation> = {}): IncomingDeviceAnnotation {
return {
datetime: '2026-06-01 21:14:03',
sourceCreatedAt: new Date('2026-06-02T03:14:03.000Z'),
datetimeUpdated: null,
drawer: 'lighten',
color: 'yellow',
@@ -43,6 +44,7 @@ function makeAnnotationRow(overrides: Record<string, unknown> = {}) {
deletedAt: null,
deviceCreatedAt: '2026-06-01 21:14:03',
deviceUpdatedAt: null,
sourceCreatedAt: null,
createdAt: new Date('2026-06-01T21:14:03Z'),
updatedAt: new Date('2026-06-01T21:14:03Z'),
...overrides,
@@ -109,6 +111,8 @@ function makeRepo(): RepoMock {
setDeviceIdentitySilent: vi.fn().mockResolvedValue(undefined),
setDeviceIdentitiesSilent: vi.fn().mockResolvedValue(undefined),
setDeviceUpdatedAtSilent: vi.fn().mockResolvedValue(undefined),
setSourceCreatedAtSilent: vi.fn().mockResolvedValue(undefined),
setSourceCreatedAtsSilent: vi.fn().mockResolvedValue(undefined),
bumpVersion: vi.fn().mockResolvedValue(5),
softDeleteById: vi.fn().mockResolvedValue(undefined),
setDeleteAcked: vi.fn().mockResolvedValue(undefined),
@@ -180,6 +184,7 @@ describe('AnnotationSyncService', () => {
color: '#84CC16',
version: 1,
deviceCreatedAt: '2026-06-01 21:14:03',
sourceCreatedAt: new Date('2026-06-02T03:14:03.000Z'),
});
expect(position).toMatchObject({
bookFileId: BOOK_FILE_ID,
@@ -296,6 +301,7 @@ describe('AnnotationSyncService', () => {
const result = await ingest(service, [makeIncoming()]);
expect(result).toMatchObject({ unchanged: 1, updated: 0, created: 0 });
expect(repo.setSourceCreatedAtSilent).toHaveBeenCalledWith(100, new Date('2026-06-02T03:14:03.000Z'), TX);
expect(repo.touchState).toHaveBeenCalled();
expect(repo.applyContentPatch).not.toHaveBeenCalled();
});
@@ -444,9 +450,20 @@ describe('AnnotationSyncService', () => {
});
describe('detectDeviceDeletions', () => {
const detect = (presentKeys: { k: string; dt: string }[]) =>
const detect = (presentKeys: { k: string; dt: string; sourceCreatedAt?: Date | null }[]) =>
service.detectDeviceDeletions({ userId: USER_ID, source: 'koreader', deviceId: DEVICE_ID, bookId: BOOK_ID, presentKeys });
it('backfills source creation times from a complete key set without changing annotation versions', async () => {
const state = makeStateRow();
repo.findStatesForDeviceBook.mockResolvedValue([{ state, annotation: makeAnnotationRow() }]);
const sourceCreatedAt = new Date('2026-06-02T03:14:03.000Z');
await detect([{ k: state.externalKey as string, dt: state.externalCreatedAt as string, sourceCreatedAt }]);
expect(repo.setSourceCreatedAtsSilent).toHaveBeenCalledWith([{ annotationId: 100, sourceCreatedAt }], TX);
expect(repo.applyContentPatch).not.toHaveBeenCalled();
});
it('soft-deletes annotations missing from the device key set and acks this device', async () => {
repo.findStatesForDeviceBook.mockResolvedValue([{ state: makeStateRow(), annotation: makeAnnotationRow() }]);
@@ -29,6 +29,7 @@ export interface IncomingDeviceAnnotation {
*/
externalKey?: string;
datetime: string;
sourceCreatedAt?: Date | null;
datetimeUpdated?: string | null;
drawer?: string;
color?: string | null;
@@ -163,6 +164,7 @@ export class AnnotationSyncService {
result.skippedDeleted += 1;
return;
}
await this.backfillSourceCreatedAt(annotation, incoming, source, tx);
const changed = await this.applyIncomingContent(annotation, incoming, bookFileId, tx);
if (changed) {
await this.syncRepo.updateState(deviceState.id, { lastAppliedVersion: changed.newVersion }, tx);
@@ -192,6 +194,7 @@ export class AnnotationSyncService {
result.skippedDeleted += 1;
return;
}
await this.backfillSourceCreatedAt(crossDevice.annotation, incoming, source, tx);
const state = await this.syncRepo.insertState(
{
annotationId: crossDevice.annotation.id,
@@ -240,6 +243,8 @@ export class AnnotationSyncService {
return;
}
await this.backfillSourceCreatedAt(annotation, incoming, source, tx);
const isMove = !exact && position != null && position.pos0 !== incoming.pos0;
let version = annotation.version;
if (isMove) {
@@ -287,6 +292,7 @@ export class AnnotationSyncService {
version: 1,
deviceCreatedAt: incoming.datetime,
deviceUpdatedAt: incoming.datetimeUpdated ?? null,
sourceCreatedAt: incoming.sourceCreatedAt ?? null,
},
{
bookFileId,
@@ -309,6 +315,16 @@ export class AnnotationSyncService {
result.created += 1;
}
private async backfillSourceCreatedAt(
annotation: AnnotationRow,
incoming: IncomingDeviceAnnotation,
source: AnnotationSyncSource,
tx: DbTx,
): Promise<void> {
if (annotation.origin !== source || annotation.sourceCreatedAt != null || incoming.sourceCreatedAt == null) return;
await this.syncRepo.setSourceCreatedAtSilent(annotation.id, incoming.sourceCreatedAt, tx);
}
/**
* Applies device content only when the device actually made a new edit since the last
* one we ingested (device-clock comparison, no cross-clock skew). Style and color use
@@ -547,10 +563,22 @@ export class AnnotationSyncService {
}): Promise<number> {
const present = new Set(params.presentKeys.map((entry) => entry.k));
const presentDatetimes = new Set(params.presentKeys.map((entry) => entry.dt));
const sourceCreatedAtByKey = new Map(
params.presentKeys.flatMap((entry) => (entry.sourceCreatedAt ? [[entry.k, entry.sourceCreatedAt] as const] : [])),
);
let deletedCount = 0;
await this.syncRepo.transaction(async (tx) => {
const states = await this.syncRepo.findStatesForDeviceBook(params.userId, params.source, params.deviceId, params.bookId, tx);
await this.syncRepo.setSourceCreatedAtsSilent(
states.flatMap(({ state, annotation }) => {
const sourceCreatedAt = sourceCreatedAtByKey.get(state.externalKey);
return annotation.origin === params.source && annotation.sourceCreatedAt == null && sourceCreatedAt
? [{ annotationId: annotation.id, sourceCreatedAt }]
: [];
}),
tx,
);
for (const { state, annotation } of states) {
if (annotation.deletedAt) continue;
if (present.has(state.externalKey)) continue;
@@ -747,6 +775,7 @@ export class AnnotationSyncService {
export interface DeviceKeyEntry {
k: string;
dt: string;
sourceCreatedAt?: Date | null;
}
export interface PushDownSets {
@@ -1,4 +1,6 @@
import { Test } from '@nestjs/testing';
import type { SQL } from 'drizzle-orm';
import { PgDialect } from 'drizzle-orm/pg-core';
import { DB } from '../../db';
import { AnnotationRepository } from './annotation.repository';
@@ -20,12 +22,18 @@ function makeRow(overrides?: Record<string, unknown>) {
deletedAt: null,
deviceCreatedAt: null,
deviceUpdatedAt: null,
sourceCreatedAt: null,
createdAt: new Date('2026-01-01T00:00:00Z'),
updatedAt: new Date('2026-01-01T00:00:00Z'),
...overrides,
};
}
function compileSql(fragments: unknown[]): string {
const dialect = new PgDialect();
return fragments.map((fragment) => dialect.sqlToQuery(fragment as SQL).sql).join(' ');
}
type ChainQuery = Record<string, ReturnType<typeof vi.fn>> & {
then: (resolve: (value: unknown) => unknown, reject?: (error: unknown) => unknown) => Promise<unknown>;
};
@@ -92,6 +100,7 @@ describe('AnnotationRepository', () => {
expect(db.select).toHaveBeenCalled();
expect(db._queries[0].leftJoin).toHaveBeenCalled();
expect(db._queries[0].orderBy).toHaveBeenCalled();
expect(compileSql(db._queries[0].orderBy.mock.calls[0])).toContain('coalesce("annotations"."source_created_at", "annotations"."created_at")');
expect(result).toEqual(rows);
});
@@ -524,7 +533,10 @@ describe('AnnotationRepository', () => {
expect(result.items).toHaveLength(1);
expect(result.items[0].jumpFileFormat).toBe('mobi');
expect(db.select.mock.calls[0][0]).toHaveProperty('jumpFileFormat');
expect(db.select.mock.calls[0][0]).toHaveProperty('xpointer');
expect(db._queries[0].where).toHaveBeenCalled();
expect(compileSql(db._queries[0].where.mock.calls[0])).toContain('coalesce("annotations"."source_created_at", "annotations"."created_at")');
expect(compileSql(db._queries[0].orderBy.mock.calls[0])).toContain('coalesce("annotations"."source_created_at", "annotations"."created_at")');
});
it('orders by book title when sort.by is book', async () => {
@@ -580,6 +592,8 @@ describe('AnnotationRepository', () => {
});
expect(rows).toHaveLength(2);
expect(db.select.mock.calls[0][0]).toHaveProperty('xpointer');
expect(compileSql(db._queries[0].orderBy.mock.calls[0])).toContain('coalesce("annotations"."source_created_at", "annotations"."created_at")');
});
});
@@ -48,7 +48,16 @@ export type AnnotationWithCfi = AnnotationRow & {
pdfPos0?: string | null;
pdfStatus?: string | null;
};
export type HubAnnotationRow = AnnotationWithCfi & { bookTitle: string | null; author: string | null; jumpFileFormat: string | null };
export type HubAnnotationRow = AnnotationWithCfi & {
bookTitle: string | null;
author: string | null;
jumpFileFormat: string | null;
xpointer: string | null;
};
function highlightedAt(): SQL<Date> {
return sql<Date>`coalesce(${annotations.sourceCreatedAt}, ${annotations.createdAt})`;
}
export interface HubFilters {
bookId?: number;
@@ -142,6 +151,9 @@ export class AnnotationRepository {
pageno: sql<
number | null
>`(select (ap3.extras ->> 'pageno')::int from annotation_positions ap3 where ap3.annotation_id = ${annotations.id} and ap3.format in ('xpointer', 'pdf') limit 1)`,
xpointer: sql<
string | null
>`(select ap4.pos0 from annotation_positions ap4 where ap4.annotation_id = ${annotations.id} and ap4.format = 'xpointer' limit 1)`,
};
}
@@ -173,7 +185,7 @@ export class AnnotationRepository {
async findByBookId(bookId: number, userId: number): Promise<AnnotationWithCfi[]> {
return this.selectWithCfi()
.where(and(...this.baseConditions(bookId, userId)))
.orderBy(asc(annotations.createdAt));
.orderBy(asc(highlightedAt()));
}
async findById(bookId: number, annotationId: number, userId: number): Promise<AnnotationWithCfi | null> {
@@ -256,7 +268,7 @@ export class AnnotationRepository {
// The step after `/6` in an epub CFI addresses the spine itemref. Kept raw here and
// converted to a spine index in foldChapterRows, where it can be tested.
cfiSpineStep: sql<number | null>`min(nullif(substring(${annotationPositions.pos0} from 'epubcfi[(]/6/([0-9]+)'), '')::int)`,
firstCreatedAt: sql<Date>`min(${annotations.createdAt})`,
firstCreatedAt: sql<Date>`min(${highlightedAt()})`,
})
.from(annotations)
.leftJoin(annotationPositions, cfiJoin)
@@ -264,7 +276,7 @@ export class AnnotationRepository {
.groupBy(annotations.chapterTitle, annotations.color),
this.db
.select({
day: sql<string>`to_char(${annotations.createdAt} at time zone 'UTC', 'YYYY-MM-DD')`,
day: sql<string>`to_char(${highlightedAt()} at time zone 'UTC', 'YYYY-MM-DD')`,
origin: annotations.origin,
count: count(),
})
@@ -421,7 +433,7 @@ export class AnnotationRepository {
// The hub groups a page at a time, so a grouped run only ever lands whole when the
// rows arrive already ordered by the grouping key. Every non-date sort therefore
// falls back to newest-first inside the group.
const withinGroup = [desc(annotations.createdAt), desc(annotations.id)];
const withinGroup = [desc(highlightedAt()), desc(annotations.id)];
const orderBy =
sort.by === 'book'
? [direction(bookMetadata.title), ...withinGroup]
@@ -429,7 +441,7 @@ export class AnnotationRepository {
? [direction(annotations.color), ...withinGroup]
: sort.by === 'origin'
? [direction(annotations.origin), ...withinGroup]
: [direction(annotations.createdAt), direction(annotations.id)];
: [direction(highlightedAt()), direction(annotations.id)];
const offset = (page - 1) * pageSize;
const [items, totalResult] = await Promise.all([
@@ -483,7 +495,7 @@ export class AnnotationRepository {
.leftJoin(bookMetadata, eq(bookMetadata.bookId, annotations.bookId))
.leftJoin(books, eq(books.id, annotations.bookId))
.where(and(...conditions))
.orderBy(asc(bookMetadata.title), asc(annotations.chapterTitle), asc(annotations.createdAt))
.orderBy(asc(bookMetadata.title), asc(annotations.chapterTitle), asc(highlightedAt()))
.limit(limit);
return rows as HubAnnotationRow[];
}
@@ -546,12 +558,12 @@ export class AnnotationRepository {
async getHubActivityWeeks(userId: number, filters: HubFilters, since: Date) {
return this.db
.select({
weekStart: sql<string>`to_char(date_trunc('week', ${annotations.createdAt} at time zone 'UTC'), 'YYYY-MM-DD')`,
weekStart: sql<string>`to_char(date_trunc('week', ${highlightedAt()} at time zone 'UTC'), 'YYYY-MM-DD')`,
origin: annotations.origin,
count: count(),
})
.from(annotations)
.where(and(...this.buildHubConditions(userId, filters), gte(annotations.createdAt, since)))
.where(and(...this.buildHubConditions(userId, filters), gte(highlightedAt(), since)))
.groupBy(sql`1`, annotations.origin)
.orderBy(sql`1`);
}
@@ -613,7 +625,7 @@ export class AnnotationRepository {
.leftJoin(bookMetadata, eq(bookMetadata.bookId, annotations.bookId))
.where(and(...conditions))
.groupBy(annotations.bookId, bookMetadata.title)
.orderBy(...(params.order === 'count' ? [desc(count()), asc(bookMetadata.title)] : [desc(max(annotations.createdAt)), asc(bookMetadata.title)]))
.orderBy(...(params.order === 'count' ? [desc(count()), asc(bookMetadata.title)] : [desc(max(highlightedAt())), asc(bookMetadata.title)]))
.limit(params.limit);
}
@@ -685,8 +697,8 @@ export class AnnotationRepository {
sql`exists (select 1 from ${annotationPositions} where ${annotationPositions.annotationId} = ${annotations.id} and ${annotationPositions.format} = 'cfi' and ${annotationPositions.status} <> 'exact')`,
);
}
if (filters.dateFrom) conditions.push(gte(annotations.createdAt, filters.dateFrom));
if (filters.dateTo) conditions.push(lte(annotations.createdAt, filters.dateTo));
if (filters.dateFrom) conditions.push(gte(highlightedAt(), filters.dateFrom));
if (filters.dateTo) conditions.push(lte(highlightedAt(), filters.dateTo));
return conditions;
}
@@ -722,10 +734,10 @@ export class AnnotationRepository {
conditions.push(eq(annotations.chapterTitle, filters.chapter));
}
if (filters.dateFrom) {
conditions.push(gte(annotations.createdAt, filters.dateFrom));
conditions.push(gte(highlightedAt(), filters.dateFrom));
}
if (filters.dateTo) {
conditions.push(lte(annotations.createdAt, filters.dateTo));
conditions.push(lte(highlightedAt(), filters.dateTo));
}
if (filters.hasNote) {
conditions.push(sql`${annotations.note} is not null and ${annotations.note} <> ''`);
@@ -767,6 +779,6 @@ export class AnnotationRepository {
direction(annotations.id),
];
}
return [direction(annotations.createdAt), direction(annotations.id)];
return [direction(highlightedAt()), direction(annotations.id)];
}
}
@@ -54,6 +54,7 @@ function makeAnnotationRow(overrides?: Record<string, unknown>) {
deletedAt: null,
deviceCreatedAt: null,
deviceUpdatedAt: null,
sourceCreatedAt: null,
createdAt: new Date('2026-01-01T00:00:00Z'),
updatedAt: new Date('2026-01-01T00:00:00Z'),
...overrides,
@@ -559,9 +560,23 @@ describe('AnnotationService', () => {
expect(dto.style).toBe('underline');
expect(dto.note).toBe('a note');
expect(dto.chapterTitle).toBe('Intro');
expect(dto.highlightedAt).toEqual(new Date('2026-03-01T00:00:00Z'));
expect(dto.createdAt).toEqual(new Date('2026-03-01T00:00:00Z'));
});
it('maps source creation and ingestion timestamps independently', () => {
const row = makeAnnotationRow({
origin: 'koreader',
sourceCreatedAt: new Date('2026-01-15T16:30:00.000Z'),
createdAt: new Date('2026-09-06T17:10:19.332Z'),
});
const dto = AnnotationResponseDto.from(row as never);
expect(dto.highlightedAt).toEqual(new Date('2026-01-15T16:30:00.000Z'));
expect(dto.createdAt).toEqual(new Date('2026-09-06T17:10:19.332Z'));
});
it('parses pdf geometry and derives positionStatus for a pdf row', () => {
const geometry = {
page: 2,
@@ -67,6 +67,7 @@ export class AnnotationService {
origin: dto.origin as 'web' | 'koreader' | 'kobo',
positionStatus: dto.positionStatus as AnnotationListResponse['items'][number]['positionStatus'],
chapterIndex: dto.chapterIndex,
highlightedAt: dto.highlightedAt.toISOString(),
createdAt: dto.createdAt instanceof Date ? dto.createdAt.toISOString() : String(dto.createdAt),
pdf: dto.pdf,
};
@@ -16,6 +16,7 @@ export class AnnotationResponseDto {
origin!: string;
positionStatus!: string | null;
chapterIndex!: number | null;
highlightedAt!: Date;
createdAt!: Date;
pdf!: AnnotationPdfPosition | null;
@@ -36,6 +37,7 @@ export class AnnotationResponseDto {
dto.positionStatus = resolvePositionStatus(row);
const chapterIndex = (row.cfiExtras as { chapterIndex?: number } | null)?.chapterIndex;
dto.chapterIndex = typeof chapterIndex === 'number' ? chapterIndex : null;
dto.highlightedAt = row.sourceCreatedAt ?? row.createdAt;
dto.createdAt = row.createdAt;
return dto;
}
@@ -5,15 +5,15 @@ import type { RequestUser } from '../../common/types/request-user';
import type { AnnotationSyncService } from '../annotation/annotation-sync.service';
import type { PositionConverterService } from '../position-converter/position-converter.service';
import type { AnnotationExchangeAckDto, AnnotationExchangeDto } from './dto';
import { KoreaderAnnotationExchangeService } from './koreader-annotation-exchange.service';
import { KoreaderAnnotationExchangeService, parseKoreaderSourceCreatedAt } from './koreader-annotation-exchange.service';
import type { KoreaderRepository } from './koreader.repository';
const DEVICE_ID = 'device-1234';
const HASH_A = 'a'.repeat(32);
const HASH_B = 'b'.repeat(32);
function makeUser(): RequestUser {
return { id: 7, settings: {} } as unknown as RequestUser;
function makeUser(timezone = 'UTC'): RequestUser {
return { id: 7, settings: { timezone } } as unknown as RequestUser;
}
function makeAnnotationRow(overrides: Record<string, unknown> = {}) {
@@ -31,6 +31,7 @@ function makeAnnotationRow(overrides: Record<string, unknown> = {}) {
deletedAt: null,
deviceCreatedAt: null,
deviceUpdatedAt: null,
sourceCreatedAt: null,
createdAt: new Date('2026-06-08T10:00:00Z'),
updatedAt: new Date('2026-06-09T11:30:00Z'),
...overrides,
@@ -106,6 +107,33 @@ describe('KoreaderAnnotationExchangeService', () => {
return { hash: HASH_A, keys: [], keysComplete: true, changes: [], ...overrides } as AnnotationExchangeDto['books'][number];
}
it('converts KOReader wall-clock dates with the user timezone, including historical DST', () => {
expect(parseKoreaderSourceCreatedAt('2026-01-15 09:30:00', 'America/Denver')).toEqual(new Date('2026-01-15T16:30:00.000Z'));
expect(parseKoreaderSourceCreatedAt('2026-07-15 09:30:00', 'America/Denver')).toEqual(new Date('2026-07-15T15:30:00.000Z'));
});
it('rejects impossible or nonexistent KOReader wall-clock dates', () => {
expect(parseKoreaderSourceCreatedAt('2026-02-30 09:30:00', 'UTC')).toBeNull();
expect(parseKoreaderSourceCreatedAt('2026-03-08 02:30:00', 'America/Denver')).toBeNull();
});
it('passes the normalized source date into annotation ingest', async () => {
const change = {
datetime: '2026-01-15 09:30:00',
drawer: 'lighten',
posFormat: 'xpointer',
pos0: '/body/DocFragment[1]',
} as AnnotationExchangeDto['books'][number]['changes'][number];
await service.exchange(makeUser('America/Denver'), makeExchangeDto([makeBook({ changes: [change] })]));
expect(annotationSync.ingestDeviceAnnotations).toHaveBeenCalledWith(
expect.objectContaining({
annotations: [expect.objectContaining({ datetime: change.datetime, sourceCreatedAt: new Date('2026-01-15T16:30:00.000Z') })],
}),
);
});
it('reports unmatched hashes without processing them', async () => {
const response = await service.exchange(makeUser(), makeExchangeDto([makeBook({ hash: HASH_B })]));
@@ -137,7 +165,12 @@ describe('KoreaderAnnotationExchangeService', () => {
await service.exchange(makeUser(), makeExchangeDto([makeBook({ keys })]));
expect(annotationSync.detectDeviceDeletions).toHaveBeenCalledWith(
expect.objectContaining({ userId: 7, deviceId: DEVICE_ID, bookId: 20, presentKeys: keys }),
expect.objectContaining({
userId: 7,
deviceId: DEVICE_ID,
bookId: 20,
presentKeys: [expect.objectContaining({ ...keys[0], sourceCreatedAt: new Date('2026-06-01T21:14:03.000Z') })],
}),
);
});
@@ -1,7 +1,9 @@
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { formatInTimeZone, fromZonedTime } from 'date-fns-tz';
import type { RequestUser } from '../../common/types/request-user';
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
import { resolveTimeZone } from '../../common/utils/timezone.utils';
import type { AnnotationPosition, AnnotationRow } from '../../db/schema';
import { drawerFromStyle, koreaderColorFromHex } from '../annotation/annotation-style-map';
import { AnnotationSyncService, formatDeviceDatetime, type IncomingDeviceAnnotation, type IngestResult } from '../annotation/annotation-sync.service';
@@ -15,9 +17,20 @@ const ACK_EVENT = 'koreader.annotation_exchange_ack';
const MAX_CHANGES_PER_REQUEST = 50;
const PUSH_DOWN_PAGE = 100;
const CONVERSION_BUDGET_PER_REQUEST = 20;
const DEVICE_DATETIME_FORMAT = 'yyyy-MM-dd HH:mm:ss';
type DevicePositionsByFormat = { pdf?: AnnotationPosition; xpointer?: AnnotationPosition; cfi?: AnnotationPosition };
export function parseKoreaderSourceCreatedAt(datetime: string, timeZone: string): Date | null {
try {
const parsed = fromZonedTime(datetime, timeZone);
if (Number.isNaN(parsed.getTime())) return null;
return formatInTimeZone(parsed, timeZone, DEVICE_DATETIME_FORMAT) === datetime ? parsed : null;
} catch {
return null;
}
}
export interface ExchangeAddEntry {
serverId: number;
version: number;
@@ -102,6 +115,7 @@ export class KoreaderAnnotationExchangeService {
const hashes = [...new Set(dto.books.map((book) => book.hash.toLowerCase()))];
const matches = await this.koreaderRepo.resolveBookFilesByHashes(hashes, accessibleLibraryIds, user.id);
const deviceClockOffsetMs = this.deviceClockOffsetMs(dto.deviceTime);
const timeZone = resolveTimeZone((user.settings as { timezone?: unknown } | undefined)?.timezone, 'UTC');
const results: ExchangeBookResult[] = [];
const unmatched: string[] = [];
@@ -114,7 +128,7 @@ export class KoreaderAnnotationExchangeService {
unmatched.push(hash);
continue;
}
const result = await this.exchangeBook(user.id, dto.deviceId, hash, match.bookId, match.bookFileId, book, deviceClockOffsetMs);
const result = await this.exchangeBook(user.id, dto.deviceId, hash, match.bookId, match.bookFileId, book, deviceClockOffsetMs, timeZone);
pushedTotal += result.toApply.add.length + result.toApply.edit.length + result.toApply.delete.length;
results.push(result);
}
@@ -202,6 +216,7 @@ export class KoreaderAnnotationExchangeService {
bookFileId: number,
book: ExchangeBookDto,
deviceClockOffsetMs: number,
timeZone: string,
): Promise<ExchangeBookResult> {
const ingest = await this.annotationSync.ingestDeviceAnnotations({
userId,
@@ -209,7 +224,7 @@ export class KoreaderAnnotationExchangeService {
deviceId,
bookId,
bookFileId,
annotations: book.changes.map((change) => this.toIncoming(change)),
annotations: book.changes.map((change) => this.toIncoming(change, timeZone)),
});
let deviceDeleted = 0;
@@ -219,7 +234,10 @@ export class KoreaderAnnotationExchangeService {
source: 'koreader',
deviceId,
bookId,
presentKeys: book.keys,
presentKeys: book.keys.map((entry) => ({
...entry,
sourceCreatedAt: parseKoreaderSourceCreatedAt(entry.dt, timeZone),
})),
});
}
@@ -427,9 +445,10 @@ export class KoreaderAnnotationExchangeService {
return candidate;
}
private toIncoming(change: KoreaderAnnotationDto): IncomingDeviceAnnotation {
private toIncoming(change: KoreaderAnnotationDto, timeZone: string): IncomingDeviceAnnotation {
return {
datetime: change.datetime,
sourceCreatedAt: parseKoreaderSourceCreatedAt(change.datetime, timeZone),
datetimeUpdated: change.datetimeUpdated ?? null,
drawer: change.drawer,
color: change.color ?? null,