diff --git a/server/src/modules/kobo/services/kobo-proxy.service.test.ts b/server/src/modules/kobo/services/kobo-proxy.service.test.ts index e5a2dd295..c53046233 100644 --- a/server/src/modules/kobo/services/kobo-proxy.service.test.ts +++ b/server/src/modules/kobo/services/kobo-proxy.service.test.ts @@ -1,5 +1,6 @@ import { BadRequestException } from '@nestjs/common'; +import type { KoboProxyRequestOptions } from './kobo-proxy.service'; import { KoboProxyService } from './kobo-proxy.service'; function makeReply() { @@ -43,6 +44,9 @@ describe('KoboProxyService', () => { accept: 'application/json', host: 'localhost:3000', 'x-kobo-deviceid': 'dev123', + 'x-kobo-userkey': 'user-secret', + 'x-kobo-synctoken': 'raw-kobo-token', + 'content-length': '17', }, body: { hello: 'world' }, }; @@ -50,14 +54,24 @@ describe('KoboProxyService', () => { await service.forward(req as never, reply as never, 'device-1'); - expect(fetchMock).toHaveBeenCalledWith('https://storeapi.kobo.com/v1/library/sync?since=1', { - method: 'POST', - headers: { - accept: 'application/json', - 'x-kobo-deviceid': 'dev123', - }, - body: '{"hello":"world"}', - }); + expect(fetchMock).toHaveBeenCalledWith( + 'https://storeapi.kobo.com/v1/library/sync?since=1', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + accept: 'application/json', + 'x-kobo-deviceid': 'dev123', + 'x-kobo-userkey': 'user-secret', + 'x-kobo-synctoken': 'raw-kobo-token', + }), + body: '{"hello":"world"}', + }), + ); + const sentHeaders = fetchMock.mock.calls[0][1].headers as Record; + expect(sentHeaders).not.toHaveProperty('host'); + expect(sentHeaders).not.toHaveProperty('content-length'); + // Raw Kobo tokens (no PX. prefix) reach Kobo verbatim, so the device's own cursor is forwarded. + expect(sentHeaders['x-kobo-synctoken']).toBe('raw-kobo-token'); expect(reply.status).toHaveBeenCalledWith(200); expect(reply.header).toHaveBeenCalledWith('content-type', 'application/json'); expect(reply.header).toHaveBeenCalledWith('x-custom', 'ok'); @@ -119,18 +133,21 @@ describe('KoboProxyService', () => { await service.forward(req as never, reply as never, 'device-1'); - expect(fetchMock).toHaveBeenCalledWith(`https://storeapi.kobo.com/v1/library/${entitlementId}/state`, { - method: 'PUT', - headers: { - accept: 'application/json', - authorization: 'Bearer kobo-oauth-token', - 'content-type': 'application/json', - 'user-agent': 'Kobo Touch', - 'x-kobo-appversion': '4.45.23697', - 'x-kobo-deviceid': 'device-id', - }, - body: JSON.stringify(body), - }); + expect(fetchMock).toHaveBeenCalledWith( + `https://storeapi.kobo.com/v1/library/${entitlementId}/state`, + expect.objectContaining({ + method: 'PUT', + headers: expect.objectContaining({ + accept: 'application/json', + authorization: 'Bearer kobo-oauth-token', + 'content-type': 'application/json', + 'user-agent': 'Kobo Touch', + 'x-kobo-appversion': '4.45.23697', + 'x-kobo-deviceid': 'device-id', + }), + body: JSON.stringify(body), + }), + ); expect(reply.status).toHaveBeenCalledWith(200); expect(reply.send).toHaveBeenCalledWith(Buffer.from('{"RequestResult":"Success"}')); }); @@ -154,6 +171,236 @@ describe('KoboProxyService', () => { expect(reply.send).toHaveBeenCalledWith({ message: 'Upstream Kobo API unavailable' }); }); + describe('over-the-device-borrow flow', () => { + it('forwards the Kobo userkey and decodes the BookOrbit composite sync token before proxying a borrow', async () => { + const service = new KoboProxyService(); + const upstream = { + status: 200, + headers: new Headers({ 'content-type': 'application/json' }), + arrayBuffer: vi.fn().mockResolvedValue(new TextEncoder().encode('{"success":true}').buffer), + }; + const fetchMock = vi.fn().mockResolvedValue(upstream); + vi.stubGlobal('fetch', fetchMock); + + const composite = `PX.${Buffer.from(JSON.stringify({ snapshotId: 4, koboSyncToken: 'kobo-cursor-9' })).toString('base64')}`; + const req = { + method: 'POST', + url: '/api/v1/kobo/dev/v1/library/borrow', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + 'x-kobo-userkey': 'device-user-key', + 'x-kobo-deviceid': 'device-id', + 'x-kobo-synctoken': composite, + }, + body: { BookId: '9780123456789' }, + }; + const reply = makeReply(); + + await service.forward(req as never, reply as never, 'dev'); + + expect(fetchMock).toHaveBeenCalledWith( + 'https://storeapi.kobo.com/v1/library/borrow', + expect.objectContaining({ + headers: expect.objectContaining({ + 'x-kobo-userkey': 'device-user-key', + 'x-kobo-synctoken': 'kobo-cursor-9', + }), + body: JSON.stringify({ BookId: '9780123456789' }), + }), + ); + }); + + it('omits the sync token entirely when the device sent no Kobo cursor', async () => { + const service = new KoboProxyService(); + const upstream = { + status: 200, + headers: new Headers(), + arrayBuffer: vi.fn().mockResolvedValue(new ArrayBuffer(0)), + }; + const fetchMock = vi.fn().mockResolvedValue(upstream); + vi.stubGlobal('fetch', fetchMock); + + const composite = `PX.${Buffer.from(JSON.stringify({ snapshotId: 4 })).toString('base64')}`; + const req = { + method: 'POST', + url: '/api/v1/kobo/dev/v1/library/borrow', + headers: { + 'x-kobo-userkey': 'device-user-key', + 'x-kobo-synctoken': composite, + }, + body: {}, + }; + + await service.forward(req as never, makeReply() as never, 'dev'); + + const sentHeaders = fetchMock.mock.calls[0][1].headers as Record; + expect(sentHeaders).not.toHaveProperty('x-kobo-synctoken'); + expect(sentHeaders['x-kobo-userkey']).toBe('device-user-key'); + }); + }); + + describe('credentials that must not reach Kobo', () => { + function stubFetch() { + const fetchMock = vi.fn().mockResolvedValue({ + status: 200, + headers: new Headers(), + arrayBuffer: vi.fn().mockResolvedValue(new ArrayBuffer(0)), + }); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + async function sentHeaders(headers: Record, options?: KoboProxyRequestOptions): Promise> { + const fetchMock = stubFetch(); + await new KoboProxyService().request({ method: 'GET', url: '/api/v1/kobo/dev/v1/library/sync', headers } as never, 'dev', options); + return fetchMock.mock.calls[0][1].headers as Record; + } + + const composite = `PX.${Buffer.from(JSON.stringify({ snapshotId: 4, koboSyncToken: 'kobo-cursor-9' })).toString('base64')}`; + + // Every caller gets the substitution, not just the ones that remember to ask: forwardTagDelete + // relays through request() with no options and must not hand Kobo a token it cannot read. + it('substitutes the Kobo cursor for the composite token even with no options', async () => { + expect((await sentHeaders({ 'x-kobo-synctoken': composite }))['x-kobo-synctoken']).toBe('kobo-cursor-9'); + }); + + it('drops a composite token it cannot decode rather than forwarding it', async () => { + expect(await sentHeaders({ 'x-kobo-synctoken': 'PX.not-base64-json' })).not.toHaveProperty('x-kobo-synctoken'); + }); + + it('still lets omitHeaders drop the token the substitution would have written', async () => { + const headers = await sentHeaders({ 'x-kobo-synctoken': composite }, { omitHeaders: ['x-kobo-synctoken'] }); + expect(headers).not.toHaveProperty('x-kobo-synctoken'); + }); + + it('still lets extraHeaders override the token the substitution would have written', async () => { + const headers = await sentHeaders({ 'x-kobo-synctoken': composite }, { extraHeaders: { 'x-kobo-synctoken': 'server-cursor' } }); + expect(headers['x-kobo-synctoken']).toBe('server-cursor'); + }); + + // These routes sit on the BookOrbit origin, so a browser carrying a session reaches them with + // our auth cookies attached. Kobo must never receive those; cookies it set itself still ride. + it('strips the BookOrbit session cookies while keeping the rest of the jar', async () => { + const headers = await sentHeaders({ cookie: 'access_token=secret; kobo_session=keep-me; refresh_token=secret; wsid=abc' }); + expect(headers.cookie).toBe('kobo_session=keep-me; wsid=abc'); + }); + + it('sends no cookie header at all when only BookOrbit cookies were present', async () => { + expect(await sentHeaders({ cookie: 'access_token=secret; refresh_token=secret' })).not.toHaveProperty('cookie'); + }); + + it('drops the reverse proxy forwarding headers that describe our hop', async () => { + const headers = await sentHeaders({ + forwarded: 'for=10.0.0.5', + 'x-forwarded-for': '10.0.0.5', + 'x-forwarded-host': 'books.example', + 'x-forwarded-proto': 'https', + 'x-real-ip': '10.0.0.5', + 'x-kobo-userkey': 'device-user-key', + }); + + expect(headers).not.toHaveProperty('forwarded'); + expect(headers).not.toHaveProperty('x-forwarded-for'); + expect(headers).not.toHaveProperty('x-forwarded-host'); + expect(headers).not.toHaveProperty('x-forwarded-proto'); + expect(headers).not.toHaveProperty('x-real-ip'); + expect(headers['x-kobo-userkey']).toBe('device-user-key'); + }); + }); + + describe('hop-by-hop header handling', () => { + it('strips the singular Trailer header field on the way out', async () => { + const service = new KoboProxyService(); + const upstream = { + status: 200, + headers: new Headers(), + arrayBuffer: vi.fn().mockResolvedValue(new ArrayBuffer(0)), + }; + const fetchMock = vi.fn().mockResolvedValue(upstream); + vi.stubGlobal('fetch', fetchMock); + + await service.forward( + { + method: 'GET', + url: '/api/v1/kobo/dev/v1/library/sync', + headers: { + trailer: 'x-internal-state', + 'x-kobo-deviceid': 'device-id', + }, + } as never, + makeReply() as never, + 'dev', + ); + + const sentHeaders = fetchMock.mock.calls[0][1].headers as Record; + expect(sentHeaders).not.toHaveProperty('trailer'); + expect(sentHeaders['x-kobo-deviceid']).toBe('device-id'); + }); + + it('drops header fields nominated by the device Connection header before forwarding', async () => { + const service = new KoboProxyService(); + const upstream = { + status: 200, + headers: new Headers(), + arrayBuffer: vi.fn().mockResolvedValue(new ArrayBuffer(0)), + }; + const fetchMock = vi.fn().mockResolvedValue(upstream); + vi.stubGlobal('fetch', fetchMock); + + await service.forward( + { + method: 'GET', + url: '/api/v1/kobo/dev/v1/library/sync', + headers: { + connection: 'x-kobo-tracking, x-kobo-internal', + 'x-kobo-tracking': 'should-be-stripped', + 'x-kobo-internal': 'also-stripped', + 'x-kobo-deviceid': 'device-id', + }, + } as never, + makeReply() as never, + 'dev', + ); + + const sentHeaders = fetchMock.mock.calls[0][1].headers as Record; + expect(sentHeaders).not.toHaveProperty('x-kobo-tracking'); + expect(sentHeaders).not.toHaveProperty('x-kobo-internal'); + expect(sentHeaders['x-kobo-deviceid']).toBe('device-id'); + }); + + it('drops header fields nominated by the upstream Connection header before relaying the response', async () => { + const service = new KoboProxyService(); + const upstream = { + status: 200, + headers: new Headers({ + connection: 'x-kobo-tracking, x-kobo-internal', + 'x-kobo-tracking': 'should-be-stripped', + 'x-kobo-internal': 'also-stripped', + 'x-kobo-deviceid': 'device-id', + }), + arrayBuffer: vi.fn().mockResolvedValue(new TextEncoder().encode('{}').buffer), + }; + const fetchMock = vi.fn().mockResolvedValue(upstream); + vi.stubGlobal('fetch', fetchMock); + + const reply = makeReply(); + await service.forward( + { + method: 'GET', + url: '/api/v1/kobo/dev/v1/library/sync', + headers: {}, + } as never, + reply as never, + 'dev', + ); + + expect(reply.header).not.toHaveBeenCalledWith('x-kobo-tracking', expect.anything()); + expect(reply.header).not.toHaveBeenCalledWith('x-kobo-internal', expect.anything()); + expect(reply.header).toHaveBeenCalledWith('x-kobo-deviceid', 'device-id'); + }); + }); + describe('request', () => { function stubUpstream(headers: Record, body = '[]') { const fetchMock = vi.fn().mockResolvedValue({ @@ -168,7 +415,7 @@ describe('KoboProxyService', () => { const syncRequest = { method: 'GET', url: '/api/v1/kobo/dev/v1/library/sync?Filter=ALL', - headers: { authorization: 'Bearer kobo-jwt', 'x-kobo-synctoken': 'PX.composite' }, + headers: { authorization: 'Bearer kobo-jwt', 'x-kobo-synctoken': 'PX.composite', 'x-kobo-userkey': 'user-secret' }, }; it('returns the upstream response with lowercased headers instead of piping it', async () => { diff --git a/server/src/modules/kobo/services/kobo-proxy.service.ts b/server/src/modules/kobo/services/kobo-proxy.service.ts index 01204bea3..febeb43cb 100644 --- a/server/src/modules/kobo/services/kobo-proxy.service.ts +++ b/server/src/modules/kobo/services/kobo-proxy.service.ts @@ -1,26 +1,56 @@ import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import type { FastifyReply, FastifyRequest } from 'fastify'; +import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils'; +import { decodeSyncToken } from './kobo-sync-token'; + const KOBO_API_BASE = 'https://storeapi.kobo.com'; const KOBO_API_HOSTNAME = 'storeapi.kobo.com'; -const FORWARD_HEADERS = [ - 'accept', - 'accept-language', - 'authorization', - 'content-type', - 'user-agent', - 'x-kobo-affiliatename', - 'x-kobo-appversion', - 'x-kobo-deviceid', - 'x-kobo-devicemodel', - 'x-kobo-deviceos', - 'x-kobo-deviceosversion', - 'x-kobo-platformid', - 'x-kobo-synctokenversion', -]; +// Headers that must never be forwarded. Connection-specific headers are defined by RFC 7230 as +// scoped to a single hop and reset by every proxy, and `host` would otherwise point Kobo at the +// BookOrbit host that the device just spoke to. Everything else the device sends (including +// `x-kobo-userkey`, which authorizes most Kobo API calls, and `x-kobo-synctoken`, which carries +// the upstream cursor) is forwarded, with two escape hatches callers can use: +// * `extraHeaders` overrides a forwarded value (the sync-token-aware entry points use this to +// hand Kobo its own cursor instead of the BookOrbit composite). +// * `omitHeaders` strips a forwarded header (used to drop our composite when we have no Kobo +// cursor to substitute). +const DROP_HEADERS = new Set([ + 'host', + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', + 'content-length', + 'content-encoding', + // Added by the operator's reverse proxy; describes our hop, not the device's, and tells Kobo + // about the deployment's internal addressing. + 'forwarded', + 'x-forwarded-for', + 'x-forwarded-host', + 'x-forwarded-proto', + 'x-real-ip', +]); -const HOP_BY_HOP_HEADERS = ['transfer-encoding', 'connection', 'content-encoding', 'content-length']; +const SYNC_TOKEN_HEADER = 'x-kobo-synctoken'; + +// BookOrbit's own session cookies. The device token in the path is the credential for these +// routes, so a browser that carries a BookOrbit session to one must not hand it to Kobo. +const BOOKORBIT_COOKIE_NAMES = new Set(['access_token', 'refresh_token']); + +/** Drops BookOrbit's session cookies from a Cookie header, keeping any the device set itself. */ +function stripBookOrbitCookies(cookieHeader: string): string { + return cookieHeader + .split(';') + .filter((pair) => !BOOKORBIT_COOKIE_NAMES.has(pair.split('=')[0].trim())) + .join(';') + .trim(); +} export type KoboProxyResponse = { status: number; @@ -30,11 +60,26 @@ export type KoboProxyResponse = { export type KoboProxyRequestOptions = { extraHeaders?: Record; - /** Drops headers FORWARD_HEADERS would otherwise copy from the device request. */ + /** Drops headers the device would otherwise copy. */ omitHeaders?: string[]; timeoutMs?: number; }; +/** + * Reads the `Connection` header from a header bag, splits it on commas, lowercases each field + * name, and returns the result. RFC 7230 ยง6.1 says every field listed in `Connection` is + * hop-specific: a proxy must remove those fields before forwarding the message. + */ +function readConnectionFieldNames(headers: Record): string[] { + const raw = headers['connection']; + if (raw === undefined) return []; + const values = Array.isArray(raw) ? raw : [raw]; + return values + .flatMap((v) => v.split(',')) + .map((name) => name.trim().toLowerCase()) + .filter((name) => name.length > 0); +} + @Injectable() export class KoboProxyService { private readonly logger = new Logger(KoboProxyService.name); @@ -43,11 +88,34 @@ export class KoboProxyService { async request(req: FastifyRequest, deviceToken: string, options: KoboProxyRequestOptions = {}): Promise { const targetUrl = this.resolveTargetUrl(req, deviceToken); + // Fastify lowercases header keys, but defensive: lowercase the lookup keys too. + const sourceHeaders = req.headers as Record; + const connectionFields = new Set(readConnectionFieldNames(sourceHeaders)); const headers: Record = {}; - for (const key of FORWARD_HEADERS) { - const val = req.headers[key]; - if (val) headers[key] = Array.isArray(val) ? val[0] : val; + for (const [key, value] of Object.entries(sourceHeaders)) { + if (value === undefined) continue; + const lowerKey = key.toLowerCase(); + if (DROP_HEADERS.has(lowerKey)) continue; + if (connectionFields.has(lowerKey)) continue; + const raw = Array.isArray(value) ? (value[0] ?? '') : value; + + // Kobo cannot read the BookOrbit composite token, so it gets its own cursor or nothing. + // Applied before omitHeaders and extraHeaders so callers can still override either way. + if (lowerKey === SYNC_TOKEN_HEADER) { + const koboCursor = decodeSyncToken(raw).koboSyncToken; + if (koboCursor !== undefined) headers[lowerKey] = koboCursor; + continue; + } + + if (lowerKey === 'cookie') { + const kept = stripBookOrbitCookies(raw); + if (kept.length > 0) headers[lowerKey] = kept; + continue; + } + + headers[lowerKey] = raw; } + for (const key of options.omitHeaders ?? []) { delete headers[key.toLowerCase()]; } @@ -76,28 +144,49 @@ export class KoboProxyService { return { status: upstream.status, headers: responseHeaders, body: Buffer.from(await upstream.arrayBuffer()) }; } + /** + * Relays a request to Kobo, dropping the connection-specific headers and replacing the device's + * BookOrbit composite sync token with the Kobo cursor it carries. Kobo's API does not understand + * `PX.` and treats it as a fresh cursor, so a borrow call that uses it lands Kobo in a + * state where the device has an entitlement BookOrbit never sees on this iteration. + */ + async forward(req: FastifyRequest, reply: FastifyReply, deviceToken: string): Promise { + const startedAt = Date.now(); + const targetUrl = this.resolveTargetUrl(req, deviceToken); + const deviceId = (req as unknown as { koboDevice?: { deviceId: number } }).koboDevice?.deviceId; + + this.logger.debug(`[kobo.proxy] [start] deviceId=${deviceId ?? 'unknown'} - upstream Kobo relay started`); + + try { + const response = await this.request(req, deviceToken); + this.sendUpstream(reply, response); + this.logger.debug( + `[kobo.proxy] [end] deviceId=${deviceId ?? 'unknown'} durationMs=${Date.now() - startedAt} upstreamStatus=${response.status} - upstream Kobo relay completed`, + ); + } catch (err: unknown) { + const errorClass = err instanceof Error ? err.name : 'UnknownError'; + const errorMessage = sanitizeLogValue(err instanceof Error ? err.message : 'unknown error'); + const sanitizedUrl = sanitizeLogValue(targetUrl); + this.logger.warn( + `[kobo.proxy] [fail] deviceId=${deviceId ?? 'unknown'} durationMs=${Date.now() - startedAt} errorClass=${errorClass} error="${errorMessage}" targetUrl="${sanitizedUrl}" - upstream Kobo relay failed`, + ); + reply.status(502).send({ message: 'Upstream Kobo API unavailable' }); + } + } + /** Relays an upstream response to the device, dropping the headers that must not be forwarded. */ sendUpstream(reply: FastifyReply, response: KoboProxyResponse): void { reply.status(response.status); + const connectionFields = new Set(readConnectionFieldNames(response.headers)); for (const [key, value] of Object.entries(response.headers)) { - if (!HOP_BY_HOP_HEADERS.includes(key)) { - reply.header(key, value); - } + const lowerKey = key.toLowerCase(); + if (DROP_HEADERS.has(lowerKey)) continue; + if (connectionFields.has(lowerKey)) continue; + reply.header(lowerKey, value); } reply.send(response.body); } - async forward(req: FastifyRequest, reply: FastifyReply, deviceToken: string) { - const targetUrl = this.resolveTargetUrl(req, deviceToken); - - try { - this.sendUpstream(reply, await this.request(req, deviceToken)); - } catch (err) { - this.logger.warn(`Proxy failed for ${targetUrl}: ${(err as Error).message}`); - reply.status(502).send({ message: 'Upstream Kobo API unavailable' }); - } - } - private resolveTargetUrl(req: FastifyRequest, deviceToken: string): string { const rawUrl = req.url; const prefix = `/api/v1/kobo/${deviceToken}`; diff --git a/server/src/modules/maintenance/missing-resources.service.test.ts b/server/src/modules/maintenance/missing-resources.service.test.ts index ce778ccc9..3b1613860 100644 --- a/server/src/modules/maintenance/missing-resources.service.test.ts +++ b/server/src/modules/maintenance/missing-resources.service.test.ts @@ -9,6 +9,8 @@ import { MissingResourcesService } from './missing-resources.service'; const user = { id: 7, isSuperuser: true } as never; +const SWEEP_SETTLE_TIMEOUT_MS = 10_000; + describe('MissingResourcesService', () => { let appDataPath: string; let coversRoot: string; @@ -54,9 +56,13 @@ describe('MissingResourcesService', () => { async function runSweep(service: MissingResourcesService): Promise { await service.startSweep(user); - // The sweep runs detached; drain the microtask/IO queue until it settles. - for (let attempt = 0; attempt < 50 && store.isRunning(user.id); attempt += 1) { - await new Promise((resolve) => setImmediate(resolve)); + // The sweep runs detached over real filesystem I/O, so wait on the store rather than draining a + // fixed number of turns: a loaded runner outlasts any turn budget, and giving up silently left + // the sweep running into the next call, which failed as an unrelated "sweep is not complete". + const deadline = Date.now() + SWEEP_SETTLE_TIMEOUT_MS; + while (store.isRunning(user.id)) { + if (Date.now() >= deadline) throw new Error(`cover sweep did not settle within ${SWEEP_SETTLE_TIMEOUT_MS}ms`); + await new Promise((resolve) => setTimeout(resolve, 5)); } }