mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
feat(books): add read-aloud generation
Generate a synced read-aloud EPUB from a book's EPUB and audiobook through an optional STAlign worker (Docker on CPU, or native with MLX on Apple Silicon), with review, re-sync, replace in place, restart recovery and live progress on the Files tab. Book details also stack the ebook and audiobook covers when both exist. See docs/READ_ALOUD.md.
This commit is contained in:
neon
@@ -0,0 +1,68 @@
|
||||
# Debian trixie for ffmpeg 7: STAlign's audiobook probe passes `ffprobe -output_format`, which the
|
||||
# ffmpeg 5.1 in bookworm rejects, failing every job in its first stage.
|
||||
ARG NODE_IMAGE=node:26.8.1-trixie-slim
|
||||
|
||||
FROM ${NODE_IMAGE} AS builder
|
||||
WORKDIR /app
|
||||
RUN npm install -g pnpm@11.22.0
|
||||
|
||||
# The image runs on CPU only: without this, onnxruntime-node downloads its CUDA provider on
|
||||
# linux/x64, which cannot load here (no CUDA or cuDNN libraries) and only adds size.
|
||||
ENV ONNXRUNTIME_NODE_INSTALL=skip
|
||||
|
||||
COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./
|
||||
COPY patches/ ./patches/
|
||||
COPY packages/types/package.json ./packages/types/
|
||||
COPY workers/read-aloud/package.json ./workers/read-aloud/
|
||||
RUN --mount=type=cache,id=pnpm-read-aloud,target=/root/.local/share/pnpm/store \
|
||||
pnpm install --filter @bookorbit/read-aloud-worker... --frozen-lockfile
|
||||
|
||||
COPY packages/types/ ./packages/types/
|
||||
COPY workers/read-aloud/ ./workers/read-aloud/
|
||||
RUN pnpm --config.verify-deps-before-run=false --filter @bookorbit/types build && \
|
||||
pnpm --config.verify-deps-before-run=false --filter @bookorbit/read-aloud-worker build && \
|
||||
pnpm --config.allow-unused-patches=true --filter @bookorbit/read-aloud-worker deploy --prod --legacy /deploy && \
|
||||
rm -rf /deploy/dist /deploy/assets && \
|
||||
cp -r /app/workers/read-aloud/dist /app/workers/read-aloud/assets /deploy/ && \
|
||||
types_dir="$(dirname "$(cd /deploy && node -p 'require.resolve("@bookorbit/types/package.json")')")" && \
|
||||
rm -rf "$types_dir/dist" && cp -r /app/packages/types/dist "$types_dir/dist" && \
|
||||
cd /deploy && node -e 'require("@bookorbit/types").READ_ALOUD_WORKER_PROTOCOL || process.exit(1)' && \
|
||||
find node_modules/.pnpm/onnxruntime-node@*/node_modules/onnxruntime-node/bin/napi-v6 -mindepth 1 -maxdepth 1 ! -name linux -exec rm -rf {} +
|
||||
|
||||
# Speech models reach the data volume (XDG_DATA_HOME) only through the ghost-story patch; without it
|
||||
# they go to the home folder, which is read-only under Compose, and every model download fails.
|
||||
RUN cd /deploy && \
|
||||
patched="$(find node_modules/.pnpm -path '*/node_modules/@storyteller-platform/ghost-story/dist/utilities/FileSystem.js')" && \
|
||||
if [ -z "$patched" ] || grep -L 'process.env\["XDG_DATA_HOME"\]' $patched | grep -q .; then \
|
||||
echo "ghost-story in the image lacks patches/@storyteller-platform__ghost-story@0.3.4.patch" >&2; exit 1; \
|
||||
fi
|
||||
|
||||
FROM ${NODE_IMAGE} AS runtime
|
||||
WORKDIR /app
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends ca-certificates ffmpeg tini && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
mkdir -p /data/read-aloud/jobs /data/read-aloud/tmp /data/read-aloud/ghost-story && \
|
||||
chown -R node:node /data
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=8787
|
||||
ENV READ_ALOUD_WORKER_HOST=0.0.0.0
|
||||
ENV READ_ALOUD_LIBRARY_ROOTS=/books
|
||||
ENV READ_ALOUD_WORK_ROOT=/data/read-aloud/jobs
|
||||
ENV READ_ALOUD_TMP_DIR=/data/read-aloud/tmp
|
||||
ENV READ_ALOUD_CTC_DEVICE=auto
|
||||
ENV TMPDIR=/data/read-aloud/tmp
|
||||
ENV XDG_DATA_HOME=/data/read-aloud
|
||||
|
||||
COPY --from=builder --chown=node:node /deploy ./
|
||||
COPY --chown=node:node server/file-env.sh workers/read-aloud/entrypoint.sh ./
|
||||
RUN chmod +x /app/entrypoint.sh
|
||||
|
||||
# The release this image was built from, reported in health; unset falls back to the package version.
|
||||
ARG APP_VERSION
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
USER node
|
||||
EXPOSE 8787
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "/app/entrypoint.sh"]
|
||||
Binary file not shown.
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
. /app/file-env.sh
|
||||
load_file_env
|
||||
|
||||
exec node dist/main.js
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"name": "@bookorbit/read-aloud-worker",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"license": "AGPL-3.0-only",
|
||||
"type": "module",
|
||||
"files": [
|
||||
"dist",
|
||||
"assets"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=24"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "node dist/main.js",
|
||||
"pretest": "pnpm --filter @bookorbit/types build",
|
||||
"test": "pnpm build && node --test test/*.test.mjs",
|
||||
"pretype-check": "pnpm --filter @bookorbit/types build",
|
||||
"type-check": "tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bookorbit/types": "workspace:*",
|
||||
"@storyteller-platform/align": "0.2.13",
|
||||
"@storyteller-platform/ghost-story": "0.3.4",
|
||||
"onnxruntime-node": "1.30.0",
|
||||
"pino": "^10.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^26.6.3",
|
||||
"typescript": "^6.0.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Writes assets/cuda-probe.onnx: an ONNX model with one Identity node, float32 input "x" of shape [1]
|
||||
// and output "y". The accelerator probe opens it with a GPU execution provider and runs it once.
|
||||
// The protobuf is encoded by hand so the worker needs no ONNX tooling to rebuild it.
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const VARINT = 0;
|
||||
const LENGTH_DELIMITED = 2;
|
||||
const FLOAT = 1;
|
||||
|
||||
function varint(value) {
|
||||
const bytes = [];
|
||||
let remaining = BigInt(value);
|
||||
do {
|
||||
let byte = Number(remaining & 0x7fn);
|
||||
remaining >>= 7n;
|
||||
if (remaining > 0n) byte |= 0x80;
|
||||
bytes.push(byte);
|
||||
} while (remaining > 0n);
|
||||
return Buffer.from(bytes);
|
||||
}
|
||||
|
||||
function key(field, wireType) {
|
||||
return varint((field << 3) | wireType);
|
||||
}
|
||||
|
||||
function intField(field, value) {
|
||||
return Buffer.concat([key(field, VARINT), varint(value)]);
|
||||
}
|
||||
|
||||
function bytesField(field, value) {
|
||||
const payload = typeof value === "string" ? Buffer.from(value, "utf8") : value;
|
||||
return Buffer.concat([key(field, LENGTH_DELIMITED), varint(payload.length), payload]);
|
||||
}
|
||||
|
||||
function message(...fields) {
|
||||
return Buffer.concat(fields);
|
||||
}
|
||||
|
||||
// TypeProto { tensor_type: TypeProto.Tensor { elem_type: FLOAT, shape: { dim: [{ dim_value: 1 }] } } }
|
||||
const dimension = message(intField(1, 1));
|
||||
const shape = message(bytesField(1, dimension));
|
||||
const tensorType = message(intField(1, FLOAT), bytesField(2, shape));
|
||||
const type = message(bytesField(1, tensorType));
|
||||
|
||||
const valueInfo = (name) => message(bytesField(1, name), bytesField(2, type));
|
||||
|
||||
// NodeProto { input: "x", output: "y", name: "identity", op_type: "Identity" }
|
||||
const node = message(bytesField(1, "x"), bytesField(2, "y"), bytesField(3, "identity"), bytesField(4, "Identity"));
|
||||
|
||||
// GraphProto { node, name, input (11), output (12) }
|
||||
const graph = message(bytesField(1, node), bytesField(2, "probe"), bytesField(11, valueInfo("x")), bytesField(12, valueInfo("y")));
|
||||
|
||||
// OperatorSetIdProto { domain: "", version: 13 }
|
||||
const opset = message(bytesField(1, ""), intField(2, 13));
|
||||
|
||||
// ModelProto { ir_version: 7, producer_name: "bookorbit", graph (7), opset_import (8) }
|
||||
const model = message(intField(1, 7), bytesField(2, "bookorbit"), bytesField(7, graph), bytesField(8, opset));
|
||||
|
||||
const target = fileURLToPath(new URL("../assets/cuda-probe.onnx", import.meta.url));
|
||||
writeFileSync(target, model);
|
||||
process.stdout.write(`wrote ${model.length} bytes\n`);
|
||||
@@ -0,0 +1,78 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { ReadAloudWorkerDevice, ReadAloudWorkerHealth } from "@bookorbit/types";
|
||||
|
||||
export type ConfiguredCtcDevice = ReadAloudWorkerDevice;
|
||||
export type ResolvedCtcDevice = Exclude<ReadAloudWorkerDevice, "auto">;
|
||||
export type DeviceCheck = ReadAloudWorkerHealth["deviceCheck"];
|
||||
|
||||
export const CTC_DEVICES = ["auto", "cpu", "cuda", "webgpu", "mlx"] as const satisfies readonly ReadAloudWorkerDevice[];
|
||||
|
||||
type Runtime = {
|
||||
platform: NodeJS.Platform;
|
||||
architecture: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* `auto` never picks CUDA: the image ships no CUDA libraries, and a missing provider fails the job
|
||||
* instead of falling back. CUDA is only for native Linux installs that ask for it explicitly.
|
||||
*/
|
||||
export function resolveCtcDevice(configured: ConfiguredCtcDevice, runtime: Runtime = currentRuntime()): ResolvedCtcDevice {
|
||||
const appleSilicon = runtime.platform === "darwin" && runtime.architecture === "arm64";
|
||||
if (configured === "auto") return appleSilicon ? "mlx" : "cpu";
|
||||
if (configured === "mlx" && !appleSilicon) {
|
||||
throw new Error("READ_ALOUD_CTC_DEVICE=mlx requires native macOS on Apple Silicon");
|
||||
}
|
||||
if (configured === "cuda" && (runtime.platform !== "linux" || runtime.architecture !== "x64")) {
|
||||
throw new Error("READ_ALOUD_CTC_DEVICE=cuda requires native Linux x64 with CUDA 12 and cuDNN 9");
|
||||
}
|
||||
if (configured === "webgpu" && runtime.platform === "linux" && runtime.architecture === "arm64") {
|
||||
throw new Error("READ_ALOUD_CTC_DEVICE=webgpu is not available on Linux arm64");
|
||||
}
|
||||
return configured;
|
||||
}
|
||||
|
||||
/**
|
||||
* GPU devices are probed because a missing runtime does not fail until inference, hours into a job. In the Docker image
|
||||
* neither probe can pass (no CUDA libraries, no Vulkan driver for WebGPU), which keeps it on the CPU as documented.
|
||||
*/
|
||||
export function needsDeviceProbe(device: ResolvedCtcDevice): boolean {
|
||||
return device === "cuda" || device === "webgpu";
|
||||
}
|
||||
|
||||
export const PROBE_MODEL_PATH = fileURLToPath(new URL("../assets/cuda-probe.onnx", import.meta.url));
|
||||
const PROBE_SCRIPT_PATH = fileURLToPath(new URL("./probe.js", import.meta.url));
|
||||
|
||||
const PROBE_HINTS: Partial<Record<ResolvedCtcDevice, string>> = {
|
||||
cuda: "CUDA needs native Linux x64 with CUDA 12 and cuDNN 9 installed and onnxruntime-node's CUDA provider",
|
||||
webgpu: "WebGPU needs a native worker with a GPU ONNX Runtime can reach; the Docker image has none",
|
||||
};
|
||||
|
||||
/**
|
||||
* Opens a tiny identity model with the device's execution provider in a short-lived child process, so
|
||||
* a missing GPU runtime fails startup with a clear message instead of failing the first job hours in.
|
||||
*/
|
||||
export function probeDevice(device: ResolvedCtcDevice, options: { timeoutMs?: number; modelPath?: string } = {}): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
process.execPath,
|
||||
[PROBE_SCRIPT_PATH, device, options.modelPath ?? PROBE_MODEL_PATH],
|
||||
{ timeout: options.timeoutMs ?? 120_000, maxBuffer: 1024 * 1024 },
|
||||
(error, _stdout, stderr) => {
|
||||
if (!error) return resolve();
|
||||
const detail = stderr.trim().split("\n").filter(Boolean).at(-1) ?? error.message;
|
||||
const hint = PROBE_HINTS[device];
|
||||
reject(
|
||||
new Error(
|
||||
`READ_ALOUD_CTC_DEVICE=${device} could not start ONNX Runtime's ${device} provider (${detail.slice(0, 400)}). ` +
|
||||
`${hint ? `${hint}; ` : ""}use READ_ALOUD_CTC_DEVICE=cpu otherwise.`,
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function currentRuntime(): Runtime {
|
||||
return { platform: process.platform, architecture: process.arch };
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, isAbsolute, join, resolve } from "node:path";
|
||||
import { CTC_DEVICES, type ConfiguredCtcDevice } from "./accelerator.js";
|
||||
|
||||
export type WorkerConfig = {
|
||||
host: string;
|
||||
port: number;
|
||||
token: string;
|
||||
libraryRoots: string[];
|
||||
workRoot: string;
|
||||
tmpDir: string;
|
||||
configuredDevice: ConfiguredCtcDevice;
|
||||
dtype: "fp32" | "fp16" | null;
|
||||
batchSize: number | null;
|
||||
};
|
||||
|
||||
type Env = Record<string, string | undefined>;
|
||||
|
||||
/** The worker's own folder inside READ_ALOUD_TMP_DIR: runners write their scratch there, and it is the only thing emptied at startup. */
|
||||
export const WORKER_TMP_SUBDIR = "read-aloud-worker";
|
||||
|
||||
export function workerTmpDir(config: Pick<WorkerConfig, "tmpDir">): string {
|
||||
return join(config.tmpDir, WORKER_TMP_SUBDIR);
|
||||
}
|
||||
|
||||
/** The release the image was built from (its APP_VERSION build argument), else the package version for a native build. */
|
||||
export function workerVersion(env: Env): string {
|
||||
const baked = env.APP_VERSION?.trim();
|
||||
if (baked) return baked;
|
||||
return (JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { version: string }).version;
|
||||
}
|
||||
|
||||
export function loadConfig(env: Env): WorkerConfig {
|
||||
const token = env.READ_ALOUD_WORKER_TOKEN?.trim() ?? "";
|
||||
if (token.length < 16) throw new Error("READ_ALOUD_WORKER_TOKEN must contain at least 16 characters");
|
||||
const workRoot = absolutePath(env.READ_ALOUD_WORK_ROOT, "/data/read-aloud/jobs", "READ_ALOUD_WORK_ROOT");
|
||||
return {
|
||||
host: env.READ_ALOUD_WORKER_HOST?.trim() || "127.0.0.1",
|
||||
port: port(env.PORT, 8787),
|
||||
token,
|
||||
libraryRoots: parseLibraryRoots(env),
|
||||
workRoot,
|
||||
tmpDir: absolutePath(env.READ_ALOUD_TMP_DIR, resolve(dirname(workRoot), "tmp"), "READ_ALOUD_TMP_DIR"),
|
||||
configuredDevice: readEnum(env.READ_ALOUD_CTC_DEVICE, CTC_DEVICES, "auto", "READ_ALOUD_CTC_DEVICE"),
|
||||
dtype: readOptionalEnum(env.READ_ALOUD_CTC_DTYPE, ["fp32", "fp16"] as const, "READ_ALOUD_CTC_DTYPE"),
|
||||
batchSize: optionalBatchSize(env.READ_ALOUD_CTC_BATCH_SIZE),
|
||||
};
|
||||
}
|
||||
|
||||
/** READ_ALOUD_LIBRARY_ROOTS (comma-separated), else READ_ALOUD_LIBRARY_ROOT (one folder), else /books. */
|
||||
export function parseLibraryRoots(env: Env): string[] {
|
||||
const list = env.READ_ALOUD_LIBRARY_ROOTS?.trim();
|
||||
const raw = list ? list.split(",") : [env.READ_ALOUD_LIBRARY_ROOT?.trim() || "/books"];
|
||||
const roots: string[] = [];
|
||||
for (const entry of raw) {
|
||||
const value = entry.trim();
|
||||
if (!value) continue;
|
||||
if (!isAbsolute(value)) throw new Error(`Library roots must be absolute paths, got "${value}"`);
|
||||
const normalized = resolve(value);
|
||||
if (!roots.includes(normalized)) roots.push(normalized);
|
||||
}
|
||||
if (roots.length === 0) throw new Error("READ_ALOUD_LIBRARY_ROOTS must list at least one absolute path");
|
||||
return roots;
|
||||
}
|
||||
|
||||
function absolutePath(value: string | undefined, fallback: string, name: string): string {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) return resolve(fallback);
|
||||
if (!isAbsolute(trimmed)) throw new Error(`${name} must be an absolute path`);
|
||||
return resolve(trimmed);
|
||||
}
|
||||
|
||||
function port(value: string | undefined, fallback: number): number {
|
||||
if (!value?.trim()) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0 || parsed > 65535) throw new Error("PORT must be an integer from 0 to 65535");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readEnum<const T extends readonly string[]>(value: string | undefined, choices: T, fallback: T[number], name: string): T[number] {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
if (!normalized) return fallback;
|
||||
if (choices.includes(normalized)) return normalized as T[number];
|
||||
throw new Error(`${name} must be one of: ${choices.join(", ")}`);
|
||||
}
|
||||
|
||||
function readOptionalEnum<const T extends readonly string[]>(value: string | undefined, choices: T, name: string): T[number] | null {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
if (!normalized || normalized === "auto") return null;
|
||||
if (choices.includes(normalized)) return normalized as T[number];
|
||||
throw new Error(`${name} must be one of: auto, ${choices.join(", ")}`);
|
||||
}
|
||||
|
||||
function optionalBatchSize(value: string | undefined): number | null {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed || trimmed === "auto") return null;
|
||||
const parsed = Number(trimmed);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 64) {
|
||||
throw new Error("READ_ALOUD_CTC_BATCH_SIZE must be an integer from 1 to 64");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
import { fork, type ChildProcess } from "node:child_process";
|
||||
import { readFileSync, realpathSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { mkdir, statfs } from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type {
|
||||
ReadAloudErrorCode,
|
||||
ReadAloudJobEventLevel,
|
||||
ReadAloudStageDetail,
|
||||
ReadAloudWorkerEvent,
|
||||
ReadAloudWorkerJob,
|
||||
ReadAloudWorkerRequest,
|
||||
ReadAloudWorkerStage,
|
||||
} from "@bookorbit/types";
|
||||
import type { ResolvedCtcDevice } from "./accelerator.js";
|
||||
import { workerTmpDir, type WorkerConfig } from "./config.js";
|
||||
import { cleanMessage, errorClass, errorMessage, logLine } from "./log.js";
|
||||
import { ReadableRoots, RequestError, expectedWorkDir, resolveSource, type RootCheckOptions } from "./paths.js";
|
||||
import {
|
||||
MAX_EVENT_MESSAGE_LENGTH,
|
||||
type EventData,
|
||||
type Redaction,
|
||||
type RunnerJob,
|
||||
type RunnerToControl,
|
||||
type SourceFingerprint,
|
||||
} from "./protocol.js";
|
||||
import { markerStates, planStages, requiredDiskBytes, type FingerprintSource, type MarkerStage } from "./stages.js";
|
||||
|
||||
const MAX_EVENTS_PER_JOB = 1_000;
|
||||
const DEFAULT_CANCEL_GRACE_MS = 30_000;
|
||||
const DEFAULT_RETENTION_MS = 24 * 60 * 60_000;
|
||||
const SEQ_FILE = ".worker-event-seq";
|
||||
const TERMINAL = new Set<ReadAloudWorkerJob["status"]>(["completed", "failed", "cancelled"]);
|
||||
|
||||
export type RegistryOptions = {
|
||||
config: WorkerConfig;
|
||||
device: ResolvedCtcDevice;
|
||||
runnerPath?: string;
|
||||
/** Extra environment for the runner; the tests use it to load a fake pipeline. */
|
||||
runnerEnv?: Record<string, string>;
|
||||
cancelGraceMs?: number;
|
||||
retentionMs?: number;
|
||||
freeBytes?: (path: string) => Promise<number | null>;
|
||||
rootChecks?: RootCheckOptions;
|
||||
};
|
||||
|
||||
type JobRecord = {
|
||||
jobId: string;
|
||||
request: ReadAloudWorkerRequest;
|
||||
status: ReadAloudWorkerJob["status"];
|
||||
stage: ReadAloudWorkerStage;
|
||||
progress: number;
|
||||
stageDetail: ReadAloudStageDetail | null;
|
||||
errorCode: ReadAloudErrorCode | null;
|
||||
error: string | null;
|
||||
outputBytes: number | null;
|
||||
events: ReadAloudWorkerEvent[];
|
||||
lastSeq: number;
|
||||
/** Where the last event number is kept, so numbering continues after a worker restart. */
|
||||
seqFile: string;
|
||||
attempt: number;
|
||||
child: ChildProcess | null;
|
||||
result: Extract<RunnerToControl, { type: "done" | "failed" }> | null;
|
||||
/** Whether the runner has reported a stage yet; failures before that carry no stage. */
|
||||
stageReported: boolean;
|
||||
cancelRequested: boolean;
|
||||
killed: boolean;
|
||||
killTimer: NodeJS.Timeout | null;
|
||||
startedAt: number;
|
||||
finishedAt: number | null;
|
||||
};
|
||||
|
||||
export type SubmitResult = { status: 200 | 202; job: ReadAloudWorkerJob };
|
||||
|
||||
type ResolvedSources = { workDir: string; epub: SourceFingerprint; audio: SourceFingerprint[] };
|
||||
|
||||
export class JobRegistry {
|
||||
private readonly jobs = new Map<string, JobRecord>();
|
||||
private activeJobId: string | null = null;
|
||||
/** The submit between taking the slot and spawning its runner, so shutdown can wait it out. */
|
||||
private starting: Promise<unknown> | null = null;
|
||||
private stopping = false;
|
||||
private readonly runnerPath: string;
|
||||
private readonly cancelGraceMs: number;
|
||||
private readonly retentionMs: number;
|
||||
private readonly redactions: Redaction[];
|
||||
private readonly roots: ReadableRoots;
|
||||
private readonly sweepTimer: NodeJS.Timeout;
|
||||
|
||||
constructor(private readonly options: RegistryOptions) {
|
||||
this.runnerPath = options.runnerPath ?? fileURLToPath(new URL("./runner.js", import.meta.url));
|
||||
this.cancelGraceMs = options.cancelGraceMs ?? DEFAULT_CANCEL_GRACE_MS;
|
||||
this.retentionMs = options.retentionMs ?? DEFAULT_RETENTION_MS;
|
||||
this.redactions = buildRedactions(options.config);
|
||||
this.roots = new ReadableRoots(options.config.libraryRoots, options.rootChecks);
|
||||
this.sweepTimer = setInterval(() => this.sweep(), 60 * 60_000);
|
||||
this.sweepTimer.unref();
|
||||
}
|
||||
|
||||
get active(): string | null {
|
||||
return this.activeJobId;
|
||||
}
|
||||
|
||||
get redactionList(): readonly Redaction[] {
|
||||
return this.redactions;
|
||||
}
|
||||
|
||||
/** The library folders readable now; health reuses a check for a few seconds, a job always checks again. */
|
||||
libraryRoots(maxAgeMs?: number): Promise<string[]> {
|
||||
return this.roots.get(maxAgeMs);
|
||||
}
|
||||
|
||||
get(jobId: string, afterSeq = 0): ReadAloudWorkerJob | null {
|
||||
const record = this.jobs.get(jobId);
|
||||
return record ? this.view(record, afterSeq) : null;
|
||||
}
|
||||
|
||||
async submit(request: ReadAloudWorkerRequest): Promise<SubmitResult> {
|
||||
const existing = this.jobs.get(request.jobId);
|
||||
if (existing && !TERMINAL.has(existing.status)) return { status: 200, job: this.view(existing, 0) };
|
||||
// A 503 makes the server wait and resubmit to the next process instead of starting a runner here.
|
||||
if (this.stopping) throw new RequestError(503, "worker_unavailable", "The worker is shutting down");
|
||||
if (this.activeJobId) {
|
||||
throw new RequestError(503, "worker_unavailable", "The worker is busy with another job");
|
||||
}
|
||||
this.activeJobId = request.jobId;
|
||||
const starting = this.start(request, existing);
|
||||
this.starting = starting;
|
||||
try {
|
||||
return await starting;
|
||||
} finally {
|
||||
if (this.starting === starting) this.starting = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async start(request: ReadAloudWorkerRequest, existing: JobRecord | undefined): Promise<SubmitResult> {
|
||||
let resolved: ResolvedSources;
|
||||
try {
|
||||
resolved = await this.resolve(request);
|
||||
if (this.stopping) throw new RequestError(503, "worker_unavailable", "The worker is shutting down");
|
||||
} catch (error) {
|
||||
this.activeJobId = null;
|
||||
if (error instanceof RequestError) {
|
||||
logLine(
|
||||
"read_aloud.worker.request",
|
||||
"fail",
|
||||
{ jobId: request.jobId, errorCode: error.code, error: this.clean(error.message) },
|
||||
"job rejected",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const record = this.startRecord(request, existing, resolved.workDir);
|
||||
logLine(
|
||||
"read_aloud.worker.job",
|
||||
"start",
|
||||
{
|
||||
jobId: record.jobId,
|
||||
attempt: record.attempt,
|
||||
audioFiles: request.audioPaths.length,
|
||||
sourceBytes: request.sourceBytes,
|
||||
resumeFrom: request.resumeFrom ?? "none",
|
||||
},
|
||||
"job accepted",
|
||||
);
|
||||
this.event(record, "info", null, "job_accepted", "Job accepted by the worker", {
|
||||
attempt: record.attempt,
|
||||
resumeFrom: request.resumeFrom,
|
||||
device: this.options.device,
|
||||
});
|
||||
|
||||
try {
|
||||
await mkdir(resolved.workDir, { recursive: true });
|
||||
await mkdir(workerTmpDir(this.options.config), { recursive: true });
|
||||
const enough = await this.checkDisk(record, resolved);
|
||||
if (!enough) return { status: 202, job: this.view(record, 0) };
|
||||
// Shutdown began while the job was being set up: no runner may start after it.
|
||||
if (this.stopping) this.cancel(record.jobId);
|
||||
if (record.cancelRequested) {
|
||||
this.finish(record, "cancelled", "cancelled", "The job was cancelled");
|
||||
return { status: 202, job: this.view(record, 0) };
|
||||
}
|
||||
this.spawn(record, resolved);
|
||||
} catch (error) {
|
||||
this.finish(record, "failed", "unknown", `The worker could not start the job: ${this.clean(errorMessage(error))}`);
|
||||
}
|
||||
return { status: 202, job: this.view(record, 0) };
|
||||
}
|
||||
|
||||
cancel(jobId: string): ReadAloudWorkerJob | null {
|
||||
const record = this.jobs.get(jobId);
|
||||
if (!record) return null;
|
||||
if (TERMINAL.has(record.status) || record.cancelRequested) return this.view(record, 0);
|
||||
record.cancelRequested = true;
|
||||
logLine("read_aloud.worker.cancel", "start", { jobId, graceMs: this.cancelGraceMs }, "cancel requested");
|
||||
this.event(record, "info", record.stage, "cancel_requested", "Cancellation requested");
|
||||
const child = record.child;
|
||||
if (child) {
|
||||
try {
|
||||
child.send({ type: "cancel" });
|
||||
} catch {
|
||||
// The runner is already exiting; its close handler records the outcome.
|
||||
}
|
||||
record.killTimer = setTimeout(() => {
|
||||
if (record.child !== child || child.exitCode !== null || child.signalCode !== null) return;
|
||||
record.killed = true;
|
||||
logLine("read_aloud.worker.cancel", "fail", { jobId, graceMs: this.cancelGraceMs }, "runner ignored cancel, killing it");
|
||||
this.event(record, "warn", record.stage, "runner_killed", "The job did not stop in time and was killed", { graceMs: this.cancelGraceMs });
|
||||
killGroup(child);
|
||||
}, this.cancelGraceMs);
|
||||
}
|
||||
return this.view(record, 0);
|
||||
}
|
||||
|
||||
/** Cancels the running job and waits for its runner to exit, for shutdown. A job still starting never gets a runner. */
|
||||
async shutdown(): Promise<void> {
|
||||
this.stopping = true;
|
||||
clearInterval(this.sweepTimer);
|
||||
await this.starting?.catch(() => undefined);
|
||||
const record = this.activeJobId ? this.jobs.get(this.activeJobId) : undefined;
|
||||
const child = record?.child;
|
||||
if (!record || !child) return;
|
||||
const exited = new Promise<void>((resolve) => child.once("close", () => resolve()));
|
||||
this.cancel(record.jobId);
|
||||
await exited;
|
||||
}
|
||||
|
||||
private async resolve(request: ReadAloudWorkerRequest): Promise<ResolvedSources> {
|
||||
const { workRoot } = this.options.config;
|
||||
const workDir = expectedWorkDir(request.workDir, workRoot, request.jobId);
|
||||
const libraryRoots = await this.roots.get(0);
|
||||
const epub = await resolveSource(request.epubPath, libraryRoots, "EPUB");
|
||||
const audio: SourceFingerprint[] = [];
|
||||
for (const [index, path] of request.audioPaths.entries()) audio.push(await resolveSource(path, libraryRoots, `audio file ${index + 1}`));
|
||||
return { workDir, epub, audio };
|
||||
}
|
||||
|
||||
private startRecord(request: ReadAloudWorkerRequest, previous: JobRecord | undefined, workDir: string): JobRecord {
|
||||
const seqFile = join(workDir, SEQ_FILE);
|
||||
const record: JobRecord = {
|
||||
jobId: request.jobId,
|
||||
request,
|
||||
status: "queued",
|
||||
stage: "preparing_audio",
|
||||
progress: 0,
|
||||
stageDetail: null,
|
||||
errorCode: null,
|
||||
error: null,
|
||||
outputBytes: null,
|
||||
events: previous?.events ?? [],
|
||||
lastSeq: previous?.lastSeq ?? readSeq(seqFile),
|
||||
seqFile,
|
||||
attempt: (previous?.attempt ?? 0) + 1,
|
||||
child: null,
|
||||
result: null,
|
||||
stageReported: false,
|
||||
cancelRequested: false,
|
||||
killed: false,
|
||||
killTimer: null,
|
||||
startedAt: Date.now(),
|
||||
finishedAt: null,
|
||||
};
|
||||
this.jobs.set(record.jobId, record);
|
||||
return record;
|
||||
}
|
||||
|
||||
private async checkDisk(record: JobRecord, resolved: ResolvedSources): Promise<boolean> {
|
||||
const request = record.request;
|
||||
const source: FingerprintSource = {
|
||||
epub: resolved.epub,
|
||||
audio: resolved.audio,
|
||||
language: request.language,
|
||||
granularity: request.granularity,
|
||||
model: request.model,
|
||||
decisions: request.decisions,
|
||||
};
|
||||
const plan = planStages(await markerStates(resolved.workDir, source), request.resumeFrom);
|
||||
const skipped = new Set<MarkerStage>(plan.filter((entry) => !entry.run).map((entry) => entry.stage));
|
||||
const { required } = requiredDiskBytes({
|
||||
audioBytes: resolved.audio.reduce((sum, file) => sum + file.size, 0),
|
||||
epubBytes: resolved.epub.size,
|
||||
audioDurationSeconds: request.audioDurationSeconds,
|
||||
skipped,
|
||||
});
|
||||
const available = await (this.options.freeBytes ?? freeBytes)(resolved.workDir);
|
||||
const data = { neededBytes: required, availableBytes: available, cachedStages: skipped.size };
|
||||
if (available === null || available >= required) {
|
||||
this.event(record, "info", null, "disk_checked", "Enough free space for this run", data);
|
||||
return true;
|
||||
}
|
||||
logLine("read_aloud.worker.disk", "fail", { jobId: record.jobId, neededBytes: required, availableBytes: available }, "insufficient storage");
|
||||
this.event(record, "error", null, "disk_check_failed", "Not enough free space in the worker's data folder for this run", data);
|
||||
this.finish(record, "failed", "insufficient_storage", `The run needs ${required} free bytes but ${available} are available`);
|
||||
return false;
|
||||
}
|
||||
|
||||
private spawn(record: JobRecord, resolved: ResolvedSources): void {
|
||||
const { config } = this.options;
|
||||
const request = record.request;
|
||||
const job: RunnerJob = {
|
||||
jobId: record.jobId,
|
||||
attempt: record.attempt,
|
||||
workDir: resolved.workDir,
|
||||
epub: resolved.epub,
|
||||
audio: resolved.audio,
|
||||
language: request.language,
|
||||
granularity: request.granularity,
|
||||
model: request.model,
|
||||
maxOutputBytes: request.maxOutputBytes,
|
||||
decisions: request.decisions,
|
||||
resumeFrom: request.resumeFrom,
|
||||
device: this.options.device,
|
||||
dtype: config.dtype,
|
||||
batchSize: config.batchSize,
|
||||
redactions: this.redactions,
|
||||
};
|
||||
const child = fork(this.runnerPath, [], {
|
||||
execArgv: [],
|
||||
// Its own process group: STAlign runs ffmpeg through a shell, and aborting only stops the shell.
|
||||
detached: true,
|
||||
stdio: ["ignore", "inherit", "inherit", "ipc"],
|
||||
env: {
|
||||
...process.env,
|
||||
...this.options.runnerEnv,
|
||||
TMPDIR: workerTmpDir(config),
|
||||
STORYTELLER_SCRATCH_DIR: join(resolved.workDir, "scratch"),
|
||||
},
|
||||
});
|
||||
record.child = child;
|
||||
record.status = "running";
|
||||
child.on("message", (message: RunnerToControl) => this.onMessage(record, child, message));
|
||||
child.on("error", (error) => {
|
||||
logLine(
|
||||
"read_aloud.worker.runner",
|
||||
"fail",
|
||||
{ jobId: record.jobId, errorClass: errorClass(error), error: this.clean(errorMessage(error)) },
|
||||
"runner error",
|
||||
);
|
||||
});
|
||||
child.on("close", (code, signal) => this.onClose(record, child, code, signal));
|
||||
child.send({ type: "start", job });
|
||||
}
|
||||
|
||||
private onMessage(record: JobRecord, child: ChildProcess, message: RunnerToControl): void {
|
||||
if (record.child !== child) return;
|
||||
switch (message.type) {
|
||||
case "progress":
|
||||
record.stageReported = true;
|
||||
record.stage = message.stage;
|
||||
record.progress = Math.max(record.progress, Math.min(100, Math.floor(message.overall)));
|
||||
record.stageDetail = message.detail;
|
||||
return;
|
||||
case "event":
|
||||
this.event(record, message.level, message.stage, message.code, message.message, message.data);
|
||||
if (message.stage && message.code === "stage_started") {
|
||||
record.stageReported = true;
|
||||
record.stage = message.stage;
|
||||
record.stageDetail = null;
|
||||
}
|
||||
return;
|
||||
case "done":
|
||||
case "failed":
|
||||
record.result = message;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private onClose(record: JobRecord, child: ChildProcess, code: number | null, signal: NodeJS.Signals | null): void {
|
||||
if (record.child !== child) return;
|
||||
record.child = null;
|
||||
// Anything the runner left behind, such as an ffmpeg a cancel orphaned, would keep using CPU and
|
||||
// hold disk space after the job folder is gone.
|
||||
killGroup(child);
|
||||
if (record.killTimer) clearTimeout(record.killTimer);
|
||||
record.killTimer = null;
|
||||
const result = record.result;
|
||||
if (record.cancelRequested) {
|
||||
this.finish(record, "cancelled", "cancelled", record.killed ? "The job was cancelled and its runner was killed" : "The job was cancelled");
|
||||
} else if (result?.type === "done") {
|
||||
record.outputBytes = result.outputBytes;
|
||||
record.progress = 100;
|
||||
record.stage = "validating";
|
||||
record.stageDetail = null;
|
||||
this.finish(record, "completed", null, null);
|
||||
} else if (result?.type === "failed") {
|
||||
this.finish(record, result.code === "cancelled" ? "cancelled" : "failed", result.code, result.message);
|
||||
} else {
|
||||
const exit = signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`;
|
||||
this.finish(record, "failed", "worker_crashed", `The job runner exited unexpectedly (${exit})`, { exitCode: code, signal });
|
||||
}
|
||||
}
|
||||
|
||||
private finish(
|
||||
record: JobRecord,
|
||||
status: "completed" | "failed" | "cancelled",
|
||||
code: ReadAloudErrorCode | null,
|
||||
message: string | null,
|
||||
extra: EventData = {},
|
||||
): void {
|
||||
record.status = status;
|
||||
record.errorCode = code;
|
||||
record.error = message ? this.clean(message) : null;
|
||||
record.finishedAt = Date.now();
|
||||
if (this.activeJobId === record.jobId) this.activeJobId = null;
|
||||
const durationMs = record.finishedAt - record.startedAt;
|
||||
if (status === "completed") {
|
||||
logLine("read_aloud.worker.job", "end", { jobId: record.jobId, status, durationMs, outputBytes: record.outputBytes ?? 0 }, "job finished");
|
||||
this.event(record, "info", null, "job_completed", "Generation finished", { durationMs, outputBytes: record.outputBytes });
|
||||
} else if (status === "cancelled") {
|
||||
logLine("read_aloud.worker.cancel", "end", { jobId: record.jobId, durationMs, killed: record.killed }, "job cancelled");
|
||||
this.event(record, "info", null, "job_cancelled", "The job was cancelled", { durationMs, killed: record.killed });
|
||||
} else {
|
||||
logLine(
|
||||
"read_aloud.worker.job",
|
||||
"fail",
|
||||
{
|
||||
jobId: record.jobId,
|
||||
stage: record.stageReported ? record.stage : undefined,
|
||||
errorCode: code ?? "unknown",
|
||||
durationMs,
|
||||
error: record.error ?? "",
|
||||
},
|
||||
"job failed",
|
||||
);
|
||||
const stage = record.stageReported ? record.stage : null;
|
||||
this.event(record, "error", stage, "job_failed", record.error ?? "The job failed", { errorCode: code, durationMs, ...extra });
|
||||
}
|
||||
}
|
||||
|
||||
private event(
|
||||
record: JobRecord,
|
||||
level: ReadAloudJobEventLevel,
|
||||
stage: ReadAloudWorkerStage | null,
|
||||
code: string,
|
||||
message: string | null,
|
||||
data: EventData | null = null,
|
||||
): void {
|
||||
record.lastSeq += 1;
|
||||
record.events.push({
|
||||
seq: record.lastSeq,
|
||||
at: new Date().toISOString(),
|
||||
level,
|
||||
stage,
|
||||
code,
|
||||
message: message === null ? null : this.clean(message),
|
||||
data,
|
||||
});
|
||||
if (record.events.length > MAX_EVENTS_PER_JOB) record.events.splice(0, record.events.length - MAX_EVENTS_PER_JOB);
|
||||
writeSeq(record.seqFile, record.lastSeq);
|
||||
}
|
||||
|
||||
private view(record: JobRecord, afterSeq: number): ReadAloudWorkerJob {
|
||||
return {
|
||||
jobId: record.jobId,
|
||||
status: record.status,
|
||||
stage: record.stage,
|
||||
progress: record.progress,
|
||||
stageDetail: record.stageDetail,
|
||||
errorCode: record.errorCode,
|
||||
error: record.error,
|
||||
outputBytes: record.outputBytes,
|
||||
events: record.events.filter((entry) => entry.seq > afterSeq),
|
||||
};
|
||||
}
|
||||
|
||||
private clean(text: string): string {
|
||||
return cleanMessage(text, this.redactions, MAX_EVENT_MESSAGE_LENGTH);
|
||||
}
|
||||
|
||||
private sweep(): void {
|
||||
const cutoff = Date.now() - this.retentionMs;
|
||||
for (const [id, record] of this.jobs) {
|
||||
if (TERMINAL.has(record.status) && record.finishedAt !== null && record.finishedAt < cutoff) this.jobs.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Kills the runner's whole process group; ESRCH just means nothing was left in it. */
|
||||
function killGroup(child: ChildProcess): void {
|
||||
if (child.pid !== undefined) {
|
||||
try {
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
return;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ESRCH") return;
|
||||
}
|
||||
}
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
function readSeq(path: string): number {
|
||||
try {
|
||||
const value = Number(readFileSync(path, "utf8").trim());
|
||||
return Number.isSafeInteger(value) && value > 0 ? value : 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function writeSeq(path: string, seq: number): void {
|
||||
try {
|
||||
writeFileSync(`${path}.partial`, String(seq));
|
||||
renameSync(`${path}.partial`, path);
|
||||
} catch {
|
||||
// The job folder may not exist yet for the first event; the next event writes it.
|
||||
}
|
||||
}
|
||||
|
||||
export async function freeBytes(path: string): Promise<number | null> {
|
||||
try {
|
||||
const info = await statfs(path, { bigint: true });
|
||||
const bytes = info.bavail * info.bsize;
|
||||
return bytes > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(bytes);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Configured folders and their real paths (macOS resolves /tmp and /var through /private). */
|
||||
function buildRedactions(config: WorkerConfig): Redaction[] {
|
||||
const folders: Redaction[] = [
|
||||
{ path: config.workRoot, placeholder: "<work>" },
|
||||
{ path: config.tmpDir, placeholder: "<tmp>" },
|
||||
...config.libraryRoots.map((path) => ({ path, placeholder: "<library>" })),
|
||||
];
|
||||
const home = homedir();
|
||||
if (home && home !== "/") folders.push({ path: home, placeholder: "<home>" });
|
||||
const redactions: Redaction[] = [];
|
||||
for (const folder of folders) {
|
||||
redactions.push(folder);
|
||||
try {
|
||||
const real = realpathSync.native(folder.path);
|
||||
if (real !== folder.path) redactions.push({ path: real, placeholder: folder.placeholder });
|
||||
} catch {
|
||||
// A folder that does not exist yet has no other spelling.
|
||||
}
|
||||
}
|
||||
return redactions;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { Redaction } from "./protocol.js";
|
||||
|
||||
const MAX_LOG_VALUE_LENGTH = 500;
|
||||
|
||||
type LogValue = string | number | boolean | null | undefined;
|
||||
|
||||
/** Mirrors the server's sanitizeLogValue: one line, bounded, quotes escaped. */
|
||||
export function sanitizeLogValue(value: unknown, maxLength = MAX_LOG_VALUE_LENGTH): string {
|
||||
return String(value)
|
||||
.replace(/[\r\n\t]/g, " ")
|
||||
.slice(0, maxLength)
|
||||
.replace(/\\/g, "\\\\")
|
||||
.replace(/"/g, '\\"');
|
||||
}
|
||||
|
||||
/** Replaces configured absolute folders (longest first) with placeholders such as <work> or <library>. */
|
||||
export function redact(text: string, redactions: readonly Redaction[]): string {
|
||||
let result = text;
|
||||
const ordered = [...redactions].filter((entry) => entry.path.length > 1).sort((a, b) => b.path.length - a.path.length);
|
||||
for (const { path, placeholder } of ordered) result = result.split(path).join(placeholder);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function cleanMessage(text: string, redactions: readonly Redaction[], maxLength = MAX_LOG_VALUE_LENGTH): string {
|
||||
return redact(text, redactions)
|
||||
.replace(/[\u0000-\u001f\u007f]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, maxLength);
|
||||
}
|
||||
|
||||
export function logLine(event: string, phase: "start" | "end" | "fail" | null, fields: Record<string, LogValue>, message: string): void {
|
||||
const parts = [`[${event}]`];
|
||||
if (phase) parts.push(`[${phase}]`);
|
||||
for (const [key, value] of Object.entries(fields)) {
|
||||
if (value === undefined) continue;
|
||||
parts.push(typeof value === "string" && needsQuotes(key) ? `${key}="${sanitizeLogValue(value)}"` : `${key}=${formatValue(value)}`);
|
||||
}
|
||||
process.stdout.write(`${parts.join(" ")} - ${message.replace(/[\r\n]+/g, " ")}\n`);
|
||||
}
|
||||
|
||||
function needsQuotes(key: string): boolean {
|
||||
return key === "error" || key === "message" || key === "path" || key === "reason";
|
||||
}
|
||||
|
||||
function formatValue(value: LogValue): string {
|
||||
if (value === null) return "null";
|
||||
if (typeof value === "string") return value.replace(/\s+/g, "_").slice(0, 200);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
export function errorClass(error: unknown): string {
|
||||
if (error instanceof Error) return error.name || error.constructor.name || "Error";
|
||||
return typeof error;
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
if (error instanceof Error) return error.message;
|
||||
return String(error);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { needsDeviceProbe, probeDevice, resolveCtcDevice, type DeviceCheck } from "./accelerator.js";
|
||||
import { loadConfig, workerVersion } from "./config.js";
|
||||
import { errorClass, errorMessage, logLine } from "./log.js";
|
||||
import { prepareDataFolders, startWorker } from "./server.js";
|
||||
import { checkFfmpeg } from "./toolchain.js";
|
||||
|
||||
// The control process: HTTP, auth, the job slot and runner lifecycle. Pipeline code (STAlign) only
|
||||
// ever loads in the forked runner, so this process keeps answering while a job runs.
|
||||
const startedAt = Date.now();
|
||||
const version = workerVersion(process.env);
|
||||
logLine("read_aloud.worker.boot", "start", { version, pid: process.pid }, "worker starting");
|
||||
|
||||
try {
|
||||
const config = loadConfig(process.env);
|
||||
const device = resolveCtcDevice(config.configuredDevice);
|
||||
let deviceCheck: DeviceCheck = "not_checked";
|
||||
if (needsDeviceProbe(device)) {
|
||||
await probeDevice(device);
|
||||
deviceCheck = "ok";
|
||||
}
|
||||
await checkFfmpeg();
|
||||
await prepareDataFolders(config);
|
||||
const worker = await startWorker({ config, configuredDevice: config.configuredDevice, device, deviceCheck, version });
|
||||
// Logs every folder that is not readable yet; each is checked again on later health checks and jobs.
|
||||
const readableRoots = await worker.registry.libraryRoots();
|
||||
logLine(
|
||||
"read_aloud.worker.boot",
|
||||
"end",
|
||||
{
|
||||
host: config.host,
|
||||
port: worker.port,
|
||||
device,
|
||||
configuredDevice: config.configuredDevice,
|
||||
deviceCheck,
|
||||
libraryRoots: readableRoots.length,
|
||||
configuredLibraryRoots: config.libraryRoots.length,
|
||||
durationMs: Date.now() - startedAt,
|
||||
},
|
||||
"worker ready",
|
||||
);
|
||||
|
||||
let stopping = false;
|
||||
for (const signal of ["SIGINT", "SIGTERM"] as const) {
|
||||
process.on(signal, () => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
const stopStartedAt = Date.now();
|
||||
logLine("read_aloud.worker.shutdown", "start", { signal, activeJobId: worker.registry.active ?? "none" }, "worker stopping");
|
||||
void worker.close().then(() => {
|
||||
logLine("read_aloud.worker.shutdown", "end", { durationMs: Date.now() - stopStartedAt }, "worker stopped");
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logLine(
|
||||
"read_aloud.worker.boot",
|
||||
"fail",
|
||||
{ durationMs: Date.now() - startedAt, errorClass: errorClass(error), error: errorMessage(error) },
|
||||
"worker failed to start",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import { constants } from "node:fs";
|
||||
import { access, realpath, stat } from "node:fs/promises";
|
||||
import { isAbsolute, relative, resolve, sep } from "node:path";
|
||||
import type { ReadAloudErrorCode } from "@bookorbit/types";
|
||||
import { errorClass, errorMessage, logLine } from "./log.js";
|
||||
import type { SourceFingerprint } from "./protocol.js";
|
||||
|
||||
const ROOT_CHECK_MAX_AGE_MS = 5_000;
|
||||
const ROOT_CHECK_TIMEOUT_MS = 2_000;
|
||||
|
||||
export class RequestError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: ReadAloudErrorCode,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "RequestError";
|
||||
}
|
||||
}
|
||||
|
||||
export function within(path: string, root: string): boolean {
|
||||
const rel = relative(root, path);
|
||||
return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel));
|
||||
}
|
||||
|
||||
/** The job folder is always `<workRoot>/<jobId>`; anything else means the server and worker disagree on the data mount. */
|
||||
export function expectedWorkDir(input: string, workRoot: string, jobId: string): string {
|
||||
const expected = resolve(workRoot, jobId);
|
||||
if (!isAbsolute(input) || resolve(input) !== expected || !within(expected, workRoot)) {
|
||||
throw new RequestError(
|
||||
400,
|
||||
"worker_incompatible",
|
||||
"workDir does not match the worker's job folder; check that both use the same read-aloud data folder",
|
||||
);
|
||||
}
|
||||
return expected;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a source the server sent. A path outside every configured root is reported as
|
||||
* `library_not_mounted` before touching the filesystem, because an unmounted library looks exactly
|
||||
* like a missing file from inside the worker.
|
||||
*/
|
||||
export async function resolveSource(input: string, roots: readonly string[], label: string): Promise<SourceFingerprint> {
|
||||
if (!isAbsolute(input)) throw new RequestError(400, "invalid_source", `The ${label} path must be absolute`);
|
||||
const lexical = resolve(input);
|
||||
if (!roots.some((root) => within(lexical, root))) {
|
||||
throw new RequestError(400, "library_not_mounted", `The ${label} is outside every folder the worker can read`);
|
||||
}
|
||||
let real: string;
|
||||
try {
|
||||
real = await realpath(lexical);
|
||||
} catch (error) {
|
||||
throw sourceError(error, label);
|
||||
}
|
||||
const realRoots = await Promise.all(roots.map((root) => realpath(root).catch(() => null)));
|
||||
if (!realRoots.some((root) => root !== null && within(real, root))) {
|
||||
throw new RequestError(400, "library_not_mounted", `The ${label} resolves outside every folder the worker can read`);
|
||||
}
|
||||
try {
|
||||
const info = await stat(real);
|
||||
if (!info.isFile()) throw new RequestError(400, "invalid_source", `The ${label} is not a regular file`);
|
||||
await access(real, constants.R_OK);
|
||||
return { path: real, size: info.size, mtimeMs: Math.trunc(info.mtimeMs) };
|
||||
} catch (error) {
|
||||
if (error instanceof RequestError) throw error;
|
||||
throw sourceError(error, label);
|
||||
}
|
||||
}
|
||||
|
||||
function sourceError(error: unknown, label: string): RequestError {
|
||||
const code = (error as NodeJS.ErrnoException | null)?.code;
|
||||
if (code === "ENOENT" || code === "ENOTDIR") return new RequestError(400, "source_missing", `The ${label} does not exist`);
|
||||
return new RequestError(400, "invalid_source", `The ${label} cannot be read (${code ?? "unknown error"})`);
|
||||
}
|
||||
|
||||
export type RootCheckOptions = {
|
||||
/** How long health reuses the last check before the folders are checked again. */
|
||||
maxAgeMs?: number;
|
||||
/** How long one folder may take to answer before it counts as unreadable for that check. */
|
||||
timeoutMs?: number;
|
||||
/** Replaces the access check; the tests use it to stand in for a hung mount. */
|
||||
check?: (path: string) => Promise<void>;
|
||||
};
|
||||
|
||||
type RootState = { readable: boolean | null; pending: Promise<boolean> | null };
|
||||
|
||||
/**
|
||||
* The configured library folders this process can read right now. They are checked again once the last
|
||||
* check is older than `maxAgeMs`, so a folder that was not readable at startup, such as an automount that
|
||||
* comes up after the worker, is picked up without a restart. A folder whose earlier check has not answered
|
||||
* yet (a hung network mount) is not checked again until it does, so a dead mount holds at most one
|
||||
* filesystem thread.
|
||||
*/
|
||||
export class ReadableRoots {
|
||||
private readonly states = new Map<string, RootState>();
|
||||
private readable: string[] = [];
|
||||
private checkedAt: number | null = null;
|
||||
private refreshing: Promise<string[]> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly roots: readonly string[],
|
||||
private readonly options: RootCheckOptions = {},
|
||||
) {}
|
||||
|
||||
/** The readable folders in configured order; `maxAgeMs: 0` checks again unless a check is already running. */
|
||||
get(maxAgeMs = this.options.maxAgeMs ?? ROOT_CHECK_MAX_AGE_MS): Promise<string[]> {
|
||||
if (this.checkedAt !== null && Date.now() - this.checkedAt < maxAgeMs) return Promise.resolve(this.readable);
|
||||
this.refreshing ??= this.refresh().finally(() => {
|
||||
this.refreshing = null;
|
||||
});
|
||||
return this.refreshing;
|
||||
}
|
||||
|
||||
private async refresh(): Promise<string[]> {
|
||||
const results = await Promise.all(this.roots.map((root) => this.checkRoot(root)));
|
||||
this.readable = this.roots.filter((_, index) => results[index]);
|
||||
this.checkedAt = Date.now();
|
||||
return this.readable;
|
||||
}
|
||||
|
||||
private async checkRoot(root: string): Promise<boolean> {
|
||||
let state = this.states.get(root);
|
||||
if (!state) this.states.set(root, (state = { readable: null, pending: null }));
|
||||
const entry = state;
|
||||
entry.pending ??= (this.options.check ?? canRead)(root).then(
|
||||
() => this.settle(root, entry, null),
|
||||
(error: unknown) => this.settle(root, entry, error),
|
||||
);
|
||||
const timeoutMs = this.options.timeoutMs ?? ROOT_CHECK_TIMEOUT_MS;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const timedOut = new Promise<null>((resolve) => {
|
||||
timer = setTimeout(() => resolve(null), timeoutMs);
|
||||
});
|
||||
const readable = await Promise.race([entry.pending, timedOut]);
|
||||
clearTimeout(timer);
|
||||
if (readable !== null) return readable;
|
||||
this.report(root, entry, false, "TimeoutError", `no answer within ${timeoutMs} ms`);
|
||||
return false;
|
||||
}
|
||||
|
||||
private settle(root: string, state: RootState, error: unknown): boolean {
|
||||
state.pending = null;
|
||||
if (error === null) this.report(root, state, true);
|
||||
else this.report(root, state, false, errorClass(error), (error as NodeJS.ErrnoException).code ?? errorMessage(error));
|
||||
return error === null;
|
||||
}
|
||||
|
||||
/** Logs only changes, so a folder that stays unreadable is not reported on every health check. */
|
||||
private report(root: string, state: RootState, readable: boolean, errorName?: string, error?: string): void {
|
||||
if (readable && state.readable === false) {
|
||||
logLine("read_aloud.worker.library_root", "end", { path: root }, "library folder is readable again and is back in health");
|
||||
} else if (!readable && state.readable !== false) {
|
||||
logLine(
|
||||
"read_aloud.worker.library_root",
|
||||
"fail",
|
||||
{ path: root, errorClass: errorName, error },
|
||||
"library folder is not readable and is left out of health",
|
||||
);
|
||||
}
|
||||
state.readable = readable;
|
||||
}
|
||||
}
|
||||
|
||||
function canRead(path: string): Promise<void> {
|
||||
return access(path, constants.R_OK | constants.X_OK);
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { Logger } from "pino";
|
||||
import type { ReadAloudErrorCode, ReadAloudStageDetail } from "@bookorbit/types";
|
||||
import type { EventData, RunnerJob } from "./protocol.js";
|
||||
|
||||
export type WorkPaths = {
|
||||
workDir: string;
|
||||
sourceAudioDir: string;
|
||||
processedAudioDir: string;
|
||||
emissionsDir: string;
|
||||
scratchDir: string;
|
||||
upgradedEpubPath: string;
|
||||
markedEpubPath: string;
|
||||
outputPath: string;
|
||||
reportPath: string;
|
||||
};
|
||||
|
||||
export type LogListener = (level: "info" | "warn" | "error", message: string) => void;
|
||||
|
||||
export type StageContext = {
|
||||
job: RunnerJob;
|
||||
paths: WorkPaths;
|
||||
signal: AbortSignal;
|
||||
/** A pino logger at info that prints sanitized single lines; the listener sees each raw message first. */
|
||||
createLogger(listener?: LogListener): Logger;
|
||||
progress(fraction: number, detail?: ReadAloudStageDetail | null): void;
|
||||
event(level: "info" | "warn", code: string, message: string, data?: EventData | null): void;
|
||||
};
|
||||
|
||||
export type StageResult = { data?: EventData } | void;
|
||||
|
||||
/**
|
||||
* The stage implementations the runner drives. `pipeline.ts` is the STAlign one; tests swap in a fake
|
||||
* through READ_ALOUD_RUNNER_PIPELINE so the runner's caching, progress and failure handling run for real.
|
||||
*/
|
||||
export type Pipeline = {
|
||||
modelInstalled(context: StageContext): Promise<boolean>;
|
||||
prepareAudio(context: StageContext): Promise<StageResult>;
|
||||
downloadModel(context: StageContext): Promise<StageResult>;
|
||||
generateEmissions(context: StageContext): Promise<StageResult>;
|
||||
markupEbook(context: StageContext): Promise<StageResult>;
|
||||
align(context: StageContext): Promise<StageResult>;
|
||||
};
|
||||
|
||||
/** Thrown by a stage that knows a more precise code than its stage default. */
|
||||
export class StageError extends Error {
|
||||
constructor(
|
||||
readonly code: ReadAloudErrorCode,
|
||||
message: string,
|
||||
options?: { cause?: unknown },
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = "StageError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
import { existsSync, readdirSync } from "node:fs";
|
||||
import { readdir, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { ctcAlign, emit, markup, processAudiobook, upgradeEpub } from "@storyteller-platform/align";
|
||||
import { getModelDir, loadCtcModel, resolveCtcRun, type CtcRun } from "@storyteller-platform/ghost-story";
|
||||
import type { ReadAloudStageDetail, ReadAloudStageUnit } from "@bookorbit/types";
|
||||
import { StageError, type Pipeline, type StageContext } from "./pipeline-contract.js";
|
||||
|
||||
const ENGINE = "ctc-local";
|
||||
const MAX_SPLIT_HOURS = 2;
|
||||
/** Written only after every package file matched the registry and the model loaded. */
|
||||
const INSTALL_MARKER = ".bookorbit-installed";
|
||||
|
||||
type InstallModelPackage = (
|
||||
model: CtcRun["package"],
|
||||
options: { modelDir?: string; printOutput?: boolean; signal?: AbortSignal | null },
|
||||
) => Promise<string>;
|
||||
|
||||
/** STAlign's own `pipeline --ctc` stages, called with the options Storyteller's server uses. */
|
||||
export const pipeline: Pipeline = {
|
||||
async modelInstalled(context) {
|
||||
const run = ctcRun(context);
|
||||
return isInstalled(run) && existsSync(join(packageDir(run), INSTALL_MARKER));
|
||||
},
|
||||
|
||||
async prepareAudio(context) {
|
||||
const total = context.job.audio.length;
|
||||
let reencoded = 0;
|
||||
const logger = context.createLogger((_level, message) => {
|
||||
const match = /^Forcing CBR MP3 for .* at (\S+)$/.exec(message);
|
||||
if (!match) return;
|
||||
reencoded += 1;
|
||||
context.event("info", "audio_reencoded_cbr", "A variable bitrate MP3 was re-encoded to constant bitrate for precise seeking", {
|
||||
bitrate: match[1] ?? null,
|
||||
});
|
||||
});
|
||||
// `encoding` stays unset: with a codec STAlign skips its VBR MP3 to CBR re-encode.
|
||||
await processAudiobook(context.paths.sourceAudioDir, context.paths.processedAudioDir, {
|
||||
maxLength: MAX_SPLIT_HOURS,
|
||||
parallelism: 1,
|
||||
signal: context.signal,
|
||||
logger,
|
||||
// STAlign's own fraction runs ahead (a one-range file reports 2), so count finished files instead.
|
||||
onProgress: () => {
|
||||
const done = Math.min(total, splitFilesDone(context.paths.processedAudioDir));
|
||||
context.progress(done / total, { current: done, total, unit: "file" });
|
||||
},
|
||||
});
|
||||
const tracks = await audioTrackCount(context.paths.processedAudioDir);
|
||||
return { data: { files: total, tracks, reencoded } };
|
||||
},
|
||||
|
||||
async downloadModel(context) {
|
||||
const run = ctcRun(context);
|
||||
const startedAt = Date.now();
|
||||
const marker = join(packageDir(run), INSTALL_MARKER);
|
||||
context.event("info", "model_download_started", `Downloading the ${run.variant} model`, { variant: run.variant });
|
||||
await rm(marker, { force: true });
|
||||
let verified = true;
|
||||
try {
|
||||
// ghost-story's own install skips a package whose entry file exists, so an interrupted download
|
||||
// would count as installed forever. This checks every file against the registry and resumes.
|
||||
const install = await loadInstaller();
|
||||
await install(run.package, { modelDir: getModelDir(), printOutput: false, signal: context.signal });
|
||||
} catch (error) {
|
||||
if (context.signal.aborted) throw error;
|
||||
// Offline with a model already on disk: use it if it loads, and verify again on the next run.
|
||||
if (!isInstalled(run)) throw new StageError("model_download_failed", errorText(error), { cause: error });
|
||||
verified = false;
|
||||
context.event("warn", "model_unverified", "The speech model could not be checked against the registry; using the copy on disk", {
|
||||
variant: run.variant,
|
||||
});
|
||||
}
|
||||
try {
|
||||
const loaded = await loadCtcModel(run, { signal: context.signal, printOutput: false });
|
||||
await loaded[Symbol.asyncDispose]();
|
||||
} catch (error) {
|
||||
if (context.signal.aborted) throw error;
|
||||
// Every file matched the registry, so a load failure is the device, not the download.
|
||||
if (verified) throw new StageError("inference_failed", errorText(error), { cause: error });
|
||||
throw new StageError("model_download_failed", errorText(error), { cause: error });
|
||||
}
|
||||
if (verified) await writeFile(marker, JSON.stringify({ package: run.package.name, version: run.package.version, at: new Date().toISOString() }));
|
||||
const bytes = await directoryBytes(packageDir(run));
|
||||
const durationMs = Date.now() - startedAt;
|
||||
context.event("info", "model_download_finished", `Downloaded the ${run.variant} model`, { variant: run.variant, bytes, durationMs });
|
||||
return { data: { variant: run.variant, bytes } };
|
||||
},
|
||||
|
||||
async generateEmissions(context) {
|
||||
const total = await audioTrackCount(context.paths.processedAudioDir);
|
||||
let cached = 0;
|
||||
const logger = context.createLogger((_level, message) => {
|
||||
if (message.startsWith("Found existing emissions for ")) cached += 1;
|
||||
});
|
||||
const job = context.job;
|
||||
await emit(context.paths.processedAudioDir, context.paths.emissionsDir, {
|
||||
engine: ENGINE,
|
||||
device: job.device,
|
||||
model: job.model,
|
||||
dtype: job.dtype,
|
||||
batchSize: job.batchSize,
|
||||
language: new Intl.Locale(job.language),
|
||||
concurrency: 1,
|
||||
signal: context.signal,
|
||||
logger,
|
||||
onProgress: (fraction) => context.progress(fraction, countDetail(fraction, total, "track")),
|
||||
});
|
||||
if (cached > 0) {
|
||||
context.event("info", "emissions_cached", `Reused emissions for ${cached} of ${total} tracks`, { cached, tracks: total });
|
||||
}
|
||||
return { data: { tracks: total, cached, device: job.device } };
|
||||
},
|
||||
|
||||
async markupEbook(context) {
|
||||
const locale = new Intl.Locale(context.job.language);
|
||||
const logger = context.createLogger();
|
||||
await upgradeEpub(context.job.epub.path, context.paths.upgradedEpubPath, { removeNcx: false });
|
||||
context.signal.throwIfAborted();
|
||||
const chapters = new SpineCounter();
|
||||
await markup(context.paths.upgradedEpubPath, context.paths.markedEpubPath, {
|
||||
granularity: context.job.granularity,
|
||||
primaryLocale: locale,
|
||||
logger,
|
||||
onProgress: (fraction) => context.progress(fraction, chapters.detail(fraction)),
|
||||
});
|
||||
return chapters.total ? { data: { chapters: chapters.total } } : undefined;
|
||||
},
|
||||
|
||||
async align(context) {
|
||||
const job = context.job;
|
||||
const chapters = new SpineCounter();
|
||||
await ctcAlign(context.paths.markedEpubPath, context.paths.outputPath, context.paths.emissionsDir, context.paths.processedAudioDir, {
|
||||
reportsPath: context.paths.reportPath,
|
||||
granularity: job.granularity,
|
||||
textRef: "id-fragment",
|
||||
outFormat: "epub",
|
||||
primaryLocale: new Intl.Locale(job.language),
|
||||
autoExclude: true,
|
||||
decisions: job.decisions,
|
||||
signal: context.signal,
|
||||
logger: context.createLogger(),
|
||||
onProgress: (fraction) => context.progress(fraction, chapters.detail(fraction)),
|
||||
});
|
||||
return chapters.total ? { data: { chapters: chapters.total } } : undefined;
|
||||
},
|
||||
};
|
||||
|
||||
function ctcRun(context: StageContext): CtcRun {
|
||||
return resolveCtcRun({
|
||||
device: context.job.device,
|
||||
model: context.job.model,
|
||||
language: new Intl.Locale(context.job.language).language,
|
||||
dtype: context.job.dtype,
|
||||
batchSize: context.job.batchSize,
|
||||
});
|
||||
}
|
||||
|
||||
function packageDir(run: CtcRun): string {
|
||||
return join(getModelDir(), run.package.subdir);
|
||||
}
|
||||
|
||||
/** ghost-story's own installed check: the package's entry file exists in the model folder. */
|
||||
function isInstalled(run: CtcRun): boolean {
|
||||
return existsSync(join(packageDir(run), run.package.entryFile));
|
||||
}
|
||||
|
||||
/**
|
||||
* ghost-story keeps its verifying installer out of the package exports. The pinned, patched version
|
||||
* ships it beside the entry point; a release that moves it fails the download stage loudly.
|
||||
*/
|
||||
async function loadInstaller(): Promise<InstallModelPackage> {
|
||||
const entry = fileURLToPath(import.meta.resolve("@storyteller-platform/ghost-story"));
|
||||
const module = (await import(pathToFileURL(join(dirname(entry), "cli", "ctc-install.js")).href)) as { installModelPackage?: unknown };
|
||||
if (typeof module.installModelPackage !== "function") throw new Error("ghost-story no longer provides installModelPackage");
|
||||
return module.installModelPackage as InstallModelPackage;
|
||||
}
|
||||
|
||||
function errorText(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function countDetail(fraction: number, total: number, unit: ReadAloudStageUnit): ReadAloudStageDetail | null {
|
||||
if (total <= 0) return null;
|
||||
return { current: Math.min(total, Math.max(0, Math.floor(fraction * total))), total, unit };
|
||||
}
|
||||
|
||||
/**
|
||||
* markup and ctcAlign report `index / spine.length` before each spine item, so the first non-zero
|
||||
* value is exactly `1 / spine.length`.
|
||||
*/
|
||||
class SpineCounter {
|
||||
total = 0;
|
||||
|
||||
detail(fraction: number): ReadAloudStageDetail | null {
|
||||
if (!this.total && fraction > 0) this.total = Math.round(1 / fraction);
|
||||
if (!this.total) return null;
|
||||
return { current: Math.min(this.total, Math.round(fraction * this.total)), total: this.total, unit: "chapter" };
|
||||
}
|
||||
}
|
||||
|
||||
/** Source files with at least one finished range: each range leaves a `<file>-<range>.source.json` sidecar. */
|
||||
function splitFilesDone(directory: string): number {
|
||||
let names: string[];
|
||||
try {
|
||||
names = readdirSync(directory);
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
return new Set(names.filter((name) => name.endsWith(".source.json")).map((name) => name.split("-")[0])).size;
|
||||
}
|
||||
|
||||
async function audioTrackCount(directory: string): Promise<number> {
|
||||
const names = await readdir(directory).catch(() => [] as string[]);
|
||||
return names.filter((name) => !name.startsWith(".") && !name.endsWith(".json")).length;
|
||||
}
|
||||
|
||||
async function directoryBytes(directory: string): Promise<number> {
|
||||
let total = 0;
|
||||
const entries = await readdir(directory, { withFileTypes: true, recursive: true }).catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
total += await stat(join(entry.parentPath, entry.name)).then(
|
||||
(info) => info.size,
|
||||
() => 0,
|
||||
);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { InferenceSession, Tensor } from "onnxruntime-node";
|
||||
|
||||
const [device, modelPath] = process.argv.slice(2);
|
||||
if (!device || !modelPath) {
|
||||
process.stderr.write("usage: probe.js <device> <model.onnx>\n");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
try {
|
||||
const session = await InferenceSession.create(modelPath, { executionProviders: [device] });
|
||||
const input = new Tensor("float32", Float32Array.from([1.5]), [1]);
|
||||
const output = await session.run({ x: input });
|
||||
const value = (output["y"]?.data as Float32Array | undefined)?.[0];
|
||||
await session.release();
|
||||
if (value !== 1.5) throw new Error(`probe model returned ${String(value)} instead of 1.5`);
|
||||
process.stdout.write("ok\n");
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`.replace(/\n(?=.)/g, " "));
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import type {
|
||||
ReadAloudDecisions,
|
||||
ReadAloudErrorCode,
|
||||
ReadAloudGranularity,
|
||||
ReadAloudJobEventLevel,
|
||||
ReadAloudStageDetail,
|
||||
ReadAloudWorkerStage,
|
||||
} from "@bookorbit/types";
|
||||
import type { ResolvedCtcDevice } from "./accelerator.js";
|
||||
|
||||
export type EventData = Record<string, string | number | boolean | null>;
|
||||
|
||||
export type SourceFingerprint = {
|
||||
path: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
};
|
||||
|
||||
/** Everything the runner needs for one run. Paths are already resolved and checked by the control process. */
|
||||
export type RunnerJob = {
|
||||
jobId: string;
|
||||
attempt: number;
|
||||
workDir: string;
|
||||
epub: SourceFingerprint;
|
||||
audio: SourceFingerprint[];
|
||||
language: string;
|
||||
granularity: ReadAloudGranularity;
|
||||
model: "quartznet15x5";
|
||||
maxOutputBytes: number;
|
||||
decisions: ReadAloudDecisions;
|
||||
resumeFrom: ReadAloudWorkerStage | null;
|
||||
device: ResolvedCtcDevice;
|
||||
dtype: "fp32" | "fp16" | null;
|
||||
batchSize: number | null;
|
||||
/** Absolute folders replaced by placeholders in event messages and logs. */
|
||||
redactions: Redaction[];
|
||||
};
|
||||
|
||||
export type Redaction = { path: string; placeholder: string };
|
||||
|
||||
export type ControlToRunner = { type: "start"; job: RunnerJob } | { type: "cancel" };
|
||||
|
||||
export type RunnerToControl =
|
||||
| {
|
||||
type: "progress";
|
||||
stage: ReadAloudWorkerStage;
|
||||
overall: number;
|
||||
stageFraction: number;
|
||||
detail: ReadAloudStageDetail | null;
|
||||
}
|
||||
| {
|
||||
type: "event";
|
||||
level: Exclude<ReadAloudJobEventLevel, "error">;
|
||||
stage: ReadAloudWorkerStage | null;
|
||||
code: string;
|
||||
message: string;
|
||||
data: EventData | null;
|
||||
}
|
||||
| { type: "done"; outputBytes: number }
|
||||
| { type: "failed"; code: ReadAloudErrorCode; message: string };
|
||||
|
||||
export const MAX_EVENT_MESSAGE_LENGTH = 500;
|
||||
@@ -0,0 +1,441 @@
|
||||
import { mkdir, readFile, rm, stat, symlink } from "node:fs/promises";
|
||||
import { extname, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { format } from "node:util";
|
||||
import { pino, type Logger } from "pino";
|
||||
import type { ReadAloudErrorCode, ReadAloudStageDetail, ReadAloudWorkerStage } from "@bookorbit/types";
|
||||
import { cleanMessage, errorClass, errorMessage, logLine } from "./log.js";
|
||||
import { StageError, type LogListener, type Pipeline, type StageContext, type StageResult, type WorkPaths } from "./pipeline-contract.js";
|
||||
import { MAX_EVENT_MESSAGE_LENGTH, type ControlToRunner, type EventData, type RunnerJob, type RunnerToControl } from "./protocol.js";
|
||||
import {
|
||||
MARKER_STAGES,
|
||||
STAGE_DEPENDENCIES,
|
||||
fingerprint,
|
||||
markerStates,
|
||||
normalizedWeights,
|
||||
planStages,
|
||||
removeMarker,
|
||||
stageInputs,
|
||||
writeMarker,
|
||||
type FingerprintSource,
|
||||
type MarkerStage,
|
||||
type StageDecision,
|
||||
} from "./stages.js";
|
||||
|
||||
const STAGE_ERROR_CODES: Record<ReadAloudWorkerStage, ReadAloudErrorCode> = {
|
||||
preparing_audio: "audio_processing_failed",
|
||||
downloading_model: "model_download_failed",
|
||||
generating_emissions: "inference_failed",
|
||||
marking_up_ebook: "markup_failed",
|
||||
aligning: "alignment_failed",
|
||||
validating: "invalid_output",
|
||||
};
|
||||
|
||||
const STAGE_LABELS: Record<ReadAloudWorkerStage, string> = {
|
||||
preparing_audio: "Preparing audio",
|
||||
downloading_model: "Downloading the speech model",
|
||||
generating_emissions: "Generating emissions",
|
||||
marking_up_ebook: "Marking up the EPUB",
|
||||
aligning: "Aligning text and audio",
|
||||
validating: "Validating the output",
|
||||
};
|
||||
|
||||
const MAX_REPORT_BYTES = 16 * 1024 * 1024;
|
||||
const PROGRESS_INTERVAL_MS = 1_000;
|
||||
const MAX_FORWARDED_WARNINGS = 50;
|
||||
const REPORT_TOTAL_KEYS = [
|
||||
"sentences",
|
||||
"matchedSentences",
|
||||
"interpolatedSentences",
|
||||
"unmatchedSentences",
|
||||
"droppedSentences",
|
||||
"seconds",
|
||||
"matchedSeconds",
|
||||
"interpolatedSeconds",
|
||||
"looseSeconds",
|
||||
"tracks",
|
||||
"unalignedTracks",
|
||||
] as const;
|
||||
|
||||
class CancelledError extends Error {
|
||||
constructor() {
|
||||
super("The job was cancelled");
|
||||
this.name = "CancelledError";
|
||||
}
|
||||
}
|
||||
|
||||
class StageFailure extends Error {
|
||||
constructor(
|
||||
readonly stage: ReadAloudWorkerStage | null,
|
||||
readonly code: ReadAloudErrorCode,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "StageFailure";
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
let started = false;
|
||||
let consoleContext: { jobId: string; redactions: RunnerJob["redactions"] } | null = null;
|
||||
|
||||
// Libraries print to the console directly (ghost-story logs every model download with its absolute
|
||||
// destination), so console output goes through the same single-line, redacted format as everything else.
|
||||
for (const method of ["log", "info", "debug", "warn", "error"] as const) {
|
||||
console[method] = (...args: unknown[]) => {
|
||||
const level = method === "warn" || method === "error" ? method : "info";
|
||||
const message = cleanMessage(format(...args), consoleContext?.redactions ?? []);
|
||||
if (message) logLine("read_aloud.worker.console", null, { jobId: consoleContext?.jobId, level }, message);
|
||||
};
|
||||
}
|
||||
|
||||
process.on("message", (message: ControlToRunner) => {
|
||||
if (message.type === "start" && !started) {
|
||||
started = true;
|
||||
consoleContext = { jobId: message.job.jobId, redactions: message.job.redactions };
|
||||
void run(message.job);
|
||||
} else if (message.type === "cancel") {
|
||||
controller.abort(new CancelledError());
|
||||
}
|
||||
});
|
||||
|
||||
// The control process is gone, so nothing else will clean up after this run: stop it, and on the way out
|
||||
// kill the runner's process group so an ffmpeg STAlign started does not keep running.
|
||||
process.on("disconnect", () => {
|
||||
controller.abort(new CancelledError());
|
||||
process.once("exit", killOwnProcessGroup);
|
||||
setTimeout(() => process.exit(1), 5_000).unref();
|
||||
});
|
||||
|
||||
/**
|
||||
* The control process forks each runner detached, which makes it the leader of a new process group that
|
||||
* holds only the runner and what it started; the control process is never in it. A runner that is not a
|
||||
* group leader has no group with its own pid, so the kill fails with ESRCH and signals nothing.
|
||||
*/
|
||||
function killOwnProcessGroup(): void {
|
||||
try {
|
||||
process.kill(-process.pid, "SIGKILL");
|
||||
} catch {
|
||||
// ESRCH: this runner leads no process group.
|
||||
}
|
||||
}
|
||||
|
||||
function send(message: RunnerToControl, callback?: () => void): void {
|
||||
if (!process.connected || !process.send) {
|
||||
callback?.();
|
||||
return;
|
||||
}
|
||||
process.send(message, undefined, {}, () => callback?.());
|
||||
}
|
||||
|
||||
async function run(job: RunnerJob): Promise<void> {
|
||||
const runStartedAt = Date.now();
|
||||
const signal = controller.signal;
|
||||
const paths = workPaths(job.workDir);
|
||||
const reporter = new ProgressReporter();
|
||||
const clean = (text: string) => cleanMessage(text, job.redactions, MAX_EVENT_MESSAGE_LENGTH);
|
||||
let currentStage: ReadAloudWorkerStage | null = null;
|
||||
let forwardedWarnings = 0;
|
||||
|
||||
const event = (level: "info" | "warn", stage: ReadAloudWorkerStage | null, code: string, message: string, data: EventData | null = null) => {
|
||||
send({ type: "event", level, stage, code, message: clean(message), data });
|
||||
};
|
||||
|
||||
const createLogger = (listener?: LogListener): Logger =>
|
||||
pino(
|
||||
{ level: "info", base: null, timestamp: false },
|
||||
{
|
||||
write(line: string) {
|
||||
let record: { level?: number; msg?: unknown; err?: { message?: unknown } };
|
||||
try {
|
||||
record = JSON.parse(line);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const level = (record.level ?? 30) >= 50 ? "error" : (record.level ?? 30) >= 40 ? "warn" : "info";
|
||||
const raw = typeof record.msg === "string" ? record.msg : typeof record.err?.message === "string" ? record.err.message : "";
|
||||
listener?.(level, raw);
|
||||
const message = clean(raw);
|
||||
logLine("read_aloud.worker.stalign", null, { jobId: job.jobId, stage: currentStage ?? undefined, level }, message);
|
||||
if (level !== "info" && forwardedWarnings < MAX_FORWARDED_WARNINGS) {
|
||||
forwardedWarnings += 1;
|
||||
event("warn", currentStage, "stalign_warning", message);
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const context = (stage: ReadAloudWorkerStage): StageContext => ({
|
||||
job,
|
||||
paths,
|
||||
signal,
|
||||
createLogger,
|
||||
progress: (fraction, detail) => reporter.update(stage, fraction, detail ?? null),
|
||||
event: (level, code, message, data) => event(level, stage, code, message, data ?? null),
|
||||
});
|
||||
|
||||
const runStage = async (stage: ReadAloudWorkerStage, work: () => Promise<StageResult>, data: EventData = {}): Promise<number> => {
|
||||
currentStage = stage;
|
||||
const startedAt = Date.now();
|
||||
logLine("read_aloud.worker.stage", "start", { jobId: job.jobId, stage, ...data }, "stage started");
|
||||
event("info", stage, "stage_started", STAGE_LABELS[stage], Object.keys(data).length ? data : null);
|
||||
reporter.begin(stage);
|
||||
let result: StageResult;
|
||||
try {
|
||||
result = await work();
|
||||
} catch (error) {
|
||||
const durationMs = Date.now() - startedAt;
|
||||
const failure = toFailure(stage, error, signal);
|
||||
logLine(
|
||||
"read_aloud.worker.stage",
|
||||
"fail",
|
||||
{ jobId: job.jobId, stage, durationMs, errorCode: failure.code, errorClass: errorClass(error), error: clean(errorMessage(error)) },
|
||||
"stage failed",
|
||||
);
|
||||
if (failure.code !== "cancelled") event("warn", stage, "stage_failed", failure.message, { durationMs, errorCode: failure.code });
|
||||
throw failure;
|
||||
}
|
||||
const durationMs = Date.now() - startedAt;
|
||||
const resultData = result?.data ?? {};
|
||||
logLine("read_aloud.worker.stage", "end", { jobId: job.jobId, stage, durationMs, ...resultData }, "stage finished");
|
||||
event("info", stage, "stage_finished", `${STAGE_LABELS[stage]} finished`, { durationMs, ...resultData });
|
||||
reporter.finish(stage);
|
||||
return durationMs;
|
||||
};
|
||||
|
||||
try {
|
||||
await prepareFolders(job, paths);
|
||||
const pipeline = await loadPipeline();
|
||||
const source: FingerprintSource = {
|
||||
epub: job.epub,
|
||||
audio: job.audio,
|
||||
language: job.language,
|
||||
granularity: job.granularity,
|
||||
model: job.model,
|
||||
decisions: job.decisions,
|
||||
};
|
||||
const plan = planStages(await markerStates(job.workDir, source), job.resumeFrom);
|
||||
const decisionFor = (stage: MarkerStage): StageDecision => plan.find((entry) => entry.stage === stage)!;
|
||||
let needsModel = false;
|
||||
if (decisionFor("generating_emissions").run) {
|
||||
try {
|
||||
needsModel = !(await pipeline.modelInstalled(context("downloading_model")));
|
||||
} catch (error) {
|
||||
throw toFailure("downloading_model", error, signal);
|
||||
}
|
||||
}
|
||||
const order: ReadAloudWorkerStage[] = [
|
||||
"preparing_audio",
|
||||
...(needsModel ? (["downloading_model"] as const) : []),
|
||||
"generating_emissions",
|
||||
"marking_up_ebook",
|
||||
"aligning",
|
||||
"validating",
|
||||
];
|
||||
reporter.plan(order.filter((stage) => !isMarkerStage(stage) || decisionFor(stage).run));
|
||||
|
||||
let outputBytes = 0;
|
||||
for (const stage of order) {
|
||||
signal.throwIfAborted();
|
||||
if (isMarkerStage(stage)) {
|
||||
const decision = decisionFor(stage);
|
||||
if (!decision.run) {
|
||||
logLine("read_aloud.worker.stage", "end", { jobId: job.jobId, stage, skipped: "cached" }, "stage cached");
|
||||
event("info", stage, "stage_skipped", `${STAGE_LABELS[stage]} reused from an earlier run`, { reason: "cached" });
|
||||
continue;
|
||||
}
|
||||
await removeMarker(job.workDir, stage);
|
||||
for (const dependent of dependentsOf(stage)) await removeMarker(job.workDir, dependent);
|
||||
await clearStageOutputs(stage, decision, paths);
|
||||
const inputs = stageInputs(stage, source);
|
||||
const durationMs = await runStage(stage, () => stageWork(pipeline, stage, context(stage)), { reason: decision.reason });
|
||||
await writeMarker(job.workDir, { stage, fingerprint: fingerprint(inputs), inputs, finishedAt: new Date().toISOString(), durationMs });
|
||||
} else if (stage === "downloading_model") {
|
||||
await runStage(stage, () => pipeline.downloadModel(context(stage)));
|
||||
} else {
|
||||
await runStage(stage, async () => {
|
||||
const validated = await validateOutput(paths, job.maxOutputBytes);
|
||||
outputBytes = validated.outputBytes;
|
||||
return { data: validated.data };
|
||||
});
|
||||
}
|
||||
}
|
||||
reporter.flush();
|
||||
logLine("read_aloud.worker.run", "end", { jobId: job.jobId, durationMs: Date.now() - runStartedAt, outputBytes }, "run finished");
|
||||
send({ type: "done", outputBytes }, () => process.exit(0));
|
||||
} catch (error) {
|
||||
reporter.flush();
|
||||
const failure = error instanceof StageFailure ? error : toFailure(currentStage, error, signal);
|
||||
logLine(
|
||||
"read_aloud.worker.run",
|
||||
"fail",
|
||||
{ jobId: job.jobId, stage: failure.stage ?? undefined, durationMs: Date.now() - runStartedAt, errorCode: failure.code, error: failure.message },
|
||||
"run failed",
|
||||
);
|
||||
send({ type: "failed", code: failure.code, message: failure.message }, () => process.exit(0));
|
||||
}
|
||||
|
||||
function toFailure(stage: ReadAloudWorkerStage | null, error: unknown, abortSignal: AbortSignal): StageFailure {
|
||||
if (error instanceof StageFailure) return error;
|
||||
if (abortSignal.aborted || error instanceof CancelledError) return new StageFailure(stage, "cancelled", "The job was cancelled");
|
||||
const label = stage ? STAGE_LABELS[stage] : "The job";
|
||||
const message = clean(errorMessage(error)) || `${label} failed without an error message; the worker log has details`;
|
||||
if (error instanceof StageError) return new StageFailure(stage, error.code, message);
|
||||
if ((error as NodeJS.ErrnoException | null)?.code === "ENOSPC") {
|
||||
return new StageFailure(stage, "insufficient_storage", "The work volume ran out of free space");
|
||||
}
|
||||
return new StageFailure(stage, stage ? STAGE_ERROR_CODES[stage] : "unknown", message);
|
||||
}
|
||||
}
|
||||
|
||||
function stageWork(pipeline: Pipeline, stage: MarkerStage, context: StageContext): Promise<StageResult> {
|
||||
switch (stage) {
|
||||
case "preparing_audio":
|
||||
return pipeline.prepareAudio(context);
|
||||
case "generating_emissions":
|
||||
return pipeline.generateEmissions(context);
|
||||
case "marking_up_ebook":
|
||||
return pipeline.markupEbook(context);
|
||||
case "aligning":
|
||||
return pipeline.align(context);
|
||||
}
|
||||
}
|
||||
|
||||
function isMarkerStage(stage: ReadAloudWorkerStage): stage is MarkerStage {
|
||||
return (MARKER_STAGES as readonly string[]).includes(stage);
|
||||
}
|
||||
|
||||
function dependentsOf(stage: MarkerStage): MarkerStage[] {
|
||||
const result = new Set<MarkerStage>();
|
||||
const visit = (current: MarkerStage) => {
|
||||
for (const candidate of MARKER_STAGES) {
|
||||
if (STAGE_DEPENDENCIES[candidate].includes(current) && !result.has(candidate)) {
|
||||
result.add(candidate);
|
||||
visit(candidate);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(stage);
|
||||
return [...result];
|
||||
}
|
||||
|
||||
function workPaths(workDir: string): WorkPaths {
|
||||
return {
|
||||
workDir,
|
||||
sourceAudioDir: join(workDir, "source-audio"),
|
||||
processedAudioDir: join(workDir, "processed-audio"),
|
||||
emissionsDir: join(workDir, "emissions"),
|
||||
scratchDir: join(workDir, "scratch"),
|
||||
upgradedEpubPath: join(workDir, "upgraded.epub"),
|
||||
markedEpubPath: join(workDir, "marked.epub"),
|
||||
outputPath: join(workDir, "output.epub"),
|
||||
reportPath: join(workDir, "report.json"),
|
||||
};
|
||||
}
|
||||
|
||||
/** Never removes the job folder: cached stages are what make retries and restarts resume. */
|
||||
async function prepareFolders(job: RunnerJob, paths: WorkPaths): Promise<void> {
|
||||
await mkdir(paths.workDir, { recursive: true });
|
||||
await rm(paths.scratchDir, { recursive: true, force: true });
|
||||
await mkdir(paths.scratchDir, { recursive: true });
|
||||
await rm(paths.sourceAudioDir, { recursive: true, force: true });
|
||||
await mkdir(paths.sourceAudioDir, { recursive: true });
|
||||
for (const [index, audio] of job.audio.entries()) {
|
||||
const name = `${String(index + 1).padStart(4, "0")}${extname(audio.path).toLowerCase()}`;
|
||||
await symlink(audio.path, join(paths.sourceAudioDir, name));
|
||||
}
|
||||
}
|
||||
|
||||
async function clearStageOutputs(stage: MarkerStage, decision: StageDecision, paths: WorkPaths): Promise<void> {
|
||||
if (stage === "preparing_audio") {
|
||||
// Splitting is not resumable, and a stale split from different sources would pair with the wrong emissions.
|
||||
await rm(paths.processedAudioDir, { recursive: true, force: true });
|
||||
if (decision.state === "stale") await rm(paths.emissionsDir, { recursive: true, force: true });
|
||||
}
|
||||
if (stage === "aligning") {
|
||||
await rm(paths.outputPath, { force: true });
|
||||
await rm(paths.reportPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function validateOutput(paths: WorkPaths, maxOutputBytes: number): Promise<{ outputBytes: number; data: EventData }> {
|
||||
const output = await stat(paths.outputPath).catch(() => null);
|
||||
if (!output?.isFile() || output.size <= 0) throw new StageError("invalid_output", "The aligned EPUB is missing or empty");
|
||||
if (output.size > maxOutputBytes) {
|
||||
throw new StageError("invalid_output", `The aligned EPUB is ${output.size} bytes, above the ${maxOutputBytes} byte limit`);
|
||||
}
|
||||
const report = await stat(paths.reportPath).catch(() => null);
|
||||
if (!report?.isFile() || report.size <= 0 || report.size > MAX_REPORT_BYTES) {
|
||||
throw new StageError("invalid_output", "The alignment report is missing, empty or too large");
|
||||
}
|
||||
let parsed: { totals?: Record<string, unknown> } | null;
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(paths.reportPath, "utf8"));
|
||||
} catch {
|
||||
throw new StageError("invalid_output", "The alignment report is not valid JSON");
|
||||
}
|
||||
const totals = parsed?.totals;
|
||||
if (!totals || REPORT_TOTAL_KEYS.some((key) => typeof totals[key] !== "number" || !Number.isFinite(totals[key]))) {
|
||||
throw new StageError("invalid_output", "The alignment report has no usable totals");
|
||||
}
|
||||
return {
|
||||
outputBytes: output.size,
|
||||
data: {
|
||||
outputBytes: output.size,
|
||||
sentences: totals.sentences as number,
|
||||
matchedSentences: totals.matchedSentences as number,
|
||||
tracks: totals.tracks as number,
|
||||
unalignedTracks: totals.unalignedTracks as number,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function loadPipeline(): Promise<Pipeline> {
|
||||
// Test hook: the worker tests point this at a fake pipeline module.
|
||||
const override = process.env.READ_ALOUD_RUNNER_PIPELINE?.trim();
|
||||
const module = (await import(override ? pathToFileURL(override).href : "./pipeline.js")) as { pipeline?: Pipeline; default?: Pipeline };
|
||||
const pipeline = module.pipeline ?? module.default;
|
||||
if (!pipeline) throw new Error("The pipeline module exports no pipeline");
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
/** Maps stage fractions onto 0-100 across the stages that run, at most once per second. */
|
||||
class ProgressReporter {
|
||||
private weights = new Map<ReadAloudWorkerStage, number>();
|
||||
private completed = 0;
|
||||
private overall = 0;
|
||||
private lastSentAt = 0;
|
||||
private pending: RunnerToControl | null = null;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
|
||||
plan(stages: readonly ReadAloudWorkerStage[]): void {
|
||||
this.weights = normalizedWeights(stages);
|
||||
}
|
||||
|
||||
begin(stage: ReadAloudWorkerStage): void {
|
||||
this.update(stage, 0, null, true);
|
||||
}
|
||||
|
||||
update(stage: ReadAloudWorkerStage, fraction: number, detail: ReadAloudStageDetail | null, force = false): void {
|
||||
const bounded = Number.isFinite(fraction) ? Math.min(1, Math.max(0, fraction)) : 0;
|
||||
const weight = this.weights.get(stage) ?? 0;
|
||||
this.overall = Math.min(100, Math.max(this.overall, this.completed + weight * bounded));
|
||||
this.pending = { type: "progress", stage, overall: Math.round(this.overall * 10) / 10, stageFraction: bounded, detail };
|
||||
const wait = PROGRESS_INTERVAL_MS - (Date.now() - this.lastSentAt);
|
||||
if (force || wait <= 0) this.flush();
|
||||
else this.timer ??= setTimeout(() => this.flush(), wait);
|
||||
}
|
||||
|
||||
finish(stage: ReadAloudWorkerStage): void {
|
||||
this.completed += this.weights.get(stage) ?? 0;
|
||||
}
|
||||
|
||||
flush(): void {
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
if (!this.pending) return;
|
||||
send(this.pending);
|
||||
this.pending = null;
|
||||
this.lastSentAt = Date.now();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { timingSafeEqual } from "node:crypto";
|
||||
import { mkdir, readdir, realpath, rm } from "node:fs/promises";
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { READ_ALOUD_WORKER_PROTOCOL, type ReadAloudErrorCode, type ReadAloudWorkerError, type ReadAloudWorkerHealth } from "@bookorbit/types";
|
||||
import type { ConfiguredCtcDevice, DeviceCheck, ResolvedCtcDevice } from "./accelerator.js";
|
||||
import { workerTmpDir, type WorkerConfig } from "./config.js";
|
||||
import { JobRegistry, freeBytes, type RegistryOptions } from "./jobs.js";
|
||||
import { errorClass, errorMessage, logLine } from "./log.js";
|
||||
import { RequestError, within } from "./paths.js";
|
||||
import { isJobId, validateRequest } from "./validation.js";
|
||||
|
||||
/** Room for the largest job: every audio path at a generous length, plus chapter decisions. */
|
||||
const MAX_BODY_BYTES = 4 * 1024 * 1024;
|
||||
const JOB_PATH = /^\/v1\/jobs\/([^/]+)$/;
|
||||
|
||||
export type WorkerOptions = Omit<RegistryOptions, "config"> & {
|
||||
config: WorkerConfig;
|
||||
configuredDevice: ConfiguredCtcDevice;
|
||||
device: ResolvedCtcDevice;
|
||||
deviceCheck: DeviceCheck;
|
||||
version: string;
|
||||
};
|
||||
|
||||
export type RunningWorker = {
|
||||
server: Server;
|
||||
registry: JobRegistry;
|
||||
port: number;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export async function startWorker(options: WorkerOptions): Promise<RunningWorker> {
|
||||
const registry = new JobRegistry(options);
|
||||
const server = createServer((request, response) => {
|
||||
handle(request, response, options, registry).catch((error: unknown) => {
|
||||
logLine("read_aloud.worker.http", "fail", { errorClass: errorClass(error), error: errorMessage(error) }, "request failed");
|
||||
if (!response.headersSent) sendError(response, 500, "unknown", "Internal worker error");
|
||||
else response.destroy();
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(options.config.port, options.config.host, () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const port = (server.address() as AddressInfo).port;
|
||||
return {
|
||||
server,
|
||||
registry,
|
||||
port,
|
||||
async close() {
|
||||
await registry.shutdown();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
server.closeAllConnections();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function handle(request: IncomingMessage, response: ServerResponse, options: WorkerOptions, registry: JobRegistry): Promise<void> {
|
||||
if (!authorized(request, options.config.token)) {
|
||||
request.resume();
|
||||
return sendError(response, 401, "worker_unavailable", "Unauthorized");
|
||||
}
|
||||
const url = new URL(request.url ?? "/", "http://worker.local");
|
||||
|
||||
if (url.pathname === "/health" && request.method === "GET") {
|
||||
const health: ReadAloudWorkerHealth = {
|
||||
status: "ok",
|
||||
version: options.version,
|
||||
protocol: READ_ALOUD_WORKER_PROTOCOL,
|
||||
activeJobId: registry.active,
|
||||
activeJobs: registry.active ? 1 : 0,
|
||||
capacity: 1,
|
||||
device: options.device,
|
||||
configuredDevice: options.configuredDevice,
|
||||
deviceCheck: options.deviceCheck,
|
||||
libraryRoots: await registry.libraryRoots(),
|
||||
workRoot: options.config.workRoot,
|
||||
freeBytes: await (options.freeBytes ?? freeBytes)(options.config.workRoot),
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
};
|
||||
return sendJson(response, 200, health);
|
||||
}
|
||||
|
||||
if (url.pathname === "/v1/jobs" && request.method === "POST") {
|
||||
try {
|
||||
const body = validateRequest(await readJson(request));
|
||||
const result = await registry.submit(body);
|
||||
return sendJson(response, result.status, result.job);
|
||||
} catch (error) {
|
||||
if (error instanceof RequestError) return sendError(response, error.status, error.code, error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const match = JOB_PATH.exec(url.pathname);
|
||||
if (match && (request.method === "GET" || request.method === "DELETE")) {
|
||||
const jobId = decodePathSegment(match[1]!);
|
||||
if (jobId === null || !isJobId(jobId)) return sendError(response, 404, "worker_unavailable", "Job not found");
|
||||
const job = request.method === "GET" ? registry.get(jobId, afterSeq(url)) : registry.cancel(jobId);
|
||||
return job ? sendJson(response, 200, job) : sendError(response, 404, "worker_unavailable", "Job not found");
|
||||
}
|
||||
|
||||
request.resume();
|
||||
return sendError(response, 404, "worker_incompatible", "Not found");
|
||||
}
|
||||
|
||||
/** Bad percent-encoding cannot name a job, so it is a 404 like any other unknown id rather than a 500. */
|
||||
function decodePathSegment(segment: string): string | null {
|
||||
try {
|
||||
return decodeURIComponent(segment);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function afterSeq(url: URL): number {
|
||||
const value = Number(url.searchParams.get("afterSeq") ?? 0);
|
||||
return Number.isSafeInteger(value) && value > 0 ? value : 0;
|
||||
}
|
||||
|
||||
async function readJson(request: IncomingMessage): Promise<unknown> {
|
||||
const chunks: Buffer[] = [];
|
||||
let bytes = 0;
|
||||
for await (const chunk of request) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
bytes += buffer.length;
|
||||
if (bytes > MAX_BODY_BYTES) throw new RequestError(400, "worker_incompatible", "Request body is too large");
|
||||
chunks.push(buffer);
|
||||
}
|
||||
try {
|
||||
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
||||
} catch {
|
||||
throw new RequestError(400, "worker_incompatible", "Request body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
function authorized(request: IncomingMessage, token: string): boolean {
|
||||
const header = request.headers.authorization ?? "";
|
||||
const provided = Buffer.from(header.startsWith("Bearer ") ? header.slice(7) : "");
|
||||
const expected = Buffer.from(token);
|
||||
return provided.length === expected.length && timingSafeEqual(provided, expected);
|
||||
}
|
||||
|
||||
function sendError(response: ServerResponse, status: number, errorCode: ReadAloudErrorCode, error: string): void {
|
||||
const body: ReadAloudWorkerError = { error, errorCode };
|
||||
sendJson(response, status, body);
|
||||
}
|
||||
|
||||
function sendJson(response: ServerResponse, status: number, value: unknown): void {
|
||||
response.writeHead(status, { "Content-Type": "application/json; charset=utf-8", "Cache-Control": "no-store" });
|
||||
response.end(JSON.stringify(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the job and temp folders and empties the worker's own temp subfolder, which only ever
|
||||
* holds scratch left by a killed run. Only that subfolder is emptied, so a shared folder such as
|
||||
* `/tmp` keeps everything else in it. Paths are compared as written and with symlinks resolved, so
|
||||
* a temp folder that is a link to the data folder is refused too.
|
||||
*/
|
||||
export async function prepareDataFolders(config: WorkerConfig): Promise<void> {
|
||||
const tmpDir = config.tmpDir;
|
||||
if (dirname(tmpDir) === tmpDir) throw new Error("READ_ALOUD_TMP_DIR must not be a filesystem root");
|
||||
await mkdir(config.workRoot, { recursive: true });
|
||||
await mkdir(tmpDir, { recursive: true });
|
||||
const spellings = async (path: string) => [...new Set([resolve(path), await realpath(path).catch(() => resolve(path))])];
|
||||
const temps = await spellings(tmpDir);
|
||||
const guarded = (await Promise.all([config.workRoot, ...config.libraryRoots, homedir()].filter(Boolean).map(spellings))).flat();
|
||||
if (temps.some((temp) => guarded.some((path) => within(path, temp)))) {
|
||||
throw new Error("READ_ALOUD_TMP_DIR must not be, or contain, the job folder, a library folder or the home folder");
|
||||
}
|
||||
const libraries = (await Promise.all(config.libraryRoots.map(spellings))).flat();
|
||||
if (temps.some((temp) => libraries.some((root) => within(temp, root)))) throw new Error("READ_ALOUD_TMP_DIR must not be inside a library folder");
|
||||
const scratch = workerTmpDir(config);
|
||||
await mkdir(scratch, { recursive: true });
|
||||
for (const entry of await readdir(scratch)) await rm(join(scratch, entry), { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
estimateReadAloudEmissionsBytes,
|
||||
READ_ALOUD_ASSUMED_AUDIO_BYTES_PER_SECOND,
|
||||
READ_ALOUD_DISK_MARGIN_BYTES,
|
||||
READ_ALOUD_EMISSION_BYTES_PER_SECOND,
|
||||
READ_ALOUD_WORKER_STAGES,
|
||||
type ReadAloudDecisions,
|
||||
type ReadAloudGranularity,
|
||||
type ReadAloudWorkerStage,
|
||||
} from "@bookorbit/types";
|
||||
import type { SourceFingerprint } from "./protocol.js";
|
||||
|
||||
/** Stages whose output is cached in the job folder behind a `.done-<stage>` marker. */
|
||||
export const MARKER_STAGES = ["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning"] as const;
|
||||
export type MarkerStage = (typeof MARKER_STAGES)[number];
|
||||
|
||||
/** A stage reruns whenever a stage it reads from ran in the same run. */
|
||||
export const STAGE_DEPENDENCIES: Record<MarkerStage, readonly MarkerStage[]> = {
|
||||
preparing_audio: [],
|
||||
generating_emissions: ["preparing_audio"],
|
||||
marking_up_ebook: [],
|
||||
aligning: ["preparing_audio", "generating_emissions", "marking_up_ebook"],
|
||||
};
|
||||
|
||||
export const STAGE_WEIGHTS: Record<ReadAloudWorkerStage, number> = {
|
||||
preparing_audio: 5,
|
||||
downloading_model: 5,
|
||||
generating_emissions: 60,
|
||||
marking_up_ebook: 5,
|
||||
aligning: 20,
|
||||
validating: 5,
|
||||
};
|
||||
|
||||
/** Bump to invalidate every cached stage after a change that alters stage outputs. */
|
||||
export const MARKER_VERSION = 1;
|
||||
|
||||
/** Shared with the server and client, which estimate the same space before a job starts. */
|
||||
export const EMISSION_BYTES_PER_SECOND = READ_ALOUD_EMISSION_BYTES_PER_SECOND;
|
||||
export const ASSUMED_AUDIO_BYTES_PER_SECOND = READ_ALOUD_ASSUMED_AUDIO_BYTES_PER_SECOND;
|
||||
|
||||
export const DISK_MARGIN_BYTES = READ_ALOUD_DISK_MARGIN_BYTES;
|
||||
|
||||
export type FingerprintSource = {
|
||||
epub: SourceFingerprint;
|
||||
audio: SourceFingerprint[];
|
||||
language: string;
|
||||
granularity: ReadAloudGranularity;
|
||||
model: string;
|
||||
decisions: ReadAloudDecisions;
|
||||
};
|
||||
|
||||
export function stageInputs(stage: MarkerStage, source: FingerprintSource): Record<string, unknown> {
|
||||
const locale = normalizedLocale(source.language);
|
||||
const audio = source.audio.map(({ path, size, mtimeMs }) => ({ path, size, mtimeMs }));
|
||||
const epub = { path: source.epub.path, size: source.epub.size, mtimeMs: source.epub.mtimeMs };
|
||||
switch (stage) {
|
||||
case "preparing_audio":
|
||||
return { v: MARKER_VERSION, audio };
|
||||
case "generating_emissions":
|
||||
return { v: MARKER_VERSION, audio, model: source.model, language: new Intl.Locale(locale).language };
|
||||
case "marking_up_ebook":
|
||||
return { v: MARKER_VERSION, epub, granularity: source.granularity, locale };
|
||||
case "aligning":
|
||||
return {
|
||||
v: MARKER_VERSION,
|
||||
audio,
|
||||
epub,
|
||||
model: source.model,
|
||||
locale,
|
||||
granularity: source.granularity,
|
||||
decisions: canonicalDecisions(source.decisions),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function fingerprint(inputs: unknown): string {
|
||||
return createHash("sha256").update(canonicalJson(inputs)).digest("hex");
|
||||
}
|
||||
|
||||
export type StageMarker = {
|
||||
stage: MarkerStage;
|
||||
fingerprint: string;
|
||||
inputs: Record<string, unknown>;
|
||||
finishedAt: string;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
export type MarkerState = "valid" | "missing" | "stale";
|
||||
|
||||
export function markerPath(workDir: string, stage: MarkerStage): string {
|
||||
return join(workDir, `.done-${stage}`);
|
||||
}
|
||||
|
||||
export async function readMarker(workDir: string, stage: MarkerStage): Promise<StageMarker | null> {
|
||||
try {
|
||||
const parsed = JSON.parse(await readFile(markerPath(workDir, stage), "utf8")) as Partial<StageMarker>;
|
||||
return typeof parsed.fingerprint === "string" ? (parsed as StageMarker) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeMarker(workDir: string, marker: StageMarker): Promise<void> {
|
||||
const target = markerPath(workDir, marker.stage);
|
||||
const partial = `${target}.partial`;
|
||||
await writeFile(partial, JSON.stringify(marker), "utf8");
|
||||
await rename(partial, target);
|
||||
}
|
||||
|
||||
export async function removeMarker(workDir: string, stage: MarkerStage): Promise<void> {
|
||||
await rm(markerPath(workDir, stage), { force: true });
|
||||
}
|
||||
|
||||
export async function markerStates(workDir: string, source: FingerprintSource): Promise<Record<MarkerStage, MarkerState>> {
|
||||
const states = {} as Record<MarkerStage, MarkerState>;
|
||||
for (const stage of MARKER_STAGES) {
|
||||
const marker = await readMarker(workDir, stage);
|
||||
if (!marker) states[stage] = "missing";
|
||||
else states[stage] = marker.fingerprint === fingerprint(stageInputs(stage, source)) ? "valid" : "stale";
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
export type StageDecision = {
|
||||
stage: MarkerStage;
|
||||
run: boolean;
|
||||
reason: "cached" | "missing" | "stale" | "forced" | "dependency";
|
||||
state: MarkerState;
|
||||
};
|
||||
|
||||
/** Decides which cached stages rerun: a missing or stale marker, `resumeFrom` at or before the stage, or a dependency that reruns. */
|
||||
export function planStages(states: Record<MarkerStage, MarkerState>, resumeFrom: ReadAloudWorkerStage | null): StageDecision[] {
|
||||
const forcedFrom = resumeFrom ? READ_ALOUD_WORKER_STAGES.indexOf(resumeFrom) : Number.POSITIVE_INFINITY;
|
||||
const decisions: StageDecision[] = [];
|
||||
const running = new Set<MarkerStage>();
|
||||
for (const stage of MARKER_STAGES) {
|
||||
const state = states[stage];
|
||||
let reason: StageDecision["reason"] = "cached";
|
||||
if (state !== "valid") reason = state;
|
||||
else if (READ_ALOUD_WORKER_STAGES.indexOf(stage) >= forcedFrom) reason = "forced";
|
||||
else if (STAGE_DEPENDENCIES[stage].some((dependency) => running.has(dependency))) reason = "dependency";
|
||||
const run = reason !== "cached";
|
||||
if (run) running.add(stage);
|
||||
decisions.push({ stage, run, reason, state });
|
||||
}
|
||||
return decisions;
|
||||
}
|
||||
|
||||
export type DiskInput = {
|
||||
audioBytes: number;
|
||||
epubBytes: number;
|
||||
audioDurationSeconds: number | null;
|
||||
/** Cached stages whose outputs already sit in the job folder. */
|
||||
skipped: ReadonlySet<MarkerStage>;
|
||||
};
|
||||
|
||||
export function estimateEmissionsBytes(audioDurationSeconds: number | null, audioBytes: number): number {
|
||||
return estimateReadAloudEmissionsBytes(audioDurationSeconds, audioBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Free space a run needs on the work volume: `4 x audio + 7 x EPUB + emissions + 512 MB`.
|
||||
* Processed audio is one copy of the audio; emissions are their own share; upgraded and marked EPUBs
|
||||
* are two copies of the EPUB. At its peak alignment holds three copies of EPUB plus audio at once:
|
||||
* the unpacked working copy, the repacked scratch EPUB and the output. The unpacked copy's text
|
||||
* counts twice, since it is no longer compressed. A stage whose output is cached is not needed again.
|
||||
*/
|
||||
export function requiredDiskBytes(input: DiskInput): { required: number; shares: Record<MarkerStage, number> } {
|
||||
const shares: Record<MarkerStage, number> = {
|
||||
preparing_audio: input.audioBytes,
|
||||
generating_emissions: estimateEmissionsBytes(input.audioDurationSeconds, input.audioBytes),
|
||||
marking_up_ebook: 2 * input.epubBytes,
|
||||
aligning: 3 * input.audioBytes + 5 * input.epubBytes,
|
||||
};
|
||||
let required = DISK_MARGIN_BYTES;
|
||||
for (const stage of MARKER_STAGES) if (!input.skipped.has(stage)) required += shares[stage];
|
||||
return { required, shares };
|
||||
}
|
||||
|
||||
/** Weight of each stage that will run, scaled so the run ends at 100. */
|
||||
export function normalizedWeights(stages: readonly ReadAloudWorkerStage[]): Map<ReadAloudWorkerStage, number> {
|
||||
const total = stages.reduce((sum, stage) => sum + STAGE_WEIGHTS[stage], 0);
|
||||
return new Map(stages.map((stage) => [stage, total > 0 ? (STAGE_WEIGHTS[stage] / total) * 100 : 0]));
|
||||
}
|
||||
|
||||
function normalizedLocale(language: string): string {
|
||||
return new Intl.Locale(language).toString();
|
||||
}
|
||||
|
||||
function canonicalDecisions(decisions: ReadAloudDecisions): ReadAloudDecisions {
|
||||
const chapters: ReadAloudDecisions["chapters"] = {};
|
||||
for (const href of Object.keys(decisions.chapters).sort()) chapters[href] = decisions.chapters[href]!;
|
||||
return { chapters, audio: decisions.audio.map(({ track, start, end }) => ({ track, start, end })) };
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
const entries = Object.entries(value as Record<string, unknown>)
|
||||
.filter(([, entry]) => entry !== undefined)
|
||||
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
||||
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { execFile } from "node:child_process";
|
||||
|
||||
/**
|
||||
* STAlign shells out to `ffmpeg` and `ffprobe` from the PATH and passes `ffprobe -output_format`,
|
||||
* which ffprobe 5.x rejects. Checking at startup turns that into a clear boot error instead of every
|
||||
* job failing in its first stage.
|
||||
*/
|
||||
export async function checkFfmpeg(env: NodeJS.ProcessEnv = process.env): Promise<void> {
|
||||
await run("ffmpeg", ["-hide_banner", "-version"], env);
|
||||
const help = await run("ffprobe", ["-hide_banner", "-h"], env);
|
||||
if (!/^-output_format\b/m.test(help)) {
|
||||
throw new Error("ffprobe is too old: STAlign needs ffmpeg and ffprobe 7 or newer (ffprobe has no -output_format option)");
|
||||
}
|
||||
}
|
||||
|
||||
function run(command: string, args: string[], env: NodeJS.ProcessEnv): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(command, args, { env, timeout: 15_000, maxBuffer: 4 * 1024 * 1024 }, (error, stdout) => {
|
||||
if (!error) return resolve(stdout);
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
reject(
|
||||
new Error(
|
||||
code === "ENOENT"
|
||||
? `${command} was not found on the PATH; install ffmpeg 7 or newer`
|
||||
: `${command} could not run (${error.message.split("\n")[0]})`,
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { isCtcModelVariant } from "@storyteller-platform/ghost-story/constants";
|
||||
import {
|
||||
READ_ALOUD_MAX_AUDIO_FILES,
|
||||
READ_ALOUD_WORKER_STAGES,
|
||||
type ReadAloudChapterDecision,
|
||||
type ReadAloudDecisions,
|
||||
type ReadAloudWorkerRequest,
|
||||
type ReadAloudWorkerStage,
|
||||
} from "@bookorbit/types";
|
||||
import { RequestError } from "./paths.js";
|
||||
|
||||
const JOB_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
const CHAPTER_DECISIONS: readonly ReadAloudChapterDecision[] = ["excluded", "included", "approved"];
|
||||
const MAX_AUDIO_FILES = READ_ALOUD_MAX_AUDIO_FILES;
|
||||
const MAX_CHAPTER_DECISIONS = 10_000;
|
||||
const MAX_AUDIO_EXCLUSIONS = 1_000;
|
||||
|
||||
export function isJobId(value: string): boolean {
|
||||
return JOB_ID.test(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shape errors mean the server speaks a different protocol than this worker, so they carry
|
||||
* `worker_incompatible`. Fields added after protocol 1 shipped default instead of being required.
|
||||
*/
|
||||
export function validateRequest(value: unknown): ReadAloudWorkerRequest {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw incompatible("Request body must be a JSON object");
|
||||
const body = value as Record<string, unknown>;
|
||||
const jobId = body.jobId;
|
||||
if (typeof jobId !== "string" || !isJobId(jobId)) throw incompatible("jobId must be a UUID");
|
||||
if (typeof body.epubPath !== "string" || body.epubPath.length === 0) throw incompatible("epubPath must be a string");
|
||||
if (typeof body.workDir !== "string" || body.workDir.length === 0) throw incompatible("workDir must be a string");
|
||||
const audioPaths = body.audioPaths;
|
||||
if (
|
||||
!Array.isArray(audioPaths) ||
|
||||
audioPaths.length < 1 ||
|
||||
audioPaths.length > MAX_AUDIO_FILES ||
|
||||
audioPaths.some((path) => typeof path !== "string" || path.length === 0)
|
||||
) {
|
||||
throw incompatible(`audioPaths must list 1 to ${MAX_AUDIO_FILES} paths`);
|
||||
}
|
||||
if (typeof body.language !== "string" || body.language.length === 0 || body.language.length > 50) throw incompatible("language must be a string");
|
||||
if (body.granularity !== "sentence" && body.granularity !== "word") throw incompatible("granularity must be sentence or word");
|
||||
if (body.model !== "quartznet15x5") throw incompatible("model must be quartznet15x5");
|
||||
if (!isNonNegativeInteger(body.sourceBytes)) throw incompatible("sourceBytes must be a non-negative integer");
|
||||
if (!isNonNegativeInteger(body.maxOutputBytes) || body.maxOutputBytes < 1) throw incompatible("maxOutputBytes must be a positive integer");
|
||||
const duration = body.audioDurationSeconds ?? null;
|
||||
if (duration !== null && (typeof duration !== "number" || !Number.isFinite(duration) || duration < 0)) {
|
||||
throw incompatible("audioDurationSeconds must be null or a non-negative number");
|
||||
}
|
||||
const resumeFrom = body.resumeFrom ?? null;
|
||||
if (resumeFrom !== null && !(READ_ALOUD_WORKER_STAGES as readonly unknown[]).includes(resumeFrom)) {
|
||||
throw incompatible(`resumeFrom must be null or one of: ${READ_ALOUD_WORKER_STAGES.join(", ")}`);
|
||||
}
|
||||
assertSupportedLanguage(body.language);
|
||||
return {
|
||||
jobId,
|
||||
epubPath: body.epubPath,
|
||||
audioPaths: audioPaths as string[],
|
||||
workDir: body.workDir,
|
||||
language: body.language,
|
||||
granularity: body.granularity,
|
||||
model: body.model,
|
||||
sourceBytes: body.sourceBytes,
|
||||
audioDurationSeconds: duration,
|
||||
maxOutputBytes: body.maxOutputBytes,
|
||||
decisions: validateDecisions(body.decisions),
|
||||
resumeFrom: resumeFrom as ReadAloudWorkerStage | null,
|
||||
};
|
||||
}
|
||||
|
||||
export function quartznetLanguage(language: string): string | null {
|
||||
let subtag: string;
|
||||
try {
|
||||
subtag = new Intl.Locale(language).language;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return isCtcModelVariant(`quartznet15x5_${subtag}`) ? subtag : null;
|
||||
}
|
||||
|
||||
function assertSupportedLanguage(language: string): void {
|
||||
if (!quartznetLanguage(language)) {
|
||||
throw new RequestError(400, "unsupported_language", "The QuartzNet model has no variant for this language");
|
||||
}
|
||||
}
|
||||
|
||||
function validateDecisions(value: unknown): ReadAloudDecisions {
|
||||
if (value === undefined || value === null) return { chapters: {}, audio: [] };
|
||||
if (typeof value !== "object" || Array.isArray(value)) throw incompatible("decisions must be an object");
|
||||
const { chapters = {}, audio = [] } = value as { chapters?: unknown; audio?: unknown };
|
||||
if (!chapters || typeof chapters !== "object" || Array.isArray(chapters)) throw incompatible("decisions.chapters must be an object");
|
||||
const entries = Object.entries(chapters);
|
||||
if (entries.length > MAX_CHAPTER_DECISIONS) throw incompatible("decisions.chapters has too many entries");
|
||||
const validChapters: Record<string, ReadAloudChapterDecision> = {};
|
||||
for (const [href, decision] of entries) {
|
||||
if (!CHAPTER_DECISIONS.includes(decision as ReadAloudChapterDecision)) {
|
||||
throw incompatible("decisions.chapters values must be excluded, included or approved");
|
||||
}
|
||||
validChapters[href] = decision as ReadAloudChapterDecision;
|
||||
}
|
||||
if (!Array.isArray(audio) || audio.length > MAX_AUDIO_EXCLUSIONS) throw incompatible("decisions.audio must be an array");
|
||||
const validAudio = audio.map((entry) => {
|
||||
if (!entry || typeof entry !== "object") throw incompatible("decisions.audio entries must be objects");
|
||||
const { track, start = null, end = null } = entry as { track?: unknown; start?: unknown; end?: unknown };
|
||||
if (typeof track !== "string" || !isOptionalSeconds(start) || !isOptionalSeconds(end)) {
|
||||
throw incompatible("decisions.audio entries need a track name and optional start and end seconds");
|
||||
}
|
||||
return { track, start, end };
|
||||
});
|
||||
return { chapters: validChapters, audio: validAudio };
|
||||
}
|
||||
|
||||
function isOptionalSeconds(value: unknown): value is number | null {
|
||||
return value === null || (typeof value === "number" && Number.isFinite(value) && value >= 0);
|
||||
}
|
||||
|
||||
function isNonNegativeInteger(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
||||
}
|
||||
|
||||
function incompatible(message: string): RequestError {
|
||||
return new RequestError(400, "worker_incompatible", message);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { PROBE_MODEL_PATH, needsDeviceProbe, probeDevice, resolveCtcDevice } from "../dist/accelerator.js";
|
||||
|
||||
test("auto selects MLX on Apple Silicon", () => {
|
||||
assert.equal(resolveCtcDevice("auto", runtime("darwin", "arm64")), "mlx");
|
||||
});
|
||||
|
||||
test("auto selects CPU everywhere else, including Linux x64 with a GPU", () => {
|
||||
assert.equal(resolveCtcDevice("auto", runtime("linux", "x64")), "cpu");
|
||||
assert.equal(resolveCtcDevice("auto", runtime("linux", "arm64")), "cpu");
|
||||
assert.equal(resolveCtcDevice("auto", runtime("darwin", "x64")), "cpu");
|
||||
assert.equal(resolveCtcDevice("auto", runtime("win32", "x64")), "cpu");
|
||||
});
|
||||
|
||||
test("explicit devices are honoured where they can run", () => {
|
||||
assert.equal(resolveCtcDevice("cuda", runtime("linux", "x64")), "cuda");
|
||||
assert.equal(resolveCtcDevice("webgpu", runtime("win32", "x64")), "webgpu");
|
||||
assert.equal(resolveCtcDevice("cpu", runtime("darwin", "arm64")), "cpu");
|
||||
});
|
||||
|
||||
test("explicit devices fail fast on incompatible hosts", () => {
|
||||
assert.throws(() => resolveCtcDevice("mlx", runtime("linux", "arm64")), /native macOS on Apple Silicon/);
|
||||
assert.throws(() => resolveCtcDevice("cuda", runtime("linux", "arm64")), /Linux x64/);
|
||||
assert.throws(() => resolveCtcDevice("cuda", runtime("darwin", "arm64")), /Linux x64/);
|
||||
assert.throws(() => resolveCtcDevice("webgpu", runtime("linux", "arm64")), /not available/);
|
||||
});
|
||||
|
||||
test("CUDA and WebGPU are probed at startup", () => {
|
||||
assert.equal(needsDeviceProbe("cuda"), true);
|
||||
assert.equal(needsDeviceProbe("webgpu"), true);
|
||||
assert.equal(needsDeviceProbe("cpu"), false);
|
||||
assert.equal(needsDeviceProbe("mlx"), false);
|
||||
});
|
||||
|
||||
test("the bundled probe model is a tiny identity graph that runs on CPU", async () => {
|
||||
assert.ok(statSync(PROBE_MODEL_PATH).size < 1024);
|
||||
await probeDevice("cpu");
|
||||
});
|
||||
|
||||
test("a probe for a provider that cannot load fails with a clear message", async () => {
|
||||
if (process.platform === "linux" && process.arch === "x64" && process.env.READ_ALOUD_TEST_HAS_CUDA === "1") return;
|
||||
await assert.rejects(probeDevice("cuda"), /READ_ALOUD_CTC_DEVICE=cuda could not start .*CUDA 12 and cuDNN 9.*use READ_ALOUD_CTC_DEVICE=cpu/);
|
||||
});
|
||||
|
||||
test("a failed WebGPU probe stops startup and points at the CPU", async () => {
|
||||
await assert.rejects(
|
||||
probeDevice("webgpu", { modelPath: join(tmpdir(), `read-aloud-missing-${process.pid}.onnx`) }),
|
||||
/READ_ALOUD_CTC_DEVICE=webgpu could not start .*the Docker image has none; use READ_ALOUD_CTC_DEVICE=cpu/,
|
||||
);
|
||||
// On a host without a GPU adapter, such as the Docker image or a GPU-less CI runner, the real probe fails the same way.
|
||||
const real = await probeDevice("webgpu").then(
|
||||
() => null,
|
||||
(error) => error,
|
||||
);
|
||||
if (real) assert.match(real.message, /READ_ALOUD_CTC_DEVICE=webgpu could not start/);
|
||||
});
|
||||
|
||||
function runtime(platform, architecture) {
|
||||
return { platform, architecture };
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { WORKER_TMP_SUBDIR, loadConfig, parseLibraryRoots, workerVersion } from "../dist/config.js";
|
||||
import { prepareDataFolders } from "../dist/server.js";
|
||||
import { checkFfmpeg } from "../dist/toolchain.js";
|
||||
|
||||
const TOKEN = "0123456789abcdef0123";
|
||||
|
||||
test("library roots: the list wins, trimmed and de-duplicated", () => {
|
||||
assert.deepEqual(parseLibraryRoots({ READ_ALOUD_LIBRARY_ROOTS: " /books , /mnt/audio/,,/books", READ_ALOUD_LIBRARY_ROOT: "/old" }), [
|
||||
"/books",
|
||||
"/mnt/audio",
|
||||
]);
|
||||
});
|
||||
|
||||
test("library roots: the single-root variable is the fallback, then /books", () => {
|
||||
assert.deepEqual(parseLibraryRoots({ READ_ALOUD_LIBRARY_ROOT: "/library" }), ["/library"]);
|
||||
assert.deepEqual(parseLibraryRoots({}), ["/books"]);
|
||||
assert.deepEqual(parseLibraryRoots({ READ_ALOUD_LIBRARY_ROOTS: " " }), ["/books"]);
|
||||
});
|
||||
|
||||
test("library roots: relative paths are refused", () => {
|
||||
assert.throws(() => parseLibraryRoots({ READ_ALOUD_LIBRARY_ROOTS: "/books,media" }), /absolute/);
|
||||
assert.throws(() => parseLibraryRoots({ READ_ALOUD_LIBRARY_ROOTS: "," }), /at least one/);
|
||||
});
|
||||
|
||||
test("version: the image's release version, else the package version", () => {
|
||||
const packageVersion = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
||||
assert.equal(workerVersion({ APP_VERSION: " v3.4.0 " }), "v3.4.0");
|
||||
assert.equal(workerVersion({ APP_VERSION: "" }), packageVersion);
|
||||
assert.equal(workerVersion({}), packageVersion);
|
||||
});
|
||||
|
||||
test("config: binds to loopback and keeps temp files beside the job folder by default", () => {
|
||||
const config = loadConfig({ READ_ALOUD_WORKER_TOKEN: TOKEN, READ_ALOUD_WORK_ROOT: "/data/read-aloud/jobs" });
|
||||
assert.equal(config.host, "127.0.0.1");
|
||||
assert.equal(config.port, 8787);
|
||||
assert.equal(config.tmpDir, "/data/read-aloud/tmp");
|
||||
assert.equal(config.configuredDevice, "auto");
|
||||
assert.equal(config.dtype, null);
|
||||
assert.equal(config.batchSize, null);
|
||||
assert.equal(loadConfig({ READ_ALOUD_WORKER_TOKEN: TOKEN, READ_ALOUD_WORKER_HOST: "0.0.0.0" }).host, "0.0.0.0");
|
||||
});
|
||||
|
||||
test("config: rejects a short token and bad tuning values", () => {
|
||||
assert.throws(() => loadConfig({ READ_ALOUD_WORKER_TOKEN: "short" }), /16 characters/);
|
||||
assert.throws(() => loadConfig({ READ_ALOUD_WORKER_TOKEN: TOKEN, READ_ALOUD_CTC_DEVICE: "tpu" }), /READ_ALOUD_CTC_DEVICE/);
|
||||
assert.throws(() => loadConfig({ READ_ALOUD_WORKER_TOKEN: TOKEN, READ_ALOUD_CTC_BATCH_SIZE: "100" }), /BATCH_SIZE/);
|
||||
assert.throws(() => loadConfig({ READ_ALOUD_WORKER_TOKEN: TOKEN, READ_ALOUD_TMP_DIR: "tmp" }), /absolute/);
|
||||
});
|
||||
|
||||
test("data folders: only the worker's own temp subfolder is emptied, and a temp folder holding the jobs is refused", async () => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "read-aloud-config-test-")));
|
||||
try {
|
||||
const workRoot = join(root, "read-aloud", "jobs");
|
||||
const tmpDir = join(root, "read-aloud", "tmp");
|
||||
const library = join(root, "books");
|
||||
const scratch = join(tmpDir, WORKER_TMP_SUBDIR);
|
||||
mkdirSync(join(scratch, "stale", "deep"), { recursive: true });
|
||||
writeFileSync(join(scratch, "leftover.bin"), "x");
|
||||
writeFileSync(join(tmpDir, "someone-elses.sock"), "x");
|
||||
const config = { libraryRoots: [library], workRoot, tmpDir };
|
||||
await prepareDataFolders(config);
|
||||
assert.ok(existsSync(workRoot));
|
||||
assert.deepEqual(readdirSync(scratch), []);
|
||||
assert.ok(existsSync(join(tmpDir, "someone-elses.sock")), "a shared temp folder such as /tmp keeps everything else");
|
||||
|
||||
symlinkSync(join(root, "read-aloud"), join(root, "linked-data"));
|
||||
await assert.rejects(prepareDataFolders({ ...config, tmpDir: join(root, "linked-data") }), /must not be, or contain/);
|
||||
|
||||
await assert.rejects(prepareDataFolders({ ...config, tmpDir: join(root, "read-aloud") }), /must not be, or contain/);
|
||||
await assert.rejects(prepareDataFolders({ ...config, tmpDir: join(library, "tmp") }), /inside a library/);
|
||||
await assert.rejects(prepareDataFolders({ ...config, tmpDir: "/" }), /filesystem root/);
|
||||
assert.ok(existsSync(workRoot), "a refused temp folder deletes nothing");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("toolchain: a missing ffmpeg is a startup error", async () => {
|
||||
await assert.rejects(checkFfmpeg({ PATH: "/nonexistent" }), /ffmpeg was not found on the PATH/);
|
||||
});
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
// A stand-in for the STAlign pipeline, loaded by the runner through READ_ALOUD_RUNNER_PIPELINE.
|
||||
// It writes small placeholder outputs and records every stage it runs, so the tests exercise the
|
||||
// runner's real caching, progress, cancel and failure handling without the model.
|
||||
import { spawn } from "node:child_process";
|
||||
import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
|
||||
const mode = process.env.FAKE_PIPELINE_MODE ?? "ok";
|
||||
const logPath = process.env.FAKE_PIPELINE_LOG;
|
||||
const modelDir = process.env.FAKE_MODEL_DIR;
|
||||
const stepMs = Number(process.env.FAKE_PIPELINE_STEP_MS ?? 20);
|
||||
|
||||
function record(stage, context) {
|
||||
if (!logPath) return;
|
||||
appendFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
jobId: context.job.jobId,
|
||||
stage,
|
||||
tmpdir: process.env.TMPDIR ?? null,
|
||||
scratch: process.env.STORYTELLER_SCRATCH_DIR ?? null,
|
||||
decisions: context.job.decisions,
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
async function steps(context, total, unit) {
|
||||
for (let index = 0; index <= total; index += 1) {
|
||||
context.signal.throwIfAborted();
|
||||
context.progress(index / total, { current: index, total, unit });
|
||||
await sleep(stepMs);
|
||||
}
|
||||
}
|
||||
|
||||
export const pipeline = {
|
||||
async modelInstalled() {
|
||||
return !modelDir || existsSync(join(modelDir, "installed"));
|
||||
},
|
||||
|
||||
async prepareAudio(context) {
|
||||
record("preparing_audio", context);
|
||||
mkdirSync(context.paths.processedAudioDir, { recursive: true });
|
||||
for (const [index] of context.job.audio.entries()) {
|
||||
writeFileSync(join(context.paths.processedAudioDir, `${String(index + 1).padStart(5, "0")}-00001.mp3`), "audio");
|
||||
}
|
||||
await steps(context, context.job.audio.length, "file");
|
||||
return { data: { files: context.job.audio.length } };
|
||||
},
|
||||
|
||||
async downloadModel(context) {
|
||||
record("downloading_model", context);
|
||||
context.event("info", "model_download_started", "Downloading the fake model", { variant: "fake" });
|
||||
await sleep(stepMs);
|
||||
mkdirSync(modelDir, { recursive: true });
|
||||
writeFileSync(join(modelDir, "installed"), "yes");
|
||||
context.event("info", "model_download_finished", "Downloaded the fake model", { variant: "fake", bytes: 3 });
|
||||
},
|
||||
|
||||
async generateEmissions(context) {
|
||||
record("generating_emissions", context);
|
||||
if (mode === "crash") process.exit(7);
|
||||
if (mode === "hang") {
|
||||
await new Promise((_resolve, reject) => {
|
||||
context.signal.addEventListener("abort", () => reject(context.signal.reason), { once: true });
|
||||
});
|
||||
}
|
||||
if (mode === "orphan-child") {
|
||||
// Like STAlign aborting an ffmpeg pipeline: the shell stops, the process it started does not.
|
||||
const child = spawn("sleep", ["60"], { stdio: "ignore" });
|
||||
child.unref();
|
||||
writeFileSync(join(context.paths.workDir, "orphan.pid"), String(child.pid));
|
||||
await new Promise((_resolve, reject) => {
|
||||
context.signal.addEventListener("abort", () => reject(context.signal.reason), { once: true });
|
||||
});
|
||||
}
|
||||
if (mode === "ignore-cancel") {
|
||||
setInterval(() => {}, 1_000);
|
||||
await new Promise(() => {});
|
||||
}
|
||||
mkdirSync(context.paths.emissionsDir, { recursive: true });
|
||||
writeFileSync(join(context.paths.emissionsDir, "00001-00001.emissions"), "emissions");
|
||||
await steps(context, context.job.audio.length, "track");
|
||||
if (mode === "slow") await sleep(1_500);
|
||||
return { data: { tracks: context.job.audio.length, cached: 0 } };
|
||||
},
|
||||
|
||||
async markupEbook(context) {
|
||||
record("marking_up_ebook", context);
|
||||
if (mode === "fail-markup") throw new Error(`markup broke while reading ${context.paths.upgradedEpubPath}`);
|
||||
writeFileSync(context.paths.markedEpubPath, "marked");
|
||||
await steps(context, 3, "chapter");
|
||||
},
|
||||
|
||||
async align(context) {
|
||||
record("aligning", context);
|
||||
await steps(context, 3, "chapter");
|
||||
if (mode === "block-align") {
|
||||
// STAlign's Viterbi pass is a synchronous loop; this stands in for it.
|
||||
const until = Date.now() + 2_000;
|
||||
while (Date.now() < until);
|
||||
}
|
||||
writeFileSync(context.paths.outputPath, mode === "empty-output" ? "" : "output-epub-bytes");
|
||||
writeFileSync(
|
||||
context.paths.reportPath,
|
||||
JSON.stringify({
|
||||
version: 4,
|
||||
totals: {
|
||||
sentences: 10,
|
||||
matchedSentences: 9,
|
||||
interpolatedSentences: 1,
|
||||
unmatchedSentences: 0,
|
||||
droppedSentences: 0,
|
||||
seconds: 60,
|
||||
matchedSeconds: 58,
|
||||
interpolatedSeconds: 1,
|
||||
looseSeconds: 1,
|
||||
spine: {},
|
||||
tracks: context.job.audio.length,
|
||||
unalignedTracks: 0,
|
||||
},
|
||||
}),
|
||||
);
|
||||
return { data: { chapters: 3 } };
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,128 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { startWorker } from "../dist/server.js";
|
||||
|
||||
export const TOKEN = "test-token-0123456789abcdef";
|
||||
export const FAKE_PIPELINE = fileURLToPath(new URL("./fixtures/fake-pipeline.mjs", import.meta.url));
|
||||
|
||||
/** A throwaway library and data folder under the system temp folder. */
|
||||
export function makeSandbox() {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "read-aloud-worker-test-")));
|
||||
const library = join(root, "library");
|
||||
const book = join(library, "Author", "Book");
|
||||
const data = join(root, "data", "read-aloud");
|
||||
mkdirSync(book, { recursive: true });
|
||||
mkdirSync(join(data, "jobs"), { recursive: true });
|
||||
writeFileSync(join(book, "book.epub"), "epub-bytes");
|
||||
writeFileSync(join(book, "01.mp3"), "a".repeat(1000));
|
||||
writeFileSync(join(book, "02.mp3"), "b".repeat(2000));
|
||||
return {
|
||||
root,
|
||||
library,
|
||||
book,
|
||||
workRoot: join(data, "jobs"),
|
||||
tmpDir: join(data, "tmp"),
|
||||
logPath: join(root, "fake-pipeline.jsonl"),
|
||||
modelDir: join(root, "models"),
|
||||
cleanup: () => rmSync(root, { recursive: true, force: true }),
|
||||
};
|
||||
}
|
||||
|
||||
export async function startTestWorker(sandbox, options = {}) {
|
||||
const config = {
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
token: TOKEN,
|
||||
libraryRoots: options.libraryRoots ?? [sandbox.library],
|
||||
workRoot: sandbox.workRoot,
|
||||
tmpDir: sandbox.tmpDir,
|
||||
configuredDevice: "cpu",
|
||||
dtype: null,
|
||||
batchSize: null,
|
||||
};
|
||||
mkdirSync(sandbox.tmpDir, { recursive: true });
|
||||
const worker = await startWorker({
|
||||
config,
|
||||
configuredDevice: "cpu",
|
||||
device: "cpu",
|
||||
deviceCheck: "not_checked",
|
||||
version: "0.0.0-test",
|
||||
cancelGraceMs: options.cancelGraceMs ?? 5_000,
|
||||
freeBytes: options.freeBytes,
|
||||
rootChecks: options.rootChecks,
|
||||
runnerEnv: {
|
||||
READ_ALOUD_RUNNER_PIPELINE: FAKE_PIPELINE,
|
||||
FAKE_PIPELINE_MODE: options.mode ?? "ok",
|
||||
FAKE_PIPELINE_LOG: sandbox.logPath,
|
||||
...(options.modelDir ? { FAKE_MODEL_DIR: options.modelDir } : {}),
|
||||
},
|
||||
});
|
||||
const base = `http://127.0.0.1:${worker.port}`;
|
||||
return {
|
||||
worker,
|
||||
base,
|
||||
async call(method, path, body, token = TOKEN) {
|
||||
const response = await fetch(`${base}${path}`, {
|
||||
method,
|
||||
headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}), ...(body === undefined ? {} : { "Content-Type": "application/json" }) },
|
||||
body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body),
|
||||
});
|
||||
return { status: response.status, body: await response.json() };
|
||||
},
|
||||
close: () => worker.close(),
|
||||
};
|
||||
}
|
||||
|
||||
export function jobRequest(sandbox, overrides = {}) {
|
||||
const jobId = overrides.jobId ?? randomUUID();
|
||||
return {
|
||||
jobId,
|
||||
epubPath: join(sandbox.book, "book.epub"),
|
||||
audioPaths: [join(sandbox.book, "01.mp3"), join(sandbox.book, "02.mp3")],
|
||||
workDir: join(sandbox.workRoot, jobId),
|
||||
language: "en",
|
||||
granularity: "sentence",
|
||||
model: "quartznet15x5",
|
||||
sourceBytes: 3010,
|
||||
audioDurationSeconds: 120,
|
||||
maxOutputBytes: 1_000_000,
|
||||
decisions: { chapters: {}, audio: [] },
|
||||
resumeFrom: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export async function waitForJob(client, jobId, predicate, timeoutMs = 15_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let last;
|
||||
while (Date.now() < deadline) {
|
||||
last = await client.call("GET", `/v1/jobs/${jobId}`);
|
||||
if (last.status === 200 && predicate(last.body)) return last.body;
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`Timed out waiting for job ${jobId}; last state ${JSON.stringify(last?.body)}`);
|
||||
}
|
||||
|
||||
export function stagesRun(sandbox, jobId) {
|
||||
let text = "";
|
||||
try {
|
||||
text = readFileSync(sandbox.logPath, "utf8");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
return text
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line))
|
||||
.filter((entry) => entry.jobId === jobId);
|
||||
}
|
||||
|
||||
export function resetStageLog(sandbox) {
|
||||
rmSync(sandbox.logPath, { force: true });
|
||||
}
|
||||
|
||||
export const TERMINAL = new Set(["completed", "failed", "cancelled"]);
|
||||
@@ -0,0 +1,114 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { fork } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { describe, test } from "node:test";
|
||||
import { FAKE_PIPELINE, makeSandbox, stagesRun } from "./helpers.mjs";
|
||||
|
||||
const RUNNER = new URL("../dist/runner.js", import.meta.url);
|
||||
|
||||
describe("runner without its control process", () => {
|
||||
test("a runner whose control process goes away kills what it started", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const job = runnerJob(sandbox);
|
||||
const runner = startRunner(sandbox, "orphan-child", { detached: true });
|
||||
let orphan = null;
|
||||
try {
|
||||
runner.child.send({ type: "start", job });
|
||||
const pidFile = join(job.workDir, "orphan.pid");
|
||||
await until(() => existsSync(pidFile), "the stand-in ffmpeg to start");
|
||||
orphan = Number(readFileSync(pidFile, "utf8"));
|
||||
assert.ok(isAlive(orphan), "the stand-in ffmpeg is running");
|
||||
|
||||
runner.child.disconnect();
|
||||
const exit = await runner.exited;
|
||||
assert.equal(exit.signal, "SIGKILL", "the runner went down with its own process group");
|
||||
await until(() => !isAlive(orphan), "the orphaned child to be killed");
|
||||
} finally {
|
||||
runner.stop();
|
||||
if (orphan !== null && isAlive(orphan)) process.kill(orphan, "SIGKILL");
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a runner that leads no process group signals nothing outside itself", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const job = runnerJob(sandbox);
|
||||
// Not detached: the runner shares this test's process group, as it would if it were started by hand.
|
||||
const runner = startRunner(sandbox, "hang", { detached: false });
|
||||
try {
|
||||
runner.child.send({ type: "start", job });
|
||||
await until(() => stagesRun(sandbox, job.jobId).some((entry) => entry.stage === "generating_emissions"), "the runner to reach inference");
|
||||
|
||||
runner.child.disconnect();
|
||||
const exit = await runner.exited;
|
||||
assert.equal(exit.signal, null, "the group kill found no group of its own and signalled nothing");
|
||||
assert.ok(isAlive(process.pid));
|
||||
} finally {
|
||||
runner.stop();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function startRunner(sandbox, mode, { detached }) {
|
||||
const child = fork(RUNNER, [], {
|
||||
execArgv: [],
|
||||
detached,
|
||||
stdio: ["ignore", "ignore", "inherit", "ipc"],
|
||||
env: { ...process.env, READ_ALOUD_RUNNER_PIPELINE: FAKE_PIPELINE, FAKE_PIPELINE_MODE: mode, FAKE_PIPELINE_LOG: sandbox.logPath },
|
||||
});
|
||||
const exited = new Promise((resolve) => child.once("exit", (code, signal) => resolve({ code, signal })));
|
||||
return {
|
||||
child,
|
||||
exited,
|
||||
stop() {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runnerJob(sandbox) {
|
||||
const jobId = randomUUID();
|
||||
return {
|
||||
jobId,
|
||||
attempt: 1,
|
||||
workDir: join(sandbox.workRoot, jobId),
|
||||
epub: fingerprint(join(sandbox.book, "book.epub")),
|
||||
audio: [fingerprint(join(sandbox.book, "01.mp3"))],
|
||||
language: "en",
|
||||
granularity: "sentence",
|
||||
model: "quartznet15x5",
|
||||
maxOutputBytes: 1_000_000,
|
||||
decisions: { chapters: {}, audio: [] },
|
||||
resumeFrom: null,
|
||||
device: "cpu",
|
||||
dtype: null,
|
||||
batchSize: null,
|
||||
redactions: [],
|
||||
};
|
||||
}
|
||||
|
||||
function fingerprint(path) {
|
||||
const info = statSync(path);
|
||||
return { path, size: info.size, mtimeMs: Math.trunc(info.mtimeMs) };
|
||||
}
|
||||
|
||||
async function until(predicate, what, timeoutMs = 5_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!predicate()) {
|
||||
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${what}`);
|
||||
await sleep(25);
|
||||
}
|
||||
}
|
||||
|
||||
function isAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,702 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { after, before, describe, test } from "node:test";
|
||||
import { ReadableRoots } from "../dist/paths.js";
|
||||
import { TERMINAL, TOKEN, jobRequest, makeSandbox, resetStageLog, stagesRun, startTestWorker, waitForJob } from "./helpers.mjs";
|
||||
|
||||
describe("HTTP surface", () => {
|
||||
let sandbox;
|
||||
let client;
|
||||
|
||||
before(async () => {
|
||||
sandbox = makeSandbox();
|
||||
// A folder beside the library, reachable from inside it only through `..` or a symlink.
|
||||
const outside = join(sandbox.root, "outside");
|
||||
mkdirSync(outside);
|
||||
writeFileSync(join(outside, "secret.epub"), "secret-epub");
|
||||
writeFileSync(join(outside, "secret.mp3"), "secret-audio");
|
||||
symlinkSync(join(outside, "secret.epub"), join(sandbox.book, "linked.epub"));
|
||||
symlinkSync(outside, join(sandbox.library, "escape"));
|
||||
client = await startTestWorker(sandbox);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
});
|
||||
|
||||
test("rejects missing, wrong and differently sized tokens", async () => {
|
||||
assert.equal((await client.call("GET", "/health", undefined, null)).status, 401);
|
||||
assert.equal((await client.call("GET", "/health", undefined, "x".repeat(TOKEN.length))).status, 401);
|
||||
const wrongLength = await client.call("GET", "/health", undefined, "short");
|
||||
assert.equal(wrongLength.status, 401);
|
||||
assert.equal(wrongLength.body.errorCode, "worker_unavailable");
|
||||
});
|
||||
|
||||
test("health reports the protocol, roots, device and free space", async () => {
|
||||
const { status, body } = await client.call("GET", "/health");
|
||||
assert.equal(status, 200);
|
||||
assert.equal(body.status, "ok");
|
||||
assert.equal(body.protocol, 1);
|
||||
assert.equal(body.activeJobId, null);
|
||||
assert.equal(body.activeJobs, 0);
|
||||
assert.equal(body.capacity, 1);
|
||||
assert.equal(body.device, "cpu");
|
||||
assert.equal(body.configuredDevice, "cpu");
|
||||
assert.equal(body.deviceCheck, "not_checked");
|
||||
assert.deepEqual(body.libraryRoots, [sandbox.library]);
|
||||
assert.equal(body.workRoot, sandbox.workRoot);
|
||||
assert.equal(typeof body.freeBytes, "number");
|
||||
assert.equal(body.platform, process.platform);
|
||||
assert.equal(body.architecture, process.arch);
|
||||
});
|
||||
|
||||
test("unknown jobs and routes are 404", async () => {
|
||||
const job = await client.call("GET", "/v1/jobs/00000000-0000-4000-8000-000000000000");
|
||||
assert.equal(job.status, 404);
|
||||
assert.equal(job.body.errorCode, "worker_unavailable");
|
||||
assert.equal((await client.call("DELETE", "/v1/jobs/00000000-0000-4000-8000-000000000000")).status, 404);
|
||||
const route = await client.call("GET", "/v2/nothing");
|
||||
assert.equal(route.status, 404);
|
||||
assert.equal(route.body.errorCode, "worker_incompatible");
|
||||
});
|
||||
|
||||
test("a job id with broken percent-encoding is a 404, not a 500", async () => {
|
||||
for (const method of ["GET", "DELETE"]) {
|
||||
for (const id of ["%E0%A4%A", "%ZZ", "00000000-0000-4000-8000-%"]) {
|
||||
const response = await client.call(method, `/v1/jobs/${id}`);
|
||||
assert.equal(response.status, 404, `${method} ${id}`);
|
||||
assert.equal(response.body.errorCode, "worker_unavailable");
|
||||
}
|
||||
}
|
||||
const traversal = await client.call("GET", "/v1/jobs/..%2F..%2Fhealth");
|
||||
assert.equal(traversal.status, 404);
|
||||
assert.equal(traversal.body.errorCode, "worker_unavailable");
|
||||
});
|
||||
|
||||
const rejected = [
|
||||
["malformed JSON", () => "{", "worker_incompatible"],
|
||||
["a missing field", () => ({ ...jobRequest(sandbox), granularity: "paragraph" }), "worker_incompatible"],
|
||||
["a work folder elsewhere", () => ({ ...jobRequest(sandbox), workDir: join(sandbox.root, "elsewhere") }), "worker_incompatible"],
|
||||
["a source outside every root", () => ({ ...jobRequest(sandbox), epubPath: "/somewhere/else/book.epub" }), "library_not_mounted"],
|
||||
["a missing source", () => ({ ...jobRequest(sandbox), audioPaths: [join(sandbox.book, "03.mp3")] }), "source_missing"],
|
||||
["a folder as source", () => ({ ...jobRequest(sandbox), epubPath: sandbox.book }), "invalid_source"],
|
||||
["a relative source", () => ({ ...jobRequest(sandbox), epubPath: "book.epub" }), "invalid_source"],
|
||||
["a language QuartzNet lacks", () => ({ ...jobRequest(sandbox), language: "nl" }), "unsupported_language"],
|
||||
["an invalid language tag", () => ({ ...jobRequest(sandbox), language: "not a tag" }), "unsupported_language"],
|
||||
["a bad decision", () => ({ ...jobRequest(sandbox), decisions: { chapters: { "a.xhtml": "maybe" }, audio: [] } }), "worker_incompatible"],
|
||||
// Paths are joined by hand below: path.join would resolve the `..` before the worker sees it.
|
||||
[
|
||||
"an EPUB reached through `..`",
|
||||
() => ({ ...jobRequest(sandbox), epubPath: `${sandbox.library}/../outside/secret.epub` }),
|
||||
"library_not_mounted",
|
||||
],
|
||||
[
|
||||
"an audio file reached through `..`",
|
||||
() => ({ ...jobRequest(sandbox), audioPaths: [`${sandbox.book}/../../../outside/secret.mp3`] }),
|
||||
"library_not_mounted",
|
||||
],
|
||||
[
|
||||
"a work folder reached through `..`",
|
||||
() => {
|
||||
const request = jobRequest(sandbox);
|
||||
return { ...request, workDir: `${request.workDir}/../../escaped` };
|
||||
},
|
||||
"worker_incompatible",
|
||||
],
|
||||
[
|
||||
"a symlinked file that points outside every root",
|
||||
() => ({ ...jobRequest(sandbox), epubPath: join(sandbox.book, "linked.epub") }),
|
||||
"library_not_mounted",
|
||||
],
|
||||
[
|
||||
"a file under a symlinked folder that points outside every root",
|
||||
() => ({ ...jobRequest(sandbox), audioPaths: [join(sandbox.library, "escape", "secret.mp3")] }),
|
||||
"library_not_mounted",
|
||||
],
|
||||
];
|
||||
for (const [name, body, code] of rejected) {
|
||||
test(`400 for ${name}`, async () => {
|
||||
const response = await client.call("POST", "/v1/jobs", body());
|
||||
assert.equal(response.status, 400);
|
||||
assert.equal(response.body.errorCode, code);
|
||||
assert.equal(typeof response.body.error, "string");
|
||||
assert.equal((await client.call("GET", "/health")).body.activeJobId, null);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("paths inside a root", () => {
|
||||
test("are accepted after `..` and symlinks resolve back into the library", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
symlinkSync(join(sandbox.book, "book.epub"), join(sandbox.library, "shortcut.epub"));
|
||||
const client = await startTestWorker(sandbox);
|
||||
try {
|
||||
const request = jobRequest(sandbox, {
|
||||
epubPath: join(sandbox.library, "shortcut.epub"),
|
||||
audioPaths: [`${sandbox.book}/../Book/01.mp3`, join(sandbox.book, "02.mp3")],
|
||||
});
|
||||
const created = await client.call("POST", "/v1/jobs", request);
|
||||
assert.equal(created.status, 202, JSON.stringify(created.body));
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed", JSON.stringify(job));
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("library roots", () => {
|
||||
test("a root that is not readable at startup is picked up later without a restart", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const later = join(sandbox.root, "automount");
|
||||
const client = await startTestWorker(sandbox, { libraryRoots: [sandbox.library, later], rootChecks: { maxAgeMs: 0 } });
|
||||
try {
|
||||
assert.deepEqual((await client.call("GET", "/health")).body.libraryRoots, [sandbox.library]);
|
||||
mkdirSync(later);
|
||||
assert.deepEqual((await client.call("GET", "/health")).body.libraryRoots, [sandbox.library, later]);
|
||||
rmSync(later, { recursive: true });
|
||||
assert.deepEqual((await client.call("GET", "/health")).body.libraryRoots, [sandbox.library]);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a job checks the roots again even while health reuses its last answer", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const later = join(sandbox.root, "automount");
|
||||
const client = await startTestWorker(sandbox, { libraryRoots: [sandbox.library, later] });
|
||||
try {
|
||||
assert.deepEqual((await client.call("GET", "/health")).body.libraryRoots, [sandbox.library]);
|
||||
mkdirSync(later);
|
||||
writeFileSync(join(later, "book.epub"), "epub-bytes");
|
||||
writeFileSync(join(later, "01.mp3"), "a".repeat(1000));
|
||||
assert.deepEqual((await client.call("GET", "/health")).body.libraryRoots, [sandbox.library], "health reuses a recent check");
|
||||
|
||||
const request = jobRequest(sandbox, { epubPath: join(later, "book.epub"), audioPaths: [join(later, "01.mp3")] });
|
||||
const created = await client.call("POST", "/v1/jobs", request);
|
||||
assert.equal(created.status, 202, JSON.stringify(created.body));
|
||||
assert.deepEqual((await client.call("GET", "/health")).body.libraryRoots, [sandbox.library, later]);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed", JSON.stringify(job));
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a hung root is left out within the timeout and is not checked again until it answers", async () => {
|
||||
const calls = new Map();
|
||||
let answer;
|
||||
const hung = new Promise((resolve) => (answer = resolve));
|
||||
const roots = new ReadableRoots(["/books", "/hung"], {
|
||||
maxAgeMs: 0,
|
||||
timeoutMs: 50,
|
||||
check: async (path) => {
|
||||
calls.set(path, (calls.get(path) ?? 0) + 1);
|
||||
if (path === "/hung" && calls.get(path) === 1) await hung;
|
||||
},
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
assert.deepEqual(await roots.get(), ["/books"]);
|
||||
assert.ok(Date.now() - startedAt < 1_000, "the check is bounded by its timeout");
|
||||
assert.deepEqual(await roots.get(), ["/books"]);
|
||||
assert.equal(calls.get("/hung"), 1, "a check still waiting on the mount is reused, not repeated");
|
||||
assert.equal(calls.get("/books"), 2);
|
||||
|
||||
answer();
|
||||
await sleep(0);
|
||||
assert.deepEqual(await roots.get(), ["/books", "/hung"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("job lifecycle", () => {
|
||||
let sandbox;
|
||||
let client;
|
||||
|
||||
before(async () => {
|
||||
sandbox = makeSandbox();
|
||||
client = await startTestWorker(sandbox, { mode: "slow" });
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
});
|
||||
|
||||
test("runs a job with progress, events and the scratch environment", async () => {
|
||||
const request = jobRequest(sandbox);
|
||||
const created = await client.call("POST", "/v1/jobs", request);
|
||||
assert.equal(created.status, 202);
|
||||
assert.equal(created.body.jobId, request.jobId);
|
||||
|
||||
const again = await client.call("POST", "/v1/jobs", request);
|
||||
assert.equal(again.status, 200, "a running job is returned, not restarted");
|
||||
|
||||
const other = await client.call("POST", "/v1/jobs", jobRequest(sandbox));
|
||||
assert.equal(other.status, 503);
|
||||
assert.equal(other.body.errorCode, "worker_unavailable");
|
||||
|
||||
const health = await client.call("GET", "/health");
|
||||
assert.equal(health.body.activeJobId, request.jobId);
|
||||
|
||||
const progress = [];
|
||||
const job = await waitForJob(client, request.jobId, (body) => {
|
||||
progress.push(body.progress);
|
||||
return TERMINAL.has(body.status);
|
||||
});
|
||||
assert.equal(job.status, "completed", JSON.stringify(job));
|
||||
assert.equal(job.progress, 100);
|
||||
assert.equal(job.errorCode, null);
|
||||
assert.equal(job.outputBytes, "output-epub-bytes".length);
|
||||
assert.deepEqual(
|
||||
progress,
|
||||
[...progress].sort((a, b) => a - b),
|
||||
"progress never goes backwards",
|
||||
);
|
||||
|
||||
const seqs = job.events.map((event) => event.seq);
|
||||
assert.deepEqual(
|
||||
seqs,
|
||||
seqs.map((_, index) => index + 1),
|
||||
);
|
||||
const codes = job.events.map((event) => `${event.stage ?? "-"}:${event.code}`);
|
||||
for (const expected of [
|
||||
"-:job_accepted",
|
||||
"-:disk_checked",
|
||||
"preparing_audio:stage_started",
|
||||
"preparing_audio:stage_finished",
|
||||
"generating_emissions:stage_finished",
|
||||
"marking_up_ebook:stage_finished",
|
||||
"aligning:stage_finished",
|
||||
"validating:stage_finished",
|
||||
"-:job_completed",
|
||||
]) {
|
||||
assert.ok(codes.includes(expected), `missing ${expected} in ${codes.join(", ")}`);
|
||||
}
|
||||
const finished = job.events.find((event) => event.code === "stage_finished" && event.stage === "generating_emissions");
|
||||
assert.equal(typeof finished.data.durationMs, "number");
|
||||
|
||||
const tail = await client.call("GET", `/v1/jobs/${request.jobId}?afterSeq=${seqs.at(-2)}`);
|
||||
assert.deepEqual(
|
||||
tail.body.events.map((event) => event.seq),
|
||||
[seqs.at(-1)],
|
||||
);
|
||||
|
||||
const runs = stagesRun(sandbox, request.jobId);
|
||||
assert.deepEqual(
|
||||
runs.map((entry) => entry.stage),
|
||||
["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning"],
|
||||
);
|
||||
assert.equal(runs[0].tmpdir, join(sandbox.tmpDir, "read-aloud-worker"));
|
||||
assert.equal(runs[0].scratch, join(request.workDir, "scratch"));
|
||||
for (const stage of ["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning"]) {
|
||||
assert.ok(existsSync(join(request.workDir, `.done-${stage}`)), `marker for ${stage}`);
|
||||
}
|
||||
assert.ok(existsSync(join(request.workDir, "source-audio", "0001.mp3")));
|
||||
assert.equal((await client.call("GET", "/health")).body.activeJobId, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resume from the work folder", () => {
|
||||
let sandbox;
|
||||
let client;
|
||||
let request;
|
||||
|
||||
before(async () => {
|
||||
sandbox = makeSandbox();
|
||||
client = await startTestWorker(sandbox, { modelDir: undefined });
|
||||
request = jobRequest(sandbox);
|
||||
assert.equal((await client.call("POST", "/v1/jobs", request)).status, 202);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed");
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
});
|
||||
|
||||
async function rerun(overrides) {
|
||||
resetStageLog(sandbox);
|
||||
const response = await client.call("POST", "/v1/jobs", { ...request, ...overrides });
|
||||
assert.equal(response.status, 202, "a terminal job starts a fresh run");
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed", JSON.stringify(job));
|
||||
return { job, stages: stagesRun(sandbox, request.jobId).map((entry) => entry.stage) };
|
||||
}
|
||||
|
||||
test("a repeat run skips every cached stage and keeps the event sequence", async () => {
|
||||
const before = (await client.call("GET", `/v1/jobs/${request.jobId}`)).body.events.at(-1).seq;
|
||||
const { job, stages } = await rerun({});
|
||||
assert.deepEqual(stages, []);
|
||||
const fresh = job.events.filter((event) => event.seq > before);
|
||||
assert.equal(fresh[0].seq, before + 1);
|
||||
assert.equal(fresh[0].code, "job_accepted");
|
||||
assert.equal(fresh[0].data.attempt, 2);
|
||||
assert.deepEqual(
|
||||
fresh.filter((event) => event.code === "stage_skipped").map((event) => event.stage),
|
||||
["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning"],
|
||||
);
|
||||
assert.ok(fresh.some((event) => event.code === "stage_finished" && event.stage === "validating"));
|
||||
});
|
||||
|
||||
test("changed decisions rerun only alignment", async () => {
|
||||
const { stages, job } = await rerun({ decisions: { chapters: { "chapter-1.xhtml": "excluded" }, audio: [] } });
|
||||
assert.deepEqual(stages, ["aligning"]);
|
||||
assert.equal(job.progress, 100);
|
||||
});
|
||||
|
||||
test("resumeFrom forces that stage and every later one", async () => {
|
||||
const { stages } = await rerun({ decisions: { chapters: { "chapter-1.xhtml": "excluded" }, audio: [] }, resumeFrom: "marking_up_ebook" });
|
||||
assert.deepEqual(stages, ["marking_up_ebook", "aligning"]);
|
||||
});
|
||||
|
||||
test("a changed source reruns the audio stages and alignment, not markup", async () => {
|
||||
const later = new Date(Date.now() + 60_000);
|
||||
utimesSync(join(sandbox.book, "02.mp3"), later, later);
|
||||
const { stages } = await rerun({ decisions: { chapters: { "chapter-1.xhtml": "excluded" }, audio: [] } });
|
||||
assert.deepEqual(stages, ["preparing_audio", "generating_emissions", "aligning"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("control plane", () => {
|
||||
test("health keeps answering while the runner is blocked in a synchronous loop", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "block-align" });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
await waitForJob(client, request.jobId, (body) => body.stage === "aligning");
|
||||
let slowest = 0;
|
||||
let answered = 0;
|
||||
const until = Date.now() + 1_500;
|
||||
while (Date.now() < until) {
|
||||
const sentAt = Date.now();
|
||||
const health = await client.call("GET", "/health");
|
||||
assert.equal(health.status, 200);
|
||||
slowest = Math.max(slowest, Date.now() - sentAt);
|
||||
answered += 1;
|
||||
}
|
||||
assert.ok(answered > 10, `answered ${answered}`);
|
||||
assert.ok(slowest < 500, `slowest health answer ${slowest} ms`);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed");
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("worker restart", () => {
|
||||
test("a resubmitted job resumes from its folder and keeps numbering events", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const request = jobRequest(sandbox);
|
||||
let client = await startTestWorker(sandbox);
|
||||
try {
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
const first = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
const lastSeq = first.events.at(-1).seq;
|
||||
await client.close();
|
||||
|
||||
client = await startTestWorker(sandbox);
|
||||
assert.equal((await client.call("GET", `/v1/jobs/${request.jobId}`)).status, 404, "the new process has never seen the job");
|
||||
resetStageLog(sandbox);
|
||||
const resubmitted = await client.call("POST", "/v1/jobs", request);
|
||||
assert.equal(resubmitted.status, 202);
|
||||
assert.equal(resubmitted.body.events[0].seq, lastSeq + 1);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed");
|
||||
assert.deepEqual(stagesRun(sandbox, request.jobId), []);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("shutting down while a job is still starting cancels it before a runner starts", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
let inDiskCheck;
|
||||
const reachedDiskCheck = new Promise((resolve) => (inDiskCheck = resolve));
|
||||
let finishDiskCheck;
|
||||
const diskChecked = new Promise((resolve) => (finishDiskCheck = resolve));
|
||||
const client = await startTestWorker(sandbox, {
|
||||
freeBytes: async () => {
|
||||
inDiskCheck();
|
||||
await diskChecked;
|
||||
return null;
|
||||
},
|
||||
});
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
const submitted = client.call("POST", "/v1/jobs", request);
|
||||
await reachedDiskCheck;
|
||||
const stopped = client.worker.registry.shutdown();
|
||||
finishDiskCheck();
|
||||
await stopped;
|
||||
|
||||
const response = await submitted;
|
||||
assert.equal(response.status, 202);
|
||||
assert.equal(response.body.status, "cancelled");
|
||||
assert.equal(response.body.errorCode, "cancelled");
|
||||
assert.equal(client.worker.registry.active, null);
|
||||
await sleep(300);
|
||||
assert.equal((await client.call("GET", `/v1/jobs/${request.jobId}`)).body.status, "cancelled");
|
||||
assert.deepEqual(stagesRun(sandbox, request.jobId), [], "no runner ran a stage");
|
||||
assert.ok(!existsSync(join(request.workDir, "source-audio")), "no runner prepared the job folder");
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("shutting down while a job's sources are being checked refuses it with a 503", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
let inRootCheck;
|
||||
const reachedRootCheck = new Promise((resolve) => (inRootCheck = resolve));
|
||||
let finishRootCheck;
|
||||
const rootChecked = new Promise((resolve) => (finishRootCheck = resolve));
|
||||
const client = await startTestWorker(sandbox, {
|
||||
rootChecks: {
|
||||
timeoutMs: 10_000,
|
||||
check: async () => {
|
||||
inRootCheck();
|
||||
await rootChecked;
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
const submitted = client.call("POST", "/v1/jobs", request);
|
||||
await reachedRootCheck;
|
||||
const stopped = client.worker.registry.shutdown();
|
||||
finishRootCheck();
|
||||
await stopped;
|
||||
|
||||
const response = await submitted;
|
||||
assert.equal(response.status, 503);
|
||||
assert.equal(response.body.errorCode, "worker_unavailable");
|
||||
assert.equal((await client.call("GET", `/v1/jobs/${request.jobId}`)).status, 404, "no record, so the next process takes it");
|
||||
assert.equal(client.worker.registry.active, null);
|
||||
assert.deepEqual(stagesRun(sandbox, request.jobId), []);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("shutting down stops the running job, keeps its work, and refuses new jobs", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "hang", cancelGraceMs: 10_000 });
|
||||
try {
|
||||
const running = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", running);
|
||||
await waitForJob(client, running.jobId, (body) => body.stage === "generating_emissions");
|
||||
|
||||
await client.worker.registry.shutdown();
|
||||
const stopped = await client.call("GET", `/v1/jobs/${running.jobId}`);
|
||||
assert.equal(stopped.body.status, "cancelled");
|
||||
assert.ok(existsSync(join(running.workDir, ".done-preparing_audio")), "finished stages stay cached for the next process");
|
||||
|
||||
// The slot is free now, so only the shutdown flag keeps a fresh runner from starting.
|
||||
const refused = await client.call("POST", "/v1/jobs", jobRequest(sandbox));
|
||||
assert.equal(refused.status, 503);
|
||||
assert.equal(refused.body.errorCode, "worker_unavailable");
|
||||
const resubmitted = await client.call("POST", "/v1/jobs", running);
|
||||
assert.equal(resubmitted.status, 503);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("model download", () => {
|
||||
test("runs as its own stage only while the model is missing", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { modelDir: sandbox.modelDir });
|
||||
try {
|
||||
const first = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", first);
|
||||
const job = await waitForJob(client, first.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "completed");
|
||||
const codes = job.events.map((event) => event.code);
|
||||
assert.ok(codes.includes("model_download_started"));
|
||||
assert.ok(codes.includes("model_download_finished"));
|
||||
assert.deepEqual(
|
||||
stagesRun(sandbox, first.jobId).map((entry) => entry.stage),
|
||||
["preparing_audio", "downloading_model", "generating_emissions", "marking_up_ebook", "aligning"],
|
||||
);
|
||||
|
||||
const second = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", second);
|
||||
await waitForJob(client, second.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.ok(!stagesRun(sandbox, second.jobId).some((entry) => entry.stage === "downloading_model"));
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("cancellation and failures", () => {
|
||||
test("cancel stops a cooperative runner within the grace period", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "hang", cancelGraceMs: 10_000 });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
await waitForJob(client, request.jobId, (body) => body.stage === "generating_emissions");
|
||||
const startedAt = Date.now();
|
||||
const cancel = await client.call("DELETE", `/v1/jobs/${request.jobId}`);
|
||||
assert.equal(cancel.status, 200);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.ok(Date.now() - startedAt < 5_000, "stopped well inside the grace period");
|
||||
assert.equal(job.status, "cancelled");
|
||||
assert.equal(job.errorCode, "cancelled");
|
||||
assert.ok(!job.events.some((event) => event.code === "runner_killed"));
|
||||
assert.ok(existsSync(join(request.workDir, ".done-preparing_audio")), "finished stages stay cached");
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("cancel also stops processes the runner started, such as an ffmpeg STAlign left behind", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "orphan-child", cancelGraceMs: 10_000 });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
const pidFile = join(request.workDir, "orphan.pid");
|
||||
await waitForJob(client, request.jobId, () => existsSync(pidFile));
|
||||
const pid = Number(readFileSync(pidFile, "utf8"));
|
||||
assert.ok(isAlive(pid), "the stand-in ffmpeg is running");
|
||||
await client.call("DELETE", `/v1/jobs/${request.jobId}`);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "cancelled");
|
||||
const deadline = Date.now() + 3_000;
|
||||
while (isAlive(pid) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
assert.ok(!isAlive(pid), "the orphaned child was killed with the runner's process group");
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a runner that ignores cancel is killed after the grace period", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "ignore-cancel", cancelGraceMs: 400 });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
await waitForJob(client, request.jobId, (body) => body.stage === "generating_emissions");
|
||||
await client.call("DELETE", `/v1/jobs/${request.jobId}`);
|
||||
const running = await client.call("GET", `/v1/jobs/${request.jobId}`);
|
||||
assert.equal(running.body.status, "running", "still running until the runner exits");
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "cancelled");
|
||||
assert.ok(job.events.some((event) => event.code === "runner_killed"));
|
||||
assert.equal((await client.call("GET", "/health")).body.activeJobId, null);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a runner that exits without a result is worker_crashed and frees the slot", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "crash" });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "failed");
|
||||
assert.equal(job.errorCode, "worker_crashed");
|
||||
assert.match(job.error, /exit code 7/);
|
||||
const failed = job.events.find((event) => event.code === "job_failed");
|
||||
assert.equal(failed.level, "error");
|
||||
assert.equal(failed.data.exitCode, 7);
|
||||
assert.equal((await client.call("POST", "/v1/jobs", jobRequest(sandbox))).status, 202);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a stage failure carries its stage code and a redacted message", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "fail-markup" });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.status, "failed");
|
||||
assert.equal(job.errorCode, "markup_failed");
|
||||
assert.match(job.error, /<work>\//);
|
||||
assert.ok(!job.error.includes(sandbox.root));
|
||||
for (const event of job.events) assert.ok(!(event.message ?? "").includes(sandbox.root), event.message);
|
||||
assert.ok(job.events.some((event) => event.code === "stage_failed" && event.stage === "marking_up_ebook"));
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("an empty output fails validation as invalid_output", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { mode: "empty-output" });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
await client.call("POST", "/v1/jobs", request);
|
||||
const job = await waitForJob(client, request.jobId, (body) => TERMINAL.has(body.status));
|
||||
assert.equal(job.errorCode, "invalid_output");
|
||||
assert.equal(job.stage, "validating");
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("too little free space fails the job before the runner starts", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await startTestWorker(sandbox, { freeBytes: async () => 1_000 });
|
||||
try {
|
||||
const request = jobRequest(sandbox);
|
||||
const response = await client.call("POST", "/v1/jobs", request);
|
||||
assert.equal(response.status, 202);
|
||||
assert.equal(response.body.status, "failed");
|
||||
assert.equal(response.body.errorCode, "insufficient_storage");
|
||||
const event = response.body.events.find((entry) => entry.code === "disk_check_failed");
|
||||
assert.equal(event.data.availableBytes, 1_000);
|
||||
assert.ok(event.data.neededBytes > 512 * 1024 * 1024);
|
||||
assert.deepEqual(stagesRun(sandbox, request.jobId), []);
|
||||
assert.equal((await client.call("GET", "/health")).body.activeJobId, null);
|
||||
} finally {
|
||||
await client.close();
|
||||
sandbox.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("the control process never imports the pipeline", () => {
|
||||
for (const file of ["main", "server", "jobs", "config", "stages", "paths", "validation", "log", "accelerator", "protocol", "toolchain"]) {
|
||||
const source = readFileSync(new URL(`../dist/${file}.js`, import.meta.url), "utf8");
|
||||
assert.ok(!source.includes("@storyteller-platform/align"), `${file}.js imports STAlign`);
|
||||
assert.ok(!/from "\.\/(runner|pipeline)\.js"/.test(source), `${file}.js imports the runner`);
|
||||
}
|
||||
});
|
||||
|
||||
function isAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
// Opt-in end-to-end run of the real STAlign pipeline: READ_ALOUD_SMOKE=1 pnpm --filter
|
||||
// @bookorbit/read-aloud-worker test. It needs macOS (`say`), ffmpeg, zip and unzip, and the first run
|
||||
// downloads the English QuartzNet model. READ_ALOUD_SMOKE_FRESH_MODEL=1 points the model cache at the
|
||||
// temp folder to time a fresh download. Everything it writes lives in a temp folder.
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync, spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import test from "node:test";
|
||||
import { makeFixture } from "./smoke/make-fixture.mjs";
|
||||
|
||||
const enabled = process.env.READ_ALOUD_SMOKE === "1";
|
||||
const TOKEN = `smoke-${randomUUID()}`;
|
||||
|
||||
test(
|
||||
"the real pipeline produces a media overlay EPUB and resumes from its cache",
|
||||
{ skip: !enabled && "set READ_ALOUD_SMOKE=1 to run", timeout: 30 * 60_000 },
|
||||
async () => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "read-aloud-smoke-")));
|
||||
const workRoot = join(root, "data", "read-aloud", "jobs");
|
||||
const fixture = makeFixture(root);
|
||||
const worker = await startMainProcess({
|
||||
READ_ALOUD_WORKER_TOKEN: TOKEN,
|
||||
PORT: "0",
|
||||
READ_ALOUD_LIBRARY_ROOTS: fixture.libraryRoot,
|
||||
READ_ALOUD_WORK_ROOT: workRoot,
|
||||
READ_ALOUD_CTC_DEVICE: process.env.READ_ALOUD_CTC_DEVICE ?? "auto",
|
||||
...(process.env.READ_ALOUD_SMOKE_FRESH_MODEL === "1" ? { HOME: join(root, "home"), XDG_DATA_HOME: join(root, "home", "share") } : {}),
|
||||
});
|
||||
const healthGaps = watchHealth(worker.base);
|
||||
try {
|
||||
const jobId = randomUUID();
|
||||
const workDir = join(workRoot, jobId);
|
||||
const durationSeconds = fixture.audioPaths.reduce((sum, path) => sum + probeDuration(path), 0);
|
||||
const request = {
|
||||
jobId,
|
||||
epubPath: fixture.epubPath,
|
||||
audioPaths: fixture.audioPaths,
|
||||
workDir,
|
||||
language: "en",
|
||||
granularity: "sentence",
|
||||
model: "quartznet15x5",
|
||||
sourceBytes: [fixture.epubPath, ...fixture.audioPaths].reduce((sum, path) => sum + statSync(path).size, 0),
|
||||
audioDurationSeconds: durationSeconds,
|
||||
maxOutputBytes: 1024 ** 3,
|
||||
decisions: { chapters: {}, audio: [] },
|
||||
resumeFrom: null,
|
||||
};
|
||||
|
||||
const startedAt = Date.now();
|
||||
const first = await runJob(worker.base, request);
|
||||
const wallMs = Date.now() - startedAt;
|
||||
assert.equal(first.status, "completed", JSON.stringify(first.events.slice(-5)));
|
||||
|
||||
const opf = execFileSync("unzip", ["-p", join(workDir, "output.epub"), "OEBPS/content.opf"], { encoding: "utf8" });
|
||||
assert.match(opf, /media-overlay="/);
|
||||
assert.match(opf, /application\/smil\+xml/);
|
||||
const entries = execFileSync("unzip", ["-l", join(workDir, "output.epub")], { encoding: "utf8" });
|
||||
assert.match(entries, /\.smil/);
|
||||
|
||||
const report = JSON.parse(readFileSync(join(workDir, "report.json"), "utf8"));
|
||||
assert.equal(report.totals.tracks, fixture.audioPaths.length);
|
||||
assert.equal(report.totals.unalignedTracks, 0);
|
||||
assert.ok(report.totals.sentences >= 20, `sentences ${report.totals.sentences}`);
|
||||
assert.ok(report.totals.matchedSeconds / report.totals.seconds >= 0.7, JSON.stringify(report.totals));
|
||||
assert.equal(report.spine.filter((item) => item.status === "aligned").length, 3, "every chapter aligned");
|
||||
|
||||
const codes = first.events.map((event) => event.code);
|
||||
assert.ok(codes.includes("audio_reencoded_cbr"), "the VBR chapter was re-encoded");
|
||||
const processed = readdirSync(join(workDir, "processed-audio")).filter((name) => name.startsWith("00002-") && name.endsWith(".mp3"));
|
||||
assert.equal(processed.length, 1);
|
||||
assert.ok(packetSizes(fixture.audioPaths[1]) > 2, "the source is VBR");
|
||||
assert.ok(packetSizes(join(workDir, "processed-audio", processed[0])) <= 2, "the processed file is CBR");
|
||||
|
||||
const emissionsBytes = readdirSync(join(workDir, "emissions"))
|
||||
.filter((name) => name.endsWith(".emissions"))
|
||||
.reduce((sum, name) => sum + statSync(join(workDir, "emissions", name)).size, 0);
|
||||
const stageMs = Object.fromEntries(
|
||||
first.events.filter((event) => event.code === "stage_finished").map((event) => [event.stage, event.data.durationMs]),
|
||||
);
|
||||
|
||||
const second = await runJob(worker.base, request);
|
||||
assert.equal(second.status, "completed");
|
||||
const fresh = second.events.filter((event) => event.seq > first.events.at(-1).seq);
|
||||
assert.deepEqual(
|
||||
fresh.filter((event) => event.code === "stage_skipped").map((event) => event.stage),
|
||||
["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning"],
|
||||
);
|
||||
|
||||
const resync = await runJob(worker.base, { ...request, decisions: { chapters: { "chapter-3.xhtml": "excluded" }, audio: [] } });
|
||||
assert.equal(resync.status, "completed");
|
||||
const resyncRan = resync.events
|
||||
.filter((event) => event.seq > second.events.at(-1).seq && event.code === "stage_finished")
|
||||
.map((event) => event.stage);
|
||||
assert.deepEqual(resyncRan, ["aligning", "validating"]);
|
||||
const resyncReport = JSON.parse(readFileSync(join(workDir, "report.json"), "utf8"));
|
||||
assert.equal(resyncReport.spine.find((item) => item.href.endsWith("chapter-3.xhtml"))?.status, "excluded");
|
||||
|
||||
const summary = {
|
||||
device: (await health(worker.base)).device,
|
||||
audioSeconds: Math.round(durationSeconds * 10) / 10,
|
||||
wallMs,
|
||||
stageMs,
|
||||
emissionsBytes,
|
||||
emissionsBytesPerSecond: Math.round(emissionsBytes / durationSeconds),
|
||||
outputBytes: first.outputBytes,
|
||||
modelDownload: first.events.find((event) => event.code === "model_download_finished")?.data ?? null,
|
||||
totals: report.totals,
|
||||
longestHealthResponseMs: healthGaps.longestResponseMs,
|
||||
longestHealthGapMs: healthGaps.longestGapMs,
|
||||
};
|
||||
process.stdout.write(`READ_ALOUD_SMOKE_SUMMARY ${JSON.stringify(summary)}\n`);
|
||||
if (process.env.READ_ALOUD_SMOKE_REPORT) writeFileSync(process.env.READ_ALOUD_SMOKE_REPORT, JSON.stringify(summary, null, 2));
|
||||
} finally {
|
||||
healthGaps.stop();
|
||||
await worker.stop();
|
||||
if (process.env.READ_ALOUD_SMOKE_KEEP === "1") process.stdout.write(`READ_ALOUD_SMOKE_KEPT ${root}\n`);
|
||||
else rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
async function startMainProcess(env) {
|
||||
const child = spawn(process.execPath, [new URL("../dist/main.js", import.meta.url).pathname], {
|
||||
env: { ...process.env, ...env },
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
const port = await new Promise((resolve, reject) => {
|
||||
let buffer = "";
|
||||
child.stdout.on("data", (chunk) => {
|
||||
const text = chunk.toString();
|
||||
process.stdout.write(text);
|
||||
buffer += text;
|
||||
const match = /\[read_aloud\.worker\.boot\] \[end\] .*port=(\d+)/.exec(buffer);
|
||||
if (match) resolve(Number(match[1]));
|
||||
});
|
||||
child.once("exit", (code) => reject(new Error(`worker exited with ${code}`)));
|
||||
});
|
||||
return {
|
||||
base: `http://127.0.0.1:${port}`,
|
||||
stop: async () => {
|
||||
child.kill("SIGTERM");
|
||||
await new Promise((resolve) => child.once("exit", resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function call(base, method, path, body) {
|
||||
const response = await fetch(`${base}${path}`, {
|
||||
method,
|
||||
headers: { Authorization: `Bearer ${TOKEN}`, ...(body ? { "Content-Type": "application/json" } : {}) },
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
return { status: response.status, body: await response.json() };
|
||||
}
|
||||
|
||||
async function health(base) {
|
||||
return (await call(base, "GET", "/health")).body;
|
||||
}
|
||||
|
||||
async function runJob(base, request) {
|
||||
const created = await call(base, "POST", "/v1/jobs", request);
|
||||
assert.equal(created.status, 202, JSON.stringify(created.body));
|
||||
for (;;) {
|
||||
const { body } = await call(base, "GET", `/v1/jobs/${request.jobId}`);
|
||||
if (["completed", "failed", "cancelled"].includes(body.status)) return body;
|
||||
await sleep(250);
|
||||
}
|
||||
}
|
||||
|
||||
/** Polls /health every 100 ms during the run and records the slowest answer and the longest silence. */
|
||||
function watchHealth(base) {
|
||||
const state = { longestResponseMs: 0, longestGapMs: 0, running: true };
|
||||
let lastOk = Date.now();
|
||||
let warmedUp = false;
|
||||
void (async () => {
|
||||
while (state.running) {
|
||||
const sentAt = Date.now();
|
||||
try {
|
||||
await health(base);
|
||||
const now = Date.now();
|
||||
// The first request includes the client's own connection setup, so it is not counted.
|
||||
if (warmedUp) {
|
||||
state.longestResponseMs = Math.max(state.longestResponseMs, now - sentAt);
|
||||
state.longestGapMs = Math.max(state.longestGapMs, now - lastOk);
|
||||
}
|
||||
warmedUp = true;
|
||||
lastOk = now;
|
||||
} catch {
|
||||
// A refused request counts toward the gap.
|
||||
}
|
||||
await sleep(100);
|
||||
}
|
||||
})();
|
||||
return {
|
||||
get longestResponseMs() {
|
||||
return state.longestResponseMs;
|
||||
},
|
||||
get longestGapMs() {
|
||||
return state.longestGapMs;
|
||||
},
|
||||
stop() {
|
||||
state.running = false;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function probeDuration(path) {
|
||||
return Number(execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path], { encoding: "utf8" }).trim());
|
||||
}
|
||||
|
||||
/** Distinct MP3 packet sizes: CBR frames differ only by the padding byte, VBR frames vary widely. */
|
||||
function packetSizes(path) {
|
||||
const output = execFileSync("ffprobe", ["-v", "error", "-select_streams", "a:0", "-show_entries", "packet=size", "-of", "csv=p=0", path], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
return new Set(output.split("\n").filter(Boolean).slice(1, -1)).size;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
// Builds the smoke-test book at test time: a three-chapter EPUB 3 with front matter, and its
|
||||
// narration spoken by macOS `say`, one MP3 per chapter. Chapter two is a VBR MP3 so the run also
|
||||
// covers STAlign's VBR to CBR re-encode. Nothing here is tracked; the caller passes a temp folder.
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
export const CHAPTERS = [
|
||||
{
|
||||
title: "Chapter One: The Lighthouse Keeper",
|
||||
paragraphs: [
|
||||
"Mara had kept the lighthouse on the northern point for eleven winters. Each evening she climbed the narrow stairs, counted the steps aloud, and trimmed the wick before the sun touched the water.",
|
||||
"The ships that passed never knew her name, but they knew her light. On the night of the great storm, the wind tore the shutters from the lower windows and the sea climbed higher than she had ever seen it.",
|
||||
"Still she climbed, and still she counted, and the lamp burned steady until morning.",
|
||||
],
|
||||
},
|
||||
{
|
||||
title: "Chapter Two: The Letter",
|
||||
paragraphs: [
|
||||
"In the spring a letter arrived with the supply boat. It was written in a careful hand and sealed with green wax.",
|
||||
"The sender was a young sailor who had seen her light during the storm and had steered his small boat safely into the harbor. He wrote that he had no way to repay her, except to say thank you, and to promise that he would keep a light burning in his own window for anyone lost at sea.",
|
||||
"Mara read the letter twice, then placed it on the shelf beside the oil lamps.",
|
||||
],
|
||||
},
|
||||
{
|
||||
title: "Chapter Three: The Visitor",
|
||||
paragraphs: [
|
||||
"Many years later, an old man walked along the cliff path and knocked on the lighthouse door. He carried a lantern, though it was the middle of the day.",
|
||||
"Mara did not recognize his face, but she recognized the green wax on the letter he held out to her. They sat together in the small kitchen and drank tea while the gulls cried outside.",
|
||||
"When the sun went down, they climbed the stairs together, counting each step aloud, and lit the lamp side by side.",
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const TITLE = "The Keeper of the Northern Light";
|
||||
|
||||
/** Returns the EPUB path and the MP3 paths in listening order. */
|
||||
export function makeFixture(directory) {
|
||||
const book = join(directory, "library", "Anonymous", TITLE);
|
||||
const build = join(directory, "build");
|
||||
mkdirSync(book, { recursive: true });
|
||||
mkdirSync(join(build, "META-INF"), { recursive: true });
|
||||
mkdirSync(join(build, "OEBPS"), { recursive: true });
|
||||
|
||||
writeFileSync(join(build, "mimetype"), "application/epub+zip");
|
||||
writeFileSync(
|
||||
join(build, "META-INF", "container.xml"),
|
||||
`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
|
||||
</container>
|
||||
`,
|
||||
);
|
||||
writeFileSync(join(build, "OEBPS", "title.xhtml"), xhtml("Title", `<h1>${TITLE}</h1>\n<p>Anonymous</p>`));
|
||||
writeFileSync(
|
||||
join(build, "OEBPS", "nav.xhtml"),
|
||||
`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" xml:lang="en" lang="en">
|
||||
<head><title>Contents</title></head>
|
||||
<body><nav epub:type="toc" id="toc"><ol>
|
||||
${CHAPTERS.map((chapter, index) => `<li><a href="chapter-${index + 1}.xhtml">${chapter.title}</a></li>`).join("\n")}
|
||||
</ol></nav></body>
|
||||
</html>
|
||||
`,
|
||||
);
|
||||
for (const [index, chapter] of CHAPTERS.entries()) {
|
||||
const body = [`<h1>${chapter.title}</h1>`, ...chapter.paragraphs.map((text) => `<p>${text}</p>`)].join("\n");
|
||||
writeFileSync(join(build, "OEBPS", `chapter-${index + 1}.xhtml`), xhtml(chapter.title, body));
|
||||
}
|
||||
writeFileSync(
|
||||
join(build, "OEBPS", "content.opf"),
|
||||
`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid" xml:lang="en">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="uid">urn:uuid:5d0a3c1e-7b7e-4c1a-9d43-2f1e8f0a6b21</dc:identifier>
|
||||
<dc:title>${TITLE}</dc:title>
|
||||
<dc:creator>Anonymous</dc:creator>
|
||||
<dc:language>en</dc:language>
|
||||
<meta property="dcterms:modified">2026-01-01T00:00:00Z</meta>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>
|
||||
<item id="title" href="title.xhtml" media-type="application/xhtml+xml"/>
|
||||
${CHAPTERS.map((_, index) => ` <item id="chapter-${index + 1}" href="chapter-${index + 1}.xhtml" media-type="application/xhtml+xml"/>`).join("\n")}
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="title"/>
|
||||
${CHAPTERS.map((_, index) => ` <itemref idref="chapter-${index + 1}"/>`).join("\n")}
|
||||
</spine>
|
||||
</package>
|
||||
`,
|
||||
);
|
||||
const epubPath = join(book, `${TITLE}.epub`);
|
||||
execFileSync("zip", ["-X0q", epubPath, "mimetype"], { cwd: build });
|
||||
execFileSync("zip", ["-Xr9Dq", epubPath, "META-INF", "OEBPS"], { cwd: build });
|
||||
|
||||
const audioPaths = [];
|
||||
for (const [index, chapter] of CHAPTERS.entries()) {
|
||||
const textPath = join(build, `chapter-${index + 1}.txt`);
|
||||
const aiffPath = join(build, `chapter-${index + 1}.aiff`);
|
||||
writeFileSync(textPath, [chapter.title, ...chapter.paragraphs].join("\n\n"));
|
||||
execFileSync("say", ["-r", "175", "-o", aiffPath, "-f", textPath]);
|
||||
const mp3Path = join(book, `${String(index + 1).padStart(2, "0")} - ${chapter.title.split(":")[0]}.mp3`);
|
||||
const quality = index === 1 ? ["-q:a", "4"] : ["-b:a", "64k"];
|
||||
execFileSync("ffmpeg", [
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-i",
|
||||
aiffPath,
|
||||
"-ar",
|
||||
"44100",
|
||||
"-ac",
|
||||
"1",
|
||||
"-c:a",
|
||||
"libmp3lame",
|
||||
...quality,
|
||||
mp3Path,
|
||||
]);
|
||||
audioPaths.push(mp3Path);
|
||||
}
|
||||
return { epubPath, audioPaths, libraryRoot: join(directory, "library") };
|
||||
}
|
||||
|
||||
function xhtml(title, body) {
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head><title>${title}</title></head>
|
||||
<body>
|
||||
${body}
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { estimateReadAloudWorkBytes } from "@bookorbit/types";
|
||||
import {
|
||||
DISK_MARGIN_BYTES,
|
||||
EMISSION_BYTES_PER_SECOND,
|
||||
ASSUMED_AUDIO_BYTES_PER_SECOND,
|
||||
fingerprint,
|
||||
normalizedWeights,
|
||||
planStages,
|
||||
requiredDiskBytes,
|
||||
stageInputs,
|
||||
} from "../dist/stages.js";
|
||||
|
||||
const GB = 1024 ** 3;
|
||||
const ALL_VALID = { preparing_audio: "valid", generating_emissions: "valid", marking_up_ebook: "valid", aligning: "valid" };
|
||||
|
||||
function source(overrides = {}) {
|
||||
return {
|
||||
epub: { path: "/books/a/book.epub", size: 1000, mtimeMs: 1 },
|
||||
audio: [{ path: "/books/a/01.mp3", size: 5000, mtimeMs: 2 }],
|
||||
language: "en",
|
||||
granularity: "sentence",
|
||||
model: "quartznet15x5",
|
||||
decisions: { chapters: {}, audio: [] },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("disk rule: 4 x audio + 7 x EPUB + emissions + 512 MB for a fresh run", () => {
|
||||
const { required } = requiredDiskBytes({ audioBytes: 2 * GB, epubBytes: 10_000_000, audioDurationSeconds: 36_000, skipped: new Set() });
|
||||
assert.equal(required, 4 * 2 * GB + 7 * 10_000_000 + 36_000 * EMISSION_BYTES_PER_SECOND + DISK_MARGIN_BYTES);
|
||||
});
|
||||
|
||||
test("disk rule: the server's estimate for a fresh run matches the worker's", () => {
|
||||
const input = { audioBytes: 2 * GB, epubBytes: 10_000_000, audioDurationSeconds: 36_000 };
|
||||
assert.equal(estimateReadAloudWorkBytes(input), requiredDiskBytes({ ...input, skipped: new Set() }).required);
|
||||
});
|
||||
|
||||
test("disk rule: cached stages no longer count", () => {
|
||||
const input = { audioBytes: 2 * GB, epubBytes: 10_000_000, audioDurationSeconds: 36_000 };
|
||||
const resync = requiredDiskBytes({ ...input, skipped: new Set(["preparing_audio", "generating_emissions", "marking_up_ebook"]) });
|
||||
// Re-aligning holds the unpacked copy, the scratch EPUB and the output at once.
|
||||
assert.equal(resync.required, 3 * 2 * GB + 5 * 10_000_000 + DISK_MARGIN_BYTES);
|
||||
const everything = requiredDiskBytes({ ...input, skipped: new Set(["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning"]) });
|
||||
assert.equal(everything.required, DISK_MARGIN_BYTES);
|
||||
});
|
||||
|
||||
test("disk rule: an unknown duration is estimated from the audio size", () => {
|
||||
const { shares } = requiredDiskBytes({ audioBytes: 800_000_000, epubBytes: 0, audioDurationSeconds: null, skipped: new Set() });
|
||||
assert.equal(shares.generating_emissions, Math.ceil((800_000_000 / ASSUMED_AUDIO_BYTES_PER_SECOND) * EMISSION_BYTES_PER_SECOND));
|
||||
});
|
||||
|
||||
test("plan: every valid marker is skipped", () => {
|
||||
assert.ok(planStages(ALL_VALID, null).every((entry) => !entry.run && entry.reason === "cached"));
|
||||
});
|
||||
|
||||
test("plan: resumeFrom forces that stage and every later one", () => {
|
||||
const plan = planStages(ALL_VALID, "marking_up_ebook");
|
||||
assert.deepEqual(
|
||||
plan.filter((entry) => entry.run).map((entry) => [entry.stage, entry.reason]),
|
||||
[
|
||||
["marking_up_ebook", "forced"],
|
||||
["aligning", "forced"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
planStages(ALL_VALID, "validating").filter((entry) => entry.run),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("plan: a rerun stage drags its dependents with it, not unrelated stages", () => {
|
||||
const plan = planStages({ ...ALL_VALID, preparing_audio: "missing" }, null);
|
||||
assert.deepEqual(
|
||||
plan.filter((entry) => entry.run).map((entry) => [entry.stage, entry.reason]),
|
||||
[
|
||||
["preparing_audio", "missing"],
|
||||
["generating_emissions", "dependency"],
|
||||
["aligning", "dependency"],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("fingerprints: changed decisions invalidate only alignment", () => {
|
||||
const before = source();
|
||||
const after = source({ decisions: { chapters: { "chapter.xhtml": "excluded" }, audio: [] } });
|
||||
for (const stage of ["preparing_audio", "generating_emissions", "marking_up_ebook"]) {
|
||||
assert.equal(fingerprint(stageInputs(stage, before)), fingerprint(stageInputs(stage, after)), stage);
|
||||
}
|
||||
assert.notEqual(fingerprint(stageInputs("aligning", before)), fingerprint(stageInputs("aligning", after)));
|
||||
});
|
||||
|
||||
test("fingerprints: decision key order does not matter, granularity and sources do", () => {
|
||||
const a = source({ decisions: { chapters: { "a.xhtml": "excluded", "b.xhtml": "included" }, audio: [] } });
|
||||
const b = source({ decisions: { chapters: { "b.xhtml": "included", "a.xhtml": "excluded" }, audio: [] } });
|
||||
assert.equal(fingerprint(stageInputs("aligning", a)), fingerprint(stageInputs("aligning", b)));
|
||||
const word = source({ granularity: "word" });
|
||||
assert.notEqual(fingerprint(stageInputs("marking_up_ebook", source())), fingerprint(stageInputs("marking_up_ebook", word)));
|
||||
assert.equal(fingerprint(stageInputs("generating_emissions", source())), fingerprint(stageInputs("generating_emissions", word)));
|
||||
const touched = source({ audio: [{ path: "/books/a/01.mp3", size: 5000, mtimeMs: 3 }] });
|
||||
assert.notEqual(fingerprint(stageInputs("preparing_audio", source())), fingerprint(stageInputs("preparing_audio", touched)));
|
||||
assert.equal(fingerprint(stageInputs("marking_up_ebook", source())), fingerprint(stageInputs("marking_up_ebook", touched)));
|
||||
});
|
||||
|
||||
test("weights: skipped stages are renormalized so a run still ends at 100", () => {
|
||||
const all = normalizedWeights(["preparing_audio", "generating_emissions", "marking_up_ebook", "aligning", "validating"]);
|
||||
assert.equal(Math.round([...all.values()].reduce((sum, value) => sum + value, 0)), 100);
|
||||
assert.equal(all.get("generating_emissions"), (60 / 95) * 100);
|
||||
const resync = normalizedWeights(["aligning", "validating"]);
|
||||
assert.equal(resync.get("aligning"), 80);
|
||||
assert.equal(resync.get("validating"), 20);
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2024",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"rootDir": "src",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
Reference in New Issue
Block a user