feat(metadata): add external HTTP metadata providers

Register book-metadata-provider services by address, with no code change per
provider; Settings lists ready-made ones to install. Ranking prefers a record of
the book's own medium. LubimyCzytac moves out to its own provider and Aladin,
whose API closes, is retired; their ids stay. Adds migrations 0129 and 0130.
See docs/METADATA_PROVIDERS.md.

Ref #1405

packages/types/src/audit.ts

@@ -107,6 +107,10 @@ export enum AuditAction {
   RequestIndexerPluginInstall = "request_indexer.plugin.install",
   RequestIndexerPluginRemove = "request_indexer.plugin.remove",
 
+  MetadataHttpProviderCreate = "metadata_http_provider.create",
+  MetadataHttpProviderUpdate = "metadata_http_provider.update",
+  MetadataHttpProviderDelete = "metadata_http_provider.delete",
+
   AuthorUpdate = "author.update",
   AuthorDelete = "author.delete",
   AuthorMerge = "author.merge",
@@ -167,6 +171,7 @@ export enum AuditResource {
   BookRequest = "book_request",
   DownloadClient = "download_client",
   RequestIndexer = "request_indexer",
+  MetadataHttpProvider = "metadata_http_provider",
   Author = "author",
   AppSettings = "app_settings",
   Genre = "genre",
@@ -243,7 +248,7 @@ export const AUDIT_CATEGORY_PREFIXES: Record<Exclude<AuditCategory, "other">, re
   requests: ["book_request.", "download_client.", "request_indexer."],
   collections: ["collection.", "smart_scope."],
   devices: ["kobo.", "email."],
-  settings: ["app_settings.", "entity_manager.", "author.", "maintenance."],
+  settings: ["app_settings.", "entity_manager.", "author.", "maintenance.", "metadata_http_provider."],
 };
 
 export function getAuditCategory(action: string): AuditCategory {
@@ -273,7 +278,12 @@ export const AUDIT_SECURITY_GROUPS = {
     AuditAction.UserUnlock,
     AuditAction.OidcUserProvisioned,
   ],
-  passwords: [AuditAction.AuthPasswordChange, AuditAction.AuthPasswordResetRequest, AuditAction.AuthPasswordReset, AuditAction.AuthPasswordAdminReset],
+  passwords: [
+    AuditAction.AuthPasswordChange,
+    AuditAction.AuthPasswordResetRequest,
+    AuditAction.AuthPasswordReset,
+    AuditAction.AuthPasswordAdminReset,
+  ],
   permissions: [AuditAction.UserPermissionSet, AuditAction.UserSuperuserEnable, AuditAction.UserSuperuserDisable],
   access: [
     AuditAction.UserContentFiltersSet,

packages/types/src/book-dock.ts

@@ -65,6 +65,8 @@ export interface BookDockMetadata {
   ranobedbId?: string | null;
   lubimyczytacId?: string | null;
   aladinId?: string | null;
+  /** Ids from external providers, keyed by provider. Absent on older servers. */
+  externalProviderIds?: Partial<Record<string, string>> | null;
   comicMetadata?: ComicMetadataFields | null;
 }
 

packages/types/src/book.ts

@@ -335,6 +335,12 @@ export type BookDetail = {
   coverVersion: string;
   hardcoverEditionId: string | null;
   providerIds: ProviderIds;
+  /**
+   * Record links for external providers' ids, keyed like `providerIds`: the page the id was applied
+   * from, or the provider's own URL template. Built-in links stay with the client's templates.
+   * Absent on older servers.
+   */
+  providerLinks?: Partial<Record<string, string>>;
   authors: { id: number; name: string; sortName: string | null }[];
   genres: string[];
   tags: string[];
@@ -398,6 +404,8 @@ export type BookMetadataRefreshPreviewFields = {
   ranobedbId?: string | null;
   lubimyczytacId?: string | null;
   aladinId?: string | null;
+  /** Ids from external providers, keyed by provider. Absent on older servers. */
+  externalProviderIds?: Partial<Record<string, string | null>>;
   audioMetadata?: {
     narrators?: string[];
     durationSeconds?: number | null;

packages/types/src/index.ts

@@ -29,6 +29,7 @@ export * from "./format-priority";
 export * from "./maintenance";
 export * from "./metadata-fetch";
 export * from "./metadata-cache";
+export * from "./metadata-http-provider";
 export * from "./pattern-resolver";
 export * from "./query";
 export * from "./jump-buckets";

packages/types/src/metadata-fetch.ts

@@ -1,8 +1,13 @@
 import type { AudiobookChapter, NarratorRef } from "./audiobook";
-import type { CoverMedium } from "./book";
+import type { ConcreteBookMediaKind, CoverMedium } from "./book";
 import type { MetadataCacheServedProvider, MetadataCandidateCacheInfo } from "./metadata-cache";
 import type { SeriesIndex } from "./series-index";
 
+/**
+ * Every provider key BookOrbit has a dedicated id field for. Most are built in; the ones in
+ * `MOVED_METADATA_PROVIDERS` are not, and keep their id columns and the API fields released clients
+ * send.
+ */
 export const MetadataProviderKey = {
   GOOGLE: "google",
   GOODREADS: "goodreads",
@@ -31,7 +36,11 @@ export const COMMUNITY_RATING_PROVIDER_KEYS = [
   MetadataProviderKey.AUDIBLE,
 ] as const;
 
-export type CommunityRatingProviderKey = (typeof COMMUNITY_RATING_PROVIDER_KEYS)[number];
+/**
+ * Any provider whose ratings can be filtered on. External providers are named at runtime, so the
+ * list above only seeds defaults; it is not the set of valid values.
+ */
+export type CommunityRatingProviderKey = MetadataProviderKey;
 
 export interface ComicMetadataFields {
   issueNumber?: string;
@@ -47,7 +56,111 @@ export interface ComicMetadataFields {
   storyArcs?: string[];
 }
 
-export type MetadataProviderKey = (typeof MetadataProviderKey)[keyof typeof MetadataProviderKey];
+/**
+ * Providers that used to be compiled into BookOrbit. One with an `image` now runs as its own service
+ * under the same key, so stored ids, ratings and field rules carry over once an admin adds it. One
+ * without has closed: its ids and links stay on books, and there is nothing to run.
+ */
+export const MOVED_METADATA_PROVIDERS: Readonly<Record<"lubimyczytac" | "aladin", { label: string; image: string | null }>> = {
+  lubimyczytac: { label: "LubimyCzytac", image: "ghcr.io/bookorbit/lubimyczytac-provider" },
+  // Aladin stopped issuing Open API keys on 2026-09-04 and ends the API on 2026-10-30, with no replacement.
+  aladin: { label: "Aladin", image: null },
+};
+
+export type MovedMetadataProviderKey = keyof typeof MOVED_METADATA_PROVIDERS;
+
+/** A key with its own id column and API field, built in or moved out. */
+export type KnownMetadataProviderKey = (typeof MetadataProviderKey)[keyof typeof MetadataProviderKey];
+
+/** A provider compiled into BookOrbit. Maps that only make sense for these are keyed by this type. */
+export type BuiltInMetadataProviderKey = Exclude<KnownMetadataProviderKey, MovedMetadataProviderKey>;
+
+/**
+ * A provider an admin registered at runtime: an HTTP service speaking the book-metadata-provider
+ * spec. Its key is only known once registered, so any string that passes
+ * `EXTERNAL_METADATA_PROVIDER_KEY_PATTERN` and is not reserved can be one.
+ */
+export type ExternalMetadataProviderKey = string & { readonly __externalMetadataProviderKey?: never };
+
+/**
+ * Any provider, built-in or external. A `Record` over this type is not exhaustive (it collapses to a
+ * string index), so a map that must list every provider is keyed by `BuiltInMetadataProviderKey`
+ * instead, and everything else is a `Partial` with a fallback. A guard test enforces it.
+ */
+export type MetadataProviderKey = KnownMetadataProviderKey | ExternalMetadataProviderKey;
+
+/** Display names of the built-in providers, the one list every surface reads. */
+export const BUILT_IN_METADATA_PROVIDER_LABELS: Record<BuiltInMetadataProviderKey, string> = {
+  google: "Google Books",
+  goodreads: "Goodreads",
+  amazon: "Amazon",
+  hardcover: "Hardcover",
+  openLibrary: "Open Library",
+  itunes: "iTunes",
+  audible: "Audible",
+  audnexus: "AudNexus",
+  librofm: "Libro.fm",
+  comicvine: "ComicVine",
+  ranobedb: "RanobeDB",
+  kobo: "Kobo",
+};
+
+/** At most 32 characters, which fits every column and DTO that carries a provider key. */
+export const EXTERNAL_METADATA_PROVIDER_KEY_PATTERN = /^[a-z][a-z0-9-]{1,31}$/;
+
+const KNOWN_METADATA_PROVIDER_KEYS: ReadonlySet<string> = new Set(Object.values(MetadataProviderKey));
+const BUILT_IN_METADATA_PROVIDER_KEYS: ReadonlySet<string> = new Set(
+  Object.values(MetadataProviderKey).filter((key) => !(key in MOVED_METADATA_PROVIDERS)),
+);
+
+/**
+ * Keys an external provider may not take: every built-in, the author and cover-search sources, and
+ * pseudo-providers the clients use for themselves (the iOS editor files auto-fill results under
+ * `autofill`, Book Dock uses `auto`, the match view `fetched`).
+ */
+export const RESERVED_METADATA_PROVIDER_KEYS: readonly string[] = [
+  ...BUILT_IN_METADATA_PROVIDER_KEYS,
+  "openlibrary",
+  "googlebooks",
+  "duckduckgo",
+  "any",
+  "all",
+  "none",
+  "auto",
+  "autofill",
+  "fetched",
+  "file",
+  "manual",
+  "custom",
+  "unknown",
+  "builtin",
+  "external",
+];
+
+const RESERVED_KEY_SET: ReadonlySet<string> = new Set(RESERVED_METADATA_PROVIDER_KEYS.map((key) => key.toLowerCase()));
+
+export function isBuiltInMetadataProviderKey(key: string): key is BuiltInMetadataProviderKey {
+  return BUILT_IN_METADATA_PROVIDER_KEYS.has(key);
+}
+
+/** Whether the key's ids live in their own column and API field, rather than with other external ids. */
+export function isKnownMetadataProviderKey(key: string): key is KnownMetadataProviderKey {
+  return KNOWN_METADATA_PROVIDER_KEYS.has(key);
+}
+
+export function isMovedMetadataProviderKey(key: string): key is MovedMetadataProviderKey {
+  return Object.prototype.hasOwnProperty.call(MOVED_METADATA_PROVIDERS, key);
+}
+
+/** Syntax only: whether `key` could name an external provider. Registration also checks collisions. */
+export function isExternalMetadataProviderKeySyntax(key: string): boolean {
+  return EXTERNAL_METADATA_PROVIDER_KEY_PATTERN.test(key) && !RESERVED_KEY_SET.has(key);
+}
+
+/** Whether `key` is a built-in or could name an external provider. Says nothing about registration. */
+export function isMetadataProviderKeySyntax(key: unknown): key is MetadataProviderKey {
+  return typeof key === "string" && (isBuiltInMetadataProviderKey(key) || isExternalMetadataProviderKeySyntax(key));
+}
 
 export interface MetadataSeriesMembership {
   seriesName: string;
@@ -125,6 +238,11 @@ export interface MetadataCandidate {
   comicMetadata?: ComicMetadataFields;
   communityRating?: number;
   communityRatingCount?: number;
+  /**
+   * The medium the provider says this record is, where it says: a light novel and its manga share a
+   * title, and this tells them apart. Absent from older servers and from providers that do not say.
+   */
+  mediaKind?: ConcreteBookMediaKind;
 }
 
 /**
@@ -133,10 +251,21 @@ export interface MetadataCandidate {
  */
 export type MetadataSearchCandidateEvent = MetadataCandidate & { cache?: MetadataCandidateCacheInfo };
 
+/** Whether a provider is compiled into BookOrbit or registered at runtime over HTTP. */
+export type MetadataProviderSource = "builtin" | "external";
+
 export interface MetadataProviderInfo {
   key: MetadataProviderKey;
   label: string;
   identifiable: boolean;
+  /** Absent on servers older than external providers. */
+  source?: MetadataProviderSource;
+  /**
+   * An external provider's icon, served by BookOrbit, and its brand colour (`#rrggbb`). Absent for
+   * built-ins, whose clients bundle their own, and for a provider that declares none.
+   */
+  iconUrl?: string;
+  color?: string;
   selectedByFieldRules?: boolean;
   /** Zero-based priority for the effective Cover field rule. Absent when the provider is not used for covers. */
   coverPriority?: number;

packages/types/src/metadata-http-provider.ts

@@ -0,0 +1,127 @@
+import type { ExternalMetadataProviderKey } from "./metadata-fetch";
+import type { MetadataField } from "./metadata-preferences";
+
+/**
+ * An external metadata provider: an HTTP service an admin registered, speaking the community
+ * book-metadata-provider spec (https://github.com/FlyinPancake/book-metadata-provider) plus the
+ * optional extensions BookOrbit reads. BookOrbit never runs its code; the spec is the only coupling.
+ */
+
+export const METADATA_HTTP_PROVIDER_STATUSES = ["ok", "stale", "unreachable", "key_changed", "incompatible"] as const;
+
+/**
+ * - `ok`: the last `/provider/info` answered and validated.
+ * - `stale`: the last refresh failed; the provider keeps serving with the info it had.
+ * - `unreachable`: several refreshes in a row failed; it is out of searches until one succeeds.
+ * - `key_changed`: the provider now reports a different key; out of searches until re-registered,
+ *   because its stored ids, ratings and field rules are filed under the old key.
+ * - `incompatible`: it answered with info BookOrbit cannot use (no usable key, bad types).
+ */
+export type MetadataHttpProviderStatus = (typeof METADATA_HTTP_PROVIDER_STATUSES)[number];
+
+export type MetadataHttpProviderMediaType = "book" | "audiobook" | "comic";
+
+/** What the provider said about itself, normalized and bounded. Unknown fields are dropped. */
+export interface MetadataHttpProviderInfo {
+  key: ExternalMetadataProviderKey;
+  name: string;
+  version: string | null;
+  specVersion: string;
+  avatarUrl: string | null;
+  authenticationRequired: boolean;
+  mediaTypes: MetadataHttpProviderMediaType[];
+  lookup: boolean;
+  /** Field-rule fields it fills, when it says. Null means "every field the spec maps". */
+  supportedFields: MetadataField[] | null;
+  searchParams: string[];
+  idMaxLength: number;
+  bookUrlTemplate: string | null;
+  rateLimit: { maxConcurrency: number; minIntervalMs: number };
+  coverShape: "portrait" | "square" | null;
+  /** Brand colour as `#rrggbb`. Info stored before colours existed has none. */
+  color?: string | null;
+}
+
+/** Icons BookOrbit downloads from a provider's `avatarUrl` are at most this size. SVG is never accepted. */
+export const METADATA_HTTP_PROVIDER_ICON_MAX_BYTES = 256 * 1024;
+
+/** One registered provider as the settings page sees it. The authorization header is never sent back. */
+export interface MetadataHttpProviderView {
+  id: number;
+  key: ExternalMetadataProviderKey;
+  baseUrl: string;
+  hasAuthorization: boolean;
+  /** A header is saved but no longer decrypts (the encryption key changed), so it is never searched. */
+  authorizationUnreadable: boolean;
+  enabled: boolean;
+  allowPrivateAddress: boolean;
+  timeoutMs: number;
+  status: MetadataHttpProviderStatus;
+  lastError: string | null;
+  info: MetadataHttpProviderInfo | null;
+  /** BookOrbit's own copy of the provider's icon, versioned; null when it declares none or it could not be used. */
+  iconUrl: string | null;
+  infoFetchedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface CreateMetadataHttpProviderPayload {
+  baseUrl: string;
+  /** The full `Authorization` header value, sent as given. */
+  authorization?: string | null;
+  allowPrivateAddress?: boolean;
+  timeoutMs?: number;
+}
+
+export interface UpdateMetadataHttpProviderPayload {
+  baseUrl?: string;
+  /** A string replaces the stored header, null clears it, absent keeps it. */
+  authorization?: string | null;
+  allowPrivateAddress?: boolean;
+  timeoutMs?: number;
+  enabled?: boolean;
+}
+
+export interface MetadataHttpProviderTestResult {
+  ok: boolean;
+  name: string | null;
+  version: string | null;
+  latencyMs: number;
+  resultCount: number | null;
+  sampleTitle: string | null;
+  error: string | null;
+}
+
+export const METADATA_HTTP_PROVIDER_TIMEOUT_MS = { min: 2_000, default: 10_000, max: 30_000 } as const;
+
+/**
+ * A ready-made provider an admin can run beside BookOrbit. The list is read from the
+ * bookorbit-metadata-providers repository, so a provider published after a release still shows up.
+ */
+export interface MetadataProviderDirectoryEntry {
+  /** The key it registers under, and the Compose service name the snippet uses. */
+  key: string;
+  name: string;
+  /** One or two sentences, in English: the directory is published content, not app copy. */
+  summary: string;
+  /** Without a tag, such as `ghcr.io/bookorbit/mangabaka-provider`. */
+  image: string;
+  /** The release to pin in the snippet. */
+  version: string;
+  port: number;
+  docsUrl: string;
+  homepage: string | null;
+  color: string | null;
+  mediaTypes: MetadataHttpProviderMediaType[];
+  /** BCP 47 codes of the languages its books are mostly in; empty when it is not tied to one. */
+  languages: string[];
+}
+
+export interface MetadataProviderDirectory {
+  providers: MetadataProviderDirectoryEntry[];
+  /** `bundled` is the copy shipped with this build: the online list is turned off or unreachable. */
+  source: "remote" | "bundled";
+  /** When the online list was last read; null for the bundled copy. */
+  fetchedAt: string | null;
+}

packages/types/src/metadata-preferences.ts

@@ -1,4 +1,4 @@
-import { MetadataProviderKey } from "./metadata-fetch";
+import { isBuiltInMetadataProviderKey, MetadataProviderKey, type BuiltInMetadataProviderKey, type MetadataProviderSource } from "./metadata-fetch";
 
 export type MetadataField =
   | "title"
@@ -128,11 +128,6 @@ export interface KoboProviderConfig {
   language: string;
 }
 
-export interface AladinProviderConfig {
-  enabled: boolean;
-  ttbKey: string;
-}
-
 export interface ProviderConfigurations {
   google: GoogleProviderConfig;
   amazon: AmazonProviderConfig;
@@ -146,8 +141,6 @@ export interface ProviderConfigurations {
   comicvine: ComicVineProviderConfig;
   ranobedb: SimpleProviderConfig;
   kobo: KoboProviderConfig;
-  lubimyczytac: SimpleProviderConfig;
-  aladin: AladinProviderConfig;
 }
 
 export interface ProviderStatus {
@@ -156,6 +149,18 @@ export interface ProviderStatus {
   configured: boolean;
   enabled: boolean;
   hint?: string;
+  /** Absent on servers older than external providers, which only had built-ins. */
+  source?: MetadataProviderSource;
+  /** An external provider's declared fields; built-ins use `METADATA_PROVIDER_SUPPORTED_FIELDS`. */
+  supportedFields?: MetadataField[] | null;
+  /** An external provider's icon and colour, as on `MetadataProviderInfo`. */
+  iconUrl?: string;
+  color?: string;
+  /**
+   * Set for a provider that used to be built in and is not registered: the image to run, or null when
+   * the source closed and there is none, and whether field rules or stored ids still use it.
+   */
+  movedOut?: { image: string | null; inUse: boolean };
 }
 
 export type ProviderConnectionTestStatus = "success" | "warning" | "fail";
@@ -176,7 +181,7 @@ export interface ProviderConnectionTestResult {
  * The two cover rules list a provider only where its art can be that slot's shape. Audible's art is
  * always square, so it is not under `cover`; it still fills an empty ebook slot as a last resort.
  */
-export const METADATA_PROVIDER_SUPPORTED_FIELDS: Record<MetadataProviderKey, readonly MetadataField[]> = {
+export const METADATA_PROVIDER_SUPPORTED_FIELDS: Record<BuiltInMetadataProviderKey, readonly MetadataField[]> = {
   [MetadataProviderKey.GOOGLE]: [
     "title",
     "subtitle",
@@ -347,41 +352,22 @@ export const METADATA_PROVIDER_SUPPORTED_FIELDS: Record<MetadataProviderKey, rea
     "seriesIndex",
     "genres",
   ],
-  [MetadataProviderKey.LUBIMYCZYTAC]: [
-    "title",
-    "description",
-    "cover",
-    "authors",
-    "publisher",
-    "isbn",
-    "publishedYear",
-    "language",
-    "pageCount",
-    "seriesName",
-    "seriesIndex",
-    "genres",
-  ],
-  [MetadataProviderKey.ALADIN]: [
-    "title",
-    "description",
-    "cover",
-    "authors",
-    "publisher",
-    "isbn",
-    "publishedYear",
-    "language",
-    "pageCount",
-    "seriesName",
-    "genres",
-  ],
 };
 
 /** How a provider stands against one field: it can fill it, its cover art is the other shape, or never. */
 export type ProviderFieldSupport = "supplies" | "otherShape" | "never";
 
-export function providerFieldSupport(provider: MetadataProviderKey, field: MetadataField): ProviderFieldSupport {
-  const fields = METADATA_PROVIDER_SUPPORTED_FIELDS[provider];
-  // An unknown key is stale data, and a hint about it would be a guess.
+/**
+ * `declaredFields` is what an external provider says it fills; built-ins are answered from
+ * `METADATA_PROVIDER_SUPPORTED_FIELDS`.
+ */
+export function providerFieldSupport(
+  provider: MetadataProviderKey,
+  field: MetadataField,
+  declaredFields?: readonly MetadataField[] | null,
+): ProviderFieldSupport {
+  const fields = declaredFields ?? (isBuiltInMetadataProviderKey(provider) ? METADATA_PROVIDER_SUPPORTED_FIELDS[provider] : undefined);
+  // An unknown key, or a provider that did not say, would only get a guess.
   if (!fields || fields.includes(field)) return "supplies";
   if (field === "cover" || field === "audioCover") return "otherShape";
   return "never";