Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: optimize docker image and scanner performance

Browse Source
This commit is contained in:
neon
2026-04-08 20:31:38 -06:00
parent 6de8e8bc60
commit a8b203ef4a
11 changed files with 103 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+7
View File
@@ -6,3 +6,10 @@
client/.env client/.env
server/.env server/.env
docker/postgres/data docker/postgres/data
/data
/books
/book-bucket
/local
/coverage
/test-results
**/*.tsbuildinfo
.env.example
+4 -8
View File
@@ -25,18 +25,14 @@ APP_URL=https://projectx.example.com
# DATABASE_URL=postgres://user:password@yourhost:5432/projectx # DATABASE_URL=postgres://user:password@yourhost:5432/projectx
# Or set individual parts (POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB) # Or set individual parts (POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB)
# JWT token lifetimes
# JWT_EXPIRES_IN=15m
# JWT_REFRESH_EXPIRES_IN=7d
# CORS origin for the frontend when served from a different domain than APP_URL # CORS origin for the frontend when served from a different domain than APP_URL
# CLIENT_URL=https://app.example.com # CLIENT_URL=https://app.example.com
# Book Bucket directory (defaults to {BOOKS_PATH}/book-bucket inside the container) # Recommended if you store SMTP provider credentials (generate: openssl rand -hex 32)
# BOOK_BUCKET_PATH=/book-bucket
# AES-256 key for encrypting stored email credentials (generate: openssl rand -hex 32)
# EMAIL_ENCRYPTION_KEY= # EMAIL_ENCRYPTION_KEY=
# Recommended if you store migration source credentials (generate: openssl rand -hex 32)
# MIGRATION_ENCRYPTION_KEY=
# Log level (default: info). Set to debug for verbose output. # Log level (default: info). Set to debug for verbose output.
# LOG_LEVEL=info # LOG_LEVEL=info
Dockerfile
+11 -7
View File
@@ -1,30 +1,33 @@
ARG NODE_IMAGE=node:24.11-alpine ARG NODE_IMAGE=node:24.11-alpine
# Stage 1: Build client FROM ${NODE_IMAGE} AS base
FROM ${NODE_IMAGE} AS client-builder
RUN npm install -g pnpm@10 RUN npm install -g pnpm@10
# Stage 1: Build client
FROM base AS client-builder
WORKDIR /app WORKDIR /app
COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./
COPY packages/ packages/ COPY packages/types/package.json ./packages/types/
COPY client/package.json ./client/ COPY client/package.json ./client/
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \ RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
pnpm install --filter client... --frozen-lockfile pnpm install --filter client... --frozen-lockfile
COPY packages/ ./packages/
COPY client/ ./client/ COPY client/ ./client/
RUN pnpm --filter client run build-only RUN pnpm --filter client run build-only
# Stage 2: Build server + create deploy bundle # Stage 2: Build server + create deploy bundle
FROM ${NODE_IMAGE} AS server-builder FROM base AS server-builder
RUN npm install -g pnpm@10
WORKDIR /app WORKDIR /app
COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./
COPY packages/ packages/ COPY packages/types/package.json ./packages/types/
COPY server/package.json ./server/ COPY server/package.json ./server/
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \ RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
pnpm install --filter server... --frozen-lockfile pnpm install --filter server... --frozen-lockfile
COPY packages/ ./packages/
COPY server/ ./server/ COPY server/ ./server/
RUN pnpm --filter server run build RUN pnpm --filter server run build
@@ -38,6 +41,7 @@ FROM ${NODE_IMAGE} AS runtime
WORKDIR /app WORKDIR /app
RUN apk upgrade --no-cache && \ RUN apk upgrade --no-cache && \
apk add --no-cache poppler-utils && \
rm -rf /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx rm -rf /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx
ENV NODE_ENV=production ENV NODE_ENV=production
@@ -56,4 +60,4 @@ EXPOSE 3000
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
CMD node -e "const p=process.env.PORT||3000;fetch('http://127.0.0.1:'+p+'/api/v1/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" CMD node -e "const p=process.env.PORT||3000;fetch('http://127.0.0.1:'+p+'/api/v1/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"
CMD ["sh", "entrypoint.sh"] CMD ["sh", "/app/entrypoint.sh"]
README.md
+19 -27
View File
@@ -69,14 +69,11 @@ projectx/
│ └── types/ Shared TypeScript types (@projectx/types) │ └── types/ Shared TypeScript types (@projectx/types)
├── docker/ ├── docker/
│ └── postgres/ Postgres init scripts (extensions) │ └── postgres/ Postgres init scripts (extensions)
├── docs/
│ └── production.md Production deploy/backup/restore runbook
├── local/ Local dev data (covers, docs) - gitignored ├── local/ Local dev data (covers, docs) - gitignored
├── docker-compose.yml Dev: runs Postgres only (app runs natively) ├── docker-compose.dev.yml Dev: runs Postgres only (app runs natively)
├── docker-compose.prod.yml Prod: app + Postgres + migration job ├── docker-compose.yml Prod: app + Postgres
├── .env.prod.example Template env file for production compose ├── .env.example Template env file for production compose
├── Dockerfile Production multi-stage build └── Dockerfile Production multi-stage build
└── Dockerfile.dev Runs entire stack in Docker (not used for normal dev)
``` ```
### Monorepo layout ### Monorepo layout
@@ -212,39 +209,34 @@ The fastest way to learn the patterns is to follow an existing module:
## Environment Variables ## Environment Variables
Server environment is configured in `server/.env` (created from `.env.example` during setup). Server environment is configured in `server/.env` for local development. Production compose uses the repo-root `.env` created from `.env.example`.
| Variable | Default | Description | | Variable | Default | Description |
| ------------------------ | ------------------------------------------------------ | ---------------------------------------------------------- | | ----------------------- | ------------------------------------------------------ | ---------------------------------------------------------- |
| `DATABASE_URL` | `postgres://projectx:projectx@localhost:5432/projectx` | PostgreSQL connection string | | `DATABASE_URL` | `postgres://projectx:projectx@localhost:5432/projectx` | PostgreSQL connection string |
| `PORT` | `3000` | Server port | | `PORT` | `3000` | Server port |
| `NODE_ENV` | `development` | Environment mode | | `NODE_ENV` | `development` | Environment mode |
| `JWT_SECRET` | `change-me-in-production` | JWT signing secret | | `JWT_SECRET` | `change-me-in-production` | JWT signing secret |
| `JWT_EXPIRES_IN` | `15m` | Access token lifetime | | `SETUP_BOOTSTRAP_TOKEN` | (empty) | Required in production for initial `/api/v1/auth/setup` |
| `JWT_REFRESH_EXPIRES_IN` | `7d` | Refresh token lifetime | | `BOOKS_PATH` | `../local/data` | App data directory for cover images (not where books live) |
| `SETUP_BOOTSTRAP_TOKEN` | (empty) | Required in production for initial `/api/v1/auth/setup` | | `APP_URL` | `http://localhost:5173` | Client URL (used in emails) |
| `BOOKS_PATH` | `../local/data` | App data directory for cover images (not where books live) |
| `SMTP_*` | (empty) | Optional SMTP config for password-reset emails |
| `APP_URL` | `http://localhost:5173` | Client URL (used in emails) |
## Production Deploy ## Production Deploy
Production uses a separate compose file and env file: Production uses the root compose file and env file:
- Compose: `docker-compose.prod.yml` - Compose: `docker-compose.yml`
- Env file: `.env.prod` (from `.env.prod.example`) - Env file: `.env` (from `.env.example`)
- Image source: immutable images built in CI (`.github/workflows/container-image.yml`) - Image source: immutable images built in CI (`.github/workflows/container-image.yml`)
Quick start: Quick start:
```bash ```bash
cp .env.prod.example .env.prod cp .env.example .env
# edit .env.prod and set APP_IMAGE + secrets # edit .env and set APP_IMAGE + secrets
pnpm prod:up pnpm prod:up
``` ```
Detailed runbook (deploy, backup, restore, restore drills): [docs/production.md](docs/production.md)
--- ---
## Database Workflow ## Database Workflow
client/public/assets/foliate/paginator.js
+1
View File
@@ -292,6 +292,7 @@ class View {
} }
render(layout) { render(layout) {
if (!layout) return if (!layout) return
if (!this.document) return
this.#column = layout.flow !== 'scrolled' this.#column = layout.flow !== 'scrolled'
this.#layout = layout this.#layout = layout
if (this.#column) this.columnize(layout) if (this.#column) this.columnize(layout)
docker-compose.dev.yml
+7 -5
View File
@@ -2,20 +2,22 @@ name: projectx-dev
services: services:
postgres: postgres:
container_name: projectx-dev-db
image: pgvector/pgvector:pg16 image: pgvector/pgvector:pg16
restart: unless-stopped restart: unless-stopped
environment: environment:
POSTGRES_USER: ${POSTGRES_USER:-projectx} POSTGRES_USER: ${DEV_POSTGRES_USER:-projectx}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-projectx} POSTGRES_PASSWORD: ${DEV_POSTGRES_PASSWORD:-projectx}
POSTGRES_DB: ${POSTGRES_DB:-projectx} POSTGRES_DB: ${DEV_POSTGRES_DB:-projectx}
POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_HOST_AUTH_METHOD: trust
ports: ports:
- "5432:5432" - "${DEV_POSTGRES_PORT:-5432}:5432"
volumes: volumes:
- postgres_data:/var/lib/postgresql/data - postgres_data:/var/lib/postgresql/data
- ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql - ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U projectx"] test:
["CMD-SHELL", 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"']
interval: 5s interval: 5s
timeout: 5s timeout: 5s
retries: 5 retries: 5
docker-compose.yml
+30 -23
View File
@@ -1,46 +1,35 @@
name: projectx name: projectx
services: services:
postgres:
image: pgvector/pgvector:pg16@sha256:7d400e340efb42f4d8c9c12c6427adb253f726881a9985d2a471bf0eed824dff
restart: unless-stopped
environment:
POSTGRES_USER: ${POSTGRES_USER?required}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?required}
POSTGRES_DB: ${POSTGRES_DB?required}
volumes:
- ./data/postgres:/var/lib/postgresql/data
healthcheck:
test:
[
"CMD-SHELL",
"pg_isready -U ${POSTGRES_USER?required} -d ${POSTGRES_DB?required}",
]
interval: 10s
timeout: 5s
retries: 10
start_period: 20s
app: app:
container_name: projectx-app
image: ${APP_IMAGE?required} image: ${APP_IMAGE?required}
restart: unless-stopped restart: unless-stopped
init: true init: true
env_file:
- .env
ports: ports:
- "${APP_PORT:-3000}:3000" - "${APP_PORT:-3000}:${PORT:-3000}"
environment: environment:
NODE_ENV: production
PORT: ${PORT:-3000}
DATABASE_URL: ${DATABASE_URL:-}
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
POSTGRES_USER: ${POSTGRES_USER?required} POSTGRES_USER: ${POSTGRES_USER?required}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?required} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?required}
POSTGRES_DB: ${POSTGRES_DB?required} POSTGRES_DB: ${POSTGRES_DB?required}
JWT_SECRET: ${JWT_SECRET?required} JWT_SECRET: ${JWT_SECRET?required}
SETUP_BOOTSTRAP_TOKEN: ${SETUP_BOOTSTRAP_TOKEN?required} SETUP_BOOTSTRAP_TOKEN: ${SETUP_BOOTSTRAP_TOKEN?required}
APP_URL: ${APP_URL?required} APP_URL: ${APP_URL?required}
CLIENT_URL: ${CLIENT_URL:-${APP_URL?required}}
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy
volumes: volumes:
- ./books:/books - ./books:/books
- ./book-bucket:/book-bucket - ./data/app:/data
- ./data:/data - ./book-bucket:/data/book-bucket
read_only: true read_only: true
tmpfs: tmpfs:
- /tmp - /tmp
@@ -59,3 +48,21 @@ services:
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 20s start_period: 20s
postgres:
container_name: projectx-db
image: pgvector/pgvector:pg16@sha256:7d400e340efb42f4d8c9c12c6427adb253f726881a9985d2a471bf0eed824dff
restart: unless-stopped
environment:
POSTGRES_USER: ${POSTGRES_USER?required}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?required}
POSTGRES_DB: ${POSTGRES_DB?required}
volumes:
- ./data/postgres:/var/lib/postgresql/data
healthcheck:
test:
["CMD-SHELL", 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"']
interval: 10s
timeout: 5s
retries: 10
start_period: 20s
server/entrypoint.sh
+1 -1
View File
@@ -9,4 +9,4 @@ export BOOKS_PATH="${BOOKS_PATH:-/data}"
export BOOK_BUCKET_PATH="${BOOK_BUCKET_PATH:-/book-bucket}" export BOOK_BUCKET_PATH="${BOOK_BUCKET_PATH:-/book-bucket}"
node dist/scripts/migrate.js node dist/scripts/migrate.js
exec node dist/main.js exec node --max-old-space-size=512 dist/main.js
server/package.json
+7 -8
View File
@@ -37,8 +37,7 @@
"@fastify/cookie": "^11.0.2", "@fastify/cookie": "^11.0.2",
"@fastify/formbody": "^8.0.2", "@fastify/formbody": "^8.0.2",
"@fastify/helmet": "^13.0.2", "@fastify/helmet": "^13.0.2",
"@fastify/multipart": "^9.4.0", "@fastify/multipart": "^10.0.0",
"@fastify/rate-limit": "^10.3.0",
"@fastify/static": "^9.1.0", "@fastify/static": "^9.1.0",
"@nestjs/common": "^11.1.18", "@nestjs/common": "^11.1.18",
"@nestjs/config": "^4.0.3", "@nestjs/config": "^4.0.3",
@@ -59,7 +58,7 @@
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
"class-validator": "^0.15.1", "class-validator": "^0.15.1",
"drizzle-orm": "^0.45.2", "drizzle-orm": "^0.45.2",
"fast-xml-parser": "^5.5.10", "fast-xml-parser": "^5.5.11",
"fastest-levenshtein": "^1.0.16", "fastest-levenshtein": "^1.0.16",
"fastify": "^5.8.4", "fastify": "^5.8.4",
"jose": "^6.2.2", "jose": "^6.2.2",
@@ -69,7 +68,7 @@
"mysql2": "^3.20.0", "mysql2": "^3.20.0",
"nestjs-pino": "^4.6.1", "nestjs-pino": "^4.6.1",
"node-unrar-js": "^2.0.2", "node-unrar-js": "^2.0.2",
"nodemailer": "^8.0.4", "nodemailer": "^8.0.5",
"passport": "^0.7.0", "passport": "^0.7.0",
"passport-jwt": "^4.0.1", "passport-jwt": "^4.0.1",
"pdf-lib": "^1.17.1", "pdf-lib": "^1.17.1",
@@ -85,7 +84,7 @@
"devDependencies": { "devDependencies": {
"@eslint/eslintrc": "^3.3.5", "@eslint/eslintrc": "^3.3.5",
"@eslint/js": "^10.0.1", "@eslint/js": "^10.0.1",
"@nestjs/cli": "^11.0.17", "@nestjs/cli": "^11.0.18",
"@nestjs/schematics": "^11.0.10", "@nestjs/schematics": "^11.0.10",
"@nestjs/testing": "^11.1.18", "@nestjs/testing": "^11.1.18",
"@swc/cli": "^0.8.1", "@swc/cli": "^0.8.1",
@@ -98,7 +97,7 @@
"@types/pg": "^8.18.0", "@types/pg": "^8.18.0",
"@types/supertest": "^7.2.0", "@types/supertest": "^7.2.0",
"@types/unzipper": "^0.10.11", "@types/unzipper": "^0.10.11",
"@vitest/coverage-v8": "^4.1.2", "@vitest/coverage-v8": "^4.1.3",
"drizzle-kit": "^0.31.10", "drizzle-kit": "^0.31.10",
"eslint": "^10.2.0", "eslint": "^10.2.0",
"eslint-config-prettier": "^10.1.8", "eslint-config-prettier": "^10.1.8",
@@ -113,7 +112,7 @@
"tsconfig-paths": "^4.2.0", "tsconfig-paths": "^4.2.0",
"tsx": "^4.21.0", "tsx": "^4.21.0",
"typescript": "^5.9.3", "typescript": "^5.9.3",
"typescript-eslint": "^8.58.0", "typescript-eslint": "^8.58.1",
"vitest": "^4.1.2" "vitest": "^4.1.3"
} }
} }
server/src/main.ts
+15 -17
View File
@@ -9,7 +9,6 @@ import { join } from 'path';
import { Readable } from 'stream'; import { Readable } from 'stream';
import fastifyCookie from '@fastify/cookie'; import fastifyCookie from '@fastify/cookie';
import fastifyMultipart from '@fastify/multipart'; import fastifyMultipart from '@fastify/multipart';
import fastifyRateLimit from '@fastify/rate-limit';
import fastifyStatic from '@fastify/static'; import fastifyStatic from '@fastify/static';
import fastifyHelmet from '@fastify/helmet'; import fastifyHelmet from '@fastify/helmet';
import fastifyCompress from '@fastify/compress'; import fastifyCompress from '@fastify/compress';
@@ -69,11 +68,12 @@ async function bootstrap() {
directives: { directives: {
defaultSrc: ["'self'"], defaultSrc: ["'self'"],
scriptSrc: ["'self'", "'unsafe-inline'", "'unsafe-eval'"], scriptSrc: ["'self'", "'unsafe-inline'", "'unsafe-eval'"],
styleSrc: ["'self'", "'unsafe-inline'"], styleSrc: ["'self'", "'unsafe-inline'", 'blob:'],
imgSrc: ["'self'", 'data:', 'blob:'], imgSrc: ["'self'", 'data:', 'blob:'],
connectSrc: ["'self'", 'ws:', 'wss:'], connectSrc: ["'self'", 'ws:', 'wss:'],
fontSrc: ["'self'", 'data:'], fontSrc: ["'self'", 'data:', 'blob:'],
objectSrc: ["'none'"], objectSrc: ["'none'"],
frameSrc: ["'self'", 'blob:'],
frameAncestors: ["'none'"], frameAncestors: ["'none'"],
}, },
}, },
@@ -84,19 +84,6 @@ async function bootstrap() {
await app.register(fastifyCookie as never); await app.register(fastifyCookie as never);
await app.register(fastifyMultipart as never, { limits: { fileSize: MAX_COVER_BYTES } }); await app.register(fastifyMultipart as never, { limits: { fileSize: MAX_COVER_BYTES } });
// Rate limit unauthenticated requests only (brute-force protection on public endpoints).
// Bypass for: JWT cookie (web app), Authorization header (OPDS Basic Auth),
// and Kobo device endpoints (token-authenticated via URL).
await app.register(fastifyRateLimit as never, {
max: 100,
timeWindow: '1 minute',
allowList: (req: { cookies?: { access_token?: string }; headers?: { authorization?: string }; url?: string }) =>
!!req.cookies?.access_token ||
!!req.headers?.authorization ||
/^\/api\/v1\/kobo\//.test(req.url ?? '') ||
/^\/api\/v1\/epub\/\d+\/file\//.test(req.url ?? ''),
});
if (process.env.NODE_ENV !== 'production') { if (process.env.NODE_ENV !== 'production') {
app.enableCors({ app.enableCors({
origin: process.env.CLIENT_URL ?? 'http://localhost:5173', origin: process.env.CLIENT_URL ?? 'http://localhost:5173',
@@ -105,10 +92,21 @@ async function bootstrap() {
} }
if (process.env.NODE_ENV === 'production') { if (process.env.NODE_ENV === 'production') {
// NestJS calls setNotFoundHandler during init — intercept it so non-API 404s
// fall back to index.html instead of returning a JSON 404 (SPA routing support).
const adapterAny = adapter as any;
adapterAny.setNotFoundHandler = (nestHandler: (req: unknown, res: unknown) => void) => {
fastify.setNotFoundHandler(async (request, reply) => {
if (request.url.startsWith('/api')) {
return nestHandler(request, reply);
}
return reply.sendFile('index.html');
});
};
await app.register(fastifyStatic as never, { await app.register(fastifyStatic as never, {
root: join(__dirname, '..', 'public'), root: join(__dirname, '..', 'public'),
prefix: '/', prefix: '/',
decorateReply: false,
}); });
} }
server/src/modules/scanner/scanner.service.ts
+1 -1
View File
@@ -16,7 +16,7 @@ import { ScannerRepository } from './scanner.repository';
import { assembleBookCards } from '../book/utils/assemble-book-cards'; import { assembleBookCards } from '../book/utils/assemble-book-cards';
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'fb2', 'pdf', 'm4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac']); const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'fb2', 'pdf', 'm4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac']);
const BATCH_SIZE = 5; const BATCH_SIZE = 3;
const BOOK_EMIT_BUFFER_SIZE = 30; const BOOK_EMIT_BUFFER_SIZE = 30;
const BOOK_EMIT_FLUSH_INTERVAL_MS = 1500; const BOOK_EMIT_FLUSH_INTERVAL_MS = 1500;
type OrganizationMode = 'book_per_file' | 'book_per_folder'; type OrganizationMode = 'book_per_file' | 'book_per_folder';