ci: make crowdin source synchronization deterministic

Upload and verify the checked-out English catalog before exporting translations so delayed webhooks cannot stall the scheduled workflow.

.github/workflows/crowdin-source-sync.yml

@@ -0,0 +1,55 @@
+name: Crowdin Source Sync
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - "client/src/locales/en.json"
+      - "crowdin.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: crowdin-synchronization
+  cancel-in-progress: false
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+  NODE_VERSION: "26"
+  PNPM_VERSION: "11"
+
+jobs:
+  synchronize:
+    name: Synchronize English source
+    if: github.repository == 'bookorbit/bookorbit'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+
+      - name: Set up Node.js and pnpm
+        uses: ./.github/actions/setup-node-pnpm
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          pnpm-version: ${{ env.PNPM_VERSION }}
+          prefetch: "true"
+
+      - name: Check synchronization credentials
+        env:
+          CROWDIN_TOKEN: ${{ secrets.CROWDIN_TOKEN }}
+        run: |
+          if [ -z "$CROWDIN_TOKEN" ]; then
+            echo "::error::CROWDIN_TOKEN is required."
+            exit 1
+          fi
+
+      - name: Synchronize English source
+        env:
+          CROWDIN_PROJECT_ID: ${{ vars.CROWDIN_PROJECT_ID }}
+          CROWDIN_TOKEN: ${{ secrets.CROWDIN_TOKEN }}
+        run: pnpm --filter client sync:crowdin-source

.github/workflows/crowdin-translations.yml

@@ -14,7 +14,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: crowdin-translation-sync
+  group: crowdin-synchronization
   cancel-in-progress: false
 
 env:

client/package.json

@@ -30,6 +30,7 @@
     "lint:oxlint:fix": "oxlint src --fix",
     "lint:eslint:check": "eslint src --cache",
     "lint:eslint:fix": "eslint src --fix --cache",
+    "sync:crowdin-source": "node ./scripts/sync-crowdin-source.mjs",
     "sync:crowdin": "node ./scripts/sync-crowdin-translations.mjs",
     "validate:locales": "node ./scripts/validate-locales.mjs",
     "validate:styles": "node ./scripts/validate-styles.mjs",

client/scripts/sync-crowdin-source.mjs

@@ -0,0 +1,9 @@
+import { syncCrowdinSource } from './sync-crowdin-translations.mjs'
+
+syncCrowdinSource({
+  token: process.env.CROWDIN_TOKEN,
+  projectId: process.env.CROWDIN_PROJECT_ID || undefined,
+}).catch((error) => {
+  console.error(error.message)
+  process.exitCode = 1
+})

client/scripts/sync-crowdin-translations.mjs

@@ -14,6 +14,7 @@ const SOURCE_PATH_SUFFIX = '/client/src/locales/en.json'
 const MAX_API_RESPONSE_BYTES = 2 * 1024 * 1024
 const MAX_CATALOG_BYTES = 10 * 1024 * 1024
 const REQUEST_TIMEOUT_MS = 30_000
+const SOURCE_SYNC_RETRY_DELAYS_MS = [0, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000]
 const scriptDirectory = import.meta.dirname ?? path.join(process.cwd(), 'scripts')
 const clientRoot = path.resolve(scriptDirectory, '..')
 const localesDirectory = path.join(clientRoot, 'src/locales')
@@ -175,14 +176,29 @@ export function createCrowdinClient({ token, projectId, fetchImpl = fetch }) {
       throw new Error(`Crowdin source file ending in ${SOURCE_PATH_SUFFIX} was not found`)
     },
 
-    async sourceIdentifiers(fileId) {
+    async sourceCatalog(fileId) {
-      const identifiers = new Set()
+      const download = await request(`/projects/${projectId}/files/${fileId}/download`)
-      for (let offset = 0; offset < 20_000; offset += 500) {
+      return downloadCatalog(fetchImpl, download.data.url)
-        const page = await request(`/projects/${projectId}/strings?fileId=${fileId}&limit=500&offset=${offset}`)
+    },
-        for (const entry of page.data) identifiers.add(entry.data.identifier)
+
-        if (page.data.length < 500) return identifiers
+    async uploadSource(filename, content) {
-      }
+      const storage = await request('/storages', {
-      throw new Error('Crowdin source contains more than 20,000 messages')
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Crowdin-API-FileName': filename,
+        },
+        body: content,
+      })
+      return storage.data.id
+    },
+
+    async updateSourceFile(fileId, storageId) {
+      await request(`/projects/${projectId}/files/${fileId}`, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ storageId, updateOption: 'keep_translations' }),
+      })
     },
 
     async exportedCatalog(fileId, languageId) {
@@ -242,10 +258,13 @@ async function mapWithConcurrency(values, concurrency, operation) {
   return results
 }
 
-export function sourceDrift(referenceMessages, identifiers) {
+export function sourceDrift(referenceMessages, crowdinMessages) {
-  const missing = [...referenceMessages.keys()].filter((key) => !identifiers.has(key))
+  const missing = [...referenceMessages.keys()].filter((key) => !crowdinMessages.has(key))
-  const unexpected = [...identifiers].filter((key) => !referenceMessages.has(key))
+  const unexpected = [...crowdinMessages.keys()].filter((key) => !referenceMessages.has(key))
-  return { missing, unexpected }
+  const changed = [...referenceMessages]
+    .filter(([key, message]) => crowdinMessages.has(key) && crowdinMessages.get(key) !== message)
+    .map(([key]) => key)
+  return { missing, unexpected, changed }
 }
 
 export function parseAllowedTranslationLosses(value = '') {
@@ -401,6 +420,70 @@ async function reportSyncIssues({ rejections, corrections, repairs, losses, repo
   }
 }
 
+function hasSourceDrift({ missing, unexpected, changed }) {
+  return missing.length > 0 || unexpected.length > 0 || changed.length > 0
+}
+
+function formatSourceDrift(drift) {
+  return [
+    ...drift.missing.slice(0, 10).map((key) => `missing in Crowdin: ${key}`),
+    ...drift.unexpected.slice(0, 10).map((key) => `missing in Git: ${key}`),
+    ...drift.changed.slice(0, 10).map((key) => `different source text: ${key}`),
+  ].join('\n')
+}
+
+async function prepareCrowdinSource({ token, projectId, fetchImpl, catalogDirectory, assertTargetConfiguration, wait }) {
+  if (!token) throw new Error('CROWDIN_TOKEN is required')
+  await assertTargetConfiguration()
+
+  const sourcePath = path.join(catalogDirectory, 'en.json')
+  const sourceContent = await readFile(sourcePath, 'utf8')
+  const reference = JSON.parse(sourceContent)
+  const referenceMessages = flattenCatalog(reference)
+  const client = createCrowdinClient({ token, projectId, fetchImpl })
+  const fileId = await client.sourceFileId()
+  let drift = sourceDrift(referenceMessages, flattenCatalog(await client.sourceCatalog(fileId)))
+  let updated = false
+
+  if (hasSourceDrift(drift)) {
+    const storageId = await client.uploadSource(path.basename(sourcePath), sourceContent)
+    await client.updateSourceFile(fileId, storageId)
+    updated = true
+
+    for (const delayMs of SOURCE_SYNC_RETRY_DELAYS_MS) {
+      if (delayMs > 0) await wait(delayMs)
+      drift = sourceDrift(referenceMessages, flattenCatalog(await client.sourceCatalog(fileId)))
+      if (!hasSourceDrift(drift)) break
+    }
+  }
+
+  if (hasSourceDrift(drift)) {
+    throw new Error(`Crowdin source did not match en.json after update\n${formatSourceDrift(drift)}`)
+  }
+
+  return { client, fileId, reference, referenceMessages, updated }
+}
+
+export async function syncCrowdinSource({
+  token,
+  projectId = '912891',
+  fetchImpl = fetch,
+  catalogDirectory = localesDirectory,
+  assertTargetConfiguration = assertCrowdinTargetConfiguration,
+  wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)),
+}) {
+  const { fileId, referenceMessages, updated } = await prepareCrowdinSource({
+    token,
+    projectId,
+    fetchImpl,
+    catalogDirectory,
+    assertTargetConfiguration,
+    wait,
+  })
+  console.log(`${updated ? 'Synchronized' : 'Verified'} ${referenceMessages.size} English source messages in Crowdin`)
+  return { fileId, messageCount: referenceMessages.size, updated }
+}
+
 export async function syncCrowdinTranslations({
   token,
   projectId = '912891',
@@ -413,12 +496,17 @@ export async function syncCrowdinTranslations({
   collectMessageKeys = collectSourceMessageKeys,
   protectedTerms = PROTECTED_SOURCE_TERMS,
   reportPath = process.env.CROWDIN_REJECTION_REPORT || '',
+  wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)),
 }) {
-  if (!token) throw new Error('CROWDIN_TOKEN is required')
+  const { client, fileId, reference, referenceMessages, updated } = await prepareCrowdinSource({
-  await assertTargetConfiguration()
+    token,
-
+    projectId,
-  const reference = JSON.parse(await readFile(path.join(catalogDirectory, 'en.json'), 'utf8'))
+    fetchImpl,
-  const referenceMessages = flattenCatalog(reference)
+    catalogDirectory,
+    assertTargetConfiguration,
+    wait,
+  })
+  console.log(`${updated ? 'Synchronized' : 'Verified'} ${referenceMessages.size} English source messages in Crowdin`)
   const currentCatalogs = new Map(
     await Promise.all(
       targetCatalogs.map(async ({ locale }) => {
@@ -427,17 +515,6 @@ export async function syncCrowdinTranslations({
       }),
     ),
   )
-  const client = createCrowdinClient({ token, projectId, fetchImpl })
-  const fileId = await client.sourceFileId()
-  const identifiers = await client.sourceIdentifiers(fileId)
-  const drift = sourceDrift(referenceMessages, identifiers)
-  if (drift.missing.length > 0 || drift.unexpected.length > 0) {
-    const details = [
-      ...drift.missing.slice(0, 10).map((key) => `missing in Crowdin: ${key}`),
-      ...drift.unexpected.slice(0, 10).map((key) => `missing in Git: ${key}`),
-    ]
-    throw new Error(`Crowdin source is not synchronized with en.json\n${details.join('\n')}`)
-  }
 
   const downloaded = await mapWithConcurrency(targetCatalogs, 4, async ({ languageId, locale }) => ({
     locale,

client/scripts/sync-crowdin-translations.spec.mjs

@@ -16,6 +16,7 @@ import {
   parseAllowedTranslationLosses,
   retentionLossLimit,
   sourceDrift,
+  syncCrowdinSource,
   syncCrowdinTranslations,
 } from './sync-crowdin-translations.mjs'
 
@@ -43,20 +44,38 @@ async function createCatalogFixture(targetCatalogs, currentCatalogs = new Map(),
   return directory
 }
 
-function createSynchronizationFetch({ identifiers = ['common.save'], catalogs = new Map(), onDownload = async () => {} } = {}) {
+function createSynchronizationFetch({
+  sourceCatalog = { common: { save: 'Save' } },
+  catalogs = new Map(),
+  applySourceUpdate = true,
+  onDownload = async () => {},
+} = {}) {
+  let currentSourceCatalog = sourceCatalog
+  let uploadedSourceCatalog
+
   return vi.fn(async (input, init = {}) => {
     const url = new URL(input)
-    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files')) {
+    const method = init.method ?? 'GET'
+    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files') && method === 'GET') {
       return new Response(JSON.stringify({ data: [{ data: { id: 7, path: '/client/src/locales/en.json' } }] }))
     }
-    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/strings')) {
+    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files/7/download')) {
-      return new Response(JSON.stringify({ data: identifiers.map((identifier) => ({ data: { identifier } })) }))
+      return new Response(JSON.stringify({ data: { url: 'https://downloads.example.test/source.json' } }))
+    }
+    if (url.hostname === 'api.crowdin.com' && url.pathname === '/api/v2/storages' && method === 'POST') {
+      uploadedSourceCatalog = JSON.parse(init.body)
+      return new Response(JSON.stringify({ data: { id: 9 } }))
+    }
+    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files/7') && method === 'PUT') {
+      if (applySourceUpdate) currentSourceCatalog = uploadedSourceCatalog
+      return new Response(JSON.stringify({ data: { id: 7 } }))
     }
     if (url.hostname === 'api.crowdin.com' && url.pathname.includes('/translations/builds/files/')) {
       const { targetLanguageId } = JSON.parse(init.body)
       return new Response(JSON.stringify({ data: { url: `https://downloads.example.test/${encodeURIComponent(targetLanguageId)}.json` } }))
     }
     if (url.hostname === 'downloads.example.test') {
+      if (url.pathname === '/source.json') return new Response(JSON.stringify(currentSourceCatalog))
       const languageId = decodeURIComponent(path.basename(url.pathname, '.json'))
       await onDownload(languageId)
       return new Response(JSON.stringify(catalogs.get(languageId) ?? { common: { save: `Translated ${languageId}` } }))
@@ -82,9 +101,18 @@ describe('Crowdin translation synchronization', () => {
   })
 
   it('detects when Crowdin has not synchronized the current English keys', () => {
-    expect(sourceDrift(new Map([['common.save', 'Save']]), new Set(['common.cancel']))).toEqual({
+    expect(sourceDrift(new Map([['common.save', 'Save']]), new Map([['common.cancel', 'Cancel']]))).toEqual({
       missing: ['common.save'],
       unexpected: ['common.cancel'],
+      changed: [],
+    })
+  })
+
+  it('detects changed English source text under an existing key', () => {
+    expect(sourceDrift(new Map([['common.save', 'Save now']]), new Map([['common.save', 'Save']]))).toEqual({
+      missing: [],
+      unexpected: [],
+      changed: ['common.save'],
     })
   })
 
@@ -231,6 +259,45 @@ describe('Crowdin translation synchronization', () => {
     })
   })
 
+  it('leaves an identical Crowdin source file unchanged', async () => {
+    const catalogDirectory = await createCatalogFixture([], new Map(), { common: { save: 'Save' } })
+    const fetchImpl = createSynchronizationFetch()
+
+    await expect(
+      syncCrowdinSource({
+        token: 'secret',
+        fetchImpl,
+        catalogDirectory,
+        assertTargetConfiguration: async () => {},
+      }),
+    ).resolves.toEqual({ fileId: 7, messageCount: 1, updated: false })
+    expect(fetchImpl.mock.calls.some(([, init]) => ['POST', 'PUT'].includes(init.method))).toBe(false)
+  })
+
+  it('updates changed Crowdin source text without discarding translations', async () => {
+    const catalogDirectory = await createCatalogFixture([], new Map(), { common: { save: 'Save now' } })
+    const fetchImpl = createSynchronizationFetch({ sourceCatalog: { common: { save: 'Save' } } })
+
+    await expect(
+      syncCrowdinSource({
+        token: 'secret',
+        fetchImpl,
+        catalogDirectory,
+        assertTargetConfiguration: async () => {},
+      }),
+    ).resolves.toEqual({ fileId: 7, messageCount: 1, updated: true })
+
+    const storageCall = fetchImpl.mock.calls.find(([input]) => new URL(input).pathname === '/api/v2/storages')
+    expect(storageCall[1].headers['Crowdin-API-FileName']).toBe('en.json')
+    expect(JSON.parse(storageCall[1].body)).toEqual({ common: { save: 'Save now' } })
+
+    const updateCall = fetchImpl.mock.calls.find(([input, init]) => new URL(input).pathname.endsWith('/files/7') && init.method === 'PUT')
+    expect(JSON.parse(updateCall[1].body)).toEqual({
+      storageId: 9,
+      updateOption: 'keep_translations',
+    })
+  })
+
   it('rejects non-HTTPS export URLs returned by Crowdin', () => {
     expect(() => assertSafeDownloadUrl('http://127.0.0.1/catalog.json')).toThrow('Crowdin export URL must use HTTPS')
   })
@@ -322,11 +389,11 @@ describe('Crowdin translation synchronization', () => {
     await expect(readFile(path.join(outputDirectory, 'cs.json'), 'utf8')).resolves.toContain('Translated cs')
   })
 
-  it('rejects source drift before downloading or writing catalogs', async () => {
+  it('rejects source drift that remains after updating Crowdin', async () => {
     const targetCatalogs = TARGET_CATALOGS.slice(0, 2)
     const catalogDirectory = await createCatalogFixture(targetCatalogs)
     const outputDirectory = path.join(catalogDirectory, 'output')
-    const fetchImpl = createSynchronizationFetch({ identifiers: [] })
+    const fetchImpl = createSynchronizationFetch({ sourceCatalog: {}, applySourceUpdate: false })
 
     await expect(
       syncCrowdinTranslations({
@@ -337,9 +404,10 @@ describe('Crowdin translation synchronization', () => {
         targetCatalogs,
         assertTargetConfiguration: async () => {},
         protectedTerms: [],
+        wait: async () => {},
       }),
-    ).rejects.toThrow('Crowdin source is not synchronized with en.json')
+    ).rejects.toThrow('Crowdin source did not match en.json after update')
-    expect(fetchImpl.mock.calls.some(([input]) => new URL(input).hostname === 'downloads.example.test')).toBe(false)
+    expect(fetchImpl.mock.calls.some(([input]) => new URL(input).pathname.includes('/translations/builds/files/'))).toBe(false)
     await expect(access(outputDirectory)).rejects.toThrow()
   })
 
@@ -431,7 +499,7 @@ describe('Crowdin translation synchronization', () => {
 
     const { rejections, repairs } = await syncCrowdinTranslations({
       token: 'secret',
-      fetchImpl: createSynchronizationFetch({ identifiers: ['books.count'], catalogs }),
+      fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
       catalogDirectory,
       outputDirectory,
       targetCatalogs,
@@ -460,7 +528,7 @@ describe('Crowdin translation synchronization', () => {
     await expect(
       syncCrowdinTranslations({
         token: 'secret',
-        fetchImpl: createSynchronizationFetch({ identifiers: keys.map((key) => `common.${key}`), catalogs }),
+        fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
         catalogDirectory,
         outputDirectory,
         targetCatalogs,
@@ -510,7 +578,7 @@ describe('Crowdin translation synchronization', () => {
 
     const { rejections } = await syncCrowdinTranslations({
       token: 'secret',
-      fetchImpl: createSynchronizationFetch({ identifiers: [key], catalogs }),
+      fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
       catalogDirectory,
       outputDirectory,
       targetCatalogs,
@@ -596,7 +664,7 @@ describe('Crowdin translation synchronization', () => {
 
     const { rejections, corrections } = await syncCrowdinTranslations({
       token: 'secret',
-      fetchImpl: createSynchronizationFetch({ identifiers: [key, 'annotations.hub.exportMarkdown'], catalogs }),
+      fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
       catalogDirectory,
       outputDirectory,
       targetCatalogs,

docs/LOCALIZATION.md

@@ -95,9 +95,9 @@ The component only supplies `count`, so its message must not require other argum
 
 After an English source change reaches `main`:
 
-1. Crowdin synchronizes `client/src/locales/en.json`.
+1. The `Crowdin Source Sync` workflow uploads `client/src/locales/en.json`, preserving translations while invalidating approvals for changed source messages, and verifies every source key and value.
 2. Translators update target strings in Crowdin.
-3. The scheduled or manually dispatched `Crowdin Translation Sync` workflow verifies that Crowdin's source keys match `en.json`.
+3. The scheduled or manually dispatched `Crowdin Translation Sync` workflow repeats the source upload and verification before exporting, so a missed push workflow cannot leave the export blocked on stale source data.
 4. The workflow requests only translated strings, removes the empty values Crowdin emits for untranslated nested JSON entries, and validates the resulting sparse catalogs.
 5. Before writing, the workflow normalizes punctuation the catalogs prohibit, omits messages that fail validation, and rejects an export that drops far more translations than translator churn explains.
 6. Only after validation and retention checks pass, the workflow updates `l10n_main` and opens a pull request to `main`.
@@ -242,14 +242,9 @@ Only enable scheduled export for the new language after this manual round trip s
 
 ## Crowdin Project Settings
 
-Use the native GitHub integration in **Source and translation files mode** with only `main` connected. Do not use Target file bundles mode or automatic feature-branch discovery. Leave source synchronization enabled, but disable the integration's scheduled translation synchronization and pull-request creation. The repository workflow owns translation export and delivery.
+The repository workflows own source upload, translation export, and pull-request delivery. Do not use the native GitHub integration or its repository webhook as a second synchronization path. When migrating an existing project, first manually run `Crowdin Source Sync` and verify that it succeeds without losing translations, then disconnect the native integration and remove its webhook.
 
-Initial import settings:
+Source updates use Crowdin's `keep_translations` mode. Existing translations survive an English edit, while their approvals are removed. Prefer a new message key when the English meaning changes so an old translation cannot be mistaken for current copy.
-
-- Import existing translations once.
-- Allow target translations to match the source.
-- Do not continuously import translations from GitHub afterward.
-- Keep Push Sources disabled.
 
 Export settings:
 
@@ -257,12 +252,12 @@ Export settings:
 - Skip untranslated files: off.
 - Export only approved translations: off unless the review policy changes explicitly.
 
-Crowdin preserves untranslated nested JSON keys with empty values even when untranslated strings are skipped. `client/scripts/sync-crowdin-translations.mjs` removes those empty entries, restores English source-key ordering, validates every translated message, and refuses to write catalogs if Crowdin has not synchronized the current English keys.
+Crowdin preserves untranslated nested JSON keys with empty values even when untranslated strings are skipped. `client/scripts/sync-crowdin-translations.mjs` removes those empty entries, restores English source-key ordering, validates every translated message, and refuses to export catalogs unless Crowdin matches every current English key and value.
 
-The `Crowdin Translation Sync` workflow requires:
+The Crowdin synchronization workflows require:
 
-- `CROWDIN_TOKEN`, with permission to read project files, strings, and translation builds.
+- `CROWDIN_TOKEN`, with permission to read and write project source files and read translation builds.
-- `CROWDIN_PR_TOKEN`, a fine-grained GitHub token with repository contents and pull-request write access.
+- `CROWDIN_PR_TOKEN`, used only by `Crowdin Translation Sync`, with repository contents and pull-request write access.
 - Optional repository variable `CROWDIN_PROJECT_ID`; the BookOrbit project ID is the script default.
 
 Configure variable mismatches and leading or trailing whitespace as Crowdin QA errors. Keep punctuation and length checks enabled, and include translator instructions for Vue I18n plural branches, the HTML prohibition, and the Unicode em dash prohibition.