ci: make crowdin source synchronization deterministic

Upload and verify the checked-out English catalog before exporting translations so delayed webhooks cannot stall the scheduled workflow.

client/scripts/sync-crowdin-source.mjs

@@ -0,0 +1,9 @@
+import { syncCrowdinSource } from './sync-crowdin-translations.mjs'
+
+syncCrowdinSource({
+  token: process.env.CROWDIN_TOKEN,
+  projectId: process.env.CROWDIN_PROJECT_ID || undefined,
+}).catch((error) => {
+  console.error(error.message)
+  process.exitCode = 1
+})

client/scripts/sync-crowdin-translations.mjs

@@ -14,6 +14,7 @@ const SOURCE_PATH_SUFFIX = '/client/src/locales/en.json'
 const MAX_API_RESPONSE_BYTES = 2 * 1024 * 1024
 const MAX_CATALOG_BYTES = 10 * 1024 * 1024
 const REQUEST_TIMEOUT_MS = 30_000
+const SOURCE_SYNC_RETRY_DELAYS_MS = [0, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000]
 const scriptDirectory = import.meta.dirname ?? path.join(process.cwd(), 'scripts')
 const clientRoot = path.resolve(scriptDirectory, '..')
 const localesDirectory = path.join(clientRoot, 'src/locales')
@@ -175,14 +176,29 @@ export function createCrowdinClient({ token, projectId, fetchImpl = fetch }) {
       throw new Error(`Crowdin source file ending in ${SOURCE_PATH_SUFFIX} was not found`)
     },
 
-    async sourceIdentifiers(fileId) {
-      const identifiers = new Set()
-      for (let offset = 0; offset < 20_000; offset += 500) {
-        const page = await request(`/projects/${projectId}/strings?fileId=${fileId}&limit=500&offset=${offset}`)
-        for (const entry of page.data) identifiers.add(entry.data.identifier)
-        if (page.data.length < 500) return identifiers
-      }
-      throw new Error('Crowdin source contains more than 20,000 messages')
+    async sourceCatalog(fileId) {
+      const download = await request(`/projects/${projectId}/files/${fileId}/download`)
+      return downloadCatalog(fetchImpl, download.data.url)
+    },
+
+    async uploadSource(filename, content) {
+      const storage = await request('/storages', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Crowdin-API-FileName': filename,
+        },
+        body: content,
+      })
+      return storage.data.id
+    },
+
+    async updateSourceFile(fileId, storageId) {
+      await request(`/projects/${projectId}/files/${fileId}`, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ storageId, updateOption: 'keep_translations' }),
+      })
     },
 
     async exportedCatalog(fileId, languageId) {
@@ -242,10 +258,13 @@ async function mapWithConcurrency(values, concurrency, operation) {
   return results
 }
 
-export function sourceDrift(referenceMessages, identifiers) {
-  const missing = [...referenceMessages.keys()].filter((key) => !identifiers.has(key))
-  const unexpected = [...identifiers].filter((key) => !referenceMessages.has(key))
-  return { missing, unexpected }
+export function sourceDrift(referenceMessages, crowdinMessages) {
+  const missing = [...referenceMessages.keys()].filter((key) => !crowdinMessages.has(key))
+  const unexpected = [...crowdinMessages.keys()].filter((key) => !referenceMessages.has(key))
+  const changed = [...referenceMessages]
+    .filter(([key, message]) => crowdinMessages.has(key) && crowdinMessages.get(key) !== message)
+    .map(([key]) => key)
+  return { missing, unexpected, changed }
 }
 
 export function parseAllowedTranslationLosses(value = '') {
@@ -401,6 +420,70 @@ async function reportSyncIssues({ rejections, corrections, repairs, losses, repo
   }
 }
 
+function hasSourceDrift({ missing, unexpected, changed }) {
+  return missing.length > 0 || unexpected.length > 0 || changed.length > 0
+}
+
+function formatSourceDrift(drift) {
+  return [
+    ...drift.missing.slice(0, 10).map((key) => `missing in Crowdin: ${key}`),
+    ...drift.unexpected.slice(0, 10).map((key) => `missing in Git: ${key}`),
+    ...drift.changed.slice(0, 10).map((key) => `different source text: ${key}`),
+  ].join('\n')
+}
+
+async function prepareCrowdinSource({ token, projectId, fetchImpl, catalogDirectory, assertTargetConfiguration, wait }) {
+  if (!token) throw new Error('CROWDIN_TOKEN is required')
+  await assertTargetConfiguration()
+
+  const sourcePath = path.join(catalogDirectory, 'en.json')
+  const sourceContent = await readFile(sourcePath, 'utf8')
+  const reference = JSON.parse(sourceContent)
+  const referenceMessages = flattenCatalog(reference)
+  const client = createCrowdinClient({ token, projectId, fetchImpl })
+  const fileId = await client.sourceFileId()
+  let drift = sourceDrift(referenceMessages, flattenCatalog(await client.sourceCatalog(fileId)))
+  let updated = false
+
+  if (hasSourceDrift(drift)) {
+    const storageId = await client.uploadSource(path.basename(sourcePath), sourceContent)
+    await client.updateSourceFile(fileId, storageId)
+    updated = true
+
+    for (const delayMs of SOURCE_SYNC_RETRY_DELAYS_MS) {
+      if (delayMs > 0) await wait(delayMs)
+      drift = sourceDrift(referenceMessages, flattenCatalog(await client.sourceCatalog(fileId)))
+      if (!hasSourceDrift(drift)) break
+    }
+  }
+
+  if (hasSourceDrift(drift)) {
+    throw new Error(`Crowdin source did not match en.json after update\n${formatSourceDrift(drift)}`)
+  }
+
+  return { client, fileId, reference, referenceMessages, updated }
+}
+
+export async function syncCrowdinSource({
+  token,
+  projectId = '912891',
+  fetchImpl = fetch,
+  catalogDirectory = localesDirectory,
+  assertTargetConfiguration = assertCrowdinTargetConfiguration,
+  wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)),
+}) {
+  const { fileId, referenceMessages, updated } = await prepareCrowdinSource({
+    token,
+    projectId,
+    fetchImpl,
+    catalogDirectory,
+    assertTargetConfiguration,
+    wait,
+  })
+  console.log(`${updated ? 'Synchronized' : 'Verified'} ${referenceMessages.size} English source messages in Crowdin`)
+  return { fileId, messageCount: referenceMessages.size, updated }
+}
+
 export async function syncCrowdinTranslations({
   token,
   projectId = '912891',
@@ -413,12 +496,17 @@ export async function syncCrowdinTranslations({
   collectMessageKeys = collectSourceMessageKeys,
   protectedTerms = PROTECTED_SOURCE_TERMS,
   reportPath = process.env.CROWDIN_REJECTION_REPORT || '',
+  wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)),
 }) {
-  if (!token) throw new Error('CROWDIN_TOKEN is required')
-  await assertTargetConfiguration()
-
-  const reference = JSON.parse(await readFile(path.join(catalogDirectory, 'en.json'), 'utf8'))
-  const referenceMessages = flattenCatalog(reference)
+  const { client, fileId, reference, referenceMessages, updated } = await prepareCrowdinSource({
+    token,
+    projectId,
+    fetchImpl,
+    catalogDirectory,
+    assertTargetConfiguration,
+    wait,
+  })
+  console.log(`${updated ? 'Synchronized' : 'Verified'} ${referenceMessages.size} English source messages in Crowdin`)
   const currentCatalogs = new Map(
     await Promise.all(
       targetCatalogs.map(async ({ locale }) => {
@@ -427,17 +515,6 @@ export async function syncCrowdinTranslations({
       }),
     ),
   )
-  const client = createCrowdinClient({ token, projectId, fetchImpl })
-  const fileId = await client.sourceFileId()
-  const identifiers = await client.sourceIdentifiers(fileId)
-  const drift = sourceDrift(referenceMessages, identifiers)
-  if (drift.missing.length > 0 || drift.unexpected.length > 0) {
-    const details = [
-      ...drift.missing.slice(0, 10).map((key) => `missing in Crowdin: ${key}`),
-      ...drift.unexpected.slice(0, 10).map((key) => `missing in Git: ${key}`),
-    ]
-    throw new Error(`Crowdin source is not synchronized with en.json\n${details.join('\n')}`)
-  }
 
   const downloaded = await mapWithConcurrency(targetCatalogs, 4, async ({ languageId, locale }) => ({
     locale,

client/scripts/sync-crowdin-translations.spec.mjs

@@ -16,6 +16,7 @@ import {
   parseAllowedTranslationLosses,
   retentionLossLimit,
   sourceDrift,
+  syncCrowdinSource,
   syncCrowdinTranslations,
 } from './sync-crowdin-translations.mjs'
 
@@ -43,20 +44,38 @@ async function createCatalogFixture(targetCatalogs, currentCatalogs = new Map(),
   return directory
 }
 
-function createSynchronizationFetch({ identifiers = ['common.save'], catalogs = new Map(), onDownload = async () => {} } = {}) {
+function createSynchronizationFetch({
+  sourceCatalog = { common: { save: 'Save' } },
+  catalogs = new Map(),
+  applySourceUpdate = true,
+  onDownload = async () => {},
+} = {}) {
+  let currentSourceCatalog = sourceCatalog
+  let uploadedSourceCatalog
+
   return vi.fn(async (input, init = {}) => {
     const url = new URL(input)
-    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files')) {
+    const method = init.method ?? 'GET'
+    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files') && method === 'GET') {
       return new Response(JSON.stringify({ data: [{ data: { id: 7, path: '/client/src/locales/en.json' } }] }))
     }
-    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/strings')) {
-      return new Response(JSON.stringify({ data: identifiers.map((identifier) => ({ data: { identifier } })) }))
+    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files/7/download')) {
+      return new Response(JSON.stringify({ data: { url: 'https://downloads.example.test/source.json' } }))
+    }
+    if (url.hostname === 'api.crowdin.com' && url.pathname === '/api/v2/storages' && method === 'POST') {
+      uploadedSourceCatalog = JSON.parse(init.body)
+      return new Response(JSON.stringify({ data: { id: 9 } }))
+    }
+    if (url.hostname === 'api.crowdin.com' && url.pathname.endsWith('/files/7') && method === 'PUT') {
+      if (applySourceUpdate) currentSourceCatalog = uploadedSourceCatalog
+      return new Response(JSON.stringify({ data: { id: 7 } }))
     }
     if (url.hostname === 'api.crowdin.com' && url.pathname.includes('/translations/builds/files/')) {
       const { targetLanguageId } = JSON.parse(init.body)
       return new Response(JSON.stringify({ data: { url: `https://downloads.example.test/${encodeURIComponent(targetLanguageId)}.json` } }))
     }
     if (url.hostname === 'downloads.example.test') {
+      if (url.pathname === '/source.json') return new Response(JSON.stringify(currentSourceCatalog))
       const languageId = decodeURIComponent(path.basename(url.pathname, '.json'))
       await onDownload(languageId)
       return new Response(JSON.stringify(catalogs.get(languageId) ?? { common: { save: `Translated ${languageId}` } }))
@@ -82,9 +101,18 @@ describe('Crowdin translation synchronization', () => {
   })
 
   it('detects when Crowdin has not synchronized the current English keys', () => {
-    expect(sourceDrift(new Map([['common.save', 'Save']]), new Set(['common.cancel']))).toEqual({
+    expect(sourceDrift(new Map([['common.save', 'Save']]), new Map([['common.cancel', 'Cancel']]))).toEqual({
       missing: ['common.save'],
       unexpected: ['common.cancel'],
+      changed: [],
+    })
+  })
+
+  it('detects changed English source text under an existing key', () => {
+    expect(sourceDrift(new Map([['common.save', 'Save now']]), new Map([['common.save', 'Save']]))).toEqual({
+      missing: [],
+      unexpected: [],
+      changed: ['common.save'],
     })
   })
 
@@ -231,6 +259,45 @@ describe('Crowdin translation synchronization', () => {
     })
   })
 
+  it('leaves an identical Crowdin source file unchanged', async () => {
+    const catalogDirectory = await createCatalogFixture([], new Map(), { common: { save: 'Save' } })
+    const fetchImpl = createSynchronizationFetch()
+
+    await expect(
+      syncCrowdinSource({
+        token: 'secret',
+        fetchImpl,
+        catalogDirectory,
+        assertTargetConfiguration: async () => {},
+      }),
+    ).resolves.toEqual({ fileId: 7, messageCount: 1, updated: false })
+    expect(fetchImpl.mock.calls.some(([, init]) => ['POST', 'PUT'].includes(init.method))).toBe(false)
+  })
+
+  it('updates changed Crowdin source text without discarding translations', async () => {
+    const catalogDirectory = await createCatalogFixture([], new Map(), { common: { save: 'Save now' } })
+    const fetchImpl = createSynchronizationFetch({ sourceCatalog: { common: { save: 'Save' } } })
+
+    await expect(
+      syncCrowdinSource({
+        token: 'secret',
+        fetchImpl,
+        catalogDirectory,
+        assertTargetConfiguration: async () => {},
+      }),
+    ).resolves.toEqual({ fileId: 7, messageCount: 1, updated: true })
+
+    const storageCall = fetchImpl.mock.calls.find(([input]) => new URL(input).pathname === '/api/v2/storages')
+    expect(storageCall[1].headers['Crowdin-API-FileName']).toBe('en.json')
+    expect(JSON.parse(storageCall[1].body)).toEqual({ common: { save: 'Save now' } })
+
+    const updateCall = fetchImpl.mock.calls.find(([input, init]) => new URL(input).pathname.endsWith('/files/7') && init.method === 'PUT')
+    expect(JSON.parse(updateCall[1].body)).toEqual({
+      storageId: 9,
+      updateOption: 'keep_translations',
+    })
+  })
+
   it('rejects non-HTTPS export URLs returned by Crowdin', () => {
     expect(() => assertSafeDownloadUrl('http://127.0.0.1/catalog.json')).toThrow('Crowdin export URL must use HTTPS')
   })
@@ -322,11 +389,11 @@ describe('Crowdin translation synchronization', () => {
     await expect(readFile(path.join(outputDirectory, 'cs.json'), 'utf8')).resolves.toContain('Translated cs')
   })
 
-  it('rejects source drift before downloading or writing catalogs', async () => {
+  it('rejects source drift that remains after updating Crowdin', async () => {
     const targetCatalogs = TARGET_CATALOGS.slice(0, 2)
     const catalogDirectory = await createCatalogFixture(targetCatalogs)
     const outputDirectory = path.join(catalogDirectory, 'output')
-    const fetchImpl = createSynchronizationFetch({ identifiers: [] })
+    const fetchImpl = createSynchronizationFetch({ sourceCatalog: {}, applySourceUpdate: false })
 
     await expect(
       syncCrowdinTranslations({
@@ -337,9 +404,10 @@ describe('Crowdin translation synchronization', () => {
         targetCatalogs,
         assertTargetConfiguration: async () => {},
         protectedTerms: [],
+        wait: async () => {},
       }),
-    ).rejects.toThrow('Crowdin source is not synchronized with en.json')
-    expect(fetchImpl.mock.calls.some(([input]) => new URL(input).hostname === 'downloads.example.test')).toBe(false)
+    ).rejects.toThrow('Crowdin source did not match en.json after update')
+    expect(fetchImpl.mock.calls.some(([input]) => new URL(input).pathname.includes('/translations/builds/files/'))).toBe(false)
     await expect(access(outputDirectory)).rejects.toThrow()
   })
 
@@ -431,7 +499,7 @@ describe('Crowdin translation synchronization', () => {
 
     const { rejections, repairs } = await syncCrowdinTranslations({
       token: 'secret',
-      fetchImpl: createSynchronizationFetch({ identifiers: ['books.count'], catalogs }),
+      fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
       catalogDirectory,
       outputDirectory,
       targetCatalogs,
@@ -460,7 +528,7 @@ describe('Crowdin translation synchronization', () => {
     await expect(
       syncCrowdinTranslations({
         token: 'secret',
-        fetchImpl: createSynchronizationFetch({ identifiers: keys.map((key) => `common.${key}`), catalogs }),
+        fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
         catalogDirectory,
         outputDirectory,
         targetCatalogs,
@@ -510,7 +578,7 @@ describe('Crowdin translation synchronization', () => {
 
     const { rejections } = await syncCrowdinTranslations({
       token: 'secret',
-      fetchImpl: createSynchronizationFetch({ identifiers: [key], catalogs }),
+      fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
       catalogDirectory,
       outputDirectory,
       targetCatalogs,
@@ -596,7 +664,7 @@ describe('Crowdin translation synchronization', () => {
 
     const { rejections, corrections } = await syncCrowdinTranslations({
       token: 'secret',
-      fetchImpl: createSynchronizationFetch({ identifiers: [key, 'annotations.hub.exportMarkdown'], catalogs }),
+      fetchImpl: createSynchronizationFetch({ sourceCatalog: referenceCatalog, catalogs }),
       catalogDirectory,
       outputDirectory,
       targetCatalogs,