import { Buffer } from 'node:buffer' import { mkdir, readFile, writeFile } from 'node:fs/promises' import { isIP } from 'node:net' import { pathToFileURL } from 'node:url' import path from 'node:path' import { TARGET_CATALOGS, assertCrowdinTargetConfiguration } from './locale-configuration.mjs' import { findInvalidTargetMessages, flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs' import { collectSourceMessageKeys } from './locale-source-keys.mjs' import { PROTECTED_SOURCE_TERMS, findProtectedTermDrift } from './locale-protected-terms.mjs' import { findExportRepairs } from './locale-export-repairs.mjs' const API = 'https://api.crowdin.com/api/v2' const SOURCE_PATH_SUFFIX = '/client/src/locales/en.json' const MAX_API_RESPONSE_BYTES = 2 * 1024 * 1024 const MAX_CATALOG_BYTES = 10 * 1024 * 1024 const REQUEST_TIMEOUT_MS = 30_000 const SOURCE_SYNC_RETRY_DELAYS_MS = [0, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000] const scriptDirectory = import.meta.dirname ?? path.join(process.cwd(), 'scripts') const clientRoot = path.resolve(scriptDirectory, '..') const localesDirectory = path.join(clientRoot, 'src/locales') export { TARGET_CATALOGS } const TARGET_LOCALES = new Set(TARGET_CATALOGS.map(({ locale }) => locale)) async function responseText(response, maxBytes) { const contentLength = Number(response.headers.get('content-length')) if (Number.isFinite(contentLength) && contentLength > maxBytes) { throw new Error(`Response exceeds ${maxBytes} bytes`) } if (!response.body) return '' const reader = response.body.getReader() const chunks = [] let total = 0 while (true) { const { done, value } = await reader.read() if (done) break total += value.byteLength if (total > maxBytes) { await reader.cancel() throw new Error(`Response exceeds ${maxBytes} bytes`) } chunks.push(Buffer.from(value)) } return Buffer.concat(chunks).toString('utf8') } function parseJson(text, context) { try { return JSON.parse(text) } catch { throw new Error(`${context} returned invalid JSON`) } } function ipv4Value(hostname) { return hostname.split('.').reduce((value, octet) => value * 256 + Number(octet), 0) >>> 0 } function isInIpv4Range(value, network, prefixLength) { const mask = prefixLength === 0 ? 0 : (0xffffffff << (32 - prefixLength)) >>> 0 return (value & mask) === (network & mask) } function isBlockedIpv4(hostname) { const value = ipv4Value(hostname) return [ [ipv4Value('0.0.0.0'), 8], [ipv4Value('10.0.0.0'), 8], [ipv4Value('100.64.0.0'), 10], [ipv4Value('127.0.0.0'), 8], [ipv4Value('169.254.0.0'), 16], [ipv4Value('172.16.0.0'), 12], [ipv4Value('192.0.0.0'), 24], [ipv4Value('192.0.2.0'), 24], [ipv4Value('192.168.0.0'), 16], [ipv4Value('198.18.0.0'), 15], [ipv4Value('198.51.100.0'), 24], [ipv4Value('203.0.113.0'), 24], [ipv4Value('224.0.0.0'), 4], [ipv4Value('240.0.0.0'), 4], ].some(([network, prefixLength]) => isInIpv4Range(value, network, prefixLength)) } function ipv6Groups(hostname) { const [head, tail = ''] = hostname.split('::') const headGroups = head ? head.split(':').map((group) => Number.parseInt(group, 16)) : [] const tailGroups = tail ? tail.split(':').map((group) => Number.parseInt(group, 16)) : [] return [...headGroups, ...Array(8 - headGroups.length - tailGroups.length).fill(0), ...tailGroups] } function isBlockedIpv6(hostname) { const groups = ipv6Groups(hostname) const first = groups[0] const ipv4Mapped = groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff if (ipv4Mapped) { const embedded = `${groups[6] >> 8}.${groups[6] & 0xff}.${groups[7] >> 8}.${groups[7] & 0xff}` return isBlockedIpv4(embedded) } return ( groups.every((group) => group === 0) || (groups.slice(0, 7).every((group) => group === 0) && groups[7] === 1) || groups.slice(0, 6).every((group) => group === 0) || (first & 0xfe00) === 0xfc00 || (first & 0xffc0) === 0xfe80 || (first & 0xffc0) === 0xfec0 || (first & 0xff00) === 0xff00 ) } export function assertSafeDownloadUrl(value) { const url = new URL(value) if (url.protocol !== 'https:') throw new Error('Crowdin export URL must use HTTPS') const hostname = url.hostname .replace(/^\[|\]$/g, '') .replace(/\.$/, '') .toLowerCase() const addressType = isIP(hostname) if (hostname === 'localhost' || hostname.endsWith('.localhost') || hostname.endsWith('.local')) { throw new Error('Crowdin export URL must not target a local network host') } if ((addressType === 4 && isBlockedIpv4(hostname)) || (addressType === 6 && isBlockedIpv6(hostname))) { throw new Error('Crowdin export URL must not target a local network host') } return url } async function downloadCatalog(fetchImpl, value) { let url = assertSafeDownloadUrl(value) for (let redirect = 0; redirect <= 3; redirect += 1) { const response = await fetchImpl(url, { redirect: 'manual', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), }) if (response.status >= 300 && response.status < 400) { const location = response.headers.get('location') if (!location || redirect === 3) throw new Error('Crowdin export exceeded the redirect limit') url = assertSafeDownloadUrl(new URL(location, url).href) continue } if (!response.ok) throw new Error(`Crowdin export download failed with HTTP ${response.status}`) return parseJson(await responseText(response, MAX_CATALOG_BYTES), 'Crowdin export') } throw new Error('Crowdin export exceeded the redirect limit') } export function createCrowdinClient({ token, projectId, fetchImpl = fetch }) { async function request(endpoint, init = {}) { const response = await fetchImpl(`${API}${endpoint}`, { ...init, headers: { Authorization: `Bearer ${token}`, ...(init.headers ?? {}) }, redirect: 'error', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), }) const body = await responseText(response, MAX_API_RESPONSE_BYTES) if (!response.ok) throw new Error(`Crowdin API ${endpoint} failed with HTTP ${response.status}: ${body.slice(0, 300)}`) return body ? parseJson(body, `Crowdin API ${endpoint}`) : null } return { async sourceFileId() { for (let offset = 0; offset < 10_000; offset += 500) { const page = await request(`/projects/${projectId}/files?limit=500&offset=${offset}`) const match = page.data.find((entry) => entry.data.path.endsWith(SOURCE_PATH_SUFFIX)) if (match) return match.data.id if (page.data.length < 500) break } throw new Error(`Crowdin source file ending in ${SOURCE_PATH_SUFFIX} was not found`) }, async sourceCatalog(fileId) { const download = await request(`/projects/${projectId}/files/${fileId}/download`) return downloadCatalog(fetchImpl, download.data.url) }, async uploadSource(filename, content) { const storage = await request('/storages', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Crowdin-API-FileName': filename, }, body: content, }) return storage.data.id }, async updateSourceFile(fileId, storageId) { await request(`/projects/${projectId}/files/${fileId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ storageId, updateOption: 'keep_translations' }), }) }, async exportedCatalog(fileId, languageId) { const build = await request(`/projects/${projectId}/translations/builds/files/${fileId}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ targetLanguageId: languageId, skipUntranslatedStrings: true }), }) return downloadCatalog(fetchImpl, build.data.url) }, } } function orderedSparseCatalog(reference, messages, prefix = '') { const output = {} for (const [key, child] of Object.entries(reference)) { const messageKey = prefix ? `${prefix}.${key}` : key if (typeof child === 'string') { const translated = messages.get(messageKey) if (translated !== undefined) output[key] = translated continue } const nested = orderedSparseCatalog(child, messages, messageKey) if (Object.keys(nested).length > 0) output[key] = nested } return output } export function normalizeCrowdinCatalog(exported, reference) { const referenceMessages = flattenCatalog(reference) const exportedMessages = flattenCatalog(exported) for (const key of exportedMessages.keys()) { if (!referenceMessages.has(key)) throw new Error(`Crowdin export contains unknown key ${key}`) } for (const [key, message] of exportedMessages) { if (message.length === 0) exportedMessages.delete(key) } return orderedSparseCatalog(reference, exportedMessages) } async function mapWithConcurrency(values, concurrency, operation) { const results = new Array(values.length) let nextIndex = 0 async function worker() { while (nextIndex < values.length) { const index = nextIndex nextIndex += 1 results[index] = await operation(values[index]) } } await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, worker)) return results } export function sourceDrift(referenceMessages, crowdinMessages) { const missing = [...referenceMessages.keys()].filter((key) => !crowdinMessages.has(key)) const unexpected = [...crowdinMessages.keys()].filter((key) => !referenceMessages.has(key)) const changed = [...referenceMessages] .filter(([key, message]) => crowdinMessages.has(key) && crowdinMessages.get(key) !== message) .map(([key]) => key) return { missing, unexpected, changed } } export function parseAllowedTranslationLosses(value = '') { const allowed = new Set() for (const entry of value.split(/[\s,]+/).filter(Boolean)) { const separator = entry.indexOf(':') const locale = entry.slice(0, separator) const key = entry.slice(separator + 1) if (separator < 1 || !TARGET_LOCALES.has(locale) || !/^[A-Za-z0-9_.-]+$/.test(key)) { throw new Error(`Invalid translation loss acknowledgement ${entry}; expected locale:message.key`) } if (allowed.has(entry)) throw new Error(`Duplicate translation loss acknowledgement ${entry}`) allowed.add(entry) } return allowed } // Exports are built with skipUntranslatedStrings, so Crowdin drops a key entirely once it stops // carrying a translation. Retention therefore compares presence, never message content: a translation // that equals the English source, such as Spanish "Error" or a product name, is a real translation. // A message this sync rejected is absent by our own decision and is reported as a rejection instead. export function findTranslationLosses({ locale, reference, current, exported, rejected = new Set() }) { const losses = [] for (const key of current.keys()) { if (!reference.has(key) || exported.has(key) || rejected.has(key)) continue losses.push({ locale, key }) } return losses } // Single translations leave Crowdin as a matter of course: a reviewer unapproves a string, or an // English edit invalidates the translation attached to it. Failing the whole sync on that churn // blocks every other locale over one message and demands a hand-typed acknowledgement that only a // manual run can supply, which is why this guard kept stalling the nightly export. It now watches for // the accident it was built to catch, an export that arrives empty or truncated, and lets ordinary // churn through to the pull request body. const RETENTION_LOSS_FLOOR = 25 const RETENTION_LOSS_RATIO = 0.01 export function retentionLossLimit(translatedKeyCount) { return Math.max(RETENTION_LOSS_FLOOR, Math.ceil(translatedKeyCount * RETENTION_LOSS_RATIO)) } export function assertTranslationRetention({ reference, currentCatalogs, exportedCatalogs, allowedLosses = new Set(), targetCatalogs = TARGET_CATALOGS, rejections = [], }) { const rejectedByLocale = new Map() for (const { locale, key } of rejections) { if (!rejectedByLocale.has(locale)) rejectedByLocale.set(locale, new Set()) rejectedByLocale.get(locale).add(key) } const losses = [] const excessive = [] for (const { locale } of targetCatalogs) { const current = currentCatalogs.get(locale) const exported = exportedCatalogs.get(locale) if (!current || !exported) throw new Error(`Translation retention comparison is missing the ${locale} catalog`) const detected = findTranslationLosses({ locale, reference, current, exported, rejected: rejectedByLocale.get(locale) }) const unacknowledged = detected.filter(({ key }) => !allowedLosses.has(`${locale}:${key}`)) const limit = retentionLossLimit(current.size) if (unacknowledged.length > limit) excessive.push({ locale, count: unacknowledged.length, limit }) losses.push(...unacknowledged) } if (excessive.length > 0) { const details = excessive.map(({ locale, count, limit }) => `${locale}: ${count} translations dropped, more than the ${limit} allowed`) throw new Error(`Crowdin export would lose existing translations:\n${details.join('\n')}`) } return losses } const MAX_REPORTED_REJECTIONS = 50 export function formatRejectionReport(rejections) { if (rejections.length === 0) return '' const listed = rejections.slice(0, MAX_REPORTED_REJECTIONS) const lines = [ `### Rejected Crowdin messages (${rejections.length})`, '', 'These translations did not pass catalog validation and were omitted, so the English source renders instead. Fix them in Crowdin.', '', ...listed.map(({ errors }) => `- ${errors[0]}`), ] if (rejections.length > listed.length) lines.push(`- ...and ${rejections.length - listed.length} more`) return `${lines.join('\n')}\n` } export function formatProtectedTermReport(corrections) { if (corrections.length === 0) return '' return `${[ `### Restored protected terms (${corrections.length})`, '', 'These messages keep the English source text in their software context. Crowdin returned a translation, so the source was restored. Fix them in Crowdin.', '', ...corrections.map(({ locale, key, message }) => `- ${locale}: ${key} was "${message}"`), '', ].join('\n')}\n` } export function formatTranslationLossReport(losses) { if (losses.length === 0) return '' const listed = losses.slice(0, MAX_REPORTED_REJECTIONS) const lines = [ `### Translations no longer in Crowdin (${losses.length})`, '', 'Crowdin stopped returning a translation for these messages, so the English source renders instead.', '', ...listed.map(({ locale, key }) => `- ${locale}: ${key}`), ] if (losses.length > listed.length) lines.push(`- ...and ${losses.length - listed.length} more`) return `${lines.join('\n')}\n` } async function reportSyncIssues({ rejections, corrections, repairs, losses, reportPath }) { if (repairs.length > 0) { const kinds = repairs.flatMap(({ kinds: repaired }) => repaired) const counts = [...new Set(kinds)].map((kind) => `${kind}=${kinds.filter((entry) => entry === kind).length}`) console.log(`Repaired ${repairs.length} Crowdin messages before validation: ${counts.join(' ')}`) } if (losses.length > 0) { console.log(`Crowdin no longer translates ${losses.length} messages; the English source renders instead:`) for (const { locale, key } of losses.slice(0, MAX_REPORTED_REJECTIONS)) console.log(` ${locale}: ${key}`) if (losses.length > MAX_REPORTED_REJECTIONS) console.log(` ...and ${losses.length - MAX_REPORTED_REJECTIONS} more`) } if (corrections.length > 0) { console.log(`Restored ${corrections.length} protected terms to the English source:`) for (const { locale, key, message } of corrections) console.log(` ${locale}: ${key} was "${message}"`) } if (rejections.length > 0) { console.log(`Rejected ${rejections.length} invalid Crowdin messages; the English source renders instead:`) for (const { errors } of rejections.slice(0, MAX_REPORTED_REJECTIONS)) console.log(` ${errors[0]}`) if (rejections.length > MAX_REPORTED_REJECTIONS) console.log(` ...and ${rejections.length - MAX_REPORTED_REJECTIONS} more`) } if (reportPath) { await writeFile(reportPath, `${formatProtectedTermReport(corrections)}${formatRejectionReport(rejections)}${formatTranslationLossReport(losses)}`) } } function hasSourceDrift({ missing, unexpected, changed }) { return missing.length > 0 || unexpected.length > 0 || changed.length > 0 } function formatSourceDrift(drift) { return [ ...drift.missing.slice(0, 10).map((key) => `missing in Crowdin: ${key}`), ...drift.unexpected.slice(0, 10).map((key) => `missing in Git: ${key}`), ...drift.changed.slice(0, 10).map((key) => `different source text: ${key}`), ].join('\n') } async function prepareCrowdinSource({ token, projectId, fetchImpl, catalogDirectory, assertTargetConfiguration, wait }) { if (!token) throw new Error('CROWDIN_TOKEN is required') await assertTargetConfiguration() const sourcePath = path.join(catalogDirectory, 'en.json') const sourceContent = await readFile(sourcePath, 'utf8') const reference = JSON.parse(sourceContent) const referenceMessages = flattenCatalog(reference) const client = createCrowdinClient({ token, projectId, fetchImpl }) const fileId = await client.sourceFileId() let drift = sourceDrift(referenceMessages, flattenCatalog(await client.sourceCatalog(fileId))) let updated = false if (hasSourceDrift(drift)) { const storageId = await client.uploadSource(path.basename(sourcePath), sourceContent) await client.updateSourceFile(fileId, storageId) updated = true for (const delayMs of SOURCE_SYNC_RETRY_DELAYS_MS) { if (delayMs > 0) await wait(delayMs) drift = sourceDrift(referenceMessages, flattenCatalog(await client.sourceCatalog(fileId))) if (!hasSourceDrift(drift)) break } } if (hasSourceDrift(drift)) { throw new Error(`Crowdin source did not match en.json after update\n${formatSourceDrift(drift)}`) } return { client, fileId, reference, referenceMessages, updated } } export async function syncCrowdinSource({ token, projectId = '912891', fetchImpl = fetch, catalogDirectory = localesDirectory, assertTargetConfiguration = assertCrowdinTargetConfiguration, wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)), }) { const { fileId, referenceMessages, updated } = await prepareCrowdinSource({ token, projectId, fetchImpl, catalogDirectory, assertTargetConfiguration, wait, }) console.log(`${updated ? 'Synchronized' : 'Verified'} ${referenceMessages.size} English source messages in Crowdin`) return { fileId, messageCount: referenceMessages.size, updated } } export async function syncCrowdinTranslations({ token, projectId = '912891', fetchImpl = fetch, catalogDirectory = localesDirectory, outputDirectory = catalogDirectory, allowedLosses = new Set(), targetCatalogs = TARGET_CATALOGS, assertTargetConfiguration = assertCrowdinTargetConfiguration, collectMessageKeys = collectSourceMessageKeys, protectedTerms = PROTECTED_SOURCE_TERMS, reportPath = process.env.CROWDIN_REJECTION_REPORT || '', wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)), }) { const { client, fileId, reference, referenceMessages, updated } = await prepareCrowdinSource({ token, projectId, fetchImpl, catalogDirectory, assertTargetConfiguration, wait, }) console.log(`${updated ? 'Synchronized' : 'Verified'} ${referenceMessages.size} English source messages in Crowdin`) const currentCatalogs = new Map( await Promise.all( targetCatalogs.map(async ({ locale }) => { const catalog = JSON.parse(await readFile(path.join(catalogDirectory, `${locale}.json`), 'utf8')) return [locale, flattenCatalog(catalog)] }), ), ) const downloaded = await mapWithConcurrency(targetCatalogs, 4, async ({ languageId, locale }) => ({ locale, catalog: normalizeCrowdinCatalog(await client.exportedCatalog(fileId, languageId), reference), })) const catalogs = new Map([['en', referenceMessages]]) for (const { locale, catalog } of downloaded) catalogs.set(locale, flattenCatalog(catalog)) const corrections = findProtectedTermDrift({ catalogs, terms: protectedTerms }) for (const { locale, key, source } of corrections) catalogs.get(locale).set(key, source) const repairs = findExportRepairs({ catalogs }) for (const { locale, key, message } of repairs) catalogs.get(locale).set(key, message) const { slotCountKeys } = await collectMessageKeys() const rejections = findInvalidTargetMessages({ catalogs, slotCountKeys }) for (const { locale, key } of rejections) catalogs.get(locale).delete(key) const rewrittenLocales = new Set([...rejections, ...corrections, ...repairs].map(({ locale }) => locale)) for (const entry of downloaded) { if (rewrittenLocales.has(entry.locale)) entry.catalog = orderedSparseCatalog(reference, catalogs.get(entry.locale)) } const errors = validateCatalogs({ catalogs, slotCountKeys }) if (errors.length > 0) throw new Error(`Crowdin export validation failed:\n${errors.join('\n')}`) const losses = assertTranslationRetention({ reference: referenceMessages, currentCatalogs, exportedCatalogs: catalogs, allowedLosses, targetCatalogs, rejections, }) await mkdir(outputDirectory, { recursive: true }) await Promise.all( downloaded.map(({ locale, catalog }) => writeFile(path.join(outputDirectory, `${locale}.json`), `${JSON.stringify(catalog, null, 2)}\n`)), ) await reportSyncIssues({ rejections, corrections, repairs, losses, reportPath }) console.log(`Synchronized ${downloaded.length} sparse translation catalogs from Crowdin`) return { rejections, corrections, repairs, losses } } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { syncCrowdinTranslations({ token: process.env.CROWDIN_TOKEN, projectId: process.env.CROWDIN_PROJECT_ID || undefined, allowedLosses: parseAllowedTranslationLosses(process.env.CROWDIN_ALLOWED_TRANSLATION_LOSSES), }).catch((error) => { console.error(error.message) process.exitCode = 1 }) }