i18n(client): preserve protected technical terms during crowdin sync

Define a list of protected technical terms that must retain their English source text in target
catalogs.

Enforce protected term parity in validate-locales.mjs and restore protected terms automatically
during Crowdin synchronization, recording any corrected translations in the sync report.
This commit is contained in:
neon
2026-08-21 16:18:35 +00:00
parent ffd1a4fd0f
commit 11cb1e0f89
5 changed files with 146 additions and 18 deletions
+24
View File
@@ -0,0 +1,24 @@
// Terms that stay in English in their software context. Scoped per locale because only these
// locales have been reviewed: several others translate the term deliberately, for example zh
// renders the OIDC slug as an identifier, and that is a Crowdin decision rather than a defect.
export const PROTECTED_SOURCE_TERMS = [
{ key: 'settings.oidc.form.slug', locales: ['es', 'fr', 'it', 'pl'] },
{ key: 'annotations.hub.exportMarkdown', locales: ['es', 'fr', 'it', 'pl'] },
]
export function findProtectedTermDrift({ catalogs, terms = PROTECTED_SOURCE_TERMS }) {
const reference = catalogs.get('en')
if (!reference) throw new Error('English reference catalog is required')
const drift = []
for (const { key, locales } of terms) {
const source = reference.get(key)
if (source === undefined) throw new Error(`Protected term ${key} is missing from the English catalog`)
for (const locale of locales) {
const message = catalogs.get(locale)?.get(key)
if (message !== undefined && message !== source) drift.push({ locale, key, message, source })
}
}
return drift
}
+33 -12
View File
@@ -6,6 +6,7 @@ import path from 'node:path'
import { TARGET_CATALOGS, assertCrowdinTargetConfiguration } from './locale-configuration.mjs'
import { findInvalidTargetMessages, flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
import { collectSourceMessageKeys } from './locale-source-keys.mjs'
import { PROTECTED_SOURCE_TERMS, findProtectedTermDrift } from './locale-protected-terms.mjs'
const API = 'https://api.crowdin.com/api/v2'
const SOURCE_PATH_SUFFIX = '/client/src/locales/en.json'
@@ -330,16 +331,32 @@ export function formatRejectionReport(rejections) {
return `${lines.join('\n')}\n`
}
async function reportRejections(rejections, reportPath) {
if (rejections.length === 0) {
if (reportPath) await writeFile(reportPath, '')
return
export function formatProtectedTermReport(corrections) {
if (corrections.length === 0) return ''
return `${[
`### Restored protected terms (${corrections.length})`,
'',
'These messages keep the English source text in their software context. Crowdin returned a translation, so the source was restored. Fix them in Crowdin.',
'',
...corrections.map(({ locale, key, message }) => `- ${locale}: ${key} was "${message}"`),
'',
].join('\n')}\n`
}
async function reportSyncIssues({ rejections, corrections, reportPath }) {
if (corrections.length > 0) {
console.log(`Restored ${corrections.length} protected terms to the English source:`)
for (const { locale, key, message } of corrections) console.log(` ${locale}: ${key} was "${message}"`)
}
console.log(`Rejected ${rejections.length} invalid Crowdin messages; the English source renders instead:`)
for (const { errors } of rejections.slice(0, MAX_REPORTED_REJECTIONS)) console.log(` ${errors[0]}`)
if (rejections.length > MAX_REPORTED_REJECTIONS) console.log(` ...and ${rejections.length - MAX_REPORTED_REJECTIONS} more`)
if (reportPath) await writeFile(reportPath, formatRejectionReport(rejections))
if (rejections.length > 0) {
console.log(`Rejected ${rejections.length} invalid Crowdin messages; the English source renders instead:`)
for (const { errors } of rejections.slice(0, MAX_REPORTED_REJECTIONS)) console.log(` ${errors[0]}`)
if (rejections.length > MAX_REPORTED_REJECTIONS) console.log(` ...and ${rejections.length - MAX_REPORTED_REJECTIONS} more`)
}
if (reportPath) await writeFile(reportPath, `${formatProtectedTermReport(corrections)}${formatRejectionReport(rejections)}`)
}
export async function syncCrowdinTranslations({
@@ -352,6 +369,7 @@ export async function syncCrowdinTranslations({
targetCatalogs = TARGET_CATALOGS,
assertTargetConfiguration = assertCrowdinTargetConfiguration,
collectMessageKeys = collectSourceMessageKeys,
protectedTerms = PROTECTED_SOURCE_TERMS,
reportPath = process.env.CROWDIN_REJECTION_REPORT || '',
}) {
if (!token) throw new Error('CROWDIN_TOKEN is required')
@@ -386,12 +404,15 @@ export async function syncCrowdinTranslations({
const catalogs = new Map([['en', referenceMessages]])
for (const { locale, catalog } of downloaded) catalogs.set(locale, flattenCatalog(catalog))
const corrections = findProtectedTermDrift({ catalogs, terms: protectedTerms })
for (const { locale, key, source } of corrections) catalogs.get(locale).set(key, source)
const { slotCountKeys } = await collectMessageKeys()
const rejections = findInvalidTargetMessages({ catalogs, slotCountKeys })
const rejectedLocales = new Set(rejections.map(({ locale }) => locale))
for (const { locale, key } of rejections) catalogs.get(locale).delete(key)
const rewrittenLocales = new Set([...rejections, ...corrections].map(({ locale }) => locale))
for (const entry of downloaded) {
if (rejectedLocales.has(entry.locale)) entry.catalog = orderedSparseCatalog(reference, catalogs.get(entry.locale))
if (rewrittenLocales.has(entry.locale)) entry.catalog = orderedSparseCatalog(reference, catalogs.get(entry.locale))
}
const errors = validateCatalogs({ catalogs, slotCountKeys })
@@ -409,9 +430,9 @@ export async function syncCrowdinTranslations({
await Promise.all(
downloaded.map(({ locale, catalog }) => writeFile(path.join(outputDirectory, `${locale}.json`), `${JSON.stringify(catalog, null, 2)}\n`)),
)
await reportRejections(rejections, reportPath)
await reportSyncIssues({ rejections, corrections, reportPath })
console.log(`Synchronized ${downloaded.length} sparse translation catalogs from Crowdin`)
return { rejections }
return { rejections, corrections }
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
@@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
import { assertCrowdinTargetConfiguration, validateCrowdinTargetConfiguration } from './locale-configuration.mjs'
import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
import { collectSourceMessageKeys } from './locale-source-keys.mjs'
import { findProtectedTermDrift } from './locale-protected-terms.mjs'
import {
TARGET_CATALOGS,
assertSafeDownloadUrl,
@@ -314,6 +315,7 @@ describe('Crowdin translation synchronization', () => {
outputDirectory,
targetCatalogs,
assertTargetConfiguration: async () => {},
protectedTerms: [],
})
expect(maximumDownloads).toBe(4)
@@ -334,6 +336,7 @@ describe('Crowdin translation synchronization', () => {
outputDirectory,
targetCatalogs,
assertTargetConfiguration: async () => {},
protectedTerms: [],
}),
).rejects.toThrow('Crowdin source is not synchronized with en.json')
expect(fetchImpl.mock.calls.some(([input]) => new URL(input).hostname === 'downloads.example.test')).toBe(false)
@@ -355,6 +358,7 @@ describe('Crowdin translation synchronization', () => {
targetCatalogs,
reportPath,
assertTargetConfiguration: async () => {},
protectedTerms: [],
})
expect(rejections).toEqual([
@@ -386,6 +390,7 @@ describe('Crowdin translation synchronization', () => {
outputDirectory,
targetCatalogs,
assertTargetConfiguration: async () => {},
protectedTerms: [],
}),
).rejects.toThrow('rejected by catalog validation')
await expect(access(outputDirectory)).rejects.toThrow()
@@ -406,6 +411,7 @@ describe('Crowdin translation synchronization', () => {
outputDirectory,
targetCatalogs,
assertTargetConfiguration: async () => {},
protectedTerms: [],
}),
).rejects.toThrow('missing from Crowdin export')
await expect(access(outputDirectory)).rejects.toThrow()
@@ -426,8 +432,9 @@ describe('Crowdin translation synchronization', () => {
outputDirectory,
targetCatalogs,
assertTargetConfiguration: async () => {},
protectedTerms: [],
}),
).resolves.toEqual({ rejections: [] })
).resolves.toEqual({ rejections: [], corrections: [] })
expect(JSON.parse(await readFile(path.join(outputDirectory, `${targetCatalogs[0].locale}.json`), 'utf8'))).toEqual({
common: { save: 'Save' },
})
@@ -454,6 +461,7 @@ describe('Crowdin translation synchronization', () => {
outputDirectory,
targetCatalogs,
assertTargetConfiguration: async () => {},
protectedTerms: [],
collectMessageKeys: async () => ({ keys: new Set([key]), slotCountKeys: new Set([key]) }),
})
@@ -473,6 +481,82 @@ describe('Crowdin translation synchronization', () => {
expect(validateCatalogs({ catalogs: written, slotCountKeys: new Set([key]) })).toEqual([])
})
it('reports a protected term only for the locales that were reviewed', () => {
const catalogs = new Map([
[
'en',
new Map([
['settings.oidc.form.slug', 'Slug'],
['annotations.hub.exportMarkdown', 'Markdown'],
]),
],
['es', new Map([['settings.oidc.form.slug', 'Nombre corto de URL']])],
['da', new Map([['settings.oidc.form.slug', 'Snegl']])],
])
expect(findProtectedTermDrift({ catalogs })).toEqual([
{ locale: 'es', key: 'settings.oidc.form.slug', message: 'Nombre corto de URL', source: 'Slug' },
])
})
it('ignores a protected term a sparse catalog does not carry', () => {
const catalogs = new Map([
[
'en',
new Map([
['settings.oidc.form.slug', 'Slug'],
['annotations.hub.exportMarkdown', 'Markdown'],
]),
],
['es', new Map()],
])
expect(findProtectedTermDrift({ catalogs })).toEqual([])
})
it('fails loudly when a protected term leaves the English catalog', () => {
expect(() =>
findProtectedTermDrift({
catalogs: new Map([['en', new Map()]]),
terms: [{ key: 'settings.oidc.form.slug', locales: ['es'] }],
}),
).toThrow('Protected term settings.oidc.form.slug is missing from the English catalog')
})
it('keeps every protected term present in the shipped English catalog', async () => {
const reference = flattenCatalog(JSON.parse(await readFile(path.join(process.cwd(), 'src/locales/en.json'), 'utf8')))
expect(findProtectedTermDrift({ catalogs: new Map([['en', reference]]) })).toEqual([])
})
it('restores a protected term Crowdin translated instead of failing the run', async () => {
const key = 'settings.oidc.form.slug'
const translated = { settings: { oidc: { form: { slug: 'Slug' } } } }
const referenceCatalog = { ...translated, annotations: { hub: { exportMarkdown: 'Markdown' } } }
const targetCatalogs = TARGET_CATALOGS.filter(({ locale }) => locale === 'es')
const currentCatalogs = new Map([['es', translated]])
const catalogDirectory = await createCatalogFixture(targetCatalogs, currentCatalogs, referenceCatalog)
const outputDirectory = path.join(catalogDirectory, 'output')
const reportPath = path.join(catalogDirectory, 'rejections.md')
const catalogs = new Map([[targetCatalogs[0].languageId, { settings: { oidc: { form: { slug: 'Nombre corto de URL' } } } }]])
const { rejections, corrections } = await syncCrowdinTranslations({
token: 'secret',
fetchImpl: createSynchronizationFetch({ identifiers: [key, 'annotations.hub.exportMarkdown'], catalogs }),
catalogDirectory,
outputDirectory,
targetCatalogs,
reportPath,
assertTargetConfiguration: async () => {},
collectMessageKeys: async () => ({ keys: new Set([key]), slotCountKeys: new Set() }),
})
expect(rejections).toEqual([])
expect(corrections).toEqual([{ locale: 'es', key, message: 'Nombre corto de URL', source: 'Slug' }])
expect(JSON.parse(await readFile(path.join(outputDirectory, 'es.json'), 'utf8'))).toEqual(translated)
expect(await readFile(reportPath, 'utf8')).toContain(`es: ${key} was "Nombre corto de URL"`)
})
it('scans the same sources locale validation scans', async () => {
const directory = await mkdtemp(path.join(tmpdir(), 'bookorbit-locale-source-'))
temporaryDirectories.push(directory)
+4
View File
@@ -4,6 +4,7 @@ import path from 'node:path'
import { SUPPORTED_LOCALES } from './locale-configuration.mjs'
import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
import { collectSourceMessageKeys } from './locale-source-keys.mjs'
import { findProtectedTermDrift } from './locale-protected-terms.mjs'
const clientRoot = fileURLToPath(new URL('..', import.meta.url))
const localesDirectory = path.join(clientRoot, 'src/locales')
@@ -25,6 +26,9 @@ if (!reference) throw new Error('English reference catalog is required')
const { keys: referencedKeys, slotCountKeys } = await collectSourceMessageKeys()
const errors = validateCatalogs({ catalogs, referencedKeys, slotCountKeys })
for (const { locale, key, message } of findProtectedTermDrift({ catalogs })) {
errors.push(`${locale}: protected term ${key} must keep the English source text, found "${message}"`)
}
if (errors.length > 0) {
throw new Error(`Locale validation failed:\n${errors.join('\n')}`)
-5
View File
@@ -211,11 +211,6 @@ describe('ICU message compilation', () => {
})
it('keeps reviewed technical terms in their software context', () => {
for (const catalog of [spanish, french, italian, polish]) {
expect(catalog.settings.oidc.form.slug).toBe('Slug')
expect(catalog.annotations.hub.exportMarkdown).toBe('Markdown')
}
expect(spanish.settings.metadata.autoFetch.runNow).toBe('Ejecutar ahora')
expect(french.settings.metadata.autoFetch.runNow).toBe('Exécuter maintenant')
expect(italian.settings.metadata.autoFetch.runNow).toBe('Esegui ora')