mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
i18n(client): preserve protected technical terms during crowdin sync
Define a list of protected technical terms that must retain their English source text in target catalogs. Enforce protected term parity in validate-locales.mjs and restore protected terms automatically during Crowdin synchronization, recording any corrected translations in the sync report.
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
// Terms that stay in English in their software context. Scoped per locale because only these
|
||||
// locales have been reviewed: several others translate the term deliberately, for example zh
|
||||
// renders the OIDC slug as an identifier, and that is a Crowdin decision rather than a defect.
|
||||
export const PROTECTED_SOURCE_TERMS = [
|
||||
{ key: 'settings.oidc.form.slug', locales: ['es', 'fr', 'it', 'pl'] },
|
||||
{ key: 'annotations.hub.exportMarkdown', locales: ['es', 'fr', 'it', 'pl'] },
|
||||
]
|
||||
|
||||
export function findProtectedTermDrift({ catalogs, terms = PROTECTED_SOURCE_TERMS }) {
|
||||
const reference = catalogs.get('en')
|
||||
if (!reference) throw new Error('English reference catalog is required')
|
||||
|
||||
const drift = []
|
||||
for (const { key, locales } of terms) {
|
||||
const source = reference.get(key)
|
||||
if (source === undefined) throw new Error(`Protected term ${key} is missing from the English catalog`)
|
||||
for (const locale of locales) {
|
||||
const message = catalogs.get(locale)?.get(key)
|
||||
if (message !== undefined && message !== source) drift.push({ locale, key, message, source })
|
||||
}
|
||||
}
|
||||
|
||||
return drift
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import path from 'node:path'
|
||||
import { TARGET_CATALOGS, assertCrowdinTargetConfiguration } from './locale-configuration.mjs'
|
||||
import { findInvalidTargetMessages, flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
|
||||
import { collectSourceMessageKeys } from './locale-source-keys.mjs'
|
||||
import { PROTECTED_SOURCE_TERMS, findProtectedTermDrift } from './locale-protected-terms.mjs'
|
||||
|
||||
const API = 'https://api.crowdin.com/api/v2'
|
||||
const SOURCE_PATH_SUFFIX = '/client/src/locales/en.json'
|
||||
@@ -330,16 +331,32 @@ export function formatRejectionReport(rejections) {
|
||||
return `${lines.join('\n')}\n`
|
||||
}
|
||||
|
||||
async function reportRejections(rejections, reportPath) {
|
||||
if (rejections.length === 0) {
|
||||
if (reportPath) await writeFile(reportPath, '')
|
||||
return
|
||||
export function formatProtectedTermReport(corrections) {
|
||||
if (corrections.length === 0) return ''
|
||||
|
||||
return `${[
|
||||
`### Restored protected terms (${corrections.length})`,
|
||||
'',
|
||||
'These messages keep the English source text in their software context. Crowdin returned a translation, so the source was restored. Fix them in Crowdin.',
|
||||
'',
|
||||
...corrections.map(({ locale, key, message }) => `- ${locale}: ${key} was "${message}"`),
|
||||
'',
|
||||
].join('\n')}\n`
|
||||
}
|
||||
|
||||
async function reportSyncIssues({ rejections, corrections, reportPath }) {
|
||||
if (corrections.length > 0) {
|
||||
console.log(`Restored ${corrections.length} protected terms to the English source:`)
|
||||
for (const { locale, key, message } of corrections) console.log(` ${locale}: ${key} was "${message}"`)
|
||||
}
|
||||
|
||||
console.log(`Rejected ${rejections.length} invalid Crowdin messages; the English source renders instead:`)
|
||||
for (const { errors } of rejections.slice(0, MAX_REPORTED_REJECTIONS)) console.log(` ${errors[0]}`)
|
||||
if (rejections.length > MAX_REPORTED_REJECTIONS) console.log(` ...and ${rejections.length - MAX_REPORTED_REJECTIONS} more`)
|
||||
if (reportPath) await writeFile(reportPath, formatRejectionReport(rejections))
|
||||
if (rejections.length > 0) {
|
||||
console.log(`Rejected ${rejections.length} invalid Crowdin messages; the English source renders instead:`)
|
||||
for (const { errors } of rejections.slice(0, MAX_REPORTED_REJECTIONS)) console.log(` ${errors[0]}`)
|
||||
if (rejections.length > MAX_REPORTED_REJECTIONS) console.log(` ...and ${rejections.length - MAX_REPORTED_REJECTIONS} more`)
|
||||
}
|
||||
|
||||
if (reportPath) await writeFile(reportPath, `${formatProtectedTermReport(corrections)}${formatRejectionReport(rejections)}`)
|
||||
}
|
||||
|
||||
export async function syncCrowdinTranslations({
|
||||
@@ -352,6 +369,7 @@ export async function syncCrowdinTranslations({
|
||||
targetCatalogs = TARGET_CATALOGS,
|
||||
assertTargetConfiguration = assertCrowdinTargetConfiguration,
|
||||
collectMessageKeys = collectSourceMessageKeys,
|
||||
protectedTerms = PROTECTED_SOURCE_TERMS,
|
||||
reportPath = process.env.CROWDIN_REJECTION_REPORT || '',
|
||||
}) {
|
||||
if (!token) throw new Error('CROWDIN_TOKEN is required')
|
||||
@@ -386,12 +404,15 @@ export async function syncCrowdinTranslations({
|
||||
const catalogs = new Map([['en', referenceMessages]])
|
||||
for (const { locale, catalog } of downloaded) catalogs.set(locale, flattenCatalog(catalog))
|
||||
|
||||
const corrections = findProtectedTermDrift({ catalogs, terms: protectedTerms })
|
||||
for (const { locale, key, source } of corrections) catalogs.get(locale).set(key, source)
|
||||
|
||||
const { slotCountKeys } = await collectMessageKeys()
|
||||
const rejections = findInvalidTargetMessages({ catalogs, slotCountKeys })
|
||||
const rejectedLocales = new Set(rejections.map(({ locale }) => locale))
|
||||
for (const { locale, key } of rejections) catalogs.get(locale).delete(key)
|
||||
const rewrittenLocales = new Set([...rejections, ...corrections].map(({ locale }) => locale))
|
||||
for (const entry of downloaded) {
|
||||
if (rejectedLocales.has(entry.locale)) entry.catalog = orderedSparseCatalog(reference, catalogs.get(entry.locale))
|
||||
if (rewrittenLocales.has(entry.locale)) entry.catalog = orderedSparseCatalog(reference, catalogs.get(entry.locale))
|
||||
}
|
||||
|
||||
const errors = validateCatalogs({ catalogs, slotCountKeys })
|
||||
@@ -409,9 +430,9 @@ export async function syncCrowdinTranslations({
|
||||
await Promise.all(
|
||||
downloaded.map(({ locale, catalog }) => writeFile(path.join(outputDirectory, `${locale}.json`), `${JSON.stringify(catalog, null, 2)}\n`)),
|
||||
)
|
||||
await reportRejections(rejections, reportPath)
|
||||
await reportSyncIssues({ rejections, corrections, reportPath })
|
||||
console.log(`Synchronized ${downloaded.length} sparse translation catalogs from Crowdin`)
|
||||
return { rejections }
|
||||
return { rejections, corrections }
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { assertCrowdinTargetConfiguration, validateCrowdinTargetConfiguration } from './locale-configuration.mjs'
|
||||
import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
|
||||
import { collectSourceMessageKeys } from './locale-source-keys.mjs'
|
||||
import { findProtectedTermDrift } from './locale-protected-terms.mjs'
|
||||
import {
|
||||
TARGET_CATALOGS,
|
||||
assertSafeDownloadUrl,
|
||||
@@ -314,6 +315,7 @@ describe('Crowdin translation synchronization', () => {
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
})
|
||||
|
||||
expect(maximumDownloads).toBe(4)
|
||||
@@ -334,6 +336,7 @@ describe('Crowdin translation synchronization', () => {
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
}),
|
||||
).rejects.toThrow('Crowdin source is not synchronized with en.json')
|
||||
expect(fetchImpl.mock.calls.some(([input]) => new URL(input).hostname === 'downloads.example.test')).toBe(false)
|
||||
@@ -355,6 +358,7 @@ describe('Crowdin translation synchronization', () => {
|
||||
targetCatalogs,
|
||||
reportPath,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
})
|
||||
|
||||
expect(rejections).toEqual([
|
||||
@@ -386,6 +390,7 @@ describe('Crowdin translation synchronization', () => {
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
}),
|
||||
).rejects.toThrow('rejected by catalog validation')
|
||||
await expect(access(outputDirectory)).rejects.toThrow()
|
||||
@@ -406,6 +411,7 @@ describe('Crowdin translation synchronization', () => {
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
}),
|
||||
).rejects.toThrow('missing from Crowdin export')
|
||||
await expect(access(outputDirectory)).rejects.toThrow()
|
||||
@@ -426,8 +432,9 @@ describe('Crowdin translation synchronization', () => {
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
}),
|
||||
).resolves.toEqual({ rejections: [] })
|
||||
).resolves.toEqual({ rejections: [], corrections: [] })
|
||||
expect(JSON.parse(await readFile(path.join(outputDirectory, `${targetCatalogs[0].locale}.json`), 'utf8'))).toEqual({
|
||||
common: { save: 'Save' },
|
||||
})
|
||||
@@ -454,6 +461,7 @@ describe('Crowdin translation synchronization', () => {
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
assertTargetConfiguration: async () => {},
|
||||
protectedTerms: [],
|
||||
collectMessageKeys: async () => ({ keys: new Set([key]), slotCountKeys: new Set([key]) }),
|
||||
})
|
||||
|
||||
@@ -473,6 +481,82 @@ describe('Crowdin translation synchronization', () => {
|
||||
expect(validateCatalogs({ catalogs: written, slotCountKeys: new Set([key]) })).toEqual([])
|
||||
})
|
||||
|
||||
it('reports a protected term only for the locales that were reviewed', () => {
|
||||
const catalogs = new Map([
|
||||
[
|
||||
'en',
|
||||
new Map([
|
||||
['settings.oidc.form.slug', 'Slug'],
|
||||
['annotations.hub.exportMarkdown', 'Markdown'],
|
||||
]),
|
||||
],
|
||||
['es', new Map([['settings.oidc.form.slug', 'Nombre corto de URL']])],
|
||||
['da', new Map([['settings.oidc.form.slug', 'Snegl']])],
|
||||
])
|
||||
|
||||
expect(findProtectedTermDrift({ catalogs })).toEqual([
|
||||
{ locale: 'es', key: 'settings.oidc.form.slug', message: 'Nombre corto de URL', source: 'Slug' },
|
||||
])
|
||||
})
|
||||
|
||||
it('ignores a protected term a sparse catalog does not carry', () => {
|
||||
const catalogs = new Map([
|
||||
[
|
||||
'en',
|
||||
new Map([
|
||||
['settings.oidc.form.slug', 'Slug'],
|
||||
['annotations.hub.exportMarkdown', 'Markdown'],
|
||||
]),
|
||||
],
|
||||
['es', new Map()],
|
||||
])
|
||||
|
||||
expect(findProtectedTermDrift({ catalogs })).toEqual([])
|
||||
})
|
||||
|
||||
it('fails loudly when a protected term leaves the English catalog', () => {
|
||||
expect(() =>
|
||||
findProtectedTermDrift({
|
||||
catalogs: new Map([['en', new Map()]]),
|
||||
terms: [{ key: 'settings.oidc.form.slug', locales: ['es'] }],
|
||||
}),
|
||||
).toThrow('Protected term settings.oidc.form.slug is missing from the English catalog')
|
||||
})
|
||||
|
||||
it('keeps every protected term present in the shipped English catalog', async () => {
|
||||
const reference = flattenCatalog(JSON.parse(await readFile(path.join(process.cwd(), 'src/locales/en.json'), 'utf8')))
|
||||
|
||||
expect(findProtectedTermDrift({ catalogs: new Map([['en', reference]]) })).toEqual([])
|
||||
})
|
||||
|
||||
it('restores a protected term Crowdin translated instead of failing the run', async () => {
|
||||
const key = 'settings.oidc.form.slug'
|
||||
const translated = { settings: { oidc: { form: { slug: 'Slug' } } } }
|
||||
const referenceCatalog = { ...translated, annotations: { hub: { exportMarkdown: 'Markdown' } } }
|
||||
const targetCatalogs = TARGET_CATALOGS.filter(({ locale }) => locale === 'es')
|
||||
const currentCatalogs = new Map([['es', translated]])
|
||||
const catalogDirectory = await createCatalogFixture(targetCatalogs, currentCatalogs, referenceCatalog)
|
||||
const outputDirectory = path.join(catalogDirectory, 'output')
|
||||
const reportPath = path.join(catalogDirectory, 'rejections.md')
|
||||
const catalogs = new Map([[targetCatalogs[0].languageId, { settings: { oidc: { form: { slug: 'Nombre corto de URL' } } } }]])
|
||||
|
||||
const { rejections, corrections } = await syncCrowdinTranslations({
|
||||
token: 'secret',
|
||||
fetchImpl: createSynchronizationFetch({ identifiers: [key, 'annotations.hub.exportMarkdown'], catalogs }),
|
||||
catalogDirectory,
|
||||
outputDirectory,
|
||||
targetCatalogs,
|
||||
reportPath,
|
||||
assertTargetConfiguration: async () => {},
|
||||
collectMessageKeys: async () => ({ keys: new Set([key]), slotCountKeys: new Set() }),
|
||||
})
|
||||
|
||||
expect(rejections).toEqual([])
|
||||
expect(corrections).toEqual([{ locale: 'es', key, message: 'Nombre corto de URL', source: 'Slug' }])
|
||||
expect(JSON.parse(await readFile(path.join(outputDirectory, 'es.json'), 'utf8'))).toEqual(translated)
|
||||
expect(await readFile(reportPath, 'utf8')).toContain(`es: ${key} was "Nombre corto de URL"`)
|
||||
})
|
||||
|
||||
it('scans the same sources locale validation scans', async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'bookorbit-locale-source-'))
|
||||
temporaryDirectories.push(directory)
|
||||
|
||||
@@ -4,6 +4,7 @@ import path from 'node:path'
|
||||
import { SUPPORTED_LOCALES } from './locale-configuration.mjs'
|
||||
import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
|
||||
import { collectSourceMessageKeys } from './locale-source-keys.mjs'
|
||||
import { findProtectedTermDrift } from './locale-protected-terms.mjs'
|
||||
|
||||
const clientRoot = fileURLToPath(new URL('..', import.meta.url))
|
||||
const localesDirectory = path.join(clientRoot, 'src/locales')
|
||||
@@ -25,6 +26,9 @@ if (!reference) throw new Error('English reference catalog is required')
|
||||
|
||||
const { keys: referencedKeys, slotCountKeys } = await collectSourceMessageKeys()
|
||||
const errors = validateCatalogs({ catalogs, referencedKeys, slotCountKeys })
|
||||
for (const { locale, key, message } of findProtectedTermDrift({ catalogs })) {
|
||||
errors.push(`${locale}: protected term ${key} must keep the English source text, found "${message}"`)
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Locale validation failed:\n${errors.join('\n')}`)
|
||||
|
||||
@@ -211,11 +211,6 @@ describe('ICU message compilation', () => {
|
||||
})
|
||||
|
||||
it('keeps reviewed technical terms in their software context', () => {
|
||||
for (const catalog of [spanish, french, italian, polish]) {
|
||||
expect(catalog.settings.oidc.form.slug).toBe('Slug')
|
||||
expect(catalog.annotations.hub.exportMarkdown).toBe('Markdown')
|
||||
}
|
||||
|
||||
expect(spanish.settings.metadata.autoFetch.runNow).toBe('Ejecutar ahora')
|
||||
expect(french.settings.metadata.autoFetch.runNow).toBe('Exécuter maintenant')
|
||||
expect(italian.settings.metadata.autoFetch.runNow).toBe('Esegui ora')
|
||||
|
||||
Reference in New Issue
Block a user