fix(client): prevent Crowdin translation loss

Use sparse target catalogs and validate Crowdin exports before proposing updates. Reject translation loss unless a maintainer acknowledges each key.
2026-11-03 00:11:37 -06:00 · 2026-08-05 23:02:58 +00:00
parent 6b0f561c76
commit 8790208858
11 changed files with 749 additions and 67 deletions
@@ -0,0 +1,43 @@
+import { validateLocaleMessage, validateSlotCountMessage } from './locale-message-validation.mjs'
+
+export function flattenCatalog(value, prefix = '', output = new Map()) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(prefix || 'catalog root must be a message object')
+  }
+
+  for (const [key, child] of Object.entries(value)) {
+    const messageKey = prefix ? `${prefix}.${key}` : key
+    if (typeof child === 'string') output.set(messageKey, child)
+    else if (child && typeof child === 'object' && !Array.isArray(child)) flattenCatalog(child, messageKey, output)
+    else throw new Error(`${messageKey} must be a string or message object`)
+  }
+
+  return output
+}
+
+export function validateCatalogs({ catalogs, referencedKeys = new Set(), slotCountKeys = new Set() }) {
+  const reference = catalogs.get('en')
+  if (!reference) throw new Error('English reference catalog is required')
+
+  const errors = []
+  for (const key of referencedKeys) {
+    if (!reference.has(key)) errors.push(`en: missing key referenced in source ${key}`)
+  }
+
+  for (const [locale, catalog] of catalogs) {
+    for (const [key, message] of catalog) {
+      const referenceMessage = reference.get(key)
+      if (referenceMessage === undefined) {
+        errors.push(`${locale}: unexpected key ${key}`)
+        continue
+      }
+      if (message.length === 0) errors.push(`${locale}: empty message ${key}`)
+      if (message.includes('\u2014')) errors.push(`${locale}: Unicode em dash is not allowed in ${key}`)
+      if (/<[^>]+>/.test(message)) errors.push(`${locale}: HTML is not allowed in ${key}`)
+      errors.push(...validateLocaleMessage({ key, locale, message, referenceMessage }))
+      if (slotCountKeys.has(key)) errors.push(...validateSlotCountMessage({ key, locale, message }))
+    }
+  }
+
+  return errors
+}
@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest'
+import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
+
+function catalogs(target = {}) {
+  return new Map([
+    [
+      'en',
+      flattenCatalog({
+        common: { save: 'Save' },
+        books: { count: '{count, plural, one {# book} other {# books}}' },
+      }),
+    ],
+    ['cs', flattenCatalog(target)],
+  ])
+}
+
+describe('locale catalog validation', () => {
+  it('accepts a sparse target catalog', () => {
+    expect(validateCatalogs({ catalogs: catalogs({ common: { save: 'Uložit' } }) })).toEqual([])
+  })
+
+  it('rejects unknown and empty target messages', () => {
+    expect(validateCatalogs({ catalogs: catalogs({ common: { save: '', unknown: 'Neznámé' } }) })).toEqual([
+      'cs: empty message common.save',
+      'cs: unexpected key common.unknown',
+    ])
+  })
+
+  it('validates ICU structure for translated messages that are present', () => {
+    const target = {
+      books: { count: '{count, plural, one {# kniha} other {# knih}}' },
+    }
+
+    expect(validateCatalogs({ catalogs: catalogs(target) })).toEqual([
+      'cs: ICU plural category few missing for books.count',
+      'cs: ICU plural category many missing for books.count',
+    ])
+  })
+})
@@ -0,0 +1,342 @@
+import { Buffer } from 'node:buffer'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { pathToFileURL } from 'node:url'
+import path from 'node:path'
+import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'
+
+const API = 'https://api.crowdin.com/api/v2'
+const SOURCE_PATH_SUFFIX = '/client/src/locales/en.json'
+const MAX_API_RESPONSE_BYTES = 2 * 1024 * 1024
+const MAX_CATALOG_BYTES = 10 * 1024 * 1024
+const REQUEST_TIMEOUT_MS = 30_000
+const clientRoot = path.basename(process.cwd()) === 'client' ? process.cwd() : path.join(process.cwd(), 'client')
+const localesDirectory = path.join(clientRoot, 'src/locales')
+
+export const TARGET_CATALOGS = [
+  { languageId: 'cs', locale: 'cs' },
+  { languageId: 'da', locale: 'da' },
+  { languageId: 'de', locale: 'de' },
+  { languageId: 'es-ES', locale: 'es' },
+  { languageId: 'fi', locale: 'fi' },
+  { languageId: 'fr', locale: 'fr' },
+  { languageId: 'it', locale: 'it' },
+  { languageId: 'nl', locale: 'nl' },
+  { languageId: 'pl', locale: 'pl' },
+  { languageId: 'pt-BR', locale: 'pt' },
+  { languageId: 'ru', locale: 'ru' },
+  { languageId: 'sl', locale: 'sl' },
+  { languageId: 'sv-SE', locale: 'sv' },
+  { languageId: 'uk', locale: 'uk' },
+  { languageId: 'zh-CN', locale: 'zh' },
+]
+
+const TARGET_LOCALES = new Set(TARGET_CATALOGS.map(({ locale }) => locale))
+
+async function responseText(response, maxBytes) {
+  const contentLength = Number(response.headers.get('content-length'))
+  if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+    throw new Error(`Response exceeds ${maxBytes} bytes`)
+  }
+
+  if (!response.body) return ''
+  const reader = response.body.getReader()
+  const chunks = []
+  let total = 0
+
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) break
+    total += value.byteLength
+    if (total > maxBytes) {
+      await reader.cancel()
+      throw new Error(`Response exceeds ${maxBytes} bytes`)
+    }
+    chunks.push(Buffer.from(value))
+  }
+
+  return Buffer.concat(chunks).toString('utf8')
+}
+
+function parseJson(text, context) {
+  try {
+    return JSON.parse(text)
+  } catch {
+    throw new Error(`${context} returned invalid JSON`)
+  }
+}
+
+function assertSafeDownloadUrl(value) {
+  const url = new URL(value)
+  if (url.protocol !== 'https:') throw new Error('Crowdin export URL must use HTTPS')
+
+  const hostname = url.hostname.replace(/^\[|\]$/g, '').toLowerCase()
+  if (
+    hostname === 'localhost' ||
+    hostname.endsWith('.localhost') ||
+    hostname.endsWith('.local') ||
+    hostname === '::1' ||
+    hostname.startsWith('fc') ||
+    hostname.startsWith('fd') ||
+    hostname.startsWith('fe80:')
+  ) {
+    throw new Error('Crowdin export URL must not target a local network host')
+  }
+
+  const octets = hostname.split('.').map(Number)
+  if (octets.length === 4 && octets.every((octet) => Number.isInteger(octet) && octet >= 0 && octet <= 255)) {
+    const [first, second] = octets
+    if (
+      first === 0 ||
+      first === 10 ||
+      first === 127 ||
+      (first === 169 && second === 254) ||
+      (first === 172 && second >= 16 && second <= 31) ||
+      (first === 192 && second === 168)
+    ) {
+      throw new Error('Crowdin export URL must not target a local network host')
+    }
+  }
+
+  return url
+}
+
+async function downloadCatalog(fetchImpl, value) {
+  let url = assertSafeDownloadUrl(value)
+
+  for (let redirect = 0; redirect <= 3; redirect += 1) {
+    const response = await fetchImpl(url, {
+      redirect: 'manual',
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+    })
+    if (response.status >= 300 && response.status < 400) {
+      const location = response.headers.get('location')
+      if (!location || redirect === 3) throw new Error('Crowdin export exceeded the redirect limit')
+      url = assertSafeDownloadUrl(new URL(location, url).href)
+      continue
+    }
+    if (!response.ok) throw new Error(`Crowdin export download failed with HTTP ${response.status}`)
+    return parseJson(await responseText(response, MAX_CATALOG_BYTES), 'Crowdin export')
+  }
+
+  throw new Error('Crowdin export exceeded the redirect limit')
+}
+
+export function createCrowdinClient({ token, projectId, fetchImpl = fetch }) {
+  async function request(endpoint, init = {}) {
+    const response = await fetchImpl(`${API}${endpoint}`, {
+      ...init,
+      headers: { Authorization: `Bearer ${token}`, ...(init.headers ?? {}) },
+      redirect: 'error',
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+    })
+    const body = await responseText(response, MAX_API_RESPONSE_BYTES)
+    if (!response.ok) throw new Error(`Crowdin API ${endpoint} failed with HTTP ${response.status}: ${body.slice(0, 300)}`)
+    return body ? parseJson(body, `Crowdin API ${endpoint}`) : null
+  }
+
+  return {
+    async sourceFileId() {
+      for (let offset = 0; offset < 10_000; offset += 500) {
+        const page = await request(`/projects/${projectId}/files?limit=500&offset=${offset}`)
+        const match = page.data.find((entry) => entry.data.path.endsWith(SOURCE_PATH_SUFFIX))
+        if (match) return match.data.id
+        if (page.data.length < 500) break
+      }
+      throw new Error(`Crowdin source file ending in ${SOURCE_PATH_SUFFIX} was not found`)
+    },
+
+    async sourceIdentifiers(fileId) {
+      const identifiers = new Set()
+      for (let offset = 0; offset < 20_000; offset += 500) {
+        const page = await request(`/projects/${projectId}/strings?fileId=${fileId}&limit=500&offset=${offset}`)
+        for (const entry of page.data) identifiers.add(entry.data.identifier)
+        if (page.data.length < 500) return identifiers
+      }
+      throw new Error('Crowdin source contains more than 20,000 messages')
+    },
+
+    async exportedCatalog(fileId, languageId) {
+      const build = await request(`/projects/${projectId}/translations/builds/files/${fileId}`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ targetLanguageId: languageId, skipUntranslatedStrings: true }),
+      })
+      return downloadCatalog(fetchImpl, build.data.url)
+    },
+  }
+}
+
+function orderedSparseCatalog(reference, messages, prefix = '') {
+  const output = {}
+  for (const [key, child] of Object.entries(reference)) {
+    const messageKey = prefix ? `${prefix}.${key}` : key
+    if (typeof child === 'string') {
+      const translated = messages.get(messageKey)
+      if (translated) output[key] = translated
+      continue
+    }
+
+    const nested = orderedSparseCatalog(child, messages, messageKey)
+    if (Object.keys(nested).length > 0) output[key] = nested
+  }
+  return output
+}
+
+export function normalizeCrowdinCatalog(exported, reference) {
+  const referenceMessages = flattenCatalog(reference)
+  const exportedMessages = flattenCatalog(exported)
+
+  for (const key of exportedMessages.keys()) {
+    if (!referenceMessages.has(key)) throw new Error(`Crowdin export contains unknown key ${key}`)
+  }
+  for (const [key, message] of exportedMessages) {
+    if (message.length === 0) exportedMessages.delete(key)
+  }
+
+  return orderedSparseCatalog(reference, exportedMessages)
+}
+
+async function mapWithConcurrency(values, concurrency, operation) {
+  const results = new Array(values.length)
+  let nextIndex = 0
+
+  async function worker() {
+    while (nextIndex < values.length) {
+      const index = nextIndex
+      nextIndex += 1
+      results[index] = await operation(values[index])
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, worker))
+  return results
+}
+
+export function sourceDrift(referenceMessages, identifiers) {
+  const missing = [...referenceMessages.keys()].filter((key) => !identifiers.has(key))
+  const unexpected = [...identifiers].filter((key) => !referenceMessages.has(key))
+  return { missing, unexpected }
+}
+
+export function parseAllowedTranslationLosses(value = '') {
+  const allowed = new Set()
+  for (const entry of value.split(/[\s,]+/).filter(Boolean)) {
+    const separator = entry.indexOf(':')
+    const locale = entry.slice(0, separator)
+    const key = entry.slice(separator + 1)
+    if (separator < 1 || !TARGET_LOCALES.has(locale) || !/^[A-Za-z0-9_.-]+$/.test(key)) {
+      throw new Error(`Invalid translation loss acknowledgement ${entry}; expected locale:message.key`)
+    }
+    if (allowed.has(entry)) throw new Error(`Duplicate translation loss acknowledgement ${entry}`)
+    allowed.add(entry)
+  }
+  return allowed
+}
+
+export function findTranslationLosses({ locale, reference, current, exported }) {
+  const legacyComplete = current.size === reference.size && [...reference.keys()].every((key) => current.has(key))
+  const losses = []
+
+  for (const [key, currentMessage] of current) {
+    const referenceMessage = reference.get(key)
+    if (referenceMessage === undefined || (legacyComplete && currentMessage === referenceMessage)) continue
+
+    const exportedMessage = exported.get(key)
+    if (exportedMessage === undefined) {
+      losses.push({ locale, key, reason: 'missing from Crowdin export' })
+    } else if (currentMessage !== referenceMessage && exportedMessage === referenceMessage) {
+      losses.push({ locale, key, reason: 'replaced by English source text' })
+    }
+  }
+
+  return losses
+}
+
+export function assertTranslationRetention({ reference, currentCatalogs, exportedCatalogs, allowedLosses = new Set() }) {
+  const losses = []
+  for (const { locale } of TARGET_CATALOGS) {
+    const current = currentCatalogs.get(locale)
+    const exported = exportedCatalogs.get(locale)
+    if (!current || !exported) throw new Error(`Translation retention comparison is missing the ${locale} catalog`)
+    losses.push(...findTranslationLosses({ locale, reference, current, exported }))
+  }
+
+  const detected = new Set(losses.map(({ locale, key }) => `${locale}:${key}`))
+  const unacknowledged = losses.filter(({ locale, key }) => !allowedLosses.has(`${locale}:${key}`))
+  const unused = [...allowedLosses].filter((entry) => !detected.has(entry))
+  if (unacknowledged.length === 0 && unused.length === 0) return
+
+  const details = [
+    ...unacknowledged.slice(0, 25).map(({ locale, key, reason }) => `${locale}:${key} - ${reason}`),
+    ...unused.slice(0, 25).map((entry) => `${entry} - acknowledgement does not match an exported loss`),
+  ]
+  const remaining = unacknowledged.length + unused.length - details.length
+  if (remaining > 0) details.push(`...and ${remaining} more`)
+  throw new Error(`Crowdin export would lose existing translations:\n${details.join('\n')}`)
+}
+
+export async function syncCrowdinTranslations({
+  token,
+  projectId = '912891',
+  fetchImpl = fetch,
+  catalogDirectory = localesDirectory,
+  outputDirectory = catalogDirectory,
+  allowedLosses = new Set(),
+}) {
+  if (!token) throw new Error('CROWDIN_TOKEN is required')
+
+  const reference = JSON.parse(await readFile(path.join(catalogDirectory, 'en.json'), 'utf8'))
+  const referenceMessages = flattenCatalog(reference)
+  const currentCatalogs = new Map(
+    await Promise.all(
+      TARGET_CATALOGS.map(async ({ locale }) => {
+        const catalog = JSON.parse(await readFile(path.join(catalogDirectory, `${locale}.json`), 'utf8'))
+        return [locale, flattenCatalog(catalog)]
+      }),
+    ),
+  )
+  const client = createCrowdinClient({ token, projectId, fetchImpl })
+  const fileId = await client.sourceFileId()
+  const identifiers = await client.sourceIdentifiers(fileId)
+  const drift = sourceDrift(referenceMessages, identifiers)
+  if (drift.missing.length > 0 || drift.unexpected.length > 0) {
+    const details = [
+      ...drift.missing.slice(0, 10).map((key) => `missing in Crowdin: ${key}`),
+      ...drift.unexpected.slice(0, 10).map((key) => `missing in Git: ${key}`),
+    ]
+    throw new Error(`Crowdin source is not synchronized with en.json\n${details.join('\n')}`)
+  }
+
+  const downloaded = await mapWithConcurrency(TARGET_CATALOGS, 4, async ({ languageId, locale }) => ({
+    locale,
+    catalog: normalizeCrowdinCatalog(await client.exportedCatalog(fileId, languageId), reference),
+  }))
+  const catalogs = new Map([['en', referenceMessages]])
+  for (const { locale, catalog } of downloaded) catalogs.set(locale, flattenCatalog(catalog))
+
+  const errors = validateCatalogs({ catalogs })
+  if (errors.length > 0) throw new Error(`Crowdin export validation failed:\n${errors.join('\n')}`)
+  assertTranslationRetention({
+    reference: referenceMessages,
+    currentCatalogs,
+    exportedCatalogs: catalogs,
+    allowedLosses,
+  })
+
+  await mkdir(outputDirectory, { recursive: true })
+  await Promise.all(
+    downloaded.map(({ locale, catalog }) => writeFile(path.join(outputDirectory, `${locale}.json`), `${JSON.stringify(catalog, null, 2)}\n`)),
+  )
+  console.log(`Synchronized ${downloaded.length} sparse translation catalogs from Crowdin`)
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  syncCrowdinTranslations({
+    token: process.env.CROWDIN_TOKEN,
+    projectId: process.env.CROWDIN_PROJECT_ID || undefined,
+    allowedLosses: parseAllowedTranslationLosses(process.env.CROWDIN_ALLOWED_TRANSLATION_LOSSES),
+  }).catch((error) => {
+    console.error(error.message)
+    process.exitCode = 1
+  })
+}
@@ -0,0 +1,137 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+  TARGET_CATALOGS,
+  assertTranslationRetention,
+  createCrowdinClient,
+  findTranslationLosses,
+  normalizeCrowdinCatalog,
+  parseAllowedTranslationLosses,
+  sourceDrift,
+} from './sync-crowdin-translations.mjs'
+
+const reference = {
+  common: { save: 'Save', cancel: 'Cancel' },
+  books: { count: '{count, plural, one {# book} other {# books}}' },
+}
+
+describe('Crowdin translation synchronization', () => {
+  it('removes empty untranslated messages and follows English key order', () => {
+    const exported = {
+      books: { count: '' },
+      common: { cancel: 'Zrušit', save: 'Uložit' },
+    }
+
+    expect(normalizeCrowdinCatalog(exported, reference)).toEqual({
+      common: { save: 'Uložit', cancel: 'Zrušit' },
+    })
+  })
+
+  it('rejects keys that do not exist in English', () => {
+    expect(() => normalizeCrowdinCatalog({ common: { unknown: 'Neznámé' } }, reference)).toThrow('Crowdin export contains unknown key common.unknown')
+  })
+
+  it('detects when Crowdin has not synchronized the current English keys', () => {
+    expect(sourceDrift(new Map([['common.save', 'Save']]), new Set(['common.cancel']))).toEqual({
+      missing: ['common.save'],
+      unexpected: ['common.cancel'],
+    })
+  })
+
+  it('protects real translations in complete legacy catalogs without treating English fallbacks as translations', () => {
+    const referenceMessages = new Map([
+      ['common.save', 'Save'],
+      ['common.cancel', 'Cancel'],
+    ])
+    const current = new Map([
+      ['common.save', 'Uložit'],
+      ['common.cancel', 'Cancel'],
+    ])
+    const exported = new Map()
+
+    expect(findTranslationLosses({ locale: 'cs', reference: referenceMessages, current, exported })).toEqual([
+      { locale: 'cs', key: 'common.save', reason: 'missing from Crowdin export' },
+    ])
+  })
+
+  it('protects every existing key after a target catalog becomes sparse', () => {
+    const referenceMessages = new Map([
+      ['common.save', 'Save'],
+      ['common.cancel', 'Cancel'],
+    ])
+    const current = new Map([['common.save', 'Save']])
+
+    expect(findTranslationLosses({ locale: 'cs', reference: referenceMessages, current, exported: new Map() })).toEqual([
+      { locale: 'cs', key: 'common.save', reason: 'missing from Crowdin export' },
+    ])
+  })
+
+  it('rejects translations that Crowdin replaces with English source text', () => {
+    const referenceMessages = new Map([['common.save', 'Save']])
+    const current = new Map([['common.save', 'Uložit']])
+    const exported = new Map([['common.save', 'Save']])
+
+    expect(findTranslationLosses({ locale: 'cs', reference: referenceMessages, current, exported })).toEqual([
+      { locale: 'cs', key: 'common.save', reason: 'replaced by English source text' },
+    ])
+  })
+
+  it('requires exact acknowledgements for intentional translation losses', () => {
+    const referenceMessages = new Map([['common.save', 'Save']])
+    const currentCatalogs = new Map(TARGET_CATALOGS.map(({ locale }) => [locale, new Map()]))
+    const exportedCatalogs = new Map(TARGET_CATALOGS.map(({ locale }) => [locale, new Map()]))
+    currentCatalogs.set('cs', new Map([['common.save', 'Uložit']]))
+
+    expect(() => assertTranslationRetention({ reference: referenceMessages, currentCatalogs, exportedCatalogs })).toThrow(
+      'cs:common.save - missing from Crowdin export',
+    )
+    expect(() =>
+      assertTranslationRetention({
+        reference: referenceMessages,
+        currentCatalogs,
+        exportedCatalogs,
+        allowedLosses: parseAllowedTranslationLosses('cs:common.save'),
+      }),
+    ).not.toThrow()
+    expect(() =>
+      assertTranslationRetention({
+        reference: referenceMessages,
+        currentCatalogs,
+        exportedCatalogs,
+        allowedLosses: parseAllowedTranslationLosses('de:common.save'),
+      }),
+    ).toThrow('de:common.save - acknowledgement does not match an exported loss')
+  })
+
+  it('rejects malformed and duplicate loss acknowledgements', () => {
+    expect(() => parseAllowedTranslationLosses('xx:common.save')).toThrow('expected locale:message.key')
+    expect(() => parseAllowedTranslationLosses('cs:common.save,cs:common.save')).toThrow('Duplicate translation loss acknowledgement')
+  })
+
+  it('requests untranslated-string omission and downloads the bounded export', async () => {
+    const fetchImpl = vi
+      .fn()
+      .mockResolvedValueOnce(
+        new Response(JSON.stringify({ data: { url: 'https://downloads.example.test/cs.json' } }), {
+          headers: { 'Content-Type': 'application/json' },
+        }),
+      )
+      .mockResolvedValueOnce(new Response(JSON.stringify({ common: { save: 'Uložit' } })))
+    const client = createCrowdinClient({ token: 'secret', projectId: '42', fetchImpl })
+
+    await expect(client.exportedCatalog(7, 'cs')).resolves.toEqual({ common: { save: 'Uložit' } })
+    expect(fetchImpl).toHaveBeenCalledTimes(2)
+    expect(fetchImpl.mock.calls[0][0]).toBe('https://api.crowdin.com/api/v2/projects/42/translations/builds/files/7')
+    expect(JSON.parse(fetchImpl.mock.calls[0][1].body)).toEqual({
+      targetLanguageId: 'cs',
+      skipUntranslatedStrings: true,
+    })
+  })
+
+  it('rejects private-network export URLs returned by Crowdin', async () => {
+    const fetchImpl = vi.fn().mockResolvedValueOnce(new Response(JSON.stringify({ data: { url: 'http://127.0.0.1/catalog.json' } })))
+    const client = createCrowdinClient({ token: 'secret', projectId: '42', fetchImpl })
+
+    await expect(client.exportedCatalog(7, 'cs')).rejects.toThrow('Crowdin export URL must use HTTPS')
+    expect(fetchImpl).toHaveBeenCalledTimes(1)
+  })
+})
@@ -1,22 +1,12 @@
 import { readdir, readFile } from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
 import path from 'node:path'
-import { validateLocaleMessage, validateSlotCountMessage } from './locale-message-validation.mjs'
+import { flattenCatalog, validateCatalogs } from './locale-catalog-validation.mjs'

 const clientRoot = fileURLToPath(new URL('..', import.meta.url))
 const localesDirectory = path.join(clientRoot, 'src/locales')
 const localeTypesPath = path.join(clientRoot, '../packages/types/src/locale.ts')

-function flatten(value, prefix = '', output = new Map()) {
-  for (const [key, child] of Object.entries(value)) {
-    const messageKey = prefix ? `${prefix}.${key}` : key
-    if (typeof child === 'string') output.set(messageKey, child)
-    else if (child && typeof child === 'object' && !Array.isArray(child)) flatten(child, messageKey, output)
-    else throw new Error(`${messageKey} must be a string or message object`)
-  }
-  return output
-}
-
 async function sourceFiles(directory) {
  const entries = await readdir(directory, { withFileTypes: true })
  const files = await Promise.all(
@@ -71,38 +61,17 @@ if (JSON.stringify(localeFiles) !== JSON.stringify(expectedFiles)) {
 const catalogs = new Map()
 for (const locale of supportedLocales) {
  const raw = await readFile(path.join(localesDirectory, `${locale}.json`), 'utf8')
-  catalogs.set(locale, flatten(JSON.parse(raw)))
+  catalogs.set(locale, flattenCatalog(JSON.parse(raw)))
 }

 const reference = catalogs.get('en')
 if (!reference) throw new Error('English reference catalog is required')

-const errors = []
 const { keys: referencedKeys, slotCountKeys } = await sourceMessageKeys()
-for (const key of referencedKeys) {
-  if (!reference.has(key)) errors.push(`en: missing key referenced in source ${key}`)
-}
-
-for (const [locale, catalog] of catalogs) {
-  for (const [key, referenceMessage] of reference) {
-    const message = catalog.get(key)
-    if (message === undefined) {
-      errors.push(`${locale}: missing key ${key}`)
-      continue
-    }
-    if (message.length === 0) errors.push(`${locale}: empty message ${key}`)
-    if (message.includes('\u2014')) errors.push(`${locale}: Unicode em dash is not allowed in ${key}`)
-    if (/<[^>]+>/.test(message)) errors.push(`${locale}: HTML is not allowed in ${key}`)
-    errors.push(...validateLocaleMessage({ key, locale, message, referenceMessage }))
-    if (slotCountKeys.has(key)) errors.push(...validateSlotCountMessage({ key, locale, message }))
-  }
-  for (const key of catalog.keys()) {
-    if (!reference.has(key)) errors.push(`${locale}: unexpected key ${key}`)
-  }
-}
+const errors = validateCatalogs({ catalogs, referencedKeys, slotCountKeys })

 if (errors.length > 0) {
  throw new Error(`Locale validation failed:\n${errors.join('\n')}`)
 }

-console.log(`Validated ${supportedLocales.length} locale catalogs with ${reference.size} messages each`)
+console.log(`Validated ${supportedLocales.length} locale catalogs against ${reference.size} English messages`)