Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 46 additions & 8 deletions scripts/translation/agenticProvider.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@
// materializes them into a pre-seeded draft before the agent session — a file
// whose every unit is seeded skips the model entirely (the deterministic gate
// still runs), and a partially seeded file is handed to the agent with
// instructions to translate only the units that are still English.
// instructions to translate only the units that are still English. A pending
// unit can itself be an identity translation (pure identifier/code text), so
// the seeded prompt lets the agent keep such a draft unchanged by replying
// NO_TRANSLATION_NEEDED; the unchanged bytes must then pass the same
// deterministic gate as any translated draft.

const fs = require('node:fs')
const os = require('node:os')
Expand All @@ -44,6 +48,21 @@ const {buildRecoveryIdentity, loadChunkLimits, loadRecoveryAnalysis, loadProgres
const DEFAULT_MAX_REPAIR_TURNS = 4
const MAX_FAILURE_ERROR_LENGTH = 2000

// Reply marker exempting an unchanged seeded draft: pending "new" units can be
// pure identifier/code lines (a parameter list row, a type alias) whose correct
// zh/ja text is byte-identical to the English, so "the agent changed nothing"
// is not automatically a failure. The seeded prompt states the identity
// rule and this marker; the provider only trusts the marker after the same
// deterministic gate any translated draft must pass. Matching is line-anchored
// so a reply that merely quotes the rule inside running prose does not
// qualify. An unchanged draft WITHOUT the marker still fails hard, so a lazy
// no-op cannot slip prose through — the residual risk is a degraded model
// claiming the marker on translatable prose, which the two independent
// locale runs and the deterministic gate bound but do not eliminate; the
// unchanged-drafts summary counter keeps that route observable.
const NO_TRANSLATION_NEEDED_MARKER = 'NO_TRANSLATION_NEEDED'
const NO_TRANSLATION_NEEDED_LINE = new RegExp('^\\s*NO_TRANSLATION_NEEDED\\s*$', 'm')

// Reasoning efforts accepted by both codex and the DeepSeek native API
// (responses gateways translate the Responses `reasoning` object into
// DeepSeek's `reasoning_effort` enum). litellm releases up to v1.100 forward
Expand Down Expand Up @@ -174,6 +193,8 @@ Finish translating one ${target} documentation page from a pre-seeded draft.
Units still to translate (id, kind, opening excerpt):
${boundedUnits}${overflow}
Identity-translation rule: a remaining unit whose text is only identifiers, parameter or type names, code, paths, or markup (for example \`**privileges** (*PrivilegeEntity[]*) -\`) must stay byte-identical; there is nothing to translate in it. If EVERY remaining unit is of that kind, leave draft.md exactly as you received it and reply with exactly: ${NO_TRANSLATION_NEEDED_MARKER}. This is the only case in which an unchanged draft.md is correct.
Rules:
${protectedBytesRules()}
Expand Down Expand Up @@ -403,14 +424,29 @@ async function runAgenticFile({item, target, siteDir, callCodex, validate, maxRe
const taskPrompt = seeded
? buildSeededTaskPrompt({item, target, validatorCommand, pendingUnits: seeded.pendingUnits, seededUnits: seeded.usableCount, styleStaged: styleContent !== null})
: buildAgenticTaskPrompt({item, target, validatorCommand, styleStaged: styleContent !== null})
await runTurn('translate', taskPrompt)
const reply = await runTurn('translate', taskPrompt)
attempts.push('translate')
log.log(`[agentic-provider] turn translate ${item.sourcePath}${seeded ? ` seeded=${seeded.usableCount}/${seeded.totalUnits}` : ''}`)
const base = {...item, target, attempts, ...(model ? {model} : {}), ...(seeded ? {semanticSeedUnits: seeded.usableCount} : {})}
let outcome
if (seededDraftBytes !== null && fs.readFileSync(draftPath, 'utf8') === seededDraftBytes) {
throw new Error(`agentic translate turn left the seeded draft of ${item.targetPath} unchanged; ${seeded.pendingUnits.length} unit(s) remain untranslated`)
if (!NO_TRANSLATION_NEEDED_LINE.test(String(reply || ''))) {
throw new Error(`agentic translate turn left the seeded draft of ${item.targetPath} unchanged; ${seeded.pendingUnits.length} unit(s) remain untranslated`)
}
// The agent claims every pending unit is an identity translation; hold
// the unchanged draft to the same deterministic gate as a translated
// one. Gate failures keep the normal repair loop below — the seed draft
// itself can drift out of validity.
outcome = await validateWithRuntimeChecks({sourceContent, draftContent: fs.readFileSync(draftPath, 'utf8'), relPath: item.sourcePath, target, validate})
if (outcome.repaired !== fs.readFileSync(draftPath, 'utf8')) fs.writeFileSync(draftPath, outcome.repaired)
if (!outcome.errors.length) {
log.log(`[agentic-provider] unchanged seeded draft accepted ${item.sourcePath} (${seeded.usableCount}/${seeded.totalUnits} seeded, ${seeded.pendingUnits.length} pending unit(s) identical in ${target})`)
return {...base, status: 'translated', review: successfulReview(), validationErrors: [], unchangedDraftAccepted: true}
}
} else {
outcome = await validateWithRuntimeChecks({sourceContent, draftContent: fs.readFileSync(draftPath, 'utf8'), relPath: item.sourcePath, target, validate})
if (outcome.repaired !== fs.readFileSync(draftPath, 'utf8')) fs.writeFileSync(draftPath, outcome.repaired)
}
let outcome = await validateWithRuntimeChecks({sourceContent, draftContent: fs.readFileSync(draftPath, 'utf8'), relPath: item.sourcePath, target, validate})
if (outcome.repaired !== fs.readFileSync(draftPath, 'utf8')) fs.writeFileSync(draftPath, outcome.repaired)
let violations = outcome.errors
while (violations.length && attempts.length <= maxRepairTurns) {
await runTurn(`repair${attempts.length}`, buildRepairPrompt({item, target, violations, validatorCommand}))
Expand All @@ -420,7 +456,6 @@ async function runAgenticFile({item, target, siteDir, callCodex, validate, maxRe
if (outcome.repaired !== fs.readFileSync(draftPath, 'utf8')) fs.writeFileSync(draftPath, outcome.repaired)
violations = outcome.errors
}
const base = {...item, target, attempts, ...(model ? {model} : {}), ...(seeded ? {semanticSeedUnits: seeded.usableCount} : {})}
if (!violations.length) {
log.log(`[agentic-provider] translated ${item.sourcePath} attempts=${attempts.join(',')}`)
return {...base, status: 'translated', review: successfulReview(), validationErrors: []}
Expand Down Expand Up @@ -543,7 +578,8 @@ async function runAgenticTranslation({siteDir, manifest, callCodex, validate, ma
const agentFiles = results.filter(result => Array.isArray(result?.attempts) && result.attempts.some(attempt => attempt === 'translate' || String(attempt).startsWith('repair'))).length
const repairTurns = results.reduce((total, result) => total + (Array.isArray(result?.attempts) ? result.attempts.filter(attempt => String(attempt).startsWith('repair')).length : 0), 0)
const sourcePreflightFailed = results.filter(result => result?.status === 'failed' && typeof result.error === 'string' && result.error.startsWith('source preflight:')).length
log.log(`[agentic-provider] summary translated=${translated} failed=${failed} seeded-full=${seededFull} verified-current=${verifiedCurrent} agent-files=${agentFiles} repair-turns=${repairTurns} source-preflight-failed=${sourcePreflightFailed}`)
const unchangedDrafts = results.filter(result => result?.status === 'translated' && result.unchangedDraftAccepted === true).length
log.log(`[agentic-provider] summary translated=${translated} failed=${failed} seeded-full=${seededFull} verified-current=${verifiedCurrent} unchanged-drafts=${unchangedDrafts} agent-files=${agentFiles} repair-turns=${repairTurns} source-preflight-failed=${sourcePreflightFailed}`)
if (usageTracker) {
const usage = usageTracker.snapshot()
log.log(`[agentic-provider] tokens turns=${usage.turns} input=${usage.inputTokens} cached=${usage.cachedInputTokens} cache-write=${usage.cacheWriteInputTokens} output=${usage.outputTokens} reasoning=${usage.reasoningOutputTokens}`)
Expand Down Expand Up @@ -733,6 +769,7 @@ async function main() {
const seededFull = report.results.filter(result => result?.status === 'translated' && result.semanticSeedUnits && Array.isArray(result.attempts) && result.attempts.length === 0).length
const verifiedCurrent = report.results.filter(result => result?.status === 'translated' && Array.isArray(result.attempts) && result.attempts.length === 1 && result.attempts[0] === 'verified-current').length
const sourcePreflightFailed = report.results.filter(result => result?.status === 'failed' && typeof result.error === 'string' && result.error.startsWith('source preflight:')).length
const unchangedDrafts = report.results.filter(result => result?.status === 'translated' && result.unchangedDraftAccepted === true).length
if (process.env.GITHUB_OUTPUT) {
fs.appendFileSync(process.env.GITHUB_OUTPUT, [
`translated_count=${report.checkpoint.translated}`,
Expand All @@ -747,7 +784,7 @@ async function main() {
if (process.env.GITHUB_STEP_SUMMARY) {
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, [
'### Agentic translation cost', '',
`- Files: ${report.checkpoint.processed} (fully seeded: ${seededFull}, verified-current: ${verifiedCurrent} — both without a model call; source-preflight failed: ${sourcePreflightFailed} — skipped the model because the source itself cannot pass the gates)`,
`- Files: ${report.checkpoint.processed} (fully seeded: ${seededFull}, verified-current: ${verifiedCurrent} — both without a model call; unchanged drafts accepted: ${unchangedDrafts}; source-preflight failed: ${sourcePreflightFailed} — skipped the model because the source itself cannot pass the gates)`,
`- Agent turns: ${usage.turns} (cached input: ${Math.round(usage.cachedInputTokens / 1000)}K tokens)`,
`- Input tokens: **${usage.inputTokens.toLocaleString('en-US')}**`,
`- Output tokens: **${usage.outputTokens.toLocaleString('en-US')}** (reasoning: ${usage.reasoningOutputTokens.toLocaleString('en-US')})`,
Expand All @@ -762,6 +799,7 @@ module.exports = {
DEFAULT_MAX_REPAIR_TURNS,
AGENTIC_REASONING_EFFORTS,
DEFAULT_REASONING_EFFORT,
NO_TRANSLATION_NEEDED_MARKER,
codexThreadConfig,
createUsageTracker,
mergeTranslatedResultsIntoProgressState,
Expand Down
104 changes: 101 additions & 3 deletions scripts/translation/agenticProvider.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ function jaFixture() {
const EN_SOURCE = `---\ntitle: Limits\nslug: /limits\n---\n\n# Limits\n\nThe collection limits apply per cluster. See [docs](./other).\n`;
const JA_CLEAN = `---\ntitle: 制限\nslug: /limits\n---\n\n# 制限\n\nコレクションの制限はクラスターごとに適用されます。[docs](./other) を参照してください。\n`;
const JA_TRANSLATED_LINK = `---\ntitle: 制限\nslug: /limits\n---\n\n# 制限\n\nコレクションの制限はクラスターごとに適用されます。[ドキュメント](./その他) を参照してください。\n`;
// Identity-translation fixture mirroring the production loop on
// Authentication-listPrivilegeGroups: the pending paragraph is a pure
// identifier list row whose correct ja text is the English itself, while the
// title and heading do carry real translations.
const EN_IDENTITY_SOURCE = `---\ntitle: Limits\nslug: /limits\n---\n\n# Limits\n\n- **privileges** (*PrivilegeEntity[]*) -\n`;
const JA_IDENTITY_PUBLISHED = `---\ntitle: 制限\nslug: /limits\n---\n\n# 制限\n\n- **privileges** (*PrivilegeEntity[]*) -\n`;

test('stylePromptPathFor resolves per target without hardcoding ja', () => {
assert.equal(stylePromptPathFor('ja-JP'), 'codex-style-guide.ja-JP.md');
Expand Down Expand Up @@ -605,7 +611,7 @@ test('mergeTranslatedResultsIntoProgressState updates the Chinese Reference mani
const {createHash} = require('node:crypto');
const {loadLocaleContract} = require('./localeContract');
const {collectCurrentUnits, pairOldSourceWithTarget} = require('./semanticSeeds');
const {resolveSeededDraft} = require('./agenticProvider');
const {NO_TRANSLATION_NEEDED_MARKER, resolveSeededDraft} = require('./agenticProvider');

function unitHash(text) {
return createHash('sha256').update(text).digest('hex');
Expand Down Expand Up @@ -700,6 +706,98 @@ test('runAgenticFile rejects a turn that leaves the seeded draft unchanged', asy
});
});

test('runAgenticFile accepts an unchanged seeded draft when the agent reports no translation needed', async () => {
await withSite(async siteDir => {
const item = jaFixture();
write(siteDir, item.sourcePath, EN_IDENTITY_SOURCE);
const seedReport = buildSeedReport({item, target: 'ja-JP', source: EN_IDENTITY_SOURCE, published: JA_IDENTITY_PUBLISHED, keep: unit => unit.kind !== 'paragraph'});
const seeded = resolveSeededDraft({item, target: 'ja-JP', siteDir, chunkOptions: null, seedReport});
assert.equal(seeded.pendingUnits.length, 1);
assert.match(seeded.pendingUnits[0].source, /\*\*privileges\*\*/);
const prompts = [];
const result = await runAgenticFile({
item, target: 'ja-JP', siteDir,
callCodex: async ({prompt, workingDirectory}) => {
prompts.push(prompt);
// The pending unit is the identifier-only privileges row; the agent
// correctly leaves the draft byte-identical and claims the marker on
// its own reply line.
writeAgentDraft(workingDirectory, seeded.draft);
return `Checked the pending unit: nothing to translate.\n${NO_TRANSLATION_NEEDED_MARKER}`;
},
seedReport,
});
assert.equal(result.status, 'translated');
assert.equal(result.unchangedDraftAccepted, true);
assert.deepEqual(result.attempts, ['translate']);
assert.ok(result.semanticSeedUnits >= 1);
assert.ok(isConsistentSuccessfulReview(result.review));
assert.deepEqual(result.validationErrors, []);
// The prompt must state the identity rule and the exact marker.
assert.match(prompts[0], /Identity-translation rule/);
assert.match(prompts[0], new RegExp(NO_TRANSLATION_NEEDED_MARKER));
// The published draft is the untouched seeded draft: reused translations
// in place, the pending identifier row still byte-identical English.
const draft = fs.readFileSync(path.join(siteDir, item.targetPath), 'utf8');
assert.equal(draft, seeded.draft.endsWith('\n') ? seeded.draft : `${seeded.draft}\n`);
assert.match(draft, /title: 制限/);
assert.match(draft, /\*\*privileges\*\*/);
});
});

test('a reply that only quotes the marker inside prose still fails an unchanged draft', async () => {
await withSite(async siteDir => {
const item = jaFixture();
write(siteDir, item.sourcePath, EN_IDENTITY_SOURCE);
const seedReport = buildSeedReport({item, target: 'ja-JP', source: EN_IDENTITY_SOURCE, published: JA_IDENTITY_PUBLISHED, keep: unit => unit.kind !== 'paragraph'});
const seeded = resolveSeededDraft({item, target: 'ja-JP', siteDir, chunkOptions: null, seedReport});
const result = await runAgenticFile({
item, target: 'ja-JP', siteDir,
callCodex: async ({workingDirectory}) => {
writeAgentDraft(workingDirectory, seeded.draft);
return 'I reviewed the pending unit and considered the identity rule (NO_TRANSLATION_NEEDED), but left the draft as it was.';
},
seedReport,
}).catch(error => ({error: String(error.message)}));
assert.match(result.error, /left the seeded draft .* unchanged/);
});
});

test('an unchanged seeded draft with the marker still goes through repair when the gate fails', async () => {
await withSite(async siteDir => {
const item = jaFixture();
write(siteDir, item.sourcePath, EN_IDENTITY_SOURCE);
// Leave both the heading and the identifier row pending so the repair
// turn's draft (JA_IDENTITY_PUBLISHED) differs from the seeded draft.
const seedReport = buildSeedReport({item, target: 'ja-JP', source: EN_IDENTITY_SOURCE, published: JA_IDENTITY_PUBLISHED, keep: unit => unit.kind !== 'paragraph' && unit.kind !== 'heading'});
const seeded = resolveSeededDraft({item, target: 'ja-JP', siteDir, chunkOptions: null, seedReport});
const prompts = [];
let outcomeCalls = 0;
const result = await runAgenticFile({
item, target: 'ja-JP', siteDir,
callCodex: async ({phase, workingDirectory}) => {
prompts.push(phase);
// First turn leaves the draft unchanged; the repair turn translates.
writeAgentDraft(workingDirectory, prompts.length === 1 ? seeded.draft : JA_IDENTITY_PUBLISHED);
return prompts.length === 1 ? NO_TRANSLATION_NEEDED_MARKER : 'DONE';
},
// The stubbed validator passes the source preflight, fails the first
// unchanged-draft outcome, and passes after the repair turn.
validate: async content => {
if (content === EN_IDENTITY_SOURCE) return [];
outcomeCalls += 1;
return outcomeCalls === 1 ? ['synthetic gate violation'] : [];
},
seedReport,
});
assert.equal(result.status, 'translated');
assert.equal(result.unchangedDraftAccepted, undefined);
assert.deepEqual(result.attempts, ['translate', 'repair1']);
const draft = fs.readFileSync(path.join(siteDir, item.targetPath), 'utf8');
assert.equal(draft, JA_IDENTITY_PUBLISHED);
});
});

test('runAgenticFile falls back to a fresh translation when the seed report identity mismatches', async () => {
await withSite(async siteDir => {
const item = jaFixture();
Expand Down Expand Up @@ -822,7 +920,7 @@ test('runAgenticTranslation logs a batch overview and a cost summary with token
assert.match(batchLine, /english=\d+KB/);
assert.ok(lines.findIndex(line => line.includes('[agentic-provider] batch ')) < lines.findIndex(line => line.includes('turn translate')));
const summaryLine = lines.find(line => line.includes('[agentic-provider] summary '));
assert.match(summaryLine, /translated=2 failed=0 seeded-full=0 verified-current=0 agent-files=2 repair-turns=0/);
assert.match(summaryLine, /translated=2 failed=0 seeded-full=0 verified-current=0 unchanged-drafts=0 agent-files=2 repair-turns=0/);
const tokenLine = lines.find(line => line.includes('[agentic-provider] tokens '));
assert.match(tokenLine, /turns=2 input=9000 cached=1000 cache-write=0 output=800 reasoning=0/);
});
Expand Down Expand Up @@ -866,7 +964,7 @@ test('runAgenticTranslation counts fully seeded files and recovery reuse in the
assert.equal(report.checkpoint.translated, 2);
assert.equal(report.checkpoint.failed, 0);
const summaryLine = lines.find(line => line.includes('[agentic-provider] summary '));
assert.match(summaryLine, /translated=2 failed=0 seeded-full=1 verified-current=0 agent-files=0 repair-turns=0/);
assert.match(summaryLine, /translated=2 failed=0 seeded-full=1 verified-current=0 unchanged-drafts=0 agent-files=0 repair-turns=0/);
const batchLine = lines.find(line => line.includes('[agentic-provider] batch '));
assert.match(batchLine, /restored=1 /);
assert.match(batchLine, /model-files=1 /);
Expand Down