diff --git a/.github/workflows/i18n-translate.yml b/.github/workflows/i18n-translate.yml index ce2a622..3f52b57 100644 --- a/.github/workflows/i18n-translate.yml +++ b/.github/workflows/i18n-translate.yml @@ -1,29 +1,38 @@ name: i18n Auto-Translate -# Uses GitHub Models inference API (https://models.github.ai/inference) with the -# repository's GITHUB_TOKEN. Requires `models: read` permission. Default model is -# openai/gpt-5-mini (a fast/cheap GPT-5 in the catalog); override via repository variable -# TRANSLATE_MODEL (e.g. openai/gpt-5-nano for cheaper, openai/gpt-5 for flagship). +# Backends (select with repository variable TRANSLATE_BACKEND): +# github (default) — GitHub Models inference API (https://models.github.ai/inference) +# with the repository's GITHUB_TOKEN. Requires `models: read`. Default model +# openai/gpt-5-mini; override via repository variable TRANSLATE_MODEL. +# anthropic — Anthropic API direct. Set secret ANTHROPIC_API_KEY and +# TRANSLATE_MODEL to a Claude model id (e.g. claude-sonnet-4-6). +# openai — any OpenAI-compatible endpoint (OpenAI, DeepSeek, Qwen, one-api/ +# new-api gateways, vLLM, ...). Set secret TRANSLATE_API_KEY, secret or +# variable TRANSLATE_ENDPOINT (API base ending in /v1), and TRANSLATE_MODEL. # -# Optional: to use Anthropic Claude directly, add a repo secret ANTHROPIC_API_KEY, -# then set repository variable TRANSLATE_BACKEND=anthropic and TRANSLATE_MODEL to -# a Claude model id (e.g. claude-sonnet-4-6). Claude is NOT in the GitHub Models -# catalog as of 2026-05. +# Trigger is `pull_request_target` (NOT `pull_request`) so that fork PRs also +# get secrets and a write token — this is what allows auto-translating and +# pushing translations back to fork branches ("Allow edits by maintainers"). # -# Scope: on pull_request events, only skills whose SKILL*.md files actually changed -# in the PR are checked/translated — keeps token usage proportional to the PR. If +# SECURITY MODEL — read before editing: +# pull_request_target runs with secrets in the BASE repo context, so this +# workflow must NEVER execute code from the PR head. We therefore: +# 1. check out the trusted base branch (workflow + scripts/i18n/* run from it), +# 2. fetch the PR *merge ref* and copy in DATA files only +# (skills/, manifest.json, categories.json), dropping symlinks, +# 3. run the trusted scripts over that data, +# 4. push resulting translations to the PR head branch with the bot token. +# Any value derived from the PR (branch name, repo name) is passed to shell +# steps via `env:` — never interpolated with ${{ }} inside `run:`. +# +# Scope: on PR events, only skills whose SKILL*.md files actually changed in +# the PR are checked/translated — keeps token usage proportional to the PR. If # manifest.json or categories.json changed, falls back to full-repo scan (these # affect the i18n fallback chain globally). workflow_dispatch always does a full # scan unless `skill` input is supplied. -# -# Fork PRs: GITHUB_TOKEN is read-only and repo secrets are unavailable, so we -# can't push translations, comment, or label. We check out the PR merge ref -# (refs/pull/N/merge — the head branch only exists in the fork), run the same -# stale-translation check, and fail with instructions for the contributor to -# run scripts/i18n/translate.py locally if translations are missing. on: - pull_request: + pull_request_target: workflow_dispatch: inputs: target_locale: @@ -36,71 +45,77 @@ on: permissions: contents: write pull-requests: write + issues: write models: read concurrency: - group: i18n-translate-${{ github.ref }} + # Per-PR group (pull_request_target's github.ref is the base branch, which + # would otherwise serialize/cancel unrelated PRs). + group: i18n-translate-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: translate: - # Don't run on bot's own commits to avoid loops - if: github.actor != 'desirecore-bot[bot]' && !contains(github.event.head_commit.message, '[skip ci]') + # Don't run on the bot's own translation pushes to avoid loops + if: github.actor != 'desirecore-bot' && github.actor != 'desirecore-bot[bot]' runs-on: ubuntu-latest timeout-minutes: 30 steps: - - name: Checkout PR branch + - name: Checkout trusted base uses: actions/checkout@v4 with: - # Same-repo PR: check out the head branch so translations can be - # pushed back to it. Fork PR: the head branch doesn't exist in this - # repo, so check out the PR merge ref instead (github.ref = - # refs/pull/N/merge on pull_request events) — read-only checks still - # run, write steps are skipped below. workflow_dispatch: github.ref. - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.head.ref || github.ref }} + # No `ref:` on purpose — for pull_request_target this is the BASE + # branch (trusted scripts); for workflow_dispatch it's the chosen ref. token: ${{ secrets.DESIRECORE_BOT_TOKEN || secrets.GITHUB_TOKEN }} fetch-depth: 0 + - name: Overlay PR content (data only) + if: github.event_name == 'pull_request_target' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -e + # The merge ref is maintained by GitHub in the BASE repo (fork branches + # live elsewhere) and already contains PR + base merged. + if ! git fetch origin "refs/pull/${PR_NUMBER}/merge"; then + echo "::error::Could not fetch refs/pull/${PR_NUMBER}/merge — the PR likely has merge conflicts. Resolve them and the check will re-run." + exit 1 + fi + # Replace data files with the PR's merged state. Trusted code + # (scripts/, .github/) intentionally stays at the base version. + rm -rf skills manifest.json categories.json + git checkout FETCH_HEAD -- skills manifest.json categories.json + # Defense in depth: a malicious PR could add a symlink named SKILL.md + # pointing outside the tree; translate.py would read through it and + # could leak runner file content into a committed "translation". + find skills -type l -print -delete + # The root data files are untrusted too — reject symlinked variants + # instead of letting trusted scripts read through them. + for f in manifest.json categories.json; do + if [ -L "$f" ]; then + echo "::error::$f in the PR is a symlink; refusing to process." + exit 1 + fi + done + git add -A + git -c user.name=ci -c user.email=ci@invalid commit --allow-empty -m "overlay: PR #${PR_NUMBER} data files" + - name: Detect relevant changes id: changes - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | # GitHub Actions default shell already runs with -eo pipefail; we set # it explicitly so the `|| true` workaround on grep below is unambiguous. set -eo pipefail # workflow_dispatch: full scan (skill input handled later if provided) - if [ "${{ github.event_name }}" != "pull_request" ]; then + if [ "${{ github.event_name }}" != "pull_request_target" ]; then echo "relevant=true" >> "$GITHUB_OUTPUT" echo "skill_paths=" >> "$GITHUB_OUTPUT" exit 0 fi - # Ensure BASE_SHA is locally reachable; if we can't fetch it, fall back - # to a full scan rather than silently producing an empty diff that would - # mis-classify a real i18n PR as relevant=false. - if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then - if ! git fetch --no-tags --depth=1 origin "$BASE_SHA" 2>/dev/null; then - echo "::warning::failed to fetch base SHA ${BASE_SHA}; falling back to full scan" - echo "relevant=true" >> "$GITHUB_OUTPUT" - echo "skill_paths=" >> "$GITHUB_OUTPUT" - exit 0 - fi - fi - - # `A...B` (triple-dot) needs merge-base(A,B) in history. With - # actions/checkout@v4 + fetch-depth: 0 it normally is, but if the base - # branch advanced after the PR was opened and the merge-base isn't - # locally reachable, fall back to a full scan instead of silently - # producing a wrong diff. - if ! merge_base=$(git merge-base "${BASE_SHA}" "${HEAD_SHA}" 2>/dev/null); then - echo "::warning::could not compute merge-base for ${BASE_SHA}...${HEAD_SHA}; falling back to full scan" - echo "relevant=true" >> "$GITHUB_OUTPUT" - echo "skill_paths=" >> "$GITHUB_OUTPUT" - exit 0 - fi - changed=$(git diff --name-only "${merge_base}" "${HEAD_SHA}") + # The overlay commit (HEAD) contains exactly the PR's effective + # changes to skills/ + manifest.json + categories.json vs base. + changed=$(git diff --name-only HEAD~1 HEAD) # If manifest.json or categories.json changed, fall back to full scan # (these affect i18n fallback chain / supportedLocales globally). @@ -157,8 +172,8 @@ jobs: # workflow_dispatch with explicit skill input takes precedence ARGS+=("$INPUT_SKILL") elif [ -n "$SKILL_PATHS" ]; then - # pull_request: read space-separated paths into an array to avoid - # word-splitting / glob expansion surprises. + # pull_request_target: read space-separated paths into an array to + # avoid word-splitting / glob expansion surprises. read -r -a SKILL_ARR <<<"$SKILL_PATHS" ARGS+=("${SKILL_ARR[@]}") fi @@ -171,24 +186,13 @@ jobs: fi exit 0 - - name: Fail fork PR with stale translations - # Fork PRs get a read-only GITHUB_TOKEN and no repo secrets, so the - # workflow can't push generated translations back to the PR branch. - # Fail here (before spending model tokens) with instructions instead. - if: >- - steps.changes.outputs.relevant == 'true' && - steps.precheck.outputs.stale == 'true' && - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository - run: | - echo "::error::The i18n pre-check failed for this fork PR (stale/missing translations, or a check error — see the 'Check for stale translations' step logs above), and the workflow cannot push auto-generated translations to a fork. Please run 'uv run scripts/i18n/translate.py --check' locally to see what's wrong, then 'uv run scripts/i18n/translate.py' and commit the generated SKILL..md files to your branch." - exit 1 - - name: Translate stale locales if: steps.changes.outputs.relevant == 'true' && steps.precheck.outputs.stale == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + TRANSLATE_API_KEY: ${{ secrets.TRANSLATE_API_KEY }} + TRANSLATE_ENDPOINT: ${{ secrets.TRANSLATE_ENDPOINT || vars.TRANSLATE_ENDPOINT }} TRANSLATE_BACKEND: ${{ vars.TRANSLATE_BACKEND || 'github' }} TRANSLATE_MODEL: ${{ vars.TRANSLATE_MODEL || 'openai/gpt-5-mini' }} SKILL_PATHS: ${{ steps.changes.outputs.skill_paths }} @@ -201,7 +205,7 @@ jobs: # workflow_dispatch input takes precedence (single specific skill) ARGS+=("$INPUT_SKILL") elif [ -n "$SKILL_PATHS" ]; then - # pull_request: only translate skills touched by this PR + # pull_request_target: only translate skills touched by this PR read -r -a SKILL_ARR <<<"$SKILL_PATHS" ARGS+=("${SKILL_ARR[@]}") fi @@ -225,11 +229,56 @@ jobs: git diff --stat fi + - name: Push translations to PR branch + if: steps.diff.outputs.changed == 'true' && github.event_name == 'pull_request_target' + env: + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BOT_TOKEN: ${{ secrets.DESIRECORE_BOT_TOKEN || secrets.GITHUB_TOKEN }} + TRANSLATE_BACKEND: ${{ vars.TRANSLATE_BACKEND || 'github' }} + TRANSLATE_MODEL: ${{ vars.TRANSLATE_MODEL || 'openai/gpt-5-mini' }} + run: | + set -e + changed_files=$(git diff --name-only) + # Enforce the security model: translate.py only ever writes + # skills//SKILL*.md. Anything else in the diff means an + # unexpected write — abort rather than push it to a contributor branch. + unexpected=$(printf '%s\n' "$changed_files" \ + | { grep -vE '^skills/[a-z0-9]([a-z0-9-]*[a-z0-9])?/SKILL(\.[a-z]{2,3}(-[A-Z]{2})?)?\.md$' || true; }) + if [ -n "$unexpected" ]; then + echo "::error::Refusing to push non-translation changes to the PR branch:" + printf '%s\n' "$unexpected" + exit 1 + fi + # The working tree is base + PR overlay + translations; the PR branch + # itself may live in a fork. Apply only the translation-generated + # files onto a checkout of the actual PR head and push that. + git remote add prhead "https://x-access-token:${BOT_TOKEN}@github.com/${HEAD_REPO}.git" + git fetch prhead "refs/heads/${HEAD_REF}" + if [ "$(git rev-parse FETCH_HEAD)" != "$HEAD_SHA" ]; then + echo "::notice::PR head moved since this run started; skipping push — the newer run will translate." + exit 0 + fi + git worktree add --detach ../prpush FETCH_HEAD + while IFS= read -r f; do + mkdir -p "../prpush/$(dirname "$f")" + cp "$f" "../prpush/$f" + done <<<"$changed_files" + cd ../prpush + git config user.name "desirecore-bot" + git config user.email "bot@desirecore.net" + git add -A + git commit -m "chore(i18n): auto-translate skills [skip ci]" \ + -m "Generated by scripts/i18n/translate.py via i18n-translate workflow." \ + -m "Backend: ${TRANSLATE_BACKEND} Model: ${TRANSLATE_MODEL}" + if ! git push prhead HEAD:"refs/heads/${HEAD_REF}"; then + echo "::error::Could not push translations to ${HEAD_REPO}:${HEAD_REF}. For fork PRs, enable 'Allow edits by maintainers' on the PR, or run 'uv run scripts/i18n/translate.py' locally and commit the generated SKILL..md files." + exit 1 + fi + - name: Commit & push translations - if: >- - steps.diff.outputs.changed == 'true' && - (github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository) + if: steps.diff.outputs.changed == 'true' && github.event_name != 'pull_request_target' run: | git config user.name "desirecore-bot" git config user.email "bot@desirecore.net" @@ -243,7 +292,7 @@ jobs: TRANSLATE_MODEL: ${{ vars.TRANSLATE_MODEL || 'openai/gpt-5-mini' }} - name: Comment on PR - if: steps.diff.outputs.changed == 'true' && github.event_name == 'pull_request' + if: steps.diff.outputs.changed == 'true' && github.event_name == 'pull_request_target' uses: actions/github-script@v7 with: github-token: ${{ secrets.DESIRECORE_BOT_TOKEN || secrets.GITHUB_TOKEN }} @@ -273,12 +322,7 @@ jobs: TRANSLATE_MODEL: ${{ vars.TRANSLATE_MODEL || 'openai/gpt-5-mini' }} - name: Label on translation failure - # Skip on fork PRs: GITHUB_TOKEN is read-only there and repo secrets - # are unavailable, so addLabels would 403 ("Resource not accessible - # by integration"). The failed check itself is the signal. - if: >- - failure() && github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository + if: failure() && github.event_name == 'pull_request_target' uses: actions/github-script@v7 with: github-token: ${{ secrets.DESIRECORE_BOT_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/scripts/i18n/translate.py b/scripts/i18n/translate.py index c2f8fd9..18d98ac 100755 --- a/scripts/i18n/translate.py +++ b/scripts/i18n/translate.py @@ -10,13 +10,18 @@ manifest.json/supportedLocales. When a target locale is missing or stale (its source_hash differs from the current source body+strings hash), translate from metadata.i18n..body using an LLM. -Backends (auto-selected, in this priority): +Backends (selected via TRANSLATE_BACKEND): 1. GitHub Models (default) — uses GITHUB_TOKEN with `models: read` permission, OpenAI-compatible chat-completions API at https://models.github.ai/inference. Model defaults to `openai/gpt-5-mini` (configure with TRANSLATE_MODEL). 2. Anthropic API direct — used when ANTHROPIC_API_KEY is set AND TRANSLATE_BACKEND=anthropic. Endpoint https://api.anthropic.com/v1/messages. Model should be a Claude model id (e.g. claude-sonnet-4-6). + 3. Generic OpenAI-compatible — TRANSLATE_BACKEND=openai. Any endpoint that + speaks the OpenAI chat-completions contract (OpenAI, DeepSeek, Qwen, + one-api/new-api gateways, vLLM, Ollama, ...). Set TRANSLATE_ENDPOINT (or + OPENAI_BASE_URL) to the API base ending in /v1 (or your gateway's + equivalent) and TRANSLATE_API_KEY (or OPENAI_API_KEY) for auth. Translations preserve: - Markdown structure (heading hierarchy, list ordering, tables, fences) @@ -41,9 +46,10 @@ Usage: Env: GITHUB_TOKEN required when backend=github (CI: provided automatically) ANTHROPIC_API_KEY required when TRANSLATE_BACKEND=anthropic - TRANSLATE_BACKEND 'github' (default) | 'anthropic' + TRANSLATE_API_KEY required when TRANSLATE_BACKEND=openai (or OPENAI_API_KEY) + TRANSLATE_BACKEND 'github' (default) | 'anthropic' | 'openai' TRANSLATE_MODEL backend-specific model id; default depends on backend - TRANSLATE_ENDPOINT override endpoint URL + TRANSLATE_ENDPOINT override endpoint URL (openai backend: OPENAI_BASE_URL also works) TRANSLATE_MAX_RETRIES default 3 """ from __future__ import annotations @@ -71,10 +77,12 @@ DEFAULT_BACKEND = os.environ.get("TRANSLATE_BACKEND", "github").lower() DEFAULT_MODEL_BY_BACKEND = { "github": os.environ.get("TRANSLATE_MODEL", "openai/gpt-5-mini"), "anthropic": os.environ.get("TRANSLATE_MODEL", "claude-sonnet-4-6"), + "openai": os.environ.get("TRANSLATE_MODEL", "gpt-4o-mini"), } DEFAULT_ENDPOINT_BY_BACKEND = { "github": "https://models.github.ai/inference", "anthropic": "https://api.anthropic.com", + "openai": "https://api.openai.com/v1", } MAX_RETRIES = int(os.environ.get("TRANSLATE_MAX_RETRIES", "3")) HTTP_TIMEOUT = httpx.Timeout(connect=10, read=180, write=30, pool=10) @@ -216,6 +224,47 @@ def call_github_models(system_prompt: str, user_payload: str, model: str, endpoi return _post_with_retries(url, headers, payload, extract=_extract_openai_text) +def call_openai_compatible(system_prompt: str, user_payload: str, model: str, endpoint: str) -> str: + """Call any OpenAI-compatible chat-completions endpoint (OpenAI, DeepSeek, + Qwen, one-api/new-api gateways, vLLM, Ollama, ...). + + Endpoint base: TRANSLATE_ENDPOINT / OPENAI_BASE_URL, ending in /v1 for most + providers (the request path appends /chat/completions). + Auth: Authorization: Bearer . + """ + api_key = os.environ.get("TRANSLATE_API_KEY") or os.environ.get("OPENAI_API_KEY") + if not api_key: + raise RuntimeError("TRANSLATE_API_KEY (or OPENAI_API_KEY) not set for backend='openai'") + url = f"{endpoint.rstrip('/')}/chat/completions" + payload: dict[str, Any] = { + "model": model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_payload}, + ], + } + # GPT-5 / o-series require `max_completion_tokens` and reject non-default + # temperature; everything else (including most OpenAI-compatible providers) + # still expects the classic `max_tokens` + temperature contract. + if _is_new_openai_contract(model): + payload["max_completion_tokens"] = 8192 + else: + payload["max_tokens"] = 8192 + payload["temperature"] = 0.1 + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + return _post_with_retries(url, headers, payload, extract=_extract_openai_text) + + +def _is_new_openai_contract(model: str) -> bool: + """True for OpenAI GPT-5 / o-series ids, with or without a publisher prefix + (e.g. 'gpt-5-mini', 'openai/gpt-5', 'o1-mini', 'openai/o3').""" + bare = model.lower().rsplit("/", 1)[-1] + return "gpt-5" in bare or bool(re.match(r"^o[134](-|$)", bare)) + + def call_anthropic(system_prompt: str, user_payload: str, model: str, endpoint: str) -> str: """Call Anthropic Messages API directly.""" api_key = os.environ.get("ANTHROPIC_API_KEY") @@ -298,6 +347,8 @@ def call_llm(system_prompt: str, user_payload: str, *, backend: str, model: str, text = call_github_models(system_prompt, user_payload, model, endpoint) elif backend == "anthropic": text = call_anthropic(system_prompt, user_payload, model, endpoint) + elif backend == "openai": + text = call_openai_compatible(system_prompt, user_payload, model, endpoint) else: raise RuntimeError(f"Unknown backend: {backend}") return parse_json_response(text) @@ -472,10 +523,14 @@ def get_target_locales(args: argparse.Namespace) -> list[str]: def resolve_backend(args: argparse.Namespace) -> tuple[str, str, str]: backend = (args.backend or DEFAULT_BACKEND).lower() - if backend not in ("github", "anthropic"): - raise SystemExit(f"Unknown backend '{backend}'; choose 'github' or 'anthropic'") + if backend not in ("github", "anthropic", "openai"): + raise SystemExit(f"Unknown backend '{backend}'; choose 'github', 'anthropic' or 'openai'") model = args.model or DEFAULT_MODEL_BY_BACKEND[backend] - endpoint = args.endpoint or os.environ.get("TRANSLATE_ENDPOINT") or DEFAULT_ENDPOINT_BY_BACKEND[backend] + endpoint = args.endpoint or os.environ.get("TRANSLATE_ENDPOINT") + if not endpoint and backend == "openai": + endpoint = os.environ.get("OPENAI_BASE_URL") + if not endpoint: + endpoint = DEFAULT_ENDPOINT_BY_BACKEND[backend] return backend, model, endpoint @@ -499,7 +554,7 @@ def main(argv: list[str]) -> int: parser.add_argument("--target", help="Single target locale (default: all manifest.supportedLocales)") parser.add_argument("--check", action="store_true", help="Report stale translations; exit 1 if any") parser.add_argument("--human", action="store_true", help="Mark new translations as 'human' (locks against re-translation)") - parser.add_argument("--backend", choices=("github", "anthropic"), help="Override backend (default: env TRANSLATE_BACKEND or 'github')") + parser.add_argument("--backend", choices=("github", "anthropic", "openai"), help="Override backend (default: env TRANSLATE_BACKEND or 'github')") parser.add_argument("--model", help="Override model id") parser.add_argument("--endpoint", help="Override API endpoint") parser.add_argument("--list-models", action="store_true", help="List models in GitHub Models catalog and exit") @@ -517,6 +572,9 @@ def main(argv: list[str]) -> int: if backend == "anthropic" and not os.environ.get("ANTHROPIC_API_KEY"): sys.stderr.write("ERROR: ANTHROPIC_API_KEY not set for backend='anthropic'\n") return 2 + if backend == "openai" and not (os.environ.get("TRANSLATE_API_KEY") or os.environ.get("OPENAI_API_KEY")): + sys.stderr.write("ERROR: TRANSLATE_API_KEY (or OPENAI_API_KEY) not set for backend='openai'\n") + return 2 if args.paths: targets = [Path(p).resolve() for p in args.paths]