mirror of
https://git.openapi.site/https://github.com/desirecore/config-center.git
synced 2026-10-21 07:04:11 +08:00
feat: 建立版本化模型目录与规范字段生成链路 (#116)
This commit is contained in:
GitHub
124
scripts/catalog-v2/compile.mjs
Normal file
124
scripts/catalog-v2/compile.mjs
Normal file
@@ -0,0 +1,124 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
import Ajv from 'ajv';
|
||||
import addFormats from 'ajv-formats';
|
||||
import {resolveEffectiveModel} from './effective-model.mjs';
|
||||
const root=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'../..');
|
||||
const clone=x=>structuredClone(x);
|
||||
let canonicalValidator;
|
||||
export function validateCanonicalSchema(catalog) {
|
||||
if(!canonicalValidator){const ajv=new Ajv({allErrors:true,strict:false});addFormats(ajv);canonicalValidator=ajv.compile(JSON.parse(fs.readFileSync(path.join(root,'schemas/catalog-v2.schema.json'),'utf8')));}
|
||||
if(!canonicalValidator(catalog))throw new Error(`Invalid V2 schema: ${JSON.stringify(canonicalValidator.errors)}`);
|
||||
}
|
||||
const REQUEST_KEYS=new Set(['adaptiveThinking','constraints','reasoning','serverSideWebSearch','thinking','thinkingRoundTrip','thinkingToolTurnValidation','endpointPath','apiFormatOverride','speech','responsesOnly','preserveThinkingDefault','upstreamModelId']);
|
||||
export function stableJson(value){if(Array.isArray(value))return `[${value.map(stableJson).join(',')}]`;if(value&&typeof value==='object')return `{${Object.keys(value).sort().map(k=>`${JSON.stringify(k)}:${stableJson(value[k])}`).join(',')}}`;return JSON.stringify(value);}
|
||||
export function digest(value){return crypto.createHash('sha256').update(stableJson(value)).digest('hex');}
|
||||
export function projectV1(catalog){
|
||||
const outputs={};
|
||||
for(const [filename,header] of Object.entries(catalog.metadata.specFiles)) outputs[`compute/model-specs/${filename}`]={...clone(header),specs:[]};
|
||||
const byRef=new Map(catalog.specs.map(s=>[s.modelRef,s]));
|
||||
for(const spec of catalog.specs){
|
||||
const data={};for(const key of spec.metadata.projection.specKeys){
|
||||
if(key==='extra'){data.extra={};for(const name of spec.metadata.projection.extraKeys){const bucket=Object.hasOwn(spec.facts.constraints??{},name)?spec.facts.constraints:spec.metadata.recordedExtra;const adaptiveProfile=spec.protocolProfiles?.find(p=>p.apiFormats.includes('anthropic-messages'));data.extra[name]=clone(name==='adaptiveThinking'&&adaptiveProfile?adaptiveProfile.request.adaptiveThinking:bucket[name]);}}
|
||||
else data[key]=clone(Object.hasOwn(spec.metadata.recordedFacts??{},key)?spec.metadata.recordedFacts[key]:spec.facts[key]);
|
||||
}
|
||||
outputs[spec.metadata.sourceFile].specs.push({...clone(spec.metadata.legacyIdentity),id:spec.identity.canonicalModelId,spec:data,...(Object.keys(spec.routing).length?{routing:clone(spec.routing)}:{})});
|
||||
}
|
||||
for(const access of catalog.accesses){
|
||||
const models=access.models.map(model=>{
|
||||
const spec=byRef.get(model.modelRef), out={};
|
||||
for(const key of model.metadata.projection.modelKeys){
|
||||
if(key==='modelName')out.modelName=model.apiModelId;
|
||||
else if(key==='apiModelId')out.apiModelId=model.request.upstreamModelId;
|
||||
else if(key==='extra'){
|
||||
out.extra={};for(const name of model.metadata.projection.extraKeys){
|
||||
const price=model.billing.find(b=>b.sourceKey===`extra.${name}`);
|
||||
if(price)out.extra[name]=clone(price.amount);
|
||||
else if(Object.hasOwn(model.request,name))out.extra[name]=clone(model.request[name]);
|
||||
else if(Object.hasOwn(model.request.constraints??{},name))out.extra[name]=clone(model.request.constraints[name]);
|
||||
else if(Object.hasOwn(model.lifecycle,name))out.extra[name]=clone(model.lifecycle[name]);
|
||||
else if(Object.hasOwn(model.metadata.recordedExtra,name))out.extra[name]=clone(model.metadata.recordedExtra[name]);
|
||||
else throw new Error(`Missing projection ${model.apiModelId}.extra.${name}`);
|
||||
for(const nested of model.billing.filter(b=>b.sourceKey.startsWith(`extra.${name}.`))){const keys=nested.sourceKey.slice(`extra.${name}.`.length).split('.');let target=out.extra[name];for(const part of keys.slice(0,-1))target=target[part]??={};target[keys.at(-1)]=clone(nested.amount);}
|
||||
}
|
||||
} else if(['inputPrice','outputPrice'].includes(key))out[key]=clone(model.billing.find(b=>b.sourceKey===key)?.amount);
|
||||
else if(Object.hasOwn(model.metadata.compatibilityValues??{},key))out[key]=clone(model.metadata.compatibilityValues[key]);
|
||||
else if(Object.hasOwn(model.overrides,key))out[key]=clone(model.overrides[key].value);
|
||||
else if(Object.hasOwn(model.metadata.attributes,key))out[key]=clone(model.metadata.attributes[key]);
|
||||
else if(Object.hasOwn(spec?.facts??{},key))out[key]=clone(spec.facts[key]);
|
||||
else throw new Error(`Missing projection ${model.apiModelId}.${key}`);
|
||||
}
|
||||
return out;
|
||||
});outputs[access.metadata.sourceFile]={...clone(access.connection),models};
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
export function validateCatalog(catalog){
|
||||
validateCanonicalSchema(catalog);
|
||||
const refs=new Set();
|
||||
for(const spec of catalog.specs){if(refs.has(spec.modelRef))throw new Error(`Duplicate modelRef ${spec.modelRef}`);refs.add(spec.modelRef);if(Object.keys(spec.facts).some(k=>/price|billing/i.test(k)))throw new Error('Spec contains billing');}
|
||||
const accessIds=new Set();
|
||||
for(const access of catalog.accesses){if(accessIds.has(access.id))throw new Error(`Duplicate access ${access.id}`);accessIds.add(access.id);for(const model of access.models){
|
||||
if(model.binding.kind==='exact'&&!refs.has(model.modelRef))throw new Error(`Unknown spec ${model.modelRef}`);
|
||||
if(model.binding.kind!=='exact'&&model.modelRef!==null)throw new Error('Dynamic/unresolved cannot claim exact spec');
|
||||
for(const [k,v]of Object.entries(model.overrides))if(!v.reason||!Object.hasOwn(v,'value'))throw new Error(`Unexplained override ${k}`);
|
||||
for(const key of Object.keys(model.request))if(!REQUEST_KEYS.has(key))throw new Error(`Unknown managed request key ${key}`);
|
||||
for(const key of ['reasoningEffort','defaultReasoningEffort','thinkingDefault','supportsThinking','cachedInputPrice','cacheHitPrice']) if(Object.hasOwn(model.request,key))throw new Error(`Old runtime key ${key}`);
|
||||
for(const bill of model.billing)if(!bill.unit||!bill.currency)throw new Error('Billing dimension missing');
|
||||
}}
|
||||
if(catalog.publication.state==='published'&&!/^\d+\.\d+\.\d+$/.test(catalog.publication.requiredClientVersion??''))throw new Error('Published catalog needs real minimum client version');
|
||||
}
|
||||
export function compileCatalog(catalog){
|
||||
validateCatalog(catalog);
|
||||
const output=clone(catalog);
|
||||
// Projection-only metadata never enters the runtime contract; useful descriptive values remain non-consumed.
|
||||
delete output.metadata.specFiles;
|
||||
for(const spec of output.specs){delete spec.metadata.projection;delete spec.metadata.legacyIdentity;}
|
||||
for(const access of output.accesses)for(const model of access.models)delete model.metadata.projection;
|
||||
output.digest=digest(output);
|
||||
const projection=projectV1(catalog);
|
||||
const diagnostics={specs:catalog.specs.length,accesses:catalog.accesses.length,models:catalog.accesses.reduce((n,a)=>n+a.models.length,0),bindings:{},overrides:[],publication:catalog.publication};
|
||||
for(const a of catalog.accesses)for(const m of a.models){diagnostics.bindings[m.binding.kind]=(diagnostics.bindings[m.binding.kind]??0)+1;for(const [k,v]of Object.entries(m.overrides))diagnostics.overrides.push({accessId:a.id,apiModelId:m.apiModelId,field:k,...v});}
|
||||
const specsByRef = new Map(output.specs.map(spec => [spec.modelRef,spec]));
|
||||
const resolverParity={formatVersion:2,fixtureOnly:true,clientVersion:'999.0.0',requiredClientVersion:'0.0.0',now:'2026-10-03T00:00:00Z',models:[]};
|
||||
for(const access of output.accesses)for(const model of access.models){
|
||||
const effective=resolveEffectiveModel(specsByRef.get(model.modelRef),model,{publicationState:'published',requiredClientVersion:resolverParity.requiredClientVersion,clientVersion:resolverParity.clientVersion,now:resolverParity.now,connection:access.connection});
|
||||
resolverParity.models.push({accessId:access.id,apiModelId:model.apiModelId,digest:digest(effective),selectable:effective.selectable,effectiveApiFormat:effective.apiFormat,effectiveApiModelId:effective.apiModelId,eligibleForCall:effective.eligibleForCall,eligibleForAutoRouting:effective.eligibleForAutoRouting,manualOnly:effective.manualOnly,blockedReasons:effective.blockedReasons});
|
||||
}
|
||||
return {output,projection,diagnostics,resolverParity};
|
||||
}
|
||||
/** Root V1 files are generated publication projections, never author inputs.
|
||||
* Preserve harmless existing serialization when values match to avoid unrelated key-order churn.
|
||||
*/
|
||||
export function synchronizeRootProjection(repoRoot, projection, {check=false}={}) {
|
||||
for(const [file,data] of Object.entries(projection)){
|
||||
if(!/^compute\/(providers|coding-plans|model-specs)\/[^/]+\.json$/.test(file) || path.basename(file).startsWith('_'))throw new Error(`Invalid V1 publication path ${file}`);
|
||||
const absolute=path.join(repoRoot,file);
|
||||
let matches=false;
|
||||
if(fs.existsSync(absolute)){
|
||||
try{matches=stableJson(JSON.parse(fs.readFileSync(absolute,'utf8')))===stableJson(data);}catch{matches=false;}
|
||||
}
|
||||
if(check){if(!matches)throw new Error(`Stale V1 projection ${file}; edit catalog/v2 then run catalog:compile`);}
|
||||
else if(!matches){fs.mkdirSync(path.dirname(absolute),{recursive:true});fs.writeFileSync(absolute,JSON.stringify(data,null,2)+'\n');}
|
||||
}
|
||||
}
|
||||
export function run({check=false}={}){
|
||||
const catalog=JSON.parse(fs.readFileSync(path.join(root,'catalog/v2/catalog.json'),'utf8'));
|
||||
const retirementsPath=path.join(root,'catalog/v2/retirements.json');
|
||||
if(fs.existsSync(retirementsPath)){const registry=JSON.parse(fs.readFileSync(retirementsPath,'utf8'));catalog.retirements=Array.isArray(registry)?registry:registry.retirements??registry.rules??[];}
|
||||
const schema=JSON.parse(fs.readFileSync(path.join(root,'schemas/catalog-v2.schema.json'),'utf8'));
|
||||
const ajv=new Ajv({allErrors:true,strict:false});addFormats(ajv);const validate=ajv.compile(schema);if(!validate(catalog))throw new Error(JSON.stringify(validate.errors));
|
||||
const {output,projection,diagnostics,resolverParity}=compileCatalog(catalog);
|
||||
const providerValidator=ajv.compile(JSON.parse(fs.readFileSync(path.join(root,'schemas/provider.schema.json'),'utf8')));
|
||||
const specValidator=ajv.compile(JSON.parse(fs.readFileSync(path.join(root,'schemas/model-spec.schema.json'),'utf8')));
|
||||
for(const [file,data]of Object.entries(projection)){const validator=file.includes('/model-specs/')?specValidator:providerValidator;if(!validator(data))throw new Error(`${file}: ${JSON.stringify(validator.errors)}`);}
|
||||
synchronizeRootProjection(root,projection,{check});
|
||||
const files={'compute/v2/resolver-parity.json':resolverParity,'compute/v2/catalog.json':output,'compute/v2/diagnostics.json':diagnostics,'compute/v2/manifest.json':{formatVersion:2,state:catalog.publication.state,requiredClientVersion:catalog.publication.requiredClientVersion,catalogPath:'compute/v2/catalog.json',digest:output.digest}};
|
||||
for(const [file,data]of Object.entries(projection)) files[`compute/v2/compatibility/${file.slice('compute/'.length)}`]=data;
|
||||
for(const [file,data]of Object.entries(files)){const absolute=path.join(root,file),text=JSON.stringify(data,null,2)+'\n';if(check){if(!fs.existsSync(absolute)||fs.readFileSync(absolute,'utf8')!==text)throw new Error(`Stale generated file ${file}`);}else {fs.mkdirSync(path.dirname(absolute),{recursive:true});fs.writeFileSync(absolute,text);}}
|
||||
console.log(`V2 ${check?'verified':'compiled'}: ${diagnostics.models} models; state=${catalog.publication.state}; digest=${output.digest}`);
|
||||
return diagnostics;
|
||||
}
|
||||
if(process.argv[1]===fileURLToPath(import.meta.url))run({check:process.argv.includes('--check')});
|
||||
75
scripts/catalog-v2/effective-model.mjs
Normal file
75
scripts/catalog-v2/effective-model.mjs
Normal file
@@ -0,0 +1,75 @@
|
||||
/** Pure contract resolver. No name inference, capability union, or metadata consumption. */
|
||||
export function compareVersions(a, b) {
|
||||
const parse = value => typeof value === 'string' && /^\d+\.\d+\.\d+$/.test(value) ? value.split('.').map(Number) : null;
|
||||
const x = parse(a), y = parse(b);
|
||||
if (!x || !y) return null;
|
||||
for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return Math.sign(x[i] - y[i]);
|
||||
return 0;
|
||||
}
|
||||
const SUPPORTED_PROTOCOLS = ['openai-completions','openai-responses','anthropic-messages','google-generative-ai','openai-codex-responses','kling-task-api','stability-rest-api'];
|
||||
const HARD_CONSTRAINTS = ['thinkingOnly', 'samplingParametersDeprecated', 'forcedToolChoiceUnsupported'];
|
||||
export function resolveEffectiveModel(spec, accessModel, options = {}) {
|
||||
const blockedReasons = [], provenance = {}, pendingOverrides = [];
|
||||
if (options.publicationState !== 'published') blockedReasons.push('catalog-not-published');
|
||||
const connection = options.connection ?? {};
|
||||
const protocol = accessModel.protocol ?? options.protocol ?? {};
|
||||
let apiFormat = accessModel.request?.apiFormatOverride ?? protocol.apiFormat ?? connection.apiFormat;
|
||||
if(accessModel.request?.responsesOnly===true && apiFormat==='openai-completions')apiFormat='openai-responses';
|
||||
const exact = Boolean(spec && accessModel.binding?.kind === 'exact' && spec.modelRef === accessModel.modelRef);
|
||||
const profiles = (exact ? spec.protocolProfiles ?? [] : []).filter(profile => profile.apiFormats.includes(apiFormat));
|
||||
const required = [options.requiredClientVersion, connection.requiredClientVersion, connection.minimumClientVersion, accessModel.requiredClientVersion, protocol.requiredClientVersion, exact ? spec.requiredClientVersion : undefined, ...profiles.map(p => p.requiredClientVersion)].filter(v => v !== undefined);
|
||||
if (!required.length || required.some(v => compareVersions(options.clientVersion, v) === null)) blockedReasons.push('invalid-client-version');
|
||||
else if (required.some(v => compareVersions(options.clientVersion, v) < 0)) blockedReasons.push('client-update-required');
|
||||
const local = ['access-local','dynamic'].includes(accessModel.binding?.kind) && accessModel.modelRef === null;
|
||||
if (!exact && !local) blockedReasons.push('unresolved-model-binding');
|
||||
if (!(options.supportedApiFormats ?? SUPPORTED_PROTOCOLS).includes(apiFormat) || protocol.state === 'unknown') blockedReasons.push('unknown-adapter');
|
||||
const now = options.now === undefined ? Date.now() : typeof options.now === 'number' ? options.now : Date.parse(options.now);
|
||||
if (!Number.isFinite(now)) blockedReasons.push('invalid-lifecycle-clock');
|
||||
for (const lifecycle of [exact ? spec.lifecycle : undefined, accessModel.lifecycle]) {
|
||||
if (['retired', 'historical'].includes(lifecycle?.status)) blockedReasons.push('retired-model');
|
||||
if (lifecycle?.retiredAt !== undefined) {
|
||||
const expiry = /^\d{4}-\d{2}-\d{2}$/.test(lifecycle.retiredAt) ? Date.parse(`${lifecycle.retiredAt}T00:00:00Z`) : NaN;
|
||||
if (!Number.isFinite(expiry)) blockedReasons.push('invalid-retirement-date');
|
||||
else if (now >= expiry) blockedReasons.push('retired-model');
|
||||
}
|
||||
}
|
||||
if (protocol.state === 'unsupported') blockedReasons.push('unsupported-adapter');
|
||||
const facts = structuredClone(exact ? spec.facts : {});
|
||||
for (const key of Object.keys(facts)) provenance[key] = {owner: 'spec', modelRef: spec.modelRef};
|
||||
for (const [key, override] of Object.entries(accessModel.overrides ?? {})) {
|
||||
if (!override || !('value' in override) || !override.reason) throw new Error(`Unexplained override: ${key}`);
|
||||
const intrinsic = exact ? spec.facts[key] : undefined;
|
||||
const unverifiedExpansion = ['contextWindow','maxInputTokens','maxOutputTokens'].includes(key) && typeof intrinsic === 'number' && typeof override.value === 'number' && override.value > intrinsic && override.verification !== 'officially-verified-access-contract';
|
||||
facts[key] = unverifiedExpansion ? intrinsic : structuredClone(override.value);
|
||||
if (unverifiedExpansion) pendingOverrides.push({field:key, requested:override.value, enforced:intrinsic, reason:'unverified-budget-expansion'});
|
||||
provenance[key] = {owner: 'access', reason: override.reason, evidence: override.evidence ?? null};
|
||||
}
|
||||
// Preferences may tighten positive numeric budgets; they never elevate managed capabilities.
|
||||
for (const [key, value] of Object.entries(options.userOverrides ?? {})) {
|
||||
if (['contextWindow', 'maxInputTokens', 'maxOutputTokens'].includes(key) && Number.isFinite(value) && value > 0 && (facts[key] === undefined || facts[key] === null || value <= facts[key])) {
|
||||
facts[key] = value; provenance[key] = {owner: 'user'};
|
||||
}
|
||||
}
|
||||
const request = Object.assign({}, ...profiles.map(profile => structuredClone(profile.request)), structuredClone(accessModel.request ?? {}));
|
||||
if(accessModel.request?.responsesOnly===true && apiFormat==='openai-responses')request.apiFormatOverride='openai-responses';
|
||||
facts.constraints = {...(facts.constraints ?? {}), ...(request.constraints ?? {})};
|
||||
for (const key of HARD_CONSTRAINTS) {
|
||||
if (exact && spec.facts.constraints?.[key] === true) facts.constraints[key] = true;
|
||||
}
|
||||
// Mandatory protocol form cannot be disabled by a weaker access annotation.
|
||||
if (profiles.some(profile => profile.request.adaptiveThinking === true)) request.adaptiveThinking = true;
|
||||
request.constraints = structuredClone(facts.constraints);
|
||||
const supported = request.reasoning?.supportedEfforts, modes = exact ? spec.routing?.reasoning?.supportedModes : undefined;
|
||||
if (Array.isArray(supported) && Array.isArray(modes)) {
|
||||
request.reasoning.supportedEfforts = supported.filter(e => modes.includes(e) || e === 'none' && modes.includes('off'));
|
||||
if (!request.reasoning.supportedEfforts.includes(request.reasoning.defaultEffort)) delete request.reasoning.defaultEffort;
|
||||
}
|
||||
if (facts.constraints.thinkingOnly === true && request.reasoning) {
|
||||
request.reasoning.supportedEfforts = (request.reasoning.supportedEfforts ?? []).filter(e => !['none', 'off'].includes(e));
|
||||
if (['none', 'off'].includes(request.reasoning.defaultEffort)) delete request.reasoning.defaultEffort;
|
||||
}
|
||||
if (request.reasoning?.supportedEfforts?.length === 0) blockedReasons.push('no-compatible-reasoning-effort');
|
||||
const uniqueReasons = [...new Set(blockedReasons)], eligible = uniqueReasons.length === 0;
|
||||
const selectable = eligible && accessModel.lifecycle?.deprecated !== true && accessModel.lifecycle?.status !== 'deprecated' && accessModel.lifecycle?.selectable !== false && (!exact || spec.lifecycle?.status !== 'deprecated' && spec.lifecycle?.selectable !== false);
|
||||
return {eligible, selectable, apiFormat, eligibleForCall:eligible, eligibleForAutoRouting:selectable && exact && pendingOverrides.length === 0 && spec.routing?.eligibleForAgent === true, automaticRoutingEligible: selectable && exact && pendingOverrides.length === 0 && spec.routing?.eligibleForAgent === true, pendingOverrides, manualOnly: !exact || pendingOverrides.length > 0, blockedReasons: uniqueReasons, identity: exact ? spec.identity : null, apiModelId: request.upstreamModelId ?? accessModel.apiModelId, facts, request, billing: structuredClone(accessModel.billing ?? []), provenance};
|
||||
}
|
||||
136
scripts/catalog-v2/migrate.mjs
Normal file
136
scripts/catalog-v2/migrate.mjs
Normal file
@@ -0,0 +1,136 @@
|
||||
/** Explicit ONE-TIME importer. Compilation never reads V1. Reimport is manual and reviewed. */
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
|
||||
export const FACT_KEYS = ['contextWindow','maxInputTokens','maxOutputTokens','capabilities','serviceType','supportsReasoning','defaultTemperature','defaultTopP','defaultTopK'];
|
||||
const INTRINSIC = new Set(['thinkingOnly','samplingParametersDeprecated','forcedToolChoiceUnsupported','speech','modelOrigin','dimensions','defaultDimension']);
|
||||
const clone = x => structuredClone(x);
|
||||
const equal = (a,b) => JSON.stringify(a) === JSON.stringify(b);
|
||||
function readJson(file) { return JSON.parse(fs.readFileSync(file,'utf8')); }
|
||||
function files(dir) { return fs.readdirSync(dir).filter(n => n.endsWith('.json') && !n.startsWith('_')).sort(); }
|
||||
function leaves(obj, prefix='extra') { return Object.entries(obj ?? {}).flatMap(([k,v]) => v && typeof v === 'object' && !Array.isArray(v) ? leaves(v, `${prefix}.${k}`) : [`${prefix}.${k}`]); }
|
||||
export function classifyExtra(key, consumers = []) {
|
||||
if (/price|pricing/i.test(key)) return 'billing';
|
||||
key = key.split('.')[0];
|
||||
if(key==='responsesOnly')return 'request';
|
||||
if (['deprecated','legacy','replacedBy','deprecationNotice','retiredAt'].includes(key)) return 'lifecycle';
|
||||
if (INTRINSIC.has(key)) return 'intrinsic';
|
||||
return consumers.length && !['thinkingDefault','supportsThinking','snapshot','aliases'].includes(key) ? 'request' : 'metadata';
|
||||
}
|
||||
function unitFor(key) {
|
||||
if (key === 'cachePricing') return 'compound-source-units';
|
||||
if (/PerMillionCharacters/.test(key)) return 'million-characters';
|
||||
if (/write5m/.test(key)) return 'million-tokens-cache-write-5m';
|
||||
if (/write1h/.test(key)) return 'million-tokens-cache-write-1h';
|
||||
if (/searchRequestPriceUsd/.test(key)) return 'search-request';
|
||||
if (/PerMinute/.test(key)) return 'minute';
|
||||
if (/PerSong/.test(key)) return 'song-up-to-five-minutes';
|
||||
if (/PerGeneration/.test(key)) return 'generation';
|
||||
if (/PerImage/.test(key)) return 'image';
|
||||
if (/Per1k/.test(key)) return 'thousand-requests';
|
||||
if (/storagePerMillionTokensPerHour|StoragePricePerMillionTokenHour/.test(key)) return 'million-token-hour';
|
||||
if (/Storage|storage/.test(key)) return 'source-unit-unresolved';
|
||||
if (/Tiers/.test(key)) return 'tiered-source-unit';
|
||||
if (/Note/.test(key)) return 'annotation';
|
||||
return 'million-tokens';
|
||||
}
|
||||
export function importCatalog(repoRoot, consumers={}) {
|
||||
const catalog = {formatVersion:2, publication:{state:'candidate',requiredClientVersion:null}, specs:[], accesses:[], retirements:[], metadata:{migrationEpoch:1, authority:'catalog/v2/catalog.json', verification:'migration-preserves-existing-evidence', specFiles:{}}};
|
||||
for (const filename of files(path.join(repoRoot,'compute/model-specs'))) {
|
||||
const data = readJson(path.join(repoRoot,'compute/model-specs',filename));
|
||||
const vendor = filename.slice(0,-5);
|
||||
catalog.metadata.specFiles[filename] = Object.fromEntries(Object.entries(data).filter(([k])=>k!=='specs'));
|
||||
for (const entry of data.specs) {
|
||||
const facts = Object.fromEntries(Object.entries(entry.spec).filter(([k])=>k!=='extra'));
|
||||
const metadata = {recordedExtra:{}, projection:{specKeys:Object.keys(entry.spec),extraKeys:Object.keys(entry.spec.extra ?? {})}, sourceFile:`compute/model-specs/${filename}`};
|
||||
for (const [key,value] of Object.entries(entry.spec.extra ?? {})) (INTRINSIC.has(key) ? (facts.constraints ??= {}) : metadata.recordedExtra)[key] = clone(value);
|
||||
const rest = Object.fromEntries(Object.entries(entry).filter(([k])=> !['id','spec','routing'].includes(k)));
|
||||
catalog.specs.push({modelRef:`${vendor}:${entry.id}`,identity:{vendor,canonicalModelId:entry.id},facts,routing:entry.routing ?? {},lifecycle:{status:'unknown'},metadata:{...metadata,legacyIdentity:rest}});
|
||||
}
|
||||
}
|
||||
for(const spec of catalog.specs){
|
||||
if(Object.hasOwn(spec.metadata.recordedExtra,'adaptiveThinking')){
|
||||
spec.protocolProfiles=[{apiFormats:['anthropic-messages'],request:{adaptiveThinking:spec.metadata.recordedExtra.adaptiveThinking}}];
|
||||
delete spec.metadata.recordedExtra.adaptiveThinking;
|
||||
}
|
||||
for(const key of ['releasedAt','retiredAt'])if(Object.hasOwn(spec.facts,key)){
|
||||
(spec.metadata.recordedFacts??={})[key]=spec.facts[key];delete spec.facts[key];
|
||||
}
|
||||
}
|
||||
const exactIndex = new Map();
|
||||
for (const spec of catalog.specs) for (const name of [spec.identity.canonicalModelId,...(spec.metadata.legacyIdentity.match?.exact ?? [])]) {
|
||||
const bucket=exactIndex.get(name) ?? []; if(!bucket.includes(spec)) bucket.push(spec); exactIndex.set(name,bucket);
|
||||
}
|
||||
for (const category of ['providers','coding-plans']) for (const filename of files(path.join(repoRoot,`compute/${category}`))) {
|
||||
const data=readJson(path.join(repoRoot,`compute/${category}`,filename));
|
||||
const connection=Object.fromEntries(Object.entries(data).filter(([k])=>k!=='models'));
|
||||
const access={id:data.id,category,connection,models:[],metadata:{sourceFile:`compute/${category}/${filename}`}};
|
||||
for (const model of data.models) {
|
||||
let candidates=exactIndex.get(model.modelName) ?? [];
|
||||
if(candidates.length>1) candidates=candidates.filter(s=>s.identity.vendor===data.provider);
|
||||
const spec=candidates.length===1?candidates[0]:null;
|
||||
const dynamic=/latest$|^auto$/.test(model.modelName);
|
||||
const next={apiModelId:model.modelName,modelRef:spec?.modelRef ?? null,binding:{kind:spec?'exact':dynamic?'dynamic':'access-local',reason:spec?'exact-id-or-declared-alias':'no-proven-exact-identity'},overrides:{},request:{},billing:[],lifecycle:{status:'unknown'},metadata:{attributes:{},recordedExtra:{},projection:{modelKeys:Object.keys(model),extraKeys:Object.keys(model.extra ?? {})},sourceRecord:`${access.metadata.sourceFile}#${model.modelName}`}};
|
||||
if(data.requiredClientVersion)next.requiredClientVersion=data.requiredClientVersion;
|
||||
for(const [key,value] of Object.entries(model)) {
|
||||
if(key==='modelName'||key==='extra')continue;
|
||||
if(key==='apiModelId'){next.request.upstreamModelId=value;continue;}
|
||||
if(['inputPrice','outputPrice'].includes(key)){next.billing.push({sourceKey:key,amount:value,currency:data.priceCurrency ?? 'unknown',unit:'million-tokens',direction:key==='inputPrice'?'input':'output',verification:'inherited-not-reverified'});continue;}
|
||||
if(FACT_KEYS.includes(key)) {if(!spec||!equal(spec.facts[key],value))next.overrides[key]={value:clone(value),reason:'preserved-access-contract',evidence:next.metadata.sourceRecord,verification:'inherited-not-reverified'};} else next.metadata.attributes[key]=clone(value);
|
||||
}
|
||||
for(const [key,value] of Object.entries(model.extra ?? {})) {
|
||||
const owner=classifyExtra(key,consumers[key] ?? []);
|
||||
if(owner==='billing') next.billing.push({sourceKey:`extra.${key}`,amount:clone(value),currency:data.priceCurrency ?? 'unknown',unit:unitFor(key),verification:'inherited-not-reverified'});
|
||||
else if(owner==='request') next.request[key]=clone(value);
|
||||
else if(owner==='intrinsic') {
|
||||
// Only original proven spec facts are shared; access-only declarations remain explicit constraints.
|
||||
next.request.constraints ??= {}; next.request.constraints[key]=clone(value);
|
||||
} else if(owner==='lifecycle') next.lifecycle[key]=clone(value);
|
||||
else next.metadata.recordedExtra[key]=clone(value);
|
||||
}
|
||||
if(spec) for(const [key,value] of Object.entries(spec.metadata.recordedExtra)) {
|
||||
if(classifyExtra(key,consumers[key]??[])==='request' && !Object.hasOwn(next.request,key)) next.request[key]=clone(value);
|
||||
}
|
||||
if(next.request.serverSideWebSearch && Object.hasOwn(next.request.serverSideWebSearch,'searchRequestPriceUsd')){
|
||||
next.billing.push({sourceKey:'extra.serverSideWebSearch.searchRequestPriceUsd',amount:next.request.serverSideWebSearch.searchRequestPriceUsd,currency:'USD',unit:'search-request',verification:'inherited-not-reverified'});
|
||||
delete next.request.serverSideWebSearch.searchRequestPriceUsd;
|
||||
}
|
||||
access.models.push(next);
|
||||
}
|
||||
catalog.accesses.push(access);
|
||||
}
|
||||
const decisionsPath=path.join(repoRoot,'docs/plans/catalog-contract-migration/SPEC-DECISIONS.json');
|
||||
if(fs.existsSync(decisionsPath)){
|
||||
const decisions=readJson(decisionsPath);
|
||||
for(const access of catalog.accesses)for(const model of access.models){
|
||||
const exception=(decisions.strictExactExceptions??[]).find(d=>d.file===access.metadata.sourceFile && d.apiModelId===model.apiModelId);
|
||||
if(exception){model.modelRef=null;model.binding={kind:exception.kind?.includes('dynamic')?'dynamic':'access-local',reason:exception.reason};model.metadata.automaticRouting=false;}
|
||||
const budget=(decisions.independentInputBudgets??[]).find(d=>d.file===access.metadata.sourceFile && d.apiModelId===model.apiModelId);
|
||||
if(budget){
|
||||
model.metadata.compatibilityValues={contextWindow:model.overrides.contextWindow?.value??catalog.specs.find(s=>s.modelRef===model.modelRef)?.facts.contextWindow};
|
||||
for(const key of ['maxInputTokens','contextWindow','maxOutputTokens'])if(Object.hasOwn(budget,key))model.overrides[key]={value:budget[key],reason:budget.semanticDecision,evidence:budget.evidence,verification:'officially-verified-independent-budget'};
|
||||
model.metadata.independentBudgetEvidence=budget;
|
||||
}
|
||||
}
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
export function discoverConsumers(clientRoot) {
|
||||
const result={};
|
||||
function scan(dir){if(!fs.existsSync(dir))return; for(const ent of fs.readdirSync(dir,{withFileTypes:true})){const file=path.join(dir,ent.name); if(ent.isDirectory()){if(!['node_modules','__tests__','dist','.git'].includes(ent.name))scan(file);} else if(/\.[cm]?[tj]sx?$/.test(ent.name)&& !/test|spec\.ts/.test(ent.name)){const text=fs.readFileSync(file,'utf8');for(const m of text.matchAll(/\bextra(?:\?\.|\.)([a-zA-Z]\w*)/g)){(result[m[1]]??= []).push(path.relative(clientRoot,file));}}}}
|
||||
scan(path.join(clientRoot,'packages'));scan(path.join(clientRoot,'app'));for(const k of Object.keys(result))result[k]=[...new Set(result[k])].sort();return result;
|
||||
}
|
||||
if(process.argv[1]===fileURLToPath(import.meta.url)){
|
||||
if(!process.argv.includes('--write')) throw new Error('One-time migration requires --write; use compile.mjs thereafter');
|
||||
const clientRoot=process.argv[process.argv.indexOf('--client-root')+1];
|
||||
if(!process.argv.includes('--client-root'))throw new Error('Client source required for field consumer audit');
|
||||
const consumers=discoverConsumers(clientRoot), catalog=importCatalog(root,consumers);
|
||||
fs.mkdirSync(path.join(root,'catalog/v2'),{recursive:true});fs.writeFileSync(path.join(root,'catalog/v2/catalog.json'),JSON.stringify(catalog,null,2)+'\n');
|
||||
const inventory = fs.readFileSync(path.join(root,'docs/plans/catalog-contract-migration/FIELD-INVENTORY.md'),'utf8');
|
||||
const paths=[...inventory.matchAll(/`extra\.([^`]+)`/g)].map(m=>m[1]);
|
||||
const decisions=paths.map(p=>({path:`extra.${p}`,owner:classifyExtra(p,consumers[p.split('.')[0]]??[]),consumerCandidates:consumers[p.split('.')[0]]??[],minimumVersion:null,versionStatus:'candidate-contract-not-released',verification:'static-property-read-candidate-not-runtime-acceptance'}));
|
||||
fs.writeFileSync(path.join(root,'catalog/v2/field-decisions.json'),JSON.stringify(decisions,null,2)+'\n');
|
||||
fs.writeFileSync(path.join(root,'docs/plans/catalog-contract-migration/FIELD-DECISIONS.md'),'# 字段归属与消费核查\n\n全部既有149条路径已分域；静态消费路径是核查候选，不宣称正式发布支持。无消费证明的值进入非运行时metadata，V1投影保留，不自动扩大能力。正式V2发布前须补实际客户端测试和版本。\n\n| 字段 | owner | 消费候选 | 发布版本 |\n| --- | --- | --- | --- |\n'+decisions.map(d=>`| \`${d.path}\` | ${d.owner} | ${d.consumerCandidates.slice(0,3).join('<br>')||'未识别；记录性隔离'} | 待正式发布 |`).join('\n')+'\n');
|
||||
console.log(`Imported ${catalog.specs.length} specs, ${catalog.accesses.length} accesses, ${decisions.length} field decisions`);
|
||||
}
|
||||
Reference in New Issue
Block a user