diff --git a/packages/utils/src/GamedataLoader.ts b/packages/utils/src/GamedataLoader.ts index cf4dadc..cf2d794 100644 --- a/packages/utils/src/GamedataLoader.ts +++ b/packages/utils/src/GamedataLoader.ts @@ -1,4 +1,5 @@ -import { fetchConfigJson } from './JsonParser'; +import { ConfigJsonError, fetchConfigJson, isMissingResource } from './JsonParser'; +import { NitroLogger } from './NitroLogger'; export const DEFAULT_TIERS = [ 'core', 'custom', 'seasonal' ] as const; export type GamedataTier = typeof DEFAULT_TIERS[number] | string; @@ -28,51 +29,69 @@ const joinUrl = (base: string, path: string): string => return `${ cleanBase }${ cleanPath }`; }; -const tryFetchOrNull = async (url: string): Promise => +// Returns the parsed payload when the manifest exists, null on a clean 404. +// Re-throws on any other error (network failure, 5xx, parse error) so callers +// don't silently skip a tier because of a typo in manifest.json5. +const tryFetchManifest = async (url: string): Promise => { try { return await fetchConfigJson(url); } - catch + catch(err) { - return null; + if(isMissingResource(err)) return null; + throw err; } }; +// Try .json5 first, then .json — both treated as optional. Anything other +// than 404 on either bubbles up. +const tryFetchManifestPair = async (baseUrl: string, name: string): Promise => +{ + const json5 = await tryFetchManifest(joinUrl(baseUrl, `${ name }.json5`)); + if(json5 !== null) return json5; + + return await tryFetchManifest(joinUrl(baseUrl, `${ name }.json`)); +}; + const isPlainObject = (value: any): value is Record => !!value && typeof value === 'object' && !Array.isArray(value); -const arrayItemsLookKeyed = (arr: any[], idKeys: readonly string[]): string | null => +const arrayItemsLookKeyed = (arr: any[], idKeys: readonly string[], sourceLabel?: string): string | null => { if(!arr.length) return null; for(const key of idKeys) { - let allHave = true; + let have = 0; for(const item of arr) { - if(!isPlainObject(item) || item[key] === undefined || item[key] === null) - { - allHave = false; - break; - } + if(isPlainObject(item) && item[key] !== undefined && item[key] !== null) have++; } - if(allHave) return key; + if(have === arr.length) return key; + + // Heuristic: if most items are keyed but a few are not, the data is + // probably keyed and the outliers are bugs in the source data. + // Surface this so operators don't get silent duplicates after merge. + if(have > 0 && have / arr.length >= 0.8) + { + NitroLogger.warn(`mergeGamedata: ${ sourceLabel ? `${ sourceLabel }: ` : '' }array looks keyed by "${ key }" (${ have }/${ arr.length } items) but some entries are missing it — falling back to concat which may produce duplicates`); + } } return null; }; -export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAULT_ID_KEYS): any => +export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAULT_ID_KEYS, sourceLabel?: string): any => { if(b === undefined) return a; if(a === undefined) return b; if(Array.isArray(a) && Array.isArray(b)) { - const idKey = arrayItemsLookKeyed(a, idKeys) || arrayItemsLookKeyed(b, idKeys); + const idKey = arrayItemsLookKeyed(a, idKeys, sourceLabel) || arrayItemsLookKeyed(b, idKeys, sourceLabel); if(!idKey) return a.concat(b); @@ -92,7 +111,7 @@ export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAUL if(at !== undefined) { - out[at] = mergeGamedata(out[at], item, idKeys); + out[at] = mergeGamedata(out[at], item, idKeys, sourceLabel); } else { @@ -110,7 +129,7 @@ export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAUL for(const k of Object.keys(b)) { - out[k] = mergeGamedata(a[k], b[k], idKeys); + out[k] = mergeGamedata(a[k], b[k], idKeys, sourceLabel); } return out; @@ -130,6 +149,11 @@ interface RootManifest files?: string[]; } +// Load every file in `files` concurrently, return them in the original +// declared order so the merge step preserves override semantics. +const fetchFilesInOrder = async (baseUrl: string, files: readonly string[]): Promise => + Promise.all(files.map(file => fetchConfigJson(joinUrl(baseUrl, file)))); + export const loadGamedata = async (url: string, options: GamedataLoadOptions = {}): Promise => { if(!url) throw new Error('loadGamedata: empty URL'); @@ -140,42 +164,47 @@ export const loadGamedata = async (url: string, options: GamedataLoadOp } const idKeys = options.mergeArrayIdKeys ?? DEFAULT_ID_KEYS; - const rootManifest = await tryFetchOrNull(joinUrl(url, 'manifest.json5')) - ?? await tryFetchOrNull(joinUrl(url, 'manifest.json')); + const rootManifest = await tryFetchManifestPair(url, 'manifest'); const tiers = (rootManifest?.tiers && rootManifest.tiers.length) ? rootManifest.tiers : (options.tiers ?? DEFAULT_TIERS); + // Fetch root-level files in parallel with discovering each tier's + // manifest. Per-tier file batches stay sequenced relative to each other + // so override order (core → custom → seasonal) is preserved during + // merge, but fetches inside a tier batch run concurrently. + const [ rootParts, tierManifests ] = await Promise.all([ + rootManifest?.files?.length ? fetchFilesInOrder(url, rootManifest.files) : Promise.resolve([] as any[]), + Promise.all(tiers.map(async tier => + { + const tierUrl = joinUrl(url, `${ tier }/`); + const manifest = await tryFetchManifestPair(tierUrl, 'manifest'); + + return { tier, tierUrl, manifest }; + })) + ]); + let merged: any = undefined; - if(rootManifest?.files?.length) + for(const part of rootParts) { - for(const file of rootManifest.files) + merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys, url); + } + + for(const { tier, tierUrl, manifest } of tierManifests) + { + if(!manifest?.files?.length) continue; + + const parts = await fetchFilesInOrder(tierUrl, manifest.files); + + for(const part of parts) { - const fileUrl = joinUrl(url, file); - const part = await fetchConfigJson(fileUrl); - merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys); + merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys, `${ url } (${ tier })`); } } - for(const tier of tiers) - { - const tierUrl = joinUrl(url, `${ tier }/`); - const tierManifest = await tryFetchOrNull(joinUrl(tierUrl, 'manifest.json5')) - ?? await tryFetchOrNull(joinUrl(tierUrl, 'manifest.json')); - - if(!tierManifest?.files?.length) continue; - - for(const file of tierManifest.files) - { - const fileUrl = joinUrl(tierUrl, file); - const part = await fetchConfigJson(fileUrl); - merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys); - } - } - - if(merged === undefined) throw new Error(`loadGamedata: directory mode at "${ url }" produced no data — make sure at least one tier (core/custom/seasonal) has a manifest.json5 with a 'files' array`); + if(merged === undefined) throw new ConfigJsonError(`loadGamedata: directory mode at "${ url }" produced no data — make sure at least one tier (core/custom/seasonal) has a manifest.json5 with a 'files' array`, 'fetch', url); return merged as T; }; diff --git a/packages/utils/src/JsonParser.ts b/packages/utils/src/JsonParser.ts index c812d63..46f3631 100644 --- a/packages/utils/src/JsonParser.ts +++ b/packages/utils/src/JsonParser.ts @@ -5,6 +5,28 @@ declare const __NITRO_JSON_MODE__: 'legacy' | 'json5' | 'auto' | undefined; const JSON5_EXTENSION = /\.json5(?:[?#]|$)/i; const JSON5_MIME = /(?:application|text)\/(?:json5|x-json5)/i; +export type ConfigJsonErrorPhase = 'fetch' | 'parse'; + +export class ConfigJsonError extends Error +{ + public readonly phase: ConfigJsonErrorPhase; + public readonly sourceUrl: string; + public readonly httpStatus?: number; + + constructor(message: string, phase: ConfigJsonErrorPhase, sourceUrl: string, httpStatus?: number, cause?: unknown) + { + super(message); + this.name = 'ConfigJsonError'; + this.phase = phase; + this.sourceUrl = sourceUrl; + this.httpStatus = httpStatus; + if(cause !== undefined) (this as any).cause = cause; + } +} + +export const isMissingResource = (err: unknown): boolean => + err instanceof ConfigJsonError && err.phase === 'fetch' && err.httpStatus === 404; + const resolveJsonMode = (): 'legacy' | 'json5' | 'auto' => { try @@ -44,9 +66,7 @@ const formatStrictError = (sourceUrl: string, err: unknown): string => export const parseConfigJson = (text: string, sourceUrl: string = ''): T => { - if(text === null || text === undefined) throw new Error(`Empty response${ sourceUrl ? ` for "${ sourceUrl }"` : '' }`); - - const trimmed = text.length > 0 ? text : ''; + const trimmed = text ?? ''; const mode = resolveJsonMode(); if(mode === 'legacy') @@ -57,7 +77,7 @@ export const parseConfigJson = (text: string, sourceUrl: string = ''): } catch(err) { - throw new Error(formatStrictError(sourceUrl, err)); + throw new ConfigJsonError(formatStrictError(sourceUrl, err), 'parse', sourceUrl, undefined, err); } } @@ -69,7 +89,7 @@ export const parseConfigJson = (text: string, sourceUrl: string = ''): } catch(err) { - throw new Error(formatParseError(sourceUrl, err, err)); + throw new ConfigJsonError(formatParseError(sourceUrl, err, err), 'parse', sourceUrl, undefined, err); } } @@ -90,7 +110,7 @@ export const parseConfigJson = (text: string, sourceUrl: string = ''): } catch(json5Error) { - throw new Error(formatParseError(sourceUrl, strictError, json5Error)); + throw new ConfigJsonError(formatParseError(sourceUrl, strictError, json5Error), 'parse', sourceUrl, undefined, json5Error); } }; @@ -109,7 +129,7 @@ export const parseConfigJsonFromResponse = async (response: Response, s } catch(err) { - throw new Error(formatParseError(url, err, err)); + throw new ConfigJsonError(formatParseError(url, err, err), 'parse', url, undefined, err); } } @@ -118,9 +138,23 @@ export const parseConfigJsonFromResponse = async (response: Response, s export const fetchConfigJson = async (url: string, init?: RequestInit): Promise => { - const response = await fetch(url, init); + let response: Response | undefined; - if(!response || response.status !== 200) throw new Error(`Failed to fetch "${ url }" — server returned HTTP ${ response?.status ?? 'no response' }`); + try + { + response = await fetch(url, init); + } + catch(networkErr) + { + const message = (networkErr as Error)?.message || String(networkErr); + throw new ConfigJsonError(`Network error fetching "${ url }" — ${ message }`, 'fetch', url, undefined, networkErr); + } + + if(!response || response.status !== 200) + { + const status = response?.status; + throw new ConfigJsonError(`Failed to fetch "${ url }" — server returned HTTP ${ status ?? 'no response' }`, 'fetch', url, status); + } return parseConfigJsonFromResponse(response, url); }; diff --git a/yarn.lock b/yarn.lock index 288c306..aaeae8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -369,7 +369,7 @@ resolved "https://registry.yarnpkg.com/@standard-schema/spec/-/spec-1.1.0.tgz#a79b55dbaf8604812f52d140b2c9ab41bc150bb8" integrity sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w== -"@thumbmarkjs/thumbmarkjs@^1.8.1": +"@thumbmarkjs/thumbmarkjs@^1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@thumbmarkjs/thumbmarkjs/-/thumbmarkjs-1.9.0.tgz#a6444ac1f924f061cfc1507a21dcaf83ee705cab" integrity sha512-6LooyYk8i5L2zEZgDMLE6m2sGDcIHHBiZfxdFp0A16Q4ZXafEmhHmt+zCqQEBMiQHi+08e/v5q77IY2KhvAJwg== @@ -527,6 +527,54 @@ "@typescript-eslint/types" "8.59.3" eslint-visitor-keys "^5.0.0" +"@typescript/native-preview-darwin-arm64@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-darwin-arm64/-/native-preview-darwin-arm64-7.0.0-dev.20260519.1.tgz#deaa14299366a79917a6ec19fc7f240304c36afe" + integrity sha512-c9zdG6sGJf25Jpz04JgE23zhYeprqFypDGuqiX94yMTvR8IWXjq3R2oMnim66YLBDon/V1nCEy6cFixeSd/4fg== + +"@typescript/native-preview-darwin-x64@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-darwin-x64/-/native-preview-darwin-x64-7.0.0-dev.20260519.1.tgz#a6348c134204afdcdfa8f9a0925091788fc994d7" + integrity sha512-N16V3wiM0tsNmSSA7nZrxqXXt5OCJxBwiCVn35rnA7fr4WzJw6rJmwf9heNNhZ6Gh4ne3+Pexajf5akzuHR75Q== + +"@typescript/native-preview-linux-arm64@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-linux-arm64/-/native-preview-linux-arm64-7.0.0-dev.20260519.1.tgz#04f2f0b314e646a6921cc818db6bac92364bab24" + integrity sha512-ltf91vAwKdbu0SlRQbFgi1h5ZrLLrBn6a4qIeN2VILGbtYrCXnARHRznLBv81yUETQ7aVr/LSQcmsWo1ejCK0w== + +"@typescript/native-preview-linux-arm@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-linux-arm/-/native-preview-linux-arm-7.0.0-dev.20260519.1.tgz#ad10403fb8ae6e2ed120cb9f81a029328a00e995" + integrity sha512-8v4BExeeuCTrhaSGfeIJqm3qQkTzlZix/Qd/FkPlWoz9f7d7COvXb3Z4qhbaVolL0MMnUvQ7m005Z4kYsZ645A== + +"@typescript/native-preview-linux-x64@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-linux-x64/-/native-preview-linux-x64-7.0.0-dev.20260519.1.tgz#dde79208611da3855f995e595b9b790c622f726f" + integrity sha512-AVD0tczTtFCHNa4RQRVPvu8Hnw4P3hQ+OlUAjnz/lHowvc6o1pYB46elMqfDuaoWqIpv+EAkAPP4ipFCofJ5IA== + +"@typescript/native-preview-win32-arm64@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-win32-arm64/-/native-preview-win32-arm64-7.0.0-dev.20260519.1.tgz#74d0cf98c75e2df20772ae2fa4c8139066e2cf7f" + integrity sha512-TM+qatljyejqjHevCta3WIH53i0oGC7K8SoJ6t+mf4cGMTpZTyd7NhC1ts7e6/aydZnG53Bsta2iQi1SMIlQEw== + +"@typescript/native-preview-win32-x64@7.0.0-dev.20260519.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview-win32-x64/-/native-preview-win32-x64-7.0.0-dev.20260519.1.tgz#a8cfe9f730399cdc21a20c40a59aee15eef75a18" + integrity sha512-r9LEsoY7JC/82gXo8hlOmpQaUXcqmngCVOv+mUx1UeMt9f+1S6oNO0W48o75mlBqqC7jfcMHqw8YS4LfVxPRGw== + +"@typescript/native-preview@^7.0.0-dev.20260510.1": + version "7.0.0-dev.20260519.1" + resolved "https://registry.yarnpkg.com/@typescript/native-preview/-/native-preview-7.0.0-dev.20260519.1.tgz#0d865f5cc6d376d896fc031dfbb38ed0004153c6" + integrity sha512-VVER7vFUDdfm5k3jbH5765tVEJa7+0rTUkFeXyGYrXPxpw9BIjA0QDxdtdlRyaU8MCZV9IKZUo6doxeAQRAjPg== + optionalDependencies: + "@typescript/native-preview-darwin-arm64" "7.0.0-dev.20260519.1" + "@typescript/native-preview-darwin-x64" "7.0.0-dev.20260519.1" + "@typescript/native-preview-linux-arm" "7.0.0-dev.20260519.1" + "@typescript/native-preview-linux-arm64" "7.0.0-dev.20260519.1" + "@typescript/native-preview-linux-x64" "7.0.0-dev.20260519.1" + "@typescript/native-preview-win32-arm64" "7.0.0-dev.20260519.1" + "@typescript/native-preview-win32-x64" "7.0.0-dev.20260519.1" + "@vitest/coverage-v8@^4.0.18": version "4.1.6" resolved "https://registry.yarnpkg.com/@vitest/coverage-v8/-/coverage-v8-4.1.6.tgz#1eacee5def68dfcb08c3ed5355edbad2a4c869b3" @@ -1681,15 +1729,10 @@ typescript-eslint@^8.26.1: "@typescript-eslint/typescript-estree" "8.59.3" "@typescript-eslint/utils" "8.59.3" -typescript@~5.5.4: - version "5.5.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.4.tgz#d9852d6c82bad2d2eda4fd74a5762a8f5909e9ba" - integrity sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q== - -typescript@~5.8.2: - version "5.8.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.8.3.tgz#92f8a3e5e3cf497356f4178c34cd65a7f5e8440e" - integrity sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ== +typescript@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-6.0.3.tgz#90251dc007916e972786cb94d74d15b185577d21" + integrity sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw== undici-types@~6.21.0: version "6.21.0"