Merge pull request #70 from medievalshell/dev

feat: interactive JSON / JSON5 mode selector at build time

README.md

@@ -15,3 +15,110 @@ yarn
 ```
 yarn add @nitrots/nitro-renderer
 ```
+
+## JSON / JSON5 configuration parser
+
+Every configuration file and gamedata file loaded by the renderer (figuredata,
+furnidata, productdata, effectmap, avatar actions, etc.) goes through
+`@nitrots/utils` → `JsonParser.ts`. The parser supports three modes, selected at
+the **host build time** through the compile-time constant `__NITRO_JSON_MODE__`:
+
+| Mode     | Behaviour                                                                 |
+|----------|---------------------------------------------------------------------------|
+| `legacy` | Strict `JSON.parse` only. Comments / trailing commas raise a clear error. |
+| `json5`  | `JSON5.parse` only. Accepts comments, trailing commas, single quotes.     |
+| `auto`   | Try strict JSON first, fall back to JSON5. Default when the flag is unset.|
+
+URL hints are still honoured: files ending in `.json5` (or served with a
+`application/json5` content-type) always go through JSON5, regardless of mode.
+
+### Wiring the flag into a host
+
+The renderer does **not** ship its own build for the flag — the host application
+(typically [Nitro V3](https://github.com/duckietm/Nitro-V3.git)) defines it via
+its bundler. Example with Vite:
+
+```js
+// vite.config.mjs in the host
+export default defineConfig({
+    define: {
+        __NITRO_JSON_MODE__: JSON.stringify('json5')   // or 'legacy' / 'auto'
+    }
+});
+```
+
+If the constant is not defined the parser falls back to `auto`, which preserves
+the original behaviour of older releases — so existing hosts keep working
+without any change.
+
+### Using the parser directly
+
+```ts
+import { parseConfigJson, fetchConfigJson } from '@nitrots/utils';
+
+const data  = parseConfigJson<MyConfig>(rawText, '/configuration/ui-config.json');
+const data2 = await fetchConfigJson<MyConfig>('/configuration/ui-config.json5');
+```
+
+Errors carry the source URL and, in `legacy` mode, a hint about switching to
+JSON5 — making misconfigurations easy to diagnose in production logs.
+
+## Split-aware gamedata loader
+
+`@nitrots/utils` also exports `loadGamedata`, the loader that backs every
+gamedata consumer in the renderer (FurnitureDataLoader, ProductDataLoader,
+EffectAssetDownloadManager, AvatarRenderManager, LocalizationManager). It
+accepts either a **single-file URL** (legacy) or a **directory URL** (split
+mode) — detected automatically by the trailing slash.
+
+### Directory layout
+
+```
+<gamedata-dir>/
+  manifest.json5            # OPTIONAL — { "tiers": ["core", "custom", "seasonal"] }
+  core/
+    manifest.json5          # REQUIRED — { "files": ["a.json5", "b.json5", ...] }
+    a.json5
+    b.json5
+  custom/                   # OPTIONAL tier
+    manifest.json5
+    overrides.json5
+  seasonal/                 # OPTIONAL tier
+    manifest.json5
+    xmas.json5
+```
+
+If the directory `manifest.json5` is absent, the loader falls back to the
+default tier order `core → custom → seasonal`. Each tier is skipped silently
+if its `manifest.json5` is missing.
+
+### Merge semantics
+
+`mergeGamedata(a, b)` (also exported) implements the rules below; tiers and
+files within a tier are merged in order, with later layers overriding
+earlier ones:
+
+| Combination                              | Result                                       |
+|------------------------------------------|----------------------------------------------|
+| Two plain objects                        | recursive merge, key by key                  |
+| Two arrays of objects sharing an id key  | merged by id (later overrides earlier)       |
+| Two arrays without an id key             | concatenated                                 |
+| Anything else                            | later value wins                             |
+
+Recognised id keys (in priority order): `id`, `classname`, `name`. Pass
+`mergeArrayIdKeys` in the options object to extend or override them.
+
+### Programmatic usage
+
+```ts
+import { loadGamedata, mergeGamedata } from '@nitrots/utils';
+
+// host code never needs to care whether the URL is split or not
+const furnidata = await loadGamedata('https://example.com/gamedata/furnidata/');
+
+// merge ad-hoc if you load tiers manually
+const merged = mergeGamedata(coreData, customData);
+```
+
+A CLI splitter for legacy single-file gamedata lives in the Nitro V3 client
+repo at `scripts/split-gamedata.mjs` — see the Nitro V3 README for usage.

packages/avatar/src/AvatarRenderManager.ts

@@ -2,7 +2,7 @@ import { AvatarSetType, IAssetManager, IAvatarEffectListener, IAvatarFigureConta
 import { GetAssetManager } from '@nitrots/assets';
 import { GetConfiguration } from '@nitrots/configuration';
 import { GetEventDispatcher, NitroEventType } from '@nitrots/events';
-import { parseConfigJsonFromResponse } from '@nitrots/utils';
+import { loadGamedata } from '@nitrots/utils';
 import { AvatarAssetDownloadManager } from './AvatarAssetDownloadManager';
 import { AvatarFigureContainer } from './AvatarFigureContainer';
 import { AvatarImage } from './AvatarImage';
@@ -73,26 +73,13 @@ export class AvatarRenderManager implements IAvatarRenderManager
 
         if(!url || !url.length) throw new Error('Missing "avatar.actions.url" in config — add the URL to your renderer-config.json');
 
-        let response: Response;
-
         try
         {
-            response = await fetch(url);
+            this._structure.updateActions(await loadGamedata(url));
         }
-        catch(fetchErr)
+        catch(err)
         {
-            throw new Error(`Could not fetch avatar actions from "${ url }" — check "avatar.actions.url" in renderer-config.json (${ fetchErr.message })`);
-        }
-
-        if(response.status !== 200) throw new Error(`Failed to load avatar actions from "${ url }" — server returned HTTP ${ response.status }. Check "avatar.actions.url" in renderer-config.json`);
-
-        try
-        {
-            this._structure.updateActions(await parseConfigJsonFromResponse(response, url));
-        }
-        catch(parseErr)
-        {
-            throw new Error(`Invalid avatar actions "${ url }" — JSON/JSON5 parse failed. Check "avatar.actions.url" in renderer-config.json (${ parseErr.message })`);
+            throw new Error(`Could not load avatar actions from "${ url }" — check "avatar.actions.url" in renderer-config.json (${ err?.message || err })`);
         }
     }
 
@@ -106,26 +93,13 @@ export class AvatarRenderManager implements IAvatarRenderManager
 
         if(!url || !url.length) throw new Error('Missing "avatar.figuredata.url" in config — add the URL to your renderer-config.json');
 
-        let response: Response;
-
         try
         {
-            response = await fetch(url);
+            this._structure.figureData.appendJSON(await loadGamedata(url));
         }
-        catch(fetchErr)
+        catch(err)
         {
-            throw new Error(`Could not fetch figure data from "${ url }" — check "avatar.figuredata.url" in renderer-config.json (${ fetchErr.message })`);
-        }
-
-        if(response.status !== 200) throw new Error(`Failed to load figure data from "${ url }" — server returned HTTP ${ response.status }. Check "avatar.figuredata.url" in renderer-config.json`);
-
-        try
-        {
-            this._structure.figureData.appendJSON(await parseConfigJsonFromResponse(response, url));
-        }
-        catch(parseErr)
-        {
-            throw new Error(`Invalid figure data "${ url }" — JSON/JSON5 parse failed. Check "avatar.figuredata.url" in renderer-config.json (${ parseErr.message })`);
+            throw new Error(`Could not load figure data from "${ url }" — check "avatar.figuredata.url" in renderer-config.json (${ err?.message || err })`);
         }
 
         this._structure.init();

packages/avatar/src/EffectAssetDownloadManager.ts

@@ -1,7 +1,7 @@
 import { IAssetManager, IAvatarEffectListener } from '@nitrots/api';
 import { GetConfiguration } from '@nitrots/configuration';
 import { AvatarRenderEffectLibraryEvent, GetEventDispatcher, NitroEvent, NitroEventType } from '@nitrots/events';
-import { parseConfigJsonFromResponse } from '@nitrots/utils';
+import { loadGamedata } from '@nitrots/utils';
 import { AvatarStructure } from './AvatarStructure';
 import { EffectAssetDownloadLibrary } from './EffectAssetDownloadLibrary';
 
@@ -32,28 +32,15 @@ export class EffectAssetDownloadManager
 
         if(!url || !url.length) throw new Error('Missing "avatar.effectmap.url" in config — add the effect map URL to your renderer-config.json');
 
-        let response: Response;
-
-        try
-        {
-            response = await fetch(url);
-        }
-        catch(fetchErr)
-        {
-            throw new Error(`Could not fetch effect map from "${ url }" — check "avatar.effectmap.url" in renderer-config.json (${ fetchErr.message })`);
-        }
-
-        if(response.status !== 200) throw new Error(`Failed to load effect map from "${ url }" — server returned HTTP ${ response.status }. Check "avatar.effectmap.url" in renderer-config.json`);
-
         let responseData: any;
 
         try
         {
-            responseData = await parseConfigJsonFromResponse(response, url);
+            responseData = await loadGamedata(url);
         }
-        catch(parseErr)
+        catch(err)
         {
-            throw new Error(`Invalid effect map "${ url }" — JSON/JSON5 parse failed. Check "avatar.effectmap.url" in renderer-config.json (${ parseErr.message })`);
+            throw new Error(`Could not load effect map from "${ url }" — check "avatar.effectmap.url" in renderer-config.json (${ err?.message || err })`);
         }
 
         this.processEffectMap(responseData.effects);

packages/localization/src/LocalizationManager.ts

@@ -1,7 +1,7 @@
 import { ILocalizationManager } from '@nitrots/api';
 import { BadgePointLimitsEvent, GetCommunication } from '@nitrots/communication';
 import { GetConfiguration } from '@nitrots/configuration';
-import { parseConfigJsonFromResponse } from '@nitrots/utils';
+import { loadGamedata } from '@nitrots/utils';
 import { BadgeBaseAndLevel } from './BadgeBaseAndLevel';
 
 export class LocalizationManager implements ILocalizationManager
@@ -26,28 +26,15 @@ export class LocalizationManager implements ILocalizationManager
 
                 url = GetConfiguration().interpolate(url);
 
-                let response: Response;
-
-                try
-                {
-                    response = await fetch(url);
-                }
-                catch(fetchErr)
-                {
-                    throw new Error(`Could not fetch localization file "${ url }" — check "external.texts.url" in ui-config.json (${ fetchErr.message })`);
-                }
-
-                if(response.status !== 200) throw new Error(`Failed to load localization file "${ url }" — server returned HTTP ${ response.status }. Check "external.texts.url" in ui-config.json`);
-
                 let data: any;
 
                 try
                 {
-                    data = await parseConfigJsonFromResponse(response, url);
+                    data = await loadGamedata(url);
                 }
-                catch(parseErr)
+                catch(err)
                 {
-                    throw new Error(`Invalid localization file "${ url }" — JSON/JSON5 parse failed. Check "external.texts.url" in ui-config.json (${ parseErr.message })`);
+                    throw new Error(`Could not load localization file "${ url }" — check "external.texts.url" in ui-config.json (${ err?.message || err })`);
                 }
 
                 this.parseLocalization(data);

packages/session/src/furniture/FurnitureDataLoader.ts

@@ -1,7 +1,7 @@
 import { FurnitureType, IFurnitureData } from '@nitrots/api';
 import { GetConfiguration } from '@nitrots/configuration';
 import { GetLocalizationManager } from '@nitrots/localization';
-import { parseConfigJsonFromResponse } from '@nitrots/utils';
+import { loadGamedata } from '@nitrots/utils';
 import { FurnitureData } from './FurnitureData';
 
 export class FurnitureDataLoader
@@ -21,28 +21,15 @@ export class FurnitureDataLoader
 
         if(!url || !url.length) throw new Error('Missing "furnidata.url" in config — add the furniture data URL to your renderer-config.json');
 
-        let response: Response;
-
-        try
-        {
-            response = await fetch(url);
-        }
-        catch(fetchErr)
-        {
-            throw new Error(`Could not fetch furniture data from "${ url }" — check "furnidata.url" in renderer-config.json (${ fetchErr.message })`);
-        }
-
-        if(response.status !== 200) throw new Error(`Failed to load furniture data from "${ url }" — server returned HTTP ${ response.status }. Check "furnidata.url" in renderer-config.json`);
-
         let responseData: any;
 
         try
         {
-            responseData = await parseConfigJsonFromResponse(response, url);
+            responseData = await loadGamedata(url);
         }
-        catch(parseErr)
+        catch(err)
         {
-            throw new Error(`Invalid furniture data "${ url }" — JSON/JSON5 parse failed. Check "furnidata.url" in renderer-config.json (${ parseErr.message })`);
+            throw new Error(`Could not load furniture data from "${ url }" — check "furnidata.url" in renderer-config.json (${ err?.message || err })`);
         }
 
         if(responseData.roomitemtypes) this.parseFloorItems(responseData.roomitemtypes);

packages/session/src/product/ProductDataLoader.ts

@@ -1,6 +1,6 @@
 import { IProductData } from '@nitrots/api';
 import { GetConfiguration } from '@nitrots/configuration';
-import { parseConfigJsonFromResponse } from '@nitrots/utils';
+import { loadGamedata } from '@nitrots/utils';
 import { ProductData } from './ProductData';
 
 export class ProductDataLoader
@@ -18,28 +18,15 @@ export class ProductDataLoader
 
         if(!url || !url.length) throw new Error('Missing "productdata.url" in config — add the product data URL to your renderer-config.json');
 
-        let response: Response;
-
-        try
-        {
-            response = await fetch(url);
-        }
-        catch(fetchErr)
-        {
-            throw new Error(`Could not fetch product data from "${ url }" — check "productdata.url" in renderer-config.json (${ fetchErr.message })`);
-        }
-
-        if(response.status !== 200) throw new Error(`Failed to load product data from "${ url }" — server returned HTTP ${ response.status }. Check "productdata.url" in renderer-config.json`);
-
         let responseData: any;
 
         try
         {
-            responseData = await parseConfigJsonFromResponse(response, url);
+            responseData = await loadGamedata(url);
         }
-        catch(parseErr)
+        catch(err)
         {
-            throw new Error(`Invalid product data "${ url }" — JSON/JSON5 parse failed. Check "productdata.url" in renderer-config.json (${ parseErr.message })`);
+            throw new Error(`Could not load product data from "${ url }" — check "productdata.url" in renderer-config.json (${ err?.message || err })`);
         }
 
         this.parseProducts(responseData.productdata);

packages/utils/src/GamedataLoader.ts

@@ -0,0 +1,181 @@
+import { fetchConfigJson } from './JsonParser';
+
+export const DEFAULT_TIERS = [ 'core', 'custom', 'seasonal' ] as const;
+export type GamedataTier = typeof DEFAULT_TIERS[number] | string;
+
+export interface GamedataLoadOptions
+{
+    tiers?: readonly GamedataTier[];
+    mergeArrayIdKeys?: readonly string[];
+}
+
+const DEFAULT_ID_KEYS = [ 'id', 'classname', 'name' ] as const;
+
+const looksLikeDirectory = (url: string): boolean =>
+{
+    if(!url) return false;
+
+    const stripped = url.split('?')[0].split('#')[0];
+
+    return stripped.endsWith('/');
+};
+
+const joinUrl = (base: string, path: string): string =>
+{
+    const cleanBase = base.endsWith('/') ? base : `${ base }/`;
+    const cleanPath = path.startsWith('/') ? path.slice(1) : path;
+
+    return `${ cleanBase }${ cleanPath }`;
+};
+
+const tryFetchOrNull = async <T = any>(url: string): Promise<T | null> =>
+{
+    try
+    {
+        return await fetchConfigJson<T>(url);
+    }
+    catch
+    {
+        return null;
+    }
+};
+
+const isPlainObject = (value: any): value is Record<string, any> => !!value && typeof value === 'object' && !Array.isArray(value);
+
+const arrayItemsLookKeyed = (arr: any[], idKeys: readonly string[]): string | null =>
+{
+    if(!arr.length) return null;
+
+    for(const key of idKeys)
+    {
+        let allHave = true;
+
+        for(const item of arr)
+        {
+            if(!isPlainObject(item) || item[key] === undefined || item[key] === null)
+            {
+                allHave = false;
+                break;
+            }
+        }
+
+        if(allHave) return key;
+    }
+
+    return null;
+};
+
+export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAULT_ID_KEYS): any =>
+{
+    if(b === undefined) return a;
+    if(a === undefined) return b;
+
+    if(Array.isArray(a) && Array.isArray(b))
+    {
+        const idKey = arrayItemsLookKeyed(a, idKeys) || arrayItemsLookKeyed(b, idKeys);
+
+        if(!idKey) return a.concat(b);
+
+        const index = new Map<any, number>();
+        const out: any[] = [];
+
+        for(const item of a)
+        {
+            index.set(item[idKey], out.length);
+            out.push(item);
+        }
+
+        for(const item of b)
+        {
+            const key = item[idKey];
+            const at = index.get(key);
+
+            if(at !== undefined)
+            {
+                out[at] = mergeGamedata(out[at], item, idKeys);
+            }
+            else
+            {
+                index.set(key, out.length);
+                out.push(item);
+            }
+        }
+
+        return out;
+    }
+
+    if(isPlainObject(a) && isPlainObject(b))
+    {
+        const out: Record<string, any> = { ...a };
+
+        for(const k of Object.keys(b))
+        {
+            out[k] = mergeGamedata(a[k], b[k], idKeys);
+        }
+
+        return out;
+    }
+
+    return b;
+};
+
+interface TierManifest
+{
+    files?: string[];
+}
+
+interface RootManifest
+{
+    tiers?: GamedataTier[];
+    files?: string[];
+}
+
+export const loadGamedata = async <T = any>(url: string, options: GamedataLoadOptions = {}): Promise<T> =>
+{
+    if(!url) throw new Error('loadGamedata: empty URL');
+
+    if(!looksLikeDirectory(url))
+    {
+        return await fetchConfigJson<T>(url);
+    }
+
+    const idKeys = options.mergeArrayIdKeys ?? DEFAULT_ID_KEYS;
+    const rootManifest = await tryFetchOrNull<RootManifest>(joinUrl(url, 'manifest.json5'))
+        ?? await tryFetchOrNull<RootManifest>(joinUrl(url, 'manifest.json'));
+
+    const tiers = (rootManifest?.tiers && rootManifest.tiers.length)
+        ? rootManifest.tiers
+        : (options.tiers ?? DEFAULT_TIERS);
+
+    let merged: any = undefined;
+
+    if(rootManifest?.files?.length)
+    {
+        for(const file of rootManifest.files)
+        {
+            const fileUrl = joinUrl(url, file);
+            const part = await fetchConfigJson(fileUrl);
+            merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys);
+        }
+    }
+
+    for(const tier of tiers)
+    {
+        const tierUrl = joinUrl(url, `${ tier }/`);
+        const tierManifest = await tryFetchOrNull<TierManifest>(joinUrl(tierUrl, 'manifest.json5'))
+            ?? await tryFetchOrNull<TierManifest>(joinUrl(tierUrl, 'manifest.json'));
+
+        if(!tierManifest?.files?.length) continue;
+
+        for(const file of tierManifest.files)
+        {
+            const fileUrl = joinUrl(tierUrl, file);
+            const part = await fetchConfigJson(fileUrl);
+            merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys);
+        }
+    }
+
+    if(merged === undefined) throw new Error(`loadGamedata: directory mode at "${ url }" produced no data — make sure at least one tier (core/custom/seasonal) has a manifest.json5 with a 'files' array`);
+
+    return merged as T;
+};

packages/utils/src/JsonParser.ts

@@ -1,8 +1,24 @@
 import JSON5 from 'json5';
 
+declare const __NITRO_JSON_MODE__: 'legacy' | 'json5' | 'auto' | undefined;
+
 const JSON5_EXTENSION = /\.json5(?:[?#]|$)/i;
 const JSON5_MIME = /(?:application|text)\/(?:json5|x-json5)/i;
 
+const resolveJsonMode = (): 'legacy' | 'json5' | 'auto' =>
+{
+    try
+    {
+        if(typeof __NITRO_JSON_MODE__ !== 'undefined' && __NITRO_JSON_MODE__)
+        {
+            if(__NITRO_JSON_MODE__ === 'legacy' || __NITRO_JSON_MODE__ === 'json5' || __NITRO_JSON_MODE__ === 'auto') return __NITRO_JSON_MODE__;
+        }
+    }
+    catch {}
+
+    return 'auto';
+};
+
 const looksLikeJson5Url = (url: string): boolean => !!url && JSON5_EXTENSION.test(url);
 
 const looksLikeJson5ContentType = (contentType: string): boolean => !!contentType && JSON5_MIME.test(contentType);
@@ -18,13 +34,34 @@ const formatParseError = (sourceUrl: string, strictError: unknown, json5Error: u
     return `Failed to parse JSON/JSON5${ source } — JSON5: ${ json5Message } (strict JSON: ${ strictMessage })`;
 };
 
+const formatStrictError = (sourceUrl: string, err: unknown): string =>
+{
+    const message = (err as Error)?.message || String(err);
+    const source = sourceUrl ? ` in "${ sourceUrl }"` : '';
+
+    return `Failed to parse strict JSON${ source } — ${ message } (build is in 'legacy' mode; switch to JSON5 mode via 'yarn configure' to accept comments/trailing commas)`;
+};
+
 export const parseConfigJson = <T = any>(text: string, sourceUrl: string = ''): T =>
 {
     if(text === null || text === undefined) throw new Error(`Empty response${ sourceUrl ? ` for "${ sourceUrl }"` : '' }`);
 
     const trimmed = text.length > 0 ? text : '';
+    const mode = resolveJsonMode();
 
-    if(looksLikeJson5Url(sourceUrl))
+    if(mode === 'legacy')
+    {
+        try
+        {
+            return JSON.parse(trimmed) as T;
+        }
+        catch(err)
+        {
+            throw new Error(formatStrictError(sourceUrl, err));
+        }
+    }
+
+    if(mode === 'json5' || looksLikeJson5Url(sourceUrl))
     {
         try
         {
@@ -62,8 +99,9 @@ export const parseConfigJsonFromResponse = async <T = any>(response: Response, s
     const contentType = response.headers?.get?.('content-type') || '';
     const text = await response.text();
     const url = sourceUrl || (response as any).url || '';
+    const mode = resolveJsonMode();
 
-    if(looksLikeJson5ContentType(contentType) && !looksLikeJson5Url(url))
+    if(mode === 'auto' && looksLikeJson5ContentType(contentType) && !looksLikeJson5Url(url))
     {
         try
         {

packages/utils/src/index.ts

@@ -4,6 +4,7 @@ export * from './BinaryReader';
 export * from './BinaryWriter';
 export * from './ColorConverter';
 export * from './FurniId';
+export * from './GamedataLoader';
 export * from './GetPixi';
 export * from './GetRenderer';
 export * from './GetStage';