feat(utils): split-aware gamedata loader with tiered merge

Introduces loadGamedata(url, options?) and mergeGamedata(a, b) in
@nitrots/utils. The loader transparently accepts:

- a single-file URL (legacy) -> parsed as before
- a directory URL ending with '/' -> tier-merged from core/custom/seasonal,
  each tier driven by its own manifest.json5

Merge rules:

- arrays of objects sharing an id key (id, classname, name): merged by id,
  later layers overriding earlier ones
- arrays without an id key: concatenated
- plain objects: recursive merge per key
- anything else: later value wins

All gamedata consumers (FurnitureDataLoader, ProductDataLoader,
EffectAssetDownloadManager, AvatarRenderManager actions+figuredata,
LocalizationManager) are migrated to loadGamedata. Behaviour is unchanged
for single-file URLs, so existing deployments need no config changes;
opt-in to split mode by appending '/' to the URL once the layout is in
place.

README updated with the directory layout, merge table and programmatic
usage example. The companion CLI splitter that produces the core/ tier
from legacy files lives in the Nitro V3 client repo.
This commit is contained in:
medievalshell
2026-05-18 21:19:54 +02:00
parent 2a00365862
commit ae9bc8bfce
8 changed files with 265 additions and 101 deletions
+60
View File
@@ -62,3 +62,63 @@ const data2 = await fetchConfigJson<MyConfig>('/configuration/ui-config.json5');
Errors carry the source URL and, in `legacy` mode, a hint about switching to Errors carry the source URL and, in `legacy` mode, a hint about switching to
JSON5 — making misconfigurations easy to diagnose in production logs. JSON5 — making misconfigurations easy to diagnose in production logs.
## Split-aware gamedata loader
`@nitrots/utils` also exports `loadGamedata`, the loader that backs every
gamedata consumer in the renderer (FurnitureDataLoader, ProductDataLoader,
EffectAssetDownloadManager, AvatarRenderManager, LocalizationManager). It
accepts either a **single-file URL** (legacy) or a **directory URL** (split
mode) — detected automatically by the trailing slash.
### Directory layout
```
<gamedata-dir>/
manifest.json5 # OPTIONAL — { "tiers": ["core", "custom", "seasonal"] }
core/
manifest.json5 # REQUIRED — { "files": ["a.json5", "b.json5", ...] }
a.json5
b.json5
custom/ # OPTIONAL tier
manifest.json5
overrides.json5
seasonal/ # OPTIONAL tier
manifest.json5
xmas.json5
```
If the directory `manifest.json5` is absent, the loader falls back to the
default tier order `core → custom → seasonal`. Each tier is skipped silently
if its `manifest.json5` is missing.
### Merge semantics
`mergeGamedata(a, b)` (also exported) implements the rules below; tiers and
files within a tier are merged in order, with later layers overriding
earlier ones:
| Combination | Result |
|------------------------------------------|----------------------------------------------|
| Two plain objects | recursive merge, key by key |
| Two arrays of objects sharing an id key | merged by id (later overrides earlier) |
| Two arrays without an id key | concatenated |
| Anything else | later value wins |
Recognised id keys (in priority order): `id`, `classname`, `name`. Pass
`mergeArrayIdKeys` in the options object to extend or override them.
### Programmatic usage
```ts
import { loadGamedata, mergeGamedata } from '@nitrots/utils';
// host code never needs to care whether the URL is split or not
const furnidata = await loadGamedata('https://example.com/gamedata/furnidata/');
// merge ad-hoc if you load tiers manually
const merged = mergeGamedata(coreData, customData);
```
A CLI splitter for legacy single-file gamedata lives in the Nitro V3 client
repo at `scripts/split-gamedata.mjs` — see the Nitro V3 README for usage.
+7 -33
View File
@@ -2,7 +2,7 @@ import { AvatarSetType, IAssetManager, IAvatarEffectListener, IAvatarFigureConta
import { GetAssetManager } from '@nitrots/assets'; import { GetAssetManager } from '@nitrots/assets';
import { GetConfiguration } from '@nitrots/configuration'; import { GetConfiguration } from '@nitrots/configuration';
import { GetEventDispatcher, NitroEventType } from '@nitrots/events'; import { GetEventDispatcher, NitroEventType } from '@nitrots/events';
import { parseConfigJsonFromResponse } from '@nitrots/utils'; import { loadGamedata } from '@nitrots/utils';
import { AvatarAssetDownloadManager } from './AvatarAssetDownloadManager'; import { AvatarAssetDownloadManager } from './AvatarAssetDownloadManager';
import { AvatarFigureContainer } from './AvatarFigureContainer'; import { AvatarFigureContainer } from './AvatarFigureContainer';
import { AvatarImage } from './AvatarImage'; import { AvatarImage } from './AvatarImage';
@@ -73,26 +73,13 @@ export class AvatarRenderManager implements IAvatarRenderManager
if(!url || !url.length) throw new Error('Missing "avatar.actions.url" in config — add the URL to your renderer-config.json'); if(!url || !url.length) throw new Error('Missing "avatar.actions.url" in config — add the URL to your renderer-config.json');
let response: Response;
try try
{ {
response = await fetch(url); this._structure.updateActions(await loadGamedata(url));
} }
catch(fetchErr) catch(err)
{ {
throw new Error(`Could not fetch avatar actions from "${ url }" — check "avatar.actions.url" in renderer-config.json (${ fetchErr.message })`); throw new Error(`Could not load avatar actions from "${ url }" — check "avatar.actions.url" in renderer-config.json (${ err?.message || err })`);
}
if(response.status !== 200) throw new Error(`Failed to load avatar actions from "${ url }" — server returned HTTP ${ response.status }. Check "avatar.actions.url" in renderer-config.json`);
try
{
this._structure.updateActions(await parseConfigJsonFromResponse(response, url));
}
catch(parseErr)
{
throw new Error(`Invalid avatar actions "${ url }" — JSON/JSON5 parse failed. Check "avatar.actions.url" in renderer-config.json (${ parseErr.message })`);
} }
} }
@@ -106,26 +93,13 @@ export class AvatarRenderManager implements IAvatarRenderManager
if(!url || !url.length) throw new Error('Missing "avatar.figuredata.url" in config — add the URL to your renderer-config.json'); if(!url || !url.length) throw new Error('Missing "avatar.figuredata.url" in config — add the URL to your renderer-config.json');
let response: Response;
try try
{ {
response = await fetch(url); this._structure.figureData.appendJSON(await loadGamedata(url));
} }
catch(fetchErr) catch(err)
{ {
throw new Error(`Could not fetch figure data from "${ url }" — check "avatar.figuredata.url" in renderer-config.json (${ fetchErr.message })`); throw new Error(`Could not load figure data from "${ url }" — check "avatar.figuredata.url" in renderer-config.json (${ err?.message || err })`);
}
if(response.status !== 200) throw new Error(`Failed to load figure data from "${ url }" — server returned HTTP ${ response.status }. Check "avatar.figuredata.url" in renderer-config.json`);
try
{
this._structure.figureData.appendJSON(await parseConfigJsonFromResponse(response, url));
}
catch(parseErr)
{
throw new Error(`Invalid figure data "${ url }" — JSON/JSON5 parse failed. Check "avatar.figuredata.url" in renderer-config.json (${ parseErr.message })`);
} }
this._structure.init(); this._structure.init();
@@ -1,7 +1,7 @@
import { IAssetManager, IAvatarEffectListener } from '@nitrots/api'; import { IAssetManager, IAvatarEffectListener } from '@nitrots/api';
import { GetConfiguration } from '@nitrots/configuration'; import { GetConfiguration } from '@nitrots/configuration';
import { AvatarRenderEffectLibraryEvent, GetEventDispatcher, NitroEvent, NitroEventType } from '@nitrots/events'; import { AvatarRenderEffectLibraryEvent, GetEventDispatcher, NitroEvent, NitroEventType } from '@nitrots/events';
import { parseConfigJsonFromResponse } from '@nitrots/utils'; import { loadGamedata } from '@nitrots/utils';
import { AvatarStructure } from './AvatarStructure'; import { AvatarStructure } from './AvatarStructure';
import { EffectAssetDownloadLibrary } from './EffectAssetDownloadLibrary'; import { EffectAssetDownloadLibrary } from './EffectAssetDownloadLibrary';
@@ -32,28 +32,15 @@ export class EffectAssetDownloadManager
if(!url || !url.length) throw new Error('Missing "avatar.effectmap.url" in config — add the effect map URL to your renderer-config.json'); if(!url || !url.length) throw new Error('Missing "avatar.effectmap.url" in config — add the effect map URL to your renderer-config.json');
let response: Response;
try
{
response = await fetch(url);
}
catch(fetchErr)
{
throw new Error(`Could not fetch effect map from "${ url }" — check "avatar.effectmap.url" in renderer-config.json (${ fetchErr.message })`);
}
if(response.status !== 200) throw new Error(`Failed to load effect map from "${ url }" — server returned HTTP ${ response.status }. Check "avatar.effectmap.url" in renderer-config.json`);
let responseData: any; let responseData: any;
try try
{ {
responseData = await parseConfigJsonFromResponse(response, url); responseData = await loadGamedata(url);
} }
catch(parseErr) catch(err)
{ {
throw new Error(`Invalid effect map "${ url }" — JSON/JSON5 parse failed. Check "avatar.effectmap.url" in renderer-config.json (${ parseErr.message })`); throw new Error(`Could not load effect map from "${ url }" — check "avatar.effectmap.url" in renderer-config.json (${ err?.message || err })`);
} }
this.processEffectMap(responseData.effects); this.processEffectMap(responseData.effects);
@@ -1,7 +1,7 @@
import { ILocalizationManager } from '@nitrots/api'; import { ILocalizationManager } from '@nitrots/api';
import { BadgePointLimitsEvent, GetCommunication } from '@nitrots/communication'; import { BadgePointLimitsEvent, GetCommunication } from '@nitrots/communication';
import { GetConfiguration } from '@nitrots/configuration'; import { GetConfiguration } from '@nitrots/configuration';
import { parseConfigJsonFromResponse } from '@nitrots/utils'; import { loadGamedata } from '@nitrots/utils';
import { BadgeBaseAndLevel } from './BadgeBaseAndLevel'; import { BadgeBaseAndLevel } from './BadgeBaseAndLevel';
export class LocalizationManager implements ILocalizationManager export class LocalizationManager implements ILocalizationManager
@@ -26,28 +26,15 @@ export class LocalizationManager implements ILocalizationManager
url = GetConfiguration().interpolate(url); url = GetConfiguration().interpolate(url);
let response: Response;
try
{
response = await fetch(url);
}
catch(fetchErr)
{
throw new Error(`Could not fetch localization file "${ url }" — check "external.texts.url" in ui-config.json (${ fetchErr.message })`);
}
if(response.status !== 200) throw new Error(`Failed to load localization file "${ url }" — server returned HTTP ${ response.status }. Check "external.texts.url" in ui-config.json`);
let data: any; let data: any;
try try
{ {
data = await parseConfigJsonFromResponse(response, url); data = await loadGamedata(url);
} }
catch(parseErr) catch(err)
{ {
throw new Error(`Invalid localization file "${ url }" — JSON/JSON5 parse failed. Check "external.texts.url" in ui-config.json (${ parseErr.message })`); throw new Error(`Could not load localization file "${ url }" — check "external.texts.url" in ui-config.json (${ err?.message || err })`);
} }
this.parseLocalization(data); this.parseLocalization(data);
@@ -1,7 +1,7 @@
import { FurnitureType, IFurnitureData } from '@nitrots/api'; import { FurnitureType, IFurnitureData } from '@nitrots/api';
import { GetConfiguration } from '@nitrots/configuration'; import { GetConfiguration } from '@nitrots/configuration';
import { GetLocalizationManager } from '@nitrots/localization'; import { GetLocalizationManager } from '@nitrots/localization';
import { parseConfigJsonFromResponse } from '@nitrots/utils'; import { loadGamedata } from '@nitrots/utils';
import { FurnitureData } from './FurnitureData'; import { FurnitureData } from './FurnitureData';
export class FurnitureDataLoader export class FurnitureDataLoader
@@ -21,28 +21,15 @@ export class FurnitureDataLoader
if(!url || !url.length) throw new Error('Missing "furnidata.url" in config — add the furniture data URL to your renderer-config.json'); if(!url || !url.length) throw new Error('Missing "furnidata.url" in config — add the furniture data URL to your renderer-config.json');
let response: Response;
try
{
response = await fetch(url);
}
catch(fetchErr)
{
throw new Error(`Could not fetch furniture data from "${ url }" — check "furnidata.url" in renderer-config.json (${ fetchErr.message })`);
}
if(response.status !== 200) throw new Error(`Failed to load furniture data from "${ url }" — server returned HTTP ${ response.status }. Check "furnidata.url" in renderer-config.json`);
let responseData: any; let responseData: any;
try try
{ {
responseData = await parseConfigJsonFromResponse(response, url); responseData = await loadGamedata(url);
} }
catch(parseErr) catch(err)
{ {
throw new Error(`Invalid furniture data "${ url }" — JSON/JSON5 parse failed. Check "furnidata.url" in renderer-config.json (${ parseErr.message })`); throw new Error(`Could not load furniture data from "${ url }" — check "furnidata.url" in renderer-config.json (${ err?.message || err })`);
} }
if(responseData.roomitemtypes) this.parseFloorItems(responseData.roomitemtypes); if(responseData.roomitemtypes) this.parseFloorItems(responseData.roomitemtypes);
@@ -1,6 +1,6 @@
import { IProductData } from '@nitrots/api'; import { IProductData } from '@nitrots/api';
import { GetConfiguration } from '@nitrots/configuration'; import { GetConfiguration } from '@nitrots/configuration';
import { parseConfigJsonFromResponse } from '@nitrots/utils'; import { loadGamedata } from '@nitrots/utils';
import { ProductData } from './ProductData'; import { ProductData } from './ProductData';
export class ProductDataLoader export class ProductDataLoader
@@ -18,28 +18,15 @@ export class ProductDataLoader
if(!url || !url.length) throw new Error('Missing "productdata.url" in config — add the product data URL to your renderer-config.json'); if(!url || !url.length) throw new Error('Missing "productdata.url" in config — add the product data URL to your renderer-config.json');
let response: Response;
try
{
response = await fetch(url);
}
catch(fetchErr)
{
throw new Error(`Could not fetch product data from "${ url }" — check "productdata.url" in renderer-config.json (${ fetchErr.message })`);
}
if(response.status !== 200) throw new Error(`Failed to load product data from "${ url }" — server returned HTTP ${ response.status }. Check "productdata.url" in renderer-config.json`);
let responseData: any; let responseData: any;
try try
{ {
responseData = await parseConfigJsonFromResponse(response, url); responseData = await loadGamedata(url);
} }
catch(parseErr) catch(err)
{ {
throw new Error(`Invalid product data "${ url }" — JSON/JSON5 parse failed. Check "productdata.url" in renderer-config.json (${ parseErr.message })`); throw new Error(`Could not load product data from "${ url }" — check "productdata.url" in renderer-config.json (${ err?.message || err })`);
} }
this.parseProducts(responseData.productdata); this.parseProducts(responseData.productdata);
+181
View File
@@ -0,0 +1,181 @@
import { fetchConfigJson } from './JsonParser';
export const DEFAULT_TIERS = [ 'core', 'custom', 'seasonal' ] as const;
export type GamedataTier = typeof DEFAULT_TIERS[number] | string;
export interface GamedataLoadOptions
{
tiers?: readonly GamedataTier[];
mergeArrayIdKeys?: readonly string[];
}
const DEFAULT_ID_KEYS = [ 'id', 'classname', 'name' ] as const;
const looksLikeDirectory = (url: string): boolean =>
{
if(!url) return false;
const stripped = url.split('?')[0].split('#')[0];
return stripped.endsWith('/');
};
const joinUrl = (base: string, path: string): string =>
{
const cleanBase = base.endsWith('/') ? base : `${ base }/`;
const cleanPath = path.startsWith('/') ? path.slice(1) : path;
return `${ cleanBase }${ cleanPath }`;
};
const tryFetchOrNull = async <T = any>(url: string): Promise<T | null> =>
{
try
{
return await fetchConfigJson<T>(url);
}
catch
{
return null;
}
};
const isPlainObject = (value: any): value is Record<string, any> => !!value && typeof value === 'object' && !Array.isArray(value);
const arrayItemsLookKeyed = (arr: any[], idKeys: readonly string[]): string | null =>
{
if(!arr.length) return null;
for(const key of idKeys)
{
let allHave = true;
for(const item of arr)
{
if(!isPlainObject(item) || item[key] === undefined || item[key] === null)
{
allHave = false;
break;
}
}
if(allHave) return key;
}
return null;
};
export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAULT_ID_KEYS): any =>
{
if(b === undefined) return a;
if(a === undefined) return b;
if(Array.isArray(a) && Array.isArray(b))
{
const idKey = arrayItemsLookKeyed(a, idKeys) || arrayItemsLookKeyed(b, idKeys);
if(!idKey) return a.concat(b);
const index = new Map<any, number>();
const out: any[] = [];
for(const item of a)
{
index.set(item[idKey], out.length);
out.push(item);
}
for(const item of b)
{
const key = item[idKey];
const at = index.get(key);
if(at !== undefined)
{
out[at] = mergeGamedata(out[at], item, idKeys);
}
else
{
index.set(key, out.length);
out.push(item);
}
}
return out;
}
if(isPlainObject(a) && isPlainObject(b))
{
const out: Record<string, any> = { ...a };
for(const k of Object.keys(b))
{
out[k] = mergeGamedata(a[k], b[k], idKeys);
}
return out;
}
return b;
};
interface TierManifest
{
files?: string[];
}
interface RootManifest
{
tiers?: GamedataTier[];
files?: string[];
}
export const loadGamedata = async <T = any>(url: string, options: GamedataLoadOptions = {}): Promise<T> =>
{
if(!url) throw new Error('loadGamedata: empty URL');
if(!looksLikeDirectory(url))
{
return await fetchConfigJson<T>(url);
}
const idKeys = options.mergeArrayIdKeys ?? DEFAULT_ID_KEYS;
const rootManifest = await tryFetchOrNull<RootManifest>(joinUrl(url, 'manifest.json5'))
?? await tryFetchOrNull<RootManifest>(joinUrl(url, 'manifest.json'));
const tiers = (rootManifest?.tiers && rootManifest.tiers.length)
? rootManifest.tiers
: (options.tiers ?? DEFAULT_TIERS);
let merged: any = undefined;
if(rootManifest?.files?.length)
{
for(const file of rootManifest.files)
{
const fileUrl = joinUrl(url, file);
const part = await fetchConfigJson(fileUrl);
merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys);
}
}
for(const tier of tiers)
{
const tierUrl = joinUrl(url, `${ tier }/`);
const tierManifest = await tryFetchOrNull<TierManifest>(joinUrl(tierUrl, 'manifest.json5'))
?? await tryFetchOrNull<TierManifest>(joinUrl(tierUrl, 'manifest.json'));
if(!tierManifest?.files?.length) continue;
for(const file of tierManifest.files)
{
const fileUrl = joinUrl(tierUrl, file);
const part = await fetchConfigJson(fileUrl);
merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys);
}
}
if(merged === undefined) throw new Error(`loadGamedata: directory mode at "${ url }" produced no data — make sure at least one tier (core/custom/seasonal) has a manifest.json5 with a 'files' array`);
return merged as T;
};
+1
View File
@@ -4,6 +4,7 @@ export * from './BinaryReader';
export * from './BinaryWriter'; export * from './BinaryWriter';
export * from './ColorConverter'; export * from './ColorConverter';
export * from './FurniId'; export * from './FurniId';
export * from './GamedataLoader';
export * from './GetPixi'; export * from './GetPixi';
export * from './GetRenderer'; export * from './GetRenderer';
export * from './GetStage'; export * from './GetStage';