merge: integrate duckietm/Dev (JSON5 + split-aware gamedata loader)

# Conflicts:
#	packages/session/src/SessionDataManager.ts
#	yarn.lock

packages/utils/package.json

@@ -10,6 +10,7 @@
   "main": "./index",
   "dependencies": {
     "@nitrots/api": "1.0.0",
+    "json5": "^2.2.3",
     "pako": "^2.1.0",
     "pixi.js": "^8.8.1"
   },

packages/utils/src/GamedataLoader.ts

@@ -0,0 +1,181 @@
+import { fetchConfigJson } from './JsonParser';
+
+export const DEFAULT_TIERS = [ 'core', 'custom', 'seasonal' ] as const;
+export type GamedataTier = typeof DEFAULT_TIERS[number] | string;
+
+export interface GamedataLoadOptions
+{
+    tiers?: readonly GamedataTier[];
+    mergeArrayIdKeys?: readonly string[];
+}
+
+const DEFAULT_ID_KEYS = [ 'id', 'classname', 'name' ] as const;
+
+const looksLikeDirectory = (url: string): boolean =>
+{
+    if(!url) return false;
+
+    const stripped = url.split('?')[0].split('#')[0];
+
+    return stripped.endsWith('/');
+};
+
+const joinUrl = (base: string, path: string): string =>
+{
+    const cleanBase = base.endsWith('/') ? base : `${ base }/`;
+    const cleanPath = path.startsWith('/') ? path.slice(1) : path;
+
+    return `${ cleanBase }${ cleanPath }`;
+};
+
+const tryFetchOrNull = async <T = any>(url: string): Promise<T | null> =>
+{
+    try
+    {
+        return await fetchConfigJson<T>(url);
+    }
+    catch
+    {
+        return null;
+    }
+};
+
+const isPlainObject = (value: any): value is Record<string, any> => !!value && typeof value === 'object' && !Array.isArray(value);
+
+const arrayItemsLookKeyed = (arr: any[], idKeys: readonly string[]): string | null =>
+{
+    if(!arr.length) return null;
+
+    for(const key of idKeys)
+    {
+        let allHave = true;
+
+        for(const item of arr)
+        {
+            if(!isPlainObject(item) || item[key] === undefined || item[key] === null)
+            {
+                allHave = false;
+                break;
+            }
+        }
+
+        if(allHave) return key;
+    }
+
+    return null;
+};
+
+export const mergeGamedata = (a: any, b: any, idKeys: readonly string[] = DEFAULT_ID_KEYS): any =>
+{
+    if(b === undefined) return a;
+    if(a === undefined) return b;
+
+    if(Array.isArray(a) && Array.isArray(b))
+    {
+        const idKey = arrayItemsLookKeyed(a, idKeys) || arrayItemsLookKeyed(b, idKeys);
+
+        if(!idKey) return a.concat(b);
+
+        const index = new Map<any, number>();
+        const out: any[] = [];
+
+        for(const item of a)
+        {
+            index.set(item[idKey], out.length);
+            out.push(item);
+        }
+
+        for(const item of b)
+        {
+            const key = item[idKey];
+            const at = index.get(key);
+
+            if(at !== undefined)
+            {
+                out[at] = mergeGamedata(out[at], item, idKeys);
+            }
+            else
+            {
+                index.set(key, out.length);
+                out.push(item);
+            }
+        }
+
+        return out;
+    }
+
+    if(isPlainObject(a) && isPlainObject(b))
+    {
+        const out: Record<string, any> = { ...a };
+
+        for(const k of Object.keys(b))
+        {
+            out[k] = mergeGamedata(a[k], b[k], idKeys);
+        }
+
+        return out;
+    }
+
+    return b;
+};
+
+interface TierManifest
+{
+    files?: string[];
+}
+
+interface RootManifest
+{
+    tiers?: GamedataTier[];
+    files?: string[];
+}
+
+export const loadGamedata = async <T = any>(url: string, options: GamedataLoadOptions = {}): Promise<T> =>
+{
+    if(!url) throw new Error('loadGamedata: empty URL');
+
+    if(!looksLikeDirectory(url))
+    {
+        return await fetchConfigJson<T>(url);
+    }
+
+    const idKeys = options.mergeArrayIdKeys ?? DEFAULT_ID_KEYS;
+    const rootManifest = await tryFetchOrNull<RootManifest>(joinUrl(url, 'manifest.json5'))
+        ?? await tryFetchOrNull<RootManifest>(joinUrl(url, 'manifest.json'));
+
+    const tiers = (rootManifest?.tiers && rootManifest.tiers.length)
+        ? rootManifest.tiers
+        : (options.tiers ?? DEFAULT_TIERS);
+
+    let merged: any = undefined;
+
+    if(rootManifest?.files?.length)
+    {
+        for(const file of rootManifest.files)
+        {
+            const fileUrl = joinUrl(url, file);
+            const part = await fetchConfigJson(fileUrl);
+            merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys);
+        }
+    }
+
+    for(const tier of tiers)
+    {
+        const tierUrl = joinUrl(url, `${ tier }/`);
+        const tierManifest = await tryFetchOrNull<TierManifest>(joinUrl(tierUrl, 'manifest.json5'))
+            ?? await tryFetchOrNull<TierManifest>(joinUrl(tierUrl, 'manifest.json'));
+
+        if(!tierManifest?.files?.length) continue;
+
+        for(const file of tierManifest.files)
+        {
+            const fileUrl = joinUrl(tierUrl, file);
+            const part = await fetchConfigJson(fileUrl);
+            merged = (merged === undefined) ? part : mergeGamedata(merged, part, idKeys);
+        }
+    }
+
+    if(merged === undefined) throw new Error(`loadGamedata: directory mode at "${ url }" produced no data — make sure at least one tier (core/custom/seasonal) has a manifest.json5 with a 'files' array`);
+
+    return merged as T;
+};

packages/utils/src/JsonParser.ts

@@ -0,0 +1,126 @@
+import JSON5 from 'json5';
+
+declare const __NITRO_JSON_MODE__: 'legacy' | 'json5' | 'auto' | undefined;
+
+const JSON5_EXTENSION = /\.json5(?:[?#]|$)/i;
+const JSON5_MIME = /(?:application|text)\/(?:json5|x-json5)/i;
+
+const resolveJsonMode = (): 'legacy' | 'json5' | 'auto' =>
+{
+    try
+    {
+        if(typeof __NITRO_JSON_MODE__ !== 'undefined' && __NITRO_JSON_MODE__)
+        {
+            if(__NITRO_JSON_MODE__ === 'legacy' || __NITRO_JSON_MODE__ === 'json5' || __NITRO_JSON_MODE__ === 'auto') return __NITRO_JSON_MODE__;
+        }
+    }
+    catch {}
+
+    return 'auto';
+};
+
+const looksLikeJson5Url = (url: string): boolean => !!url && JSON5_EXTENSION.test(url);
+
+const looksLikeJson5ContentType = (contentType: string): boolean => !!contentType && JSON5_MIME.test(contentType);
+
+const formatParseError = (sourceUrl: string, strictError: unknown, json5Error: unknown): string =>
+{
+    const strictMessage = (strictError as Error)?.message || String(strictError);
+    const json5Message = (json5Error as Error)?.message || String(json5Error);
+    const source = sourceUrl ? ` in "${ sourceUrl }"` : '';
+
+    if(strictMessage === json5Message) return `Failed to parse JSON/JSON5${ source } — ${ json5Message }`;
+
+    return `Failed to parse JSON/JSON5${ source } — JSON5: ${ json5Message } (strict JSON: ${ strictMessage })`;
+};
+
+const formatStrictError = (sourceUrl: string, err: unknown): string =>
+{
+    const message = (err as Error)?.message || String(err);
+    const source = sourceUrl ? ` in "${ sourceUrl }"` : '';
+
+    return `Failed to parse strict JSON${ source } — ${ message } (build is in 'legacy' mode; switch to JSON5 mode via 'yarn configure' to accept comments/trailing commas)`;
+};
+
+export const parseConfigJson = <T = any>(text: string, sourceUrl: string = ''): T =>
+{
+    if(text === null || text === undefined) throw new Error(`Empty response${ sourceUrl ? ` for "${ sourceUrl }"` : '' }`);
+
+    const trimmed = text.length > 0 ? text : '';
+    const mode = resolveJsonMode();
+
+    if(mode === 'legacy')
+    {
+        try
+        {
+            return JSON.parse(trimmed) as T;
+        }
+        catch(err)
+        {
+            throw new Error(formatStrictError(sourceUrl, err));
+        }
+    }
+
+    if(mode === 'json5' || looksLikeJson5Url(sourceUrl))
+    {
+        try
+        {
+            return JSON5.parse<T>(trimmed);
+        }
+        catch(err)
+        {
+            throw new Error(formatParseError(sourceUrl, err, err));
+        }
+    }
+
+    let strictError: unknown;
+
+    try
+    {
+        return JSON.parse(trimmed) as T;
+    }
+    catch(err)
+    {
+        strictError = err;
+    }
+
+    try
+    {
+        return JSON5.parse<T>(trimmed);
+    }
+    catch(json5Error)
+    {
+        throw new Error(formatParseError(sourceUrl, strictError, json5Error));
+    }
+};
+
+export const parseConfigJsonFromResponse = async <T = any>(response: Response, sourceUrl: string = ''): Promise<T> =>
+{
+    const contentType = response.headers?.get?.('content-type') || '';
+    const text = await response.text();
+    const url = sourceUrl || (response as any).url || '';
+    const mode = resolveJsonMode();
+
+    if(mode === 'auto' && looksLikeJson5ContentType(contentType) && !looksLikeJson5Url(url))
+    {
+        try
+        {
+            return JSON5.parse<T>(text);
+        }
+        catch(err)
+        {
+            throw new Error(formatParseError(url, err, err));
+        }
+    }
+
+    return parseConfigJson<T>(text, url);
+};
+
+export const fetchConfigJson = async <T = any>(url: string, init?: RequestInit): Promise<T> =>
+{
+    const response = await fetch(url, init);
+
+    if(!response || response.status !== 200) throw new Error(`Failed to fetch "${ url }" — server returned HTTP ${ response?.status ?? 'no response' }`);
+
+    return parseConfigJsonFromResponse<T>(response, url);
+};

packages/utils/src/__tests__/JsonParser.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, it } from 'vitest';
+import { fetchConfigJson, parseConfigJson, parseConfigJsonFromResponse } from '../JsonParser';
+
+describe('parseConfigJson', () =>
+{
+    it('parses strict JSON', () =>
+    {
+        const result = parseConfigJson('{"a": 1, "b": [2, 3]}');
+        expect(result).toEqual({ a: 1, b: [ 2, 3 ] });
+    });
+
+    it('falls back to JSON5 for trailing commas', () =>
+    {
+        const result = parseConfigJson('{"a": 1, "b": [2, 3,],}');
+        expect(result).toEqual({ a: 1, b: [ 2, 3 ] });
+    });
+
+    it('falls back to JSON5 for comments', () =>
+    {
+        const result = parseConfigJson(`{
+            // a number
+            "a": 1,
+            /* a list */
+            "b": [2, 3]
+        }`);
+        expect(result).toEqual({ a: 1, b: [ 2, 3 ] });
+    });
+
+    it('falls back to JSON5 for unquoted keys and single quotes', () =>
+    {
+        const result = parseConfigJson("{ a: 1, b: 'hello' }");
+        expect(result).toEqual({ a: 1, b: 'hello' });
+    });
+
+    it('uses JSON5 directly for .json5 URLs', () =>
+    {
+        const result = parseConfigJson('{ a: 1, /* hi */ b: 2 }', 'https://example.com/cfg.json5');
+        expect(result).toEqual({ a: 1, b: 2 });
+    });
+
+    it('throws a helpful error when both strict and JSON5 fail', () =>
+    {
+        expect(() => parseConfigJson('{ this is :: not json ::', 'cfg.json'))
+            .toThrowError(/Failed to parse JSON\/JSON5 in "cfg\.json"/);
+    });
+});
+
+describe('parseConfigJsonFromResponse', () =>
+{
+    const buildResponse = (body: string, contentType = 'application/json', url = 'https://example.com/x.json'): Response =>
+    {
+        const headers = new Headers({ 'content-type': contentType });
+        return new Response(body, { status: 200, headers });
+    };
+
+    it('parses JSON response bodies', async () =>
+    {
+        const res = buildResponse('{"a": 1}');
+        await expect(parseConfigJsonFromResponse(res, 'https://example.com/x.json')).resolves.toEqual({ a: 1 });
+    });
+
+    it('parses JSON5 response bodies with comments', async () =>
+    {
+        const res = buildResponse('{ /* yo */ a: 1, b: 2, }');
+        await expect(parseConfigJsonFromResponse(res, 'https://example.com/x.json')).resolves.toEqual({ a: 1, b: 2 });
+    });
+
+    it('respects application/json5 content-type', async () =>
+    {
+        const res = buildResponse('{ a: 1 }', 'application/json5');
+        await expect(parseConfigJsonFromResponse(res, 'https://example.com/x.txt')).resolves.toEqual({ a: 1 });
+    });
+});
+
+describe('fetchConfigJson', () =>
+{
+    it('fetches and parses JSON or JSON5', async () =>
+    {
+        const originalFetch = globalThis.fetch;
+        globalThis.fetch = (async () => new Response('{ a: 1, b: 2, }', {
+            status: 200,
+            headers: { 'content-type': 'application/json' }
+        })) as any;
+
+        try
+        {
+            await expect(fetchConfigJson('https://example.com/cfg.json')).resolves.toEqual({ a: 1, b: 2 });
+        }
+        finally
+        {
+            globalThis.fetch = originalFetch;
+        }
+    });
+
+    it('throws for non-200 responses', async () =>
+    {
+        const originalFetch = globalThis.fetch;
+        globalThis.fetch = (async () => new Response('', { status: 404 })) as any;
+
+        try
+        {
+            await expect(fetchConfigJson('https://example.com/missing.json')).rejects.toThrowError(/HTTP 404/);
+        }
+        finally
+        {
+            globalThis.fetch = originalFetch;
+        }
+    });
+});

packages/utils/src/index.ts

@@ -4,6 +4,7 @@ export * from './BinaryReader';
 export * from './BinaryWriter';
 export * from './ColorConverter';
 export * from './FurniId';
+export * from './GamedataLoader';
 export * from './GetPixi';
 export * from './GetRenderer';
 export * from './GetStage';
@@ -13,6 +14,7 @@ export * from './GetTickerFPS';
 export * from './GetTickerTime';
 export * from './HabboWebTools';
 export * from './Int32';
+export * from './JsonParser';
 export * from './LegacyExternalInterface';
 export * from './LinkTracker';
 export * from './Matrix4x4';