refactor: load the generated JSON at runtime

The dumps were imported into the bundle, which tied a data refresh to a
rebuild and put generated content in the source tree. They now sit in
static-contents/ with the rest of the site data, as on
bsi-ni.brain.riken.jp.

- add siteData.ts, which fetches the six files in parallel
- hand them to the singletons from main.tsx before the first render, so
  components still read them synchronously
- type the four ranking lists, which have three different shapes

.gitignore

@@ -23,9 +23,5 @@ yarn-error.log*
 .env.*.local
 
 # generated site data (dumped by the scripts in etc/)
-# static-contents/ now lives next to the repository, not inside it
+# it all lives in static-contents/, next to the repository
 /dl-limit-items.csv
-/src/database/assets/*.json
-/src/credits/assets/*.json
-/src/forum/assets/*.json
-/src/pico/assets/*.json

src/credits/lib/CreditsUtils.ts

@@ -1,6 +1,5 @@
 import type { MultiLang } from '../../config';
 import Functions from '../../functions';
-import creditsJson from '../assets/credits.json';
 
 export interface CreditsPageData {
     pid: number;
@@ -9,7 +8,7 @@ export interface CreditsPageData {
     lastupdate: number;
 }
 
-interface CreditsData {
+export interface CreditsData {
     organization: string;
     pages: CreditsPageData[];
 }
@@ -20,12 +19,19 @@ export interface CreditsMenu {
 }
 
 class CreditsUtils {
-    private data: CreditsData;
+    private data: CreditsData | null = null;
 
-    constructor(json: CreditsData) {
+    init(json: CreditsData): void {
         this.data = json;
     }
 
+    private get credits(): CreditsData {
+        if (this.data === null) {
+            throw new Error('credits data was read before loadSiteData() completed');
+        }
+        return this.data;
+    }
+
     getTitle(lang: MultiLang): string {
         return Functions.mlang('[en]Site Information[/en][ja]サイト情報[/ja]', lang);
     }
@@ -40,7 +46,7 @@ class CreditsUtils {
 
     getMenu(): CreditsMenu[] {
         const menu: CreditsMenu[] = [];
-        this.data.pages.forEach((page) => {
+        this.credits.pages.forEach((page) => {
             menu.push({ title: page.title, link: this.getPageUrl(page.pid) });
         });
         menu.push({ title: this.getAboutUsTitle(), link: this.getIndexUrl() });
@@ -48,7 +54,7 @@ class CreditsUtils {
     }
 
     getPage(pageId: number): CreditsPageData | null {
-        const page = this.data.pages.find((page) => {
+        const page = this.credits.pages.find((page) => {
             return page.pid === pageId;
         });
         if (typeof page === 'undefined') {
@@ -58,7 +64,7 @@ class CreditsUtils {
     }
 
     countPages(): number {
-        return this.data.pages.length;
+        return this.credits.pages.length;
     }
 
     getAboutUsTitle(): string {
@@ -66,8 +72,8 @@ class CreditsUtils {
     }
 
     getAboutUsContent(): string {
-        return this.data.organization;
+        return this.credits.organization;
     }
 }
 
-export default new CreditsUtils(creditsJson);
+export default new CreditsUtils();

src/custom/blocks/VisiomeNewItems.tsx

@@ -2,9 +2,9 @@ import { format } from 'date-fns';
 import { Link } from 'react-router';
 import type { MultiLang } from '../../config';
 import imageStar from '../../database/assets/images/star.gif';
-import contents from '../../database/assets/recent-contents.json';
 import ItemUtil, { type ItemCore } from '../../database/lib/ItemUtil';
 import Functions from '../../functions';
+import { siteData } from '../../siteData';
 import VisiomePreviewList from './VisiomePreviewList';
 
 interface Props {
@@ -15,7 +15,7 @@ const dummyItemIds = [6412, 6975, 6748, 6448, 6735, 6894, 6981, 6722, 6721, 6177
 
 const VisiomeNewItems = (props: Props) => {
     const { lang } = props;
-    const list = contents.map((item, idx) => {
+    const list = siteData().recentContents.map((item, idx) => {
         const url = ItemUtil.getUrl(item as ItemCore);
         const title = Functions.mlang(item.title, lang);
         const date = format(new Date(item.last_update_date * 1000), 'y/M/d');
@@ -30,7 +30,7 @@ const VisiomeNewItems = (props: Props) => {
             </div>
         );
     });
-    const itemIds = contents.map((itemCore: ItemCore) => {
+    const itemIds = siteData().recentContents.map((itemCore: ItemCore) => {
         return itemCore.item_id;
     });
     return (

src/custom/blocks/VisiomeTop10Items.tsx

@@ -2,9 +2,9 @@ import { Fragment } from 'react';
 import { Link } from 'react-router';
 import type { MultiLang } from '../../config';
 import imageStar from '../../database/assets/images/star.gif';
-import rankings from '../../database/assets/rankings.json';
 import ItemUtil, { type ItemCore } from '../../database/lib/ItemUtil';
 import Functions from '../../functions';
+import { siteData } from '../../siteData';
 import VisiomePreviewList from './VisiomePreviewList';
 
 interface Props {
@@ -23,7 +23,7 @@ const VisiomeTop10Items = (props: Props) => {
     const items = [
         {
             title: Functions.mlang('[en]Frequently accessed items[/en][ja]頻繁に閲覧されるアイテム[/ja]', lang),
-            list: rankings.accessed.map((item, idx) => {
+            list: siteData().rankings.accessed.map((item, idx) => {
                 const url = ItemUtil.getUrl(item as ItemCore);
                 const title = Functions.mlang(item.title, lang);
                 const order = orderLabel(idx + 1, lang);
@@ -41,7 +41,7 @@ const VisiomeTop10Items = (props: Props) => {
         },
         {
             title: Functions.mlang('[en]Frequently downloaded items[/en][ja]頻繁にダウンロードされるアイテム[/ja]', lang),
-            list: rankings.downloaded.map((item, idx) => {
+            list: siteData().rankings.downloaded.map((item, idx) => {
                 const url = ItemUtil.getUrl(item as ItemCore);
                 const title = Functions.mlang(item.title, lang);
                 const order = orderLabel(idx + 1, lang);

src/database/blocks/Rankings.tsx

@@ -2,8 +2,8 @@ import { Fragment } from 'react';
 import { Link } from 'react-router';
 import type { MultiLang } from '../../config';
 import Functions from '../../functions';
+import { siteData } from '../../siteData';
 import imageStar from '../assets/images/star.gif';
-import rankings from '../assets/rankings.json';
 import ItemUtil, { type ItemCore } from '../lib/ItemUtil';
 import styles from './Rankings.module.css';
 
@@ -19,7 +19,7 @@ const Rankings = (props: Props) => {
     const items = [
         {
             title: Functions.mlang('[en]Frequently accessed items[/en][ja]頻繁に閲覧されるアイテム[/ja]', props.lang),
-            list: rankings.accessed.map((item, idx) => {
+            list: siteData().rankings.accessed.map((item, idx) => {
                 const url = ItemUtil.getUrl(item as ItemCore);
                 const title = Functions.mlang(item.title, props.lang);
                 const order = orderLabel(idx + 1, props.lang);
@@ -42,7 +42,7 @@ const Rankings = (props: Props) => {
         },
         {
             title: Functions.mlang('[en]Frequently downloaded items[/en][ja]頻繁にダウンロードされるアイテム[/ja]', props.lang),
-            list: rankings.downloaded.map((item, idx) => {
+            list: siteData().rankings.downloaded.map((item, idx) => {
                 const url = ItemUtil.getUrl(item as ItemCore);
                 const title = Functions.mlang(item.title, props.lang);
                 const order = orderLabel(idx + 1, props.lang);
@@ -65,7 +65,7 @@ const Rankings = (props: Props) => {
         },
         {
             title: Functions.mlang('[en]Most active contributers[/en][ja]最も多くアイテムを公開したユーザ[/ja]', props.lang),
-            list: rankings.contributed.map((item, idx) => {
+            list: siteData().rankings.contributed.map((item, idx) => {
                 const name = item.name !== '' ? item.name + ' (' + item.uname + ')' : item.uname;
                 const title = Functions.mlang(name, props.lang);
                 const order = orderLabel(idx + 1, props.lang);
@@ -85,7 +85,7 @@ const Rankings = (props: Props) => {
         },
         {
             title: Functions.mlang('[en]Frequently searched keywords[/en][ja]頻繁に検索されるキーワード[/ja]', props.lang),
-            list: rankings.searched.map((item, idx) => {
+            list: siteData().rankings.searched.map((item, idx) => {
                 const url = ItemUtil.getSearchByKeywordUrl('all', item.keyword);
                 const title = 'Search by: ' + item.keyword;
                 const order = orderLabel(idx + 1, props.lang);

src/database/blocks/RecentContents.tsx

@@ -2,8 +2,8 @@ import { format } from 'date-fns';
 import { Link } from 'react-router';
 import type { MultiLang } from '../../config';
 import Functions from '../../functions';
+import { siteData } from '../../siteData';
 import imageStar from '../assets/images/star.gif';
-import contents from '../assets/recent-contents.json';
 import ItemUtil, { type ItemCore } from '../lib/ItemUtil';
 import styles from './RecentContents.module.css';
 
@@ -13,7 +13,7 @@ interface Props {
 
 const RecentContents = (props: Props) => {
     const { lang } = props;
-    const list = contents.map((item, idx) => {
+    const list = siteData().recentContents.map((item, idx) => {
         const url = ItemUtil.getUrl(item as ItemCore);
         const title = Functions.mlang(item.title, lang);
         const date = format(new Date(item.last_update_date * 1000), 'y/M/d');

src/database/lib/IndexUtil.ts

@@ -1,5 +1,4 @@
 import loki from 'lokijs';
-import indexesJson from '../assets/tree.json';
 
 export interface Index {
     id: number;
@@ -18,15 +17,19 @@ interface IndexData {
     num_of_items: number;
     children: IndexData[];
 }
-type IndexesData = IndexData[];
+export type IndexesData = IndexData[];
 
 class IndexUtil {
     private database: loki;
     private indexes: Collection<Index>;
 
-    constructor(json: IndexesData) {
+    constructor() {
         this.database = new loki('database');
         this.indexes = this.database.addCollection('indexes');
+    }
+
+    init(json: IndexesData): void {
+        this.indexes.clear();
         this.load(json);
     }
 
@@ -91,4 +94,4 @@ class IndexUtil {
     }
 }
 
-export default new IndexUtil(indexesJson);
+export default new IndexUtil();

src/database/lib/ItemUtil.ts

@@ -2,10 +2,9 @@ import AsyncLock from 'async-lock';
 import ky from 'ky';
 import loki from 'lokijs';
 import funcs from '../../functions';
-import simpfLinksJson from '../assets/simpf-links.json';
 import AdvancedSearchQuery from './AdvancedSearchQuery';
 
-interface SimPFLink {
+export interface SimPFLink {
     id: number;
     url: string;
 }
@@ -417,7 +416,10 @@ class ItemUtil {
                     this.callbacks = [];
                 });
             });
-        const simpfLinks = simpfLinksJson as SimPFLink[];
+    }
+
+    initSimpfLinks(simpfLinks: SimPFLink[]): void {
+        this.simpfLinks.clear();
         simpfLinks.forEach((simpfLink: SimPFLink) => {
             this.simpfLinks.insert(simpfLink);
         });

src/main.tsx

@@ -2,10 +2,34 @@ import { StrictMode } from 'react';
 import { createRoot } from 'react-dom/client';
 import 'modern-normalize/modern-normalize.css';
 import App from './App';
+import CreditsUtils from './credits/lib/CreditsUtils';
+import IndexUtil from './database/lib/IndexUtil';
+import ItemUtil from './database/lib/ItemUtil';
 import './index.css';
+import PicoUtils from './pico/lib/PicoUtils';
+import { loadSiteData, setSiteData } from './siteData';
 
-createRoot(document.getElementById('root')!).render(
-    <StrictMode>
-        <App />
-    </StrictMode>,
-);
+const container = document.getElementById('root');
+if (container === null) {
+    throw new Error('Root container #root was not found in the document.');
+}
+
+// The converted XOOPS data is served with static-contents/, so it is fetched
+// before the first render. Components can then read it synchronously.
+loadSiteData()
+    .then((data) => {
+        setSiteData(data);
+        IndexUtil.init(data.tree);
+        ItemUtil.initSimpfLinks(data.simpfLinks);
+        CreditsUtils.init(data.credits);
+        PicoUtils.init(data.pico);
+        createRoot(container).render(
+            <StrictMode>
+                <App />
+            </StrictMode>,
+        );
+    })
+    .catch((error: unknown) => {
+        console.error('failed to load the site data', error);
+        container.textContent = 'Failed to load the site data.';
+    });

src/pico/lib/PicoUtils.ts

@@ -1,6 +1,5 @@
 import ky from 'ky';
 import loki from 'lokijs';
-import configJson from '../assets/config.json';
 
 export interface PicoModuleData {
     name: string;
@@ -29,7 +28,7 @@ export interface PicoContentData {
     link: string;
 }
 
-interface PicoConfigData {
+export interface PicoConfigData {
     module: PicoModuleData;
     categories: PicoCategoryData[];
     contents: PicoContentData[];
@@ -54,9 +53,12 @@ class PicoUtils {
     private database: loki;
     private modules: Map<string, PicoData>;
 
-    constructor(json: PicoConfigData[]) {
+    constructor() {
         this.database = new loki('pico');
         this.modules = new Map<string, PicoData>();
+    }
+
+    init(json: PicoConfigData[]): void {
         json.forEach((data) => {
             const name = data.module.dirname;
             const pico = {
@@ -178,4 +180,4 @@ class PicoUtils {
     }
 }
 
-export default new PicoUtils(configJson);
+export default new PicoUtils();

src/siteData.ts

@@ -0,0 +1,76 @@
+import ky from 'ky';
+import type { CreditsData } from './credits/lib/CreditsUtils';
+import type { IndexesData } from './database/lib/IndexUtil';
+import type { ItemCore, SimPFLink } from './database/lib/ItemUtil';
+import type { PicoConfigData } from './pico/lib/PicoUtils';
+
+export interface RankedItem extends ItemCore {
+    title: string;
+    count: number;
+}
+
+export interface RankedContributor {
+    uid: number;
+    uname: string;
+    name: string;
+    count: number;
+}
+
+export interface RankedKeyword {
+    keyword: string;
+    count: number;
+}
+
+export interface RankingsData {
+    accessed: RankedItem[];
+    downloaded: RankedItem[];
+    contributed: RankedContributor[];
+    searched: RankedKeyword[];
+}
+
+export interface RecentContentItem extends ItemCore {
+    title: string;
+    last_update_date: number;
+}
+
+/**
+ * Data converted from the original XOOPS modules. It is served with the rest of
+ * static-contents/ rather than bundled, so it is fetched once at start up and
+ * handed to the singletons before the app renders.
+ */
+export interface SiteData {
+    tree: IndexesData;
+    simpfLinks: SimPFLink[];
+    rankings: RankingsData;
+    recentContents: RecentContentItem[];
+    credits: CreditsData;
+    pico: PicoConfigData[];
+}
+
+// ky hands the URL to fetch()/Request, which rejects bare relative paths.
+const resolve = (path: string): string => new URL(path, window.location.origin).toString();
+
+export const loadSiteData = async (): Promise<SiteData> => {
+    const [tree, simpfLinks, rankings, recentContents, credits, pico] = await Promise.all([
+        ky.get(resolve('/database/tree.json')).json<IndexesData>(),
+        ky.get(resolve('/database/simpf-links.json')).json<SimPFLink[]>(),
+        ky.get(resolve('/database/rankings.json')).json<RankingsData>(),
+        ky.get(resolve('/database/recent-contents.json')).json<RecentContentItem[]>(),
+        ky.get(resolve('/credits/credits.json')).json<CreditsData>(),
+        ky.get(resolve('/pico/config.json')).json<PicoConfigData[]>(),
+    ]);
+    return { tree, simpfLinks, rankings, recentContents, credits, pico };
+};
+
+let loaded: SiteData | null = null;
+
+export const setSiteData = (data: SiteData): void => {
+    loaded = data;
+};
+
+export const siteData = (): SiteData => {
+    if (loaded === null) {
+        throw new Error('site data was read before loadSiteData() completed');
+    }
+    return loaded;
+};