Proposal for parser API

package.json

@@ -10,19 +10,19 @@
     "release": "rimraf dist && tsc -p tsconfig.json && yarn run copyfiles && npm publish ./dist"
   },
   "dependencies": {
-    "@types/express": "^4.17.13",
-    "axios": "^0.21.0",
+    "axios": "^0.24.0",
     "express": "^4.17.1",
-    "jsdom": "^16.4.0",
+    "jsdom": "^18.0.1",
     "lodash": "^4.17.21",
     "regex-translator": "^0.2.7",
     "typescript": "^4.4.4"
   },
   "devDependencies": {
+    "@types/express": "^4.17.13",
     "@types/jsdom": "^16.2.13",
     "@types/lodash": "^4.14.176",
     "@types/node": "^16.11.6",
     "copyfiles": "^2.4.1",
-    "rimraf": "^2.6.2"
+    "rimraf": "^3.0.2"
   }
 }

src/core/page-parser.ts

@@ -1,67 +1,96 @@
-import {JSDOM} from "jsdom";
-import {Request} from "express";
-import {CssSelectorDefinition, CssSelectorRegistry} from "./css-selector-registry";
+import {JSDOM} from 'jsdom';
+import {snakeCase} from 'lodash';
 // @ts-ignore
 import * as RegexTranslator from 'regex-translator';
+import axios from 'axios';
 
-import {snakeCase} from 'lodash';
+import {CssSelectorDefinition, CssSelectorRegistry} from "./css-selector-registry";
 
-const axios = require('axios').default;
+export type Language = 'en' | 'de' | 'ja' | 'fr';
+export type RequestFunction = (url: string, headers?: Record<string, string>) => Promise<string>;
+
+const lodestoneBaseUrls: Record<Language, string> = {
+    'en': 'https://na.finalfantasyxiv.com/lodestone',
+    'de': 'https://de.finalfantasyxiv.com/lodestone',
+    'ja': 'https://jp.finalfantasyxiv.com/lodestone',
+    'fr': 'https://fr.finalfantasyxiv.com/lodestone',
+};
 
 export abstract class PageParser {
 
-    protected abstract getURL(req: Request): string;
+    private _language: Language;
+    protected requestFunction: RequestFunction;
+    protected baseUrl: string;
+
+    public get language() {
+        return this._language;
+    }
+
+    public set language(newLanguage: Language) {
+        this._language = Object.keys(lodestoneBaseUrls).includes(newLanguage) ? newLanguage : 'en';
+        this.baseUrl = lodestoneBaseUrls[this._language];
+    }
+
+    public constructor(language: Language = 'en', requestFunction?: RequestFunction) {
+        this._language = Object.keys(lodestoneBaseUrls).includes(language) ? language : 'en';
+        this.baseUrl = lodestoneBaseUrls[this._language];
+
+        if (typeof requestFunction === 'function') {
+            this.requestFunction = requestFunction;
+        } else {
+            const axiosInstance = axios.create();
+            this.requestFunction = (url, headers) => {
+                return axiosInstance.get(url, {headers})
+                    .then(response => response.data);
+            };
+        }
+    }
+
+    protected abstract getLodestonePage(characterId: string): Promise<string>;
 
     protected abstract getCSSSelectors(): CssSelectorRegistry;
 
-    public async parse(req: Request, columnsPrefix = ''): Promise<Object> {
-        const {data} = await axios.get(this.getURL(req)).catch((err: any) => {
-            throw new Error(err.response.status);
+    protected modifyParseResult(data: Record<string, unknown>): Record<string, unknown> {
+        return data;
+    }
+
+    public async get(characterId: string, columns: string[] = []): Promise<Record<string, unknown>> {
+        const lodestonePage = await this.getLodestonePage(characterId).catch(error => {
+            if (axios.isAxiosError(error) && error.response?.status) {
+                throw new Error(String(error.response.status));
+            }
+            throw error;
         });
-        const dom = new JSDOM(data);
-        let {document} = dom.window;
-        const columnsQuery = req.query['columns'];
+
+        const dom = new JSDOM(lodestonePage);
         const selectors = this.getCSSSelectors();
-        let columns: string[];
-        if (columnsQuery && !Array.isArray(columnsQuery)) {
-            columns = [columnsQuery.toString()]
-                .filter(column => {
-                    return column.startsWith(columnsPrefix)
-                })
-                .map(column => column.replace(columnsPrefix, ''));
-        } else if (columnsQuery && Array.isArray(columnsQuery)) {
-            columns = columnsQuery.map(c => c.toString())
-                .filter(column => {
-                    return column.startsWith(columnsPrefix)
-                })
-                .map(column => column.replace(columnsPrefix, ''));
-        } else {
-            columns = Object.keys(selectors).map(key => {
-                return this.definitionNameToColumnName(key);
-            }).filter(column => column !== 'default');
-        }
-        return columns.reduce((acc, column) => {
+        const columnsToParse = columns.length > 0 ? columns : Object.keys(selectors).map(this.definitionNameToColumnName).filter(column => column !== 'default');
+
+        let {document} = dom.window;
+        let data: Record<string, unknown> = {};
+
+        for (const column of columnsToParse) {
             const definition = this.getDefinition(selectors, column);
+
             if (column === 'Root') {
                 const context = this.handleColumn(definition, document)?.data;
                 const contextDOM = new JSDOM(context);
                 document = contextDOM.window.document;
-                return {
-                    ...acc
-                }
-            }
-            const parsed = this.handleColumn(definition, document);
-            if (parsed.isPatch || column === 'Entry') {
-                return {
-                    ...acc,
-                    ...(parsed.data || {})
+            } else {
+                const parsed = this.handleColumn(definition, document);
+
+                if (parsed.isPatch || column === 'Entry') {
+                    data = {
+                        ...data,
+                        ...(parsed.data || {}),
+                    };
+                } else {
+                    data[column] = parsed.data;
                 }
             }
-            return {
-                ...acc,
-                [column]: parsed.data
-            }
-        }, {});
+        }
+
+        return this.modifyParseResult(data);
     }
 
     private handleColumn(definition: CssSelectorRegistry | CssSelectorDefinition | null, document: Document): { isPatch: boolean, data: any } {

src/profile/achievements.ts

@@ -1,4 +1,3 @@
-import {Request} from "express";
 import {CssSelectorRegistry} from "../core/css-selector-registry";
 import * as achievements from '../lib/lodestone-css-selectors/profile/achievements.json';
 import {PageParser} from "../core/page-parser";
@@ -8,8 +7,7 @@ export class Achievements extends PageParser {
         return achievements;
     }
 
-    protected getURL(req: Request): string {
-        return "https://na.finalfantasyxiv.com/lodestone/character/" + req.params.characterId + "/achievement";
+    protected getLodestonePage(characterId: string): Promise<string> {
+        return this.requestFunction(this.baseUrl + '/character/' + encodeURIComponent(characterId) + '/achievement');
     }
-
 }

src/profile/character.ts

@@ -1,14 +1,13 @@
-import {Request} from "express";
 import {PageParser} from "../core/page-parser";
 import * as character from '../lib/lodestone-css-selectors/profile/character.json'
 import {CssSelectorRegistry} from "../core/css-selector-registry";
 
 export class Character extends PageParser {
-    protected getURL(req: Request): string {
-        return "https://na.finalfantasyxiv.com/lodestone/character/" + req.params.characterId;
-    }
-
     protected getCSSSelectors(): CssSelectorRegistry {
         return character;
     }
+
+    protected getLodestonePage(characterId: string): Promise<string> {
+        return this.requestFunction(this.baseUrl + '/character/' + encodeURIComponent(characterId));
+    }
 }

src/profile/minions.ts

@@ -0,0 +1,23 @@
+import {CssSelectorRegistry} from "../core/css-selector-registry";
+import * as minions from '../lib/lodestone-css-selectors/profile/minion.json';
+import {PageParser} from "../core/page-parser";
+
+export class Minions extends PageParser {
+    protected getCSSSelectors(): CssSelectorRegistry {
+        return minions;
+    }
+
+    protected getLodestonePage(characterId: string): Promise<string> {
+        return this.requestFunction(this.baseUrl + '/character/' + encodeURIComponent(characterId) + '/minion', {
+            'User-Agent': 'Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; SCH-I535 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
+        });
+    }
+
+    protected modifyParseResult(data: Record<string, unknown>): Record<string, unknown> {
+        const { Minions, ...rest } = data;
+        return {
+            ...(Minions as object),
+            ...rest,
+        };
+    }
+}

src/profile/mounts.ts

@@ -0,0 +1,23 @@
+import {CssSelectorRegistry} from "../core/css-selector-registry";
+import * as mounts from '../lib/lodestone-css-selectors/profile/mount.json';
+import {PageParser} from "../core/page-parser";
+
+export class Mounts extends PageParser {
+    protected getCSSSelectors(): CssSelectorRegistry {
+        return mounts;
+    }
+
+    protected getLodestonePage(characterId: string): Promise<string> {
+        return this.requestFunction(this.baseUrl + '/character/' + encodeURIComponent(characterId) + '/mount', {
+            'User-Agent': 'Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; SCH-I535 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
+        });
+    }
+
+    protected modifyParseResult(data: Record<string, unknown>): Record<string, unknown> {
+        const { Mounts, ...rest } = data;
+        return {
+            ...(Mounts as object),
+            ...rest,
+        };
+    }
+}

src/server.ts

@@ -1,11 +1,27 @@
-import express from 'express';
+import * as express from 'express';
 import {Character} from "./profile/character";
 import {Achievements} from "./profile/achievements";
+import {Minions} from './profile/minions';
+import {Mounts} from './profile/mounts';
 
 const app = express();
-
 const characterParser = new Character();
 const achievementsParser = new Achievements();
+const minionsParser = new Minions();
+const mountsParser = new Mounts();
+
+function getEntriesFromQuery(query: unknown | unknown[]): string[] {
+    const entries = Array.isArray(query) ? query as unknown[] : [query];
+    return (entries.filter(entry => typeof entry === 'string') as string[])
+        .map(entry => entry.split(','))
+        .flat();
+}
+
+function getColumnsForParser(columns: string | string[], columnsPrefix: string = ''): string[] {
+    const columnsArray = Array.isArray(columns) ? columns : [columns];
+
+    return columnsArray.filter(column => column.startsWith(columnsPrefix)).map(column => column.replace(columnsPrefix, ''));
+}
 
 app.get('/Character/:characterId', async (req, res) => {
     res.set('Access-Control-Allow-Origin', '*');
@@ -16,25 +32,33 @@ app.get('/Character/:characterId', async (req, res) => {
         return res.sendStatus(200);
     }
     try {
-        const character = await characterParser.parse(req, 'Character.');
-        const parsed: any = {
-            Character: {
-                ID: +req.params.characterId,
-                ...character
-            }
-        }
-        const additionalData = Array.isArray(req.query.data) ? req.query.data : [req.query.data].filter(d => !!d);
-        if (additionalData.includes('AC')) {
-            parsed.Achievements = await achievementsParser.parse(req, 'Achievements.');
-        }
-        return res.status(200).send(parsed);
-    } catch (err: any) {
-        if (err.message === '404') {
+        const characterId = req.params.characterId;
+        const stringColumns = getEntriesFromQuery(req.query.columns);
+        const additionalData = getEntriesFromQuery(req.query.data);
+
+        const [character, achievements, minions, mounts] = await Promise.all([
+            characterParser.get(characterId, getColumnsForParser(stringColumns, 'Character.')),
+            additionalData.includes('AC') ? achievementsParser.get(characterId, getColumnsForParser(stringColumns, 'Achievements.')) : Promise.resolve(),
+            additionalData.includes('MIMO') ? minionsParser.get(characterId, getColumnsForParser(stringColumns, 'Minions.')) : Promise.resolve(),
+            additionalData.includes('MIMO') ? mountsParser.get(characterId, getColumnsForParser(stringColumns, 'Mounts.')) : Promise.resolve(),
+        ] as Promise<(Record<string, unknown> | undefined)>[]);
+
+        const result: Record<string, unknown> = {};
+        if (character != null) result.Character = {
+            ID: Number.isNaN(Number.parseInt(characterId, 10)) ? characterId : Number.parseInt(characterId, 10),
+            ...character,
+        };
+        if (achievements != null) result.Achievements = achievements
+        if (minions != null) result.Minions = minions;
+        if (mounts != null) result.Mounts = mounts;
+
+        return res.status(200).send(result);
+    } catch (error) {
+        if (error instanceof Error && error.message === '404') {
             return res.sendStatus(404)
         }
-        return res.status(500).send(err);
+        return res.status(500).send(error);
     }
-
 });
 
 const port = process.env.PORT || 8080;