feat: some small adjustments

Author: raphael-goetz

package/mapper/package.json

@@ -33,7 +33,6 @@
     "@code0-tech/tucana": "^0.0.39",
     "@protobuf-ts/runtime": "^2.11.1",
     "@protobuf-ts/runtime-rpc": "^2.11.1",
-    "fflate": "^0.8.2",
     "vite-plugin-dts": "^4.5.4"
   }
 }

package/mapper/src/definition/mapper.ts

@@ -0,0 +1,88 @@
+import { FlowType as TucanaFlowType } from "@code0-tech/tucana/pb/shared.flow_definition_pb.js";
+import { RuntimeFunctionDefinition as TucanaFunction } from "@code0-tech/tucana/pb/shared.runtime_function_pb.js";
+import { DefinitionDataType as TucanaDataType } from "@code0-tech/tucana/pb/shared.data_type_pb.js";
+
+import type { DataType, FlowType, FunctionDefinition } from "@code0-tech/sagittarius-graphql-types";
+import { getDataType } from "../mapper/dataTypeMapper.js";
+import { mapFlowType } from "../mapper/flowTypeMapper.js";
+import { mapFunction } from "../mapper/functionMapper.js";
+import {Reader} from "./reader";
+
+export interface ConstructedDataTypes {
+    scannedTucanaTypes: TucanaDataType[];
+    constructedDataTypes: DataType[];
+    id: number;
+}
+
+export interface Feature {
+    name: string;
+    dataTypes: DataType[];
+    flowTypes: FlowType[];
+    runtimeFunctions: FunctionDefinition[];
+}
+export function getID(constructedDataTypes: ConstructedDataTypes) {
+    const last = constructedDataTypes.id;
+    constructedDataTypes.id += 1;
+    return last;
+}
+
+export async function DefinitionMapper(path: string): Promise<Feature[]> {
+    const dataTypes: { feature: string; type: TucanaDataType }[] = [];
+    const flowTypes: { feature: string; flow: TucanaFlowType }[] = [];
+    const runtimeFunctions: { feature: string; func: TucanaFunction }[] = [];
+
+    const tucanaFeature = await Reader.fromPath(path)
+
+    tucanaFeature.forEach(feature => {
+        feature.data_types.forEach(dataType => {
+            dataTypes.push({
+                feature: feature.name,
+                type: dataType
+            })
+        })
+
+        feature.flow_types.forEach(flowType => {
+            flowTypes.push({
+                feature: feature.name,
+                flow: flowType
+            })
+        })
+
+        feature.runtime_functions.forEach(runtimeFunction => {
+            runtimeFunctions.push({
+                feature: feature.name,
+                func: runtimeFunction
+            })
+        })
+    })
+
+    const constructed: ConstructedDataTypes = {
+        scannedTucanaTypes: dataTypes.map((d) => d.type),
+        constructedDataTypes: [],
+        id: 0,
+    };
+
+    const features: Feature[] = [];
+    const getFeature = (name: string): Feature => {
+        let f = features.find((x) => x.name === name);
+        if (!f) {
+            f = { name, dataTypes: [], flowTypes: [], runtimeFunctions: [] };
+            features.push(f);
+        }
+        return f;
+    };
+
+    dataTypes
+        .map((f) => ({ name: f.feature, type: getDataType(f.type.identifier, constructed) }))
+        .forEach((dt) => dt.type && getFeature(dt.name).dataTypes.push(dt.type));
+
+    runtimeFunctions
+        .map((f) => ({ name: f.feature, type: mapFunction(f.func, constructed) }))
+        .forEach((rf) => rf.type && getFeature(rf.name).runtimeFunctions.push(rf.type));
+
+    flowTypes
+        .map((f) => ({ name: f.feature, type: mapFlowType(f.flow, constructed) }))
+        .forEach((ft) => ft.type && getFeature(ft.name).flowTypes.push(ft.type));
+
+    return features;
+}

package/mapper/src/definition/reader.ts

@@ -0,0 +1,112 @@
+import {readdir, readFile} from "node:fs/promises";
+import {join, extname} from "node:path";
+
+import {FlowType} from "@code0-tech/tucana/pb/shared.flow_definition_pb.js";
+import {RuntimeFunctionDefinition} from "@code0-tech/tucana/pb/shared.runtime_function_pb.js";
+import {DefinitionDataType} from "@code0-tech/tucana/pb/shared.data_type_pb.js";
+
+export const enum MetaType {
+    FlowType = "FlowType",
+    DataType = "DataType",
+    RuntimeFunction = "RuntimeFunction",
+}
+
+export interface DefinitionError {
+    definition: string;
+    definition_type: MetaType;
+    error: string;
+}
+
+export interface TucanaFeature {
+    name: string;
+    data_types: DefinitionDataType[];
+    flow_types: FlowType[];
+    runtime_functions: RuntimeFunctionDefinition[];
+    errors: DefinitionError[];
+}
+
+export class Reader {
+    static async fromPath(root: string): Promise<TucanaFeature[]> {
+        const features = new Map<string, TucanaFeature>();
+
+        for (const featureDir of await safeReadDir(root)) {
+            if (!featureDir.isDirectory()) continue;
+            const featureName = featureDir.name;
+            const featurePath = join(root, featureName);
+
+            for (const typeDir of await safeReadDir(featurePath)) {
+                const type = toMetaType(typeDir.name);
+                if (!type) continue;
+
+                const typePath = join(featurePath, typeDir.name);
+                const jsonPaths = await collectJsonFiles(typePath);
+
+                for (const file of jsonPaths) {
+                    const def = await readFile(file, "utf8");
+                    const f = features.get(featureName) ?? emptyFeature(featureName);
+                    addDefinition(f, def, type);
+                    features.set(featureName, f);
+                }
+            }
+        }
+
+        return Array.from(features.values());
+    }
+}
+
+const toMetaType = (folder: string): MetaType | null =>
+    ({
+        flow_type: MetaType.FlowType,
+        data_type: MetaType.DataType,
+        runtime_definition: MetaType.RuntimeFunction
+    } as const)[folder] ?? null;
+
+const emptyFeature = (name: string): TucanaFeature => ({
+    name,
+    data_types: [],
+    flow_types: [],
+    runtime_functions: [],
+    errors: [],
+});
+
+const safeReadDir = async (p: string) => {
+    try {
+        return await readdir(p, {withFileTypes: true});
+    } catch {
+        return [];
+    }
+};
+
+const collectJsonFiles = async (dir: string): Promise<string[]> => {
+    const entries = await safeReadDir(dir);
+    const files = entries.filter(e => e.isFile() && extname(e.name) === ".json").map(e => join(dir, e.name));
+
+    // include one nested level
+    for (const e of entries.filter(e => e.isDirectory())) {
+        const sub = (await safeReadDir(join(dir, e.name)))
+            .filter(s => s.isFile() && extname(s.name) === ".json")
+            .map(s => join(dir, e.name, s.name));
+        files.push(...sub);
+    }
+    return files;
+};
+
+const addDefinition = (feature: TucanaFeature, def: string, type: MetaType) => {
+    try {
+        if (type === MetaType.DataType) feature.data_types.push(DefinitionDataType.fromJsonString(def));
+        else if (type === MetaType.FlowType) feature.flow_types.push(FlowType.fromJsonString(def));
+        else feature.runtime_functions.push(RuntimeFunctionDefinition.fromJsonString(def));
+    } catch (err) {
+        feature.errors.push({
+            definition: extractIdentifier(def, type),
+            definition_type: type,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+};
+
+const extractIdentifier = (def: string, type: MetaType): string => {
+    const key = type === MetaType.RuntimeFunction ? "runtime_name" : "identifier";
+    return def.match(new RegExp(`"${key}"\\s*:\\s*"([^"]+)"`))?.[1] ?? def;
+};
+

package/mapper/src/definition/remote.ts

@@ -1,2 +1,16 @@
-export { DefinitionMapper } from './definition/remote.js';
-export type { Feature } from './definition/remote.js';
+import {DefinitionMapper, Feature} from "./definition/mapper.js";
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+DefinitionMapper("../../definitions").then((value: Feature[]) => {
+    const functions = value.flatMap(v => v.runtimeFunctions);
+    const types = value.flatMap(v => v.dataTypes);
+    const flows = value.flatMap(v => v.flowTypes);
+
+    const outDir = path.resolve("./export");
+    fs.mkdirSync(outDir, {recursive: true});
+
+    fs.writeFileSync(path.join(outDir, "functions.json"), JSON.stringify(functions));
+    fs.writeFileSync(path.join(outDir, "dataTypes.json"), JSON.stringify(types));
+    fs.writeFileSync(path.join(outDir, "flowTypes.json"), JSON.stringify(flows));
+});

package/mapper/src/index.ts

@@ -5,18 +5,43 @@ import {
     DataTypeRulesInputTypeConfig,
     DataTypeRulesInputTypesConfig,
     DataTypeRulesItemOfCollectionConfig, DataTypeRulesNumberRangeConfig,
-    DataTypeRulesParentTypeConfig, DataTypeRulesRegexConfig, DataTypeRulesVariant, DataTypeVariant,
-    GenericCombinationStrategyType,
+    DataTypeRulesParentTypeConfig, DataTypeRulesRegexConfig,
 } from "@code0-tech/sagittarius-graphql-types";
 import {
     DataTypeIdentifier as TucanaDataTypeIdentifier,
     DefinitionDataType_Variant, DefinitionDataTypeRule, GenericMapper_GenericCombinationStrategy
 } from "@code0-tech/tucana/pb/shared.data_type_pb.js"
 import {GenericMapper as TucanaGenericMapper} from "@code0-tech/tucana/pb/shared.data_type_pb.js"
-import {ConstructedDataTypes, getID} from "../definition/remote.js";
+import {ConstructedDataTypes, getID} from "../definition/mapper.js";
 import {getTranslationConnection} from "./translation.js";
 import {Value} from "@code0-tech/tucana/pb/shared.struct_pb.js";
 
+enum GenericCombinationStrategyType {
+    And = 'AND',
+    Or = 'OR'
+}
+
+enum DataTypeRulesVariant {
+    ContainsKey = 'CONTAINS_KEY',
+    ContainsType = 'CONTAINS_TYPE',
+    InputType = 'INPUT_TYPE',
+    ItemOfCollection = 'ITEM_OF_COLLECTION',
+    NumberRange = 'NUMBER_RANGE',
+    ParentType = 'PARENT_TYPE',
+    Regex = 'REGEX',
+    ReturnType = 'RETURN_TYPE'
+}
+
+enum DataTypeVariant {
+    Array = 'ARRAY',
+    DataType = 'DATA_TYPE',
+    Error = 'ERROR',
+    Node = 'NODE',
+    Object = 'OBJECT',
+    Primitive = 'PRIMITIVE',
+    Type = 'TYPE'
+}
+
 function getDataType(identifier: string, constructedDataTypes: ConstructedDataTypes): DataType | null {
     const dataType = constructedDataTypes.constructedDataTypes.find(dt => dt.identifier === identifier)
     if (dataType == undefined) {