make compression threaded

Author: kirjavascript

CHANGELOG

@@ -2,7 +2,9 @@
 
 ## [unreleased]
 - Change DPLC limit to 255
+- Make (de)compression threaded
 - Fix issue where image data would be cached incorrectly
+- Electron updated to v26
 
 ## [1.2.2]
 - Added sprite rotation algorithm: 3 shears

TODO

@@ -10,6 +10,13 @@
 /* https://raw.githubusercontent.com/Dandaman955/Streets-of-Rage-1-Disassembly-Redux/master/SoR.asm */
 /* http://info.sonicretro.org/Disassemblies */
 
+
+macros for mappings
+    format hooks for defining asm output
+    define macro files in script??
+
+return rotsprite
+
 ROADMAP
 look at issues related to crashes
 drawing mode bugs
@@ -31,7 +38,6 @@ pink surrounding tiles sometimes breaks
 
 FEATURES
 set colour of art tiles
-make decompression threaded
 ctrl+wheel for horiz scroll
 ==
 select multiple sprites at once to reorder

app/components/file/file-object.js

@@ -8,7 +8,7 @@ import {
     writeBIN,
     writeASM,
 } from '#formats/scripts';
-import { compressionFormats } from '#formats/compression';
+import { decompress, compress, compressionFormats } from '#formats/compression';
 import { bufferToTiles, tilesToBuffer } from '#formats/art';
 import { buffersToColors, colorsToBuffers } from '#formats/palette';
 import { environment } from '#store/environment';
@@ -37,7 +37,7 @@ export const FileObject = observer(({ obj }) => {
     const scriptArt = scriptSafe && script.art;
     const scriptPalettes = scriptSafe && script.palettes;
     const toggleDPLCs = () => (obj.dplcs.enabled = !obj.dplcs.enabled);
-    const parseASM = ((scriptSafe && script.parseASM) || parseASMInternal);
+    const parseASM = (scriptSafe && script.parseASM) || parseASMInternal;
 
     function ioWrap(filePath, setError, e, cb) {
         setError();
@@ -83,10 +83,16 @@ export const FileObject = observer(({ obj }) => {
     function loadArt(e) {
         ioWrap(obj.art.path, setArtError, e, async (path) => {
             const buffer = (await fs.readFile(path)).slice(obj.art.offset || 0);
-            const tiles = scriptArt
-                ? script.readArt(buffer)
-                : bufferToTiles(buffer, obj.art.compression);
-            environment.tiles.replace(tiles);
+
+            if (scriptArt) {
+                environment.tiles.replace(script.readArt(buffer));
+            } else {
+                const decompBuffer = await decompress(
+                    buffer,
+                    obj.art.compression,
+                );
+                environment.tiles.replace(bufferToTiles(decompBuffer));
+            }
         });
     }
 
@@ -99,6 +105,16 @@ export const FileObject = observer(({ obj }) => {
                 ? script.writeArt(tiles)
                 : tilesToBuffer(environment.tiles, obj.art.compression);
             await fs.writeFile(path, tiles);
+
+            if (scriptArt) {
+                await fs.writeFile(path, script.writeArt(tiles));
+            } else {
+                const buffer = tilesToBuffer(environment.tiles);
+                await fs.writeFile(
+                    path,
+                    Buffer.from(await compress(buffer, obj.art.compression)),
+                );
+            }
         });
     }
 
@@ -214,13 +230,9 @@ export const FileObject = observer(({ obj }) => {
                     ? palPath
                     : workspace.absolutePath(palPath);
 
-                const chunk = (scriptPalettes
-                    ? script.writePalettes
-                    : colorsToBuffers)(
-                    environment.palettes,
-                    cursor,
-                    cursor + length,
-                );
+                const chunk = (
+                    scriptPalettes ? script.writePalettes : colorsToBuffers
+                )(environment.palettes, cursor, cursor + length);
                 await fs.writeFile(path, chunk);
                 cursor += length;
             }

app/formats/art.js

@@ -1,20 +1,17 @@
-import { decompress, compress } from '#formats/compression';
 import flattenDeep from 'lodash/flattenDeep';
 import chunk from 'lodash/chunk';
 import { toJS } from 'mobx';
 
 // a duxel is two pixels / a byte
 
-export function bufferToTiles(buffer, compression) {
+export function bufferToTiles(buffer) {
     let tiles = [];
-    const data = decompress(buffer, compression);
-
-    const tileQty = data.length/0x20;
+    const tileQty = buffer.length/0x20;
 
     for (let i = 0; i < tileQty; i++) {
         let pixels = [];
         for (let duoIndex = 0; duoIndex < 0x20; duoIndex++) {
-            let duxel = data[(0x20 * i) + duoIndex];
+            let duxel = buffer[(0x20 * i) + duoIndex];
             pixels.push(duxel >> 4, duxel & 0xF);
         }
         tiles.push(pixels);
@@ -23,7 +20,7 @@ export function bufferToTiles(buffer, compression) {
     return tiles;
 }
 
-export function tilesToBuffer(tiles, compression) {
+export function tilesToBuffer(tiles) {
 
     const pixels = flattenDeep(toJS(tiles));
 
@@ -32,5 +29,5 @@ export function tilesToBuffer(tiles, compression) {
             (H << 4) + L
         ));
 
-    return Buffer.from(compress(Uint8Array.from(duxels), compression));
+    return Uint8Array.from(duxels);
 }

app/formats/compression-worker.js

@@ -0,0 +1,40 @@
+import * as Comlink from 'comlink';
+
+self.Module = {
+    locateFile: url => `../wasm/${url}`,
+};
+
+importScripts('../wasm/mdcomp_portable.js');
+
+function mdcomp(func, data) {
+    const operation = Module[func];
+    const sp = Module.stackSave();
+    try {
+        const dataPtr = _malloc(data.length);
+        try {
+            writeArrayToMemory(data, dataPtr);
+            const outputPtrPtr = Module.stackAlloc(4);
+            const outputSizePtr = Module.stackAlloc(4);
+            if (operation(dataPtr, data.length, outputPtrPtr, outputSizePtr)) {
+                const outputPtr = HEAP32[outputPtrPtr >> 2];
+                try {
+                    const outputSize = HEAP32[outputSizePtr >> 2];
+                    const output = new Uint8Array(outputSize);
+                    let outputBuffer = outputPtr;
+                    for (let i=0; i < outputSize; i++) {
+                        output[i] = HEAP8[outputBuffer++];
+                    }
+                    return output;
+                } finally {
+                    _free(outputPtr);
+                }
+            }
+        } finally {
+            _free(dataPtr);
+        }
+    } finally {
+        Module.stackRestore(sp);
+    }
+};
+
+Comlink.expose({ mdcomp });

app/formats/compression.js

@@ -1,3 +1,5 @@
+import * as Comlink from 'comlink';
+
 export const compressionFormats = {
     'Uncompressed': undefined,
     'Nemesis': 'nemesis',
@@ -12,49 +14,19 @@ export const compressionFormats = {
     'RLE': 'snkrle',
 };
 
-export function decompress(buffer, compression) {
+const worker = Comlink.wrap(new Worker('bundles/compression-worker.js'));
+
+export async function decompress(buffer, compression) {
     const operation = compressionFormats[compression];
 
     if (!operation) return new Uint8Array(buffer);
-    else return mdcomp(`_${operation}_decode`, buffer);
+    else return await worker.mdcomp(`_${operation}_decode`, buffer);
 
 }
 
-export function compress(buffer, compression) {
+export async function compress(buffer, compression) {
     const operation = compressionFormats[compression];
 
     if (!operation) return buffer;
-    else return mdcomp(`_${operation}_encode`, buffer);
-}
-
-
-function mdcomp(func, data) {
-    const operation = Module[func];
-    const sp = Module.stackSave();
-    try {
-        const dataPtr = _malloc(data.length);
-        try {
-            writeArrayToMemory(data, dataPtr);
-            const outputPtrPtr = Module.stackAlloc(4);
-            const outputSizePtr = Module.stackAlloc(4);
-            if (operation(dataPtr, data.length, outputPtrPtr, outputSizePtr)) {
-                const outputPtr = HEAP32[outputPtrPtr >> 2];
-                try {
-                    const outputSize = HEAP32[outputSizePtr >> 2];
-                    const output = new Uint8Array(outputSize);
-                    let outputBuffer = outputPtr;
-                    for (let i=0; i < outputSize; i++) {
-                        output[i] = HEAP8[outputBuffer++];
-                    }
-                    return output;
-                } finally {
-                    _free(outputPtr);
-                }
-            }
-        } finally {
-            _free(dataPtr);
-        }
-    } finally {
-        Module.stackRestore(sp);
-    }
+    else return await worker.mdcomp(`_${operation}_encode`, buffer);
 }

development/build.js

@@ -3,19 +3,22 @@ const esbuild = require('esbuild');
 const path = require('path');
 
 module.exports = (mainWindow) => {
-    const { writeFile, unlinkSync, existsSync } = fs;
+    const { writeFile, rmSync, readdirSync } = fs;
 
     mainWindow?.openDevTools();
-    if (existsSync('./static/bundles/main.js')) {
-        unlinkSync('./static/bundles/main.js');
-    }
+
+    const outdir = './static/bundles';
+    readdirSync(outdir).forEach((f) => rmSync(`${outdir}/${f}`));
 
     const devMode = !!mainWindow;
 
     esbuild
         .build({
-            entryPoints: ['./app/main.js'],
-            outdir: './static/bundles',
+            outdir,
+            entryPoints: {
+                main: './app/main.js',
+                'compression-worker': './app/formats/compression-worker.js',
+            },
             bundle: true,
             watch: devMode,
             sourcemap: devMode,