Feature/336/colormode conversion (#337)

* (#336) Introduced colormode enum

* (#336) Extended Image class with color mode conversion methods

* (#336) Set color mode on images when reading from disk

* (#336) Set colormode on images when grabbing screen content

* (#336) Limit switching of channels only to BGR images

* (#336) Export ColorMode enum

* (#336) Chasing down another async error in tests due to jimp

Author: s1hofmann

index.ts

@@ -36,6 +36,7 @@ export {Point} from "./lib/point.class";
 export {Region} from "./lib/region.class";
 export {Window} from "./lib/window.class";
 export {FileType} from "./lib/file-type.enum";
+export {ColorMode} from "./lib/colormode.enum";
 
 const lineHelper = new LineHelper();
 

lib/colormode.enum.ts

@@ -0,0 +1,7 @@
+/**
+ * The {@link ColorMode} enum is used to specify the color mode of an {@link Image}
+ */
+export enum ColorMode {
+    BGR,
+    RGB
+}

lib/image.class.spec.ts

@@ -1,30 +1,68 @@
-import { Image } from "./image.class";
+import {Image} from "./image.class";
+import {imageToJimp} from "./provider/io/imageToJimp.function";
+import {ColorMode} from "./colormode.enum";
+
+jest.mock("./provider/io/imageToJimp.function", () => {
+    return {
+        imageToJimp: jest.fn()
+    }
+});
+
+afterEach(() => {
+    jest.resetAllMocks();
+});
 
 describe("Image class", () => {
-  it("should return alphachannel = true for > 3 channels", () => {
-    const SUT = new Image(200, 200, 123, 4, "id");
-    expect(SUT.hasAlphaChannel).toBeTruthy();
-  });
-
-  it("should return alphachannel = false for <= 3 channels", () => {
-    const SUT = new Image(200, 200, 123, 3, "id");
-    expect(SUT.hasAlphaChannel).toBeFalsy();
-  });
-  it("should return alphachannel = false for <= 3 channels", () => {
-    const SUT = new Image(200, 200, 123, 2, "id");
-    expect(SUT.hasAlphaChannel).toBeFalsy();
-  });
-  it("should return alphachannel = false for <= 3 channels", () => {
-    const SUT = new Image(200, 200, 123, 1, "id");
-    expect(SUT.hasAlphaChannel).toBeFalsy();
-  });
-
-  it("should throw for <= 0 channels", () => {
-    expect(() => new Image(200, 200, 123, 0, "id")).toThrowError("Channel <= 0");
-  });
-
-  it("should have a default pixel density of 1.0", () => {
-    const SUT = new Image(200, 200, 123, 1, "id");
-    expect(SUT.pixelDensity).toEqual({ scaleX: 1.0, scaleY: 1.0 });
-  });
+    it("should return alphachannel = true for > 3 channels", () => {
+        const SUT = new Image(200, 200, 123, 4, "id");
+        expect(SUT.hasAlphaChannel).toBeTruthy();
+    });
+
+    it("should return alphachannel = false for <= 3 channels", () => {
+        const SUT = new Image(200, 200, 123, 3, "id");
+        expect(SUT.hasAlphaChannel).toBeFalsy();
+    });
+    it("should return alphachannel = false for <= 3 channels", () => {
+        const SUT = new Image(200, 200, 123, 2, "id");
+        expect(SUT.hasAlphaChannel).toBeFalsy();
+    });
+    it("should return alphachannel = false for <= 3 channels", () => {
+        const SUT = new Image(200, 200, 123, 1, "id");
+        expect(SUT.hasAlphaChannel).toBeFalsy();
+    });
+
+    it("should throw for <= 0 channels", () => {
+        expect(() => new Image(200, 200, 123, 0, "id")).toThrowError("Channel <= 0");
+    });
+
+    it("should have a default pixel density of 1.0", () => {
+        const SUT = new Image(200, 200, 123, 1, "id");
+        expect(SUT.pixelDensity).toEqual({scaleX: 1.0, scaleY: 1.0});
+    });
+
+    describe("Colormode", () => {
+        it("should not try to convert an image to BGR if it already has the correct color mode", async () => {
+            // GIVEN
+            const bgrImage = new Image(100, 100, Buffer.from([]), 3, "testImage");
+
+            // WHEN
+            const convertedImage = await bgrImage.toBGR();
+
+            // THEN
+            expect(convertedImage).toBe(bgrImage);
+            expect(imageToJimp).not.toBeCalledTimes(1)
+        });
+
+        it("should not try to convert an image to RGB if it already has the correct color mode", async () => {
+            // GIVEN
+            const rgbImage = new Image(100, 100, Buffer.from([]), 3, "testImage", ColorMode.RGB);
+
+            // WHEN
+            const convertedImage = await rgbImage.toRGB();
+
+            // THEN
+            expect(convertedImage).toBe(rgbImage);
+            expect(imageToJimp).not.toBeCalledTimes(1)
+        });
+    });
 });

lib/image.class.ts

@@ -1,3 +1,6 @@
+import {imageToJimp} from "./provider/io/imageToJimp.function";
+import {ColorMode} from "./colormode.enum";
+
 /**
  * The {@link Image} class represents generic image data
  */
@@ -9,6 +12,7 @@ export class Image {
      * @param data Generic {@link Image} data
      * @param channels Amount of {@link Image} channels
      * @param id Image identifier
+     * @param colorMode An images color mode, defaults to {@link ColorMode.BGR}
      * @param pixelDensity Object containing scale info to work with e.g. Retina display data where the reported display size and pixel size differ (Default: {scaleX: 1.0, scaleY: 1.0})
      */
     constructor(
@@ -17,6 +21,7 @@ export class Image {
         public readonly data: any,
         public readonly channels: number,
         public readonly id: string,
+        public readonly colorMode: ColorMode = ColorMode.BGR,
         public readonly pixelDensity: { scaleX: number; scaleY: number } = {
             scaleX: 1.0,
             scaleY: 1.0,
@@ -33,4 +38,35 @@ export class Image {
     public get hasAlphaChannel() {
         return this.channels > 3;
     }
+
+    /**
+     * {@link toRGB} converts an {@link Image} from BGR color mode (default within nut.js) to RGB
+     */
+    public async toRGB(): Promise<Image> {
+        if (this.colorMode === ColorMode.RGB) {
+            return this;
+        }
+        const rgbImage = imageToJimp(this);
+        return new Image(this.width, this.height, rgbImage.bitmap.data, this.channels, this.id, ColorMode.RGB, this.pixelDensity);
+    }
+
+    /**
+     * {@link toBGR} converts an {@link Image} from RGB color mode to RGB
+     */
+    public async toBGR(): Promise<Image> {
+        if (this.colorMode === ColorMode.BGR) {
+            return this;
+        }
+        const rgbImage = imageToJimp(this);
+        return new Image(this.width, this.height, rgbImage.bitmap.data, this.channels, this.id, ColorMode.BGR, this.pixelDensity);
+    }
+
+    /**
+     * {@link fromRGBData} creates an {@link Image} from provided RGB data
+     */
+    public static fromRGBData(width: number, height: number, data: Buffer, channels: number, id: string): Image {
+        const rgbImage = new Image(width, height, data, channels, id);
+        const jimpImage = imageToJimp(rgbImage);
+        return new Image(width, height, jimpImage.bitmap.data, channels, id);
+    }
 }

lib/match-request.class.spec.ts

@@ -1,6 +1,8 @@
 import {Image} from "./image.class";
 import {MatchRequest} from "./match-request.class";
 
+jest.mock('jimp', () => {});
+
 describe("MatchRequest", () => {
     it("should default to multi-scale matching", () => {
         const SUT = new MatchRequest(

lib/provider/io/imageToJimp.function.ts

@@ -1,17 +1,20 @@
 import Jimp from "jimp";
 import {Image} from "../../image.class";
+import {ColorMode} from "../../colormode.enum";
 
 export function imageToJimp(image: Image): Jimp {
     const jimpImage = new Jimp({
         data: image.data,
         width: image.width,
         height: image.height
     });
-    // Images treat data in BGR format, so we have to switch red and blue color channels
-    jimpImage.scan(0, 0, jimpImage.bitmap.width, jimpImage.bitmap.height, function (_, __, idx) {
-        const red = this.bitmap.data[idx];
-        this.bitmap.data[idx] = this.bitmap.data[idx + 2];
-        this.bitmap.data[idx + 2] = red;
-    });
+    if (image.colorMode === ColorMode.BGR) {
+        // Image treats data in BGR format, so we have to switch red and blue color channels
+        jimpImage.scan(0, 0, jimpImage.bitmap.width, jimpImage.bitmap.height, function (_, __, idx) {
+            const red = this.bitmap.data[idx];
+            this.bitmap.data[idx] = this.bitmap.data[idx + 2];
+            this.bitmap.data[idx + 2] = red;
+        });
+    }
     return jimpImage;
 }

lib/provider/io/jimp-image-reader.class.ts

@@ -1,6 +1,7 @@
 import Jimp from 'jimp';
 import {ImageReader} from "../image-reader.type";
 import {Image} from "../../image.class";
+import {ColorMode} from "../../colormode.enum";
 
 export default class implements ImageReader {
     load(parameters: string): Promise<Image> {
@@ -18,7 +19,8 @@ export default class implements ImageReader {
                         jimpImage.bitmap.height,
                         jimpImage.bitmap.data,
                         jimpImage.hasAlpha() ? 4 : 3,
-                        parameters
+                        parameters,
+                        ColorMode.BGR
                     ));
                 }).catch(err => reject(`Failed to load image from '${parameters}'. Reason: ${err}`));
         })

lib/provider/native/libnut-screen.class.spec.ts

@@ -2,6 +2,7 @@ import libnut = require("@nut-tree/libnut");
 import { Region } from "../../region.class";
 import ScreenAction from "./libnut-screen.class";
 
+jest.mock("jimp", () => {});
 jest.mock("@nut-tree/libnut");
 
 beforeEach(() => {

lib/provider/native/libnut-screen.class.ts

@@ -1,7 +1,8 @@
 import libnut = require("@nut-tree/libnut");
-import { Image } from "../../image.class";
-import { Region } from "../../region.class";
-import { ScreenProviderInterface } from "../screen-provider.interface";
+import {Image} from "../../image.class";
+import {Region} from "../../region.class";
+import {ScreenProviderInterface} from "../screen-provider.interface";
+import {ColorMode} from "../../colormode.enum";
 
 export default class ScreenAction implements ScreenProviderInterface {
 
@@ -34,6 +35,7 @@ export default class ScreenAction implements ScreenProviderInterface {
             screenShot.image,
             4,
               "grabScreenResult",
+            ColorMode.BGR,
             pixelScaling,
           ),
         );
@@ -60,6 +62,7 @@ export default class ScreenAction implements ScreenProviderInterface {
             screenShot.image,
             4,
             "grabScreenRegionResult",
+            ColorMode.BGR,
             pixelScaling,
           ),
         );