Fix issue caused by pretokenising of UTF8 strings (which get converted after the pretokenise step and cause the length to be wrong)

Even if length were correct, pretokenising the string stops the scan for UTF8 char patterns so the string wouldn't be treated correctly

Author: gfwilliams

lib/espruinotools.js

@@ -1,6 +1,6 @@
 // EspruinoTools bundle (https://github.com/espruino/EspruinoTools)
 // Created with https://github.com/espruino/EspruinoWebIDE/blob/master/extras/create_espruinotools_js.sh
-// Based on EspruinoWebIDE  0.79.12
+// Based on EspruinoWebIDE  0.79.14
 /**
  Copyright 2014 Gordon Williams (gw@pur3.co.uk)
 
@@ -108,7 +108,10 @@ var Espruino;
     var n = 0;
     var cbCalled = false;
     var cb = function(inData) {
-      if (cbCalled) throw new Error("Internal error in "+eventType+" processor. Callback is called TWICE.");
+      if (cbCalled) {
+        throw new Error("Internal error in "+eventType+" processor. Callback is called TWICE.");
+        return;
+      }
       cbCalled = true;
       if (n < p.length) {
         cbCalled = false;
@@ -7167,6 +7170,7 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});}
           case "true" : tok = lex.next(); return true;
           case "false" : tok = lex.next(); return false;
           case "null" : tok = lex.next(); return null;
+          case "NaN" : tok = lex.next(); return NaN;
         }
         if (tok.str == "[") {
           tok = lex.next();
@@ -7677,6 +7681,7 @@ To add a new serial device, you must add an object to
     // on("open", () => ... ) connection opened
     // on("close", () => ... ) connection closed
     // on("data", (data) => ... ) when data is received (as string)
+    // on("line", (line) => ... ) when a line of data is received (as string), uses /r OR /n for lines
     // on("packet", (type,data) => ... ) when a packet is received (if .parsePackets=true)
     // on("ack", () => ... ) when an ACK is received (if .parsePackets=true)
     // on("nak", () => ... ) when an ACK is received (if .parsePackets=true)
@@ -7695,6 +7700,7 @@ To add a new serial device, you must add an object to
     rxDataHandlerLastCh = 0; // used by rxDataHandler - last received character
     rxDataHandlerPacket = undefined; // used by rxDataHandler - used for parsing
     rxDataHandlerTimeout = undefined; // timeout for unfinished packet
+    rxLine = "";          // current partial line for on("line" event
     progressAmt = 0;      // When sending a file, how many bytes through are we?
     progressMax = 0;      // When sending a file, how long is it in bytes? 0 if not sending a file
 
@@ -7773,6 +7779,13 @@ To add a new serial device, you must add an object to
         // forward any data
         if (this.cb) this.cb(data);
         this.emit('data', data);
+        // look for newlines and send out a 'line' event
+        let lines = (this.rxLine + data).split(/\r\n/);
+        while (lines.length>1)
+          this.emit('line', lines.shift());
+        this.rxLine = lines[0];
+        if (this.rxLine.length > 10000) // only store last 10k characters
+          this.rxLine = this.rxLine.slice(-10000);
       }
     }
 
@@ -7786,8 +7799,9 @@ To add a new serial device, you must add an object to
       this.hadData = false;
       this.flowControlWait = 0;
       this.rxDataHandlerLastCh = 0;
-      if (this.isOpen) {
-        this.isOpen = false;
+      this.rxLine = "";
+      if (!this.isOpen) {
+        this.isOpen = true;
         this.emit("open");
       }
       // if we had any writes queued, do them now
@@ -7796,12 +7810,12 @@ To add a new serial device, you must add an object to
 
     /** Called when the connection is closed - resets any stored info/rejects promises */
     closeHandler() {
-      log(1, "Disconnected");
       this.isOpening = false;
       this.txInProgress = false;
       this.txDataQueue = [];
       this.hadData = false;
       if (this.isOpen) {
+        log(1, "Disconnected");
         this.isOpen = false;
         this.emit("close");
       }
@@ -8165,54 +8179,67 @@ To add a new serial device, you must add an object to
     return oldListener;
   };
 
-  /* Calls 'callback(port_list, shouldCallAgain)'
-   'shouldCallAgain==true' means that more devices
-   may appear later on (eg Bluetooth LE).*/
-  var getPorts=function(callback) {
-    var ports = [];
-    var newPortToDevice = [];
-    // get all devices
-    var responses = 0;
+  /**
+   * List ports available over all configured devices.
+   * `shouldCallAgain` mean that more devices may appear later on (eg. Bluetooth LE)
+   * @param {(ports, shouldCallAgain) => void} callback
+   */
+  var getPorts = function (callback) {
+    var newPortToDevice = {};
+
     var devices = Espruino.Core.Serial.devices;
-    if (!devices || devices.length==0) {
-         portToDevice = newPortToDevice;
+    if (!devices || devices.length == 0) {
+      portToDevice = newPortToDevice;
       return callback(ports, false);
     }
-    var shouldCallAgain = false;
-    devices.forEach(function (device) {
-      //console.log("getPorts -->",device.name);
-      device.getPorts(function(devicePorts, instantPorts) {
-        //console.log("getPorts <--",device.name);
-        if (instantPorts===false) shouldCallAgain = true;
-        if (devicePorts) {
-          devicePorts.forEach(function(port) {
-            var ignored = false;
-            if (Espruino.Config.SERIAL_IGNORE)
-              Espruino.Config.SERIAL_IGNORE.split("|").forEach(function(wildcard) {
-                var regexp = "^"+wildcard.replace(/\./g,"\\.").replace(/\*/g,".*")+"$";
-                if (port.path.match(new RegExp(regexp)))
-                  ignored = true;
-              });
 
-            if (!ignored) {
-              if (port.usb && port.usb[0]==0x0483 && port.usb[1]==0x5740)
-                port.description = "Espruino board";
-              ports.push(port);
-              newPortToDevice[port.path] = device;
-            }
-          });
-        }
-        responses++;
-        if (responses == devices.length) {
-          portToDevice = newPortToDevice;
-          ports.sort(function(a,b) {
+    // Test to see if a given port path is ignore or not by configuration
+    function isIgnored(path) {
+      if (!Espruino.Config.SERIAL_IGNORE) return false;
+
+      return Espruino.Config.SERIAL_IGNORE.split("|").some((wildcard) => {
+        const regexp = new RegExp(
+          `^${wildcard.replace(/\./g, "\\.").replace(/\*/g, ".*")}$`
+        );
+
+        return path.match(regexp);
+      });
+    }
+
+    // Asynchronously call 'getPorts' on all devices and map results back as a series of promises
+    Promise.all(
+      devices.map((device) =>
+        new Promise((resolve) => device.getPorts(resolve)).then(
+          (devicePorts, instantPorts) => ({
+            device: device,
+            shouldCallAgain: !instantPorts, // If the ports are not present now (eg. BLE) then call again
+            value: (devicePorts || [])
+              .filter((port) => !isIgnored(port.path)) // Filter out all the ignored ports
+              .map((port) => {
+                // Map a description for this particular Product/Vendor
+                if (port.usb && port.usb[0] == 0x0483 && port.usb[1] == 0x5740)
+                  port.description = "Espruino board";
+                return port;
+              }),
+          })
+        )
+      )
+    ).then((results) => {
+      portToDevice = results.reduce((acc, promise) => {
+        promise.value.forEach((port) => (acc[port.path] = promise.device));
+        return acc;
+      }, {});
+
+      callback(
+        results
+          .flatMap((result) => result.value)
+          .sort((a, b) => {
             if (a.unimportant && !b.unimportant) return 1;
             if (b.unimportant && !a.unimportant) return -1;
             return 0;
-          });
-          callback(ports, shouldCallAgain);
-        }
-      });
+          }),
+        results.some((result) => result.shouldCallAgain),
+      );
     });
   };
 
@@ -8223,10 +8250,10 @@ To add a new serial device, you must add an object to
 
   var openSerialInternal=function(serialPort, connectCallback, disconnectCallback, attempts) {
     /* If openSerial is called, we need to have called getPorts first
-      in order to figure out which one of the serial_ implementations
-      we must call into. */
+    in order to figure out which one of the serial_ implementations
+    we must call into. */
     if (portToDevice === undefined) {
-      portToDevice = []; // stop recursive calls if something errors
+      portToDevice = {}; // stop recursive calls if something errors
       return getPorts(function() {
         openSerialInternal(serialPort, connectCallback, disconnectCallback, attempts);
       });
@@ -8332,9 +8359,8 @@ To add a new serial device, you must add an object to
       * sent blocks to 512 because on Mac we seem to lose
       * data otherwise (not on any other platforms!) */
     if (slowWrite) blockSize=19;
-    writeData.blockSize = blockSize;
 
-    writeData.showStatus &= writeData.data.length>writeData.blockSize;
+    writeData.showStatus &= writeData.data.length>blockSize;
     if (writeData.showStatus) {
       Espruino.Core.Status.setStatus("Sending...", writeData.data.length);
       console.log("serial: ---> "+JSON.stringify(writeData.data));
@@ -8371,12 +8397,13 @@ To add a new serial device, you must add an object to
       }
 
       let isLast = writeData.data.length == 0;
-      // update status
-      if (writeData.showStatus)
-        Espruino.Core.Status.incrementProgress(d.length);
       // actually write data
       //console.log("serial: Sending block "+JSON.stringify(d)+", wait "+split.delay+"ms");
+      Espruino.Core.Serial.connection.chunkSize = blockSize;
       Espruino.Core.Serial.connection.write(d, function() { // write data, but the callback returns a promise that delays
+        // update status
+        if (writeData.showStatus)
+          Espruino.Core.Status.incrementProgress(d.length);
         return new Promise(resolve => setTimeout(function() {
           if (isLast && writeData.showStatus) {
             Espruino.Core.Status.setStatus("Sent");
@@ -8859,8 +8886,8 @@ To add a new serial device, you must add an object to
       // if we needed to load something, wait until it's loaded before resolving this
       Promise.all(newPromises).then(function(){
         // add the module to end of our array
-        if (Espruino.Config.MODULE_AS_FUNCTION)
-          loadedModuleData.push("Modules.addCached(" + JSON.stringify(module.name) + ",function(){" + module.code + "\n});");
+        if (Espruino.Config.MODULE_AS_FUNCTION) // check for '//' is just in case, most minified modules shouldn't have it
+          loadedModuleData.push("Modules.addCached(" + JSON.stringify(module.name) + ",function(){" + module.code.trim() + (module.code.includes("//")?"\n":"") + "});");
         else
           loadedModuleData.push("Modules.addCached(" + JSON.stringify(module.name) + "," + JSON.stringify(module.code) + ");");
         // We're done
@@ -37091,7 +37118,7 @@ global.esmangle = require('../lib/esmangle');
 
   function minify(code, callback, level, isModule, description) {
     (function() {
-      Espruino.Core.Status.setStatus("Minifying"+(isModule?description.substr(2):""));
+      Espruino.Core.Status.setStatus("Minifying"+(isModule?description.substr(4)/*module name*/:""));
       var _callback = callback;
       callback = function(code) {
         Espruino.Core.Status.setStatus("Minification complete");
@@ -37310,7 +37337,8 @@ global.esmangle = require('../lib/esmangle');
         resultCode += "\n";
       if (tok.str==")" || tok.str=="}" || tok.str=="]") brackets--;
       // if we have a token for something, use that - else use the string
-      if (pretokeniseStrings && tok.type == "STRING") {
+      if (pretokeniseStrings && tok.type == "STRING" &&
+          tok.value.split("").reduce((r,ch) => r&&(ch.charCodeAt(0)<256), true)/*all 8 bit*/) {
         let str = tok.value;  // get string value
         lastIdx = tok.endIdx; // get next token
         lastlastTok = lastTok;