Merge pull request #1194 from plotly/dates-utc-backend

Dates: Accept ISO-8601 format, and use UTC milliseconds as the backend

Author: alexcjohnson

circle.yml

@@ -6,6 +6,8 @@ general:
 machine:
   node:
     version: 6.1.0
+  timezone:
+    America/Anchorage
   services:
     - docker
 

src/components/rangeselector/get_update_object.js

@@ -42,13 +42,13 @@ function getXRange(axisLayout, buttonLayout) {
 
     switch(buttonLayout.stepmode) {
         case 'backward':
-            range0 = Lib.ms2DateTime(+d3.time[step].offset(base, -count));
+            range0 = Lib.ms2DateTime(+d3.time[step].utc.offset(base, -count));
             break;
 
         case 'todate':
-            var base2 = d3.time[step].offset(base, -count);
+            var base2 = d3.time[step].utc.offset(base, -count);
 
-            range0 = Lib.ms2DateTime(+d3.time[step].ceil(base2));
+            range0 = Lib.ms2DateTime(+d3.time[step].utc.ceil(base2));
             break;
     }
 

src/lib/dates.js

@@ -10,7 +10,6 @@
 'use strict';
 
 var d3 = require('d3');
-var isNumeric = require('fast-isnumeric');
 
 var logError = require('./loggers').error;
 
@@ -21,6 +20,11 @@ var ONEHOUR = constants.ONEHOUR;
 var ONEMIN = constants.ONEMIN;
 var ONESEC = constants.ONESEC;
 
+var DATETIME_REGEXP = /^\s*(-?\d\d\d\d|\d\d)(-(0?[1-9]|1[012])(-([0-3]?\d)([ Tt]([01]?\d|2[0-3])(:([0-5]\d)(:([0-5]\d(\.\d+)?))?(Z|z|[+\-]\d\d:?\d\d)?)?)?)?)?\s*$/m;
+
+// for 2-digit years, the first year we map them onto
+var YFIRST = new Date().getFullYear() - 70;
+
 // is an object a javascript date?
 exports.isJSDate = function(v) {
     return typeof v === 'object' && v !== null && typeof v.getTime === 'function';
@@ -32,20 +36,33 @@ exports.isJSDate = function(v) {
 var MIN_MS, MAX_MS;
 
 /**
- * dateTime2ms - turn a date object or string s of the form
- * YYYY-mm-dd HH:MM:SS.sss into milliseconds (relative to 1970-01-01,
- * per javascript standard)
- * may truncate after any full field, and sss can be any length
- * even >3 digits, though javascript dates truncate to milliseconds
- * returns BADNUM if it doesn't find a date
+ * dateTime2ms - turn a date object or string s into milliseconds
+ * (relative to 1970-01-01, per javascript standard)
+ *
+ * Returns BADNUM if it doesn't find a date
+ *
+ * strings should have the form:
+ *
+ *    -?YYYY-mm-dd<sep>HH:MM:SS.sss<tzInfo>?
+ *
+ * <sep>: space (our normal standard) or T or t (ISO-8601)
+ * <tzInfo>: Z, z, or [+\-]HH:?MM and we THROW IT AWAY
+ * this format comes from https://tools.ietf.org/html/rfc3339#section-5.6
+ * but we allow it even with a space as the separator
+ *
+ * May truncate after any full field, and sss can be any length
+ * even >3 digits, though javascript dates truncate to milliseconds,
+ * we keep as much as javascript numeric precision can hold, but we only
+ * report back up to 100 microsecond precision, because most dates support
+ * this precision (close to 1970 support more, very far away support less)
  *
  * Expanded to support negative years to -9999 but you must always
  * give 4 digits, except for 2-digit positive years which we assume are
  * near the present time.
  * Note that we follow ISO 8601:2004: there *is* a year 0, which
  * is 1BC/BCE, and -1===2BC etc.
  *
- * 2-digit to 4-digit year conversion, where to cut off?
+ * Where to cut off 2-digit years between 1900s and 2000s?
  * from http://support.microsoft.com/kb/244664:
  *   1930-2029 (the most retro of all...)
  * but in my mac chrome from eg. d=new Date(Date.parse('8/19/50')):
@@ -70,96 +87,36 @@ var MIN_MS, MAX_MS;
 exports.dateTime2ms = function(s) {
     // first check if s is a date object
     if(exports.isJSDate(s)) {
-        s = Number(s);
+        // Convert to the UTC milliseconds that give the same
+        // hours as this date has in the local timezone
+        s = Number(s) - s.getTimezoneOffset() * ONEMIN;
         if(s >= MIN_MS && s <= MAX_MS) return s;
         return BADNUM;
     }
     // otherwise only accept strings and numbers
     if(typeof s !== 'string' && typeof s !== 'number') return BADNUM;
 
-    var y, m, d, h;
-    // split date and time parts
-    // TODO: we strip leading/trailing whitespace but not other
-    // characters like we do for numbers - do we want to?
-    var datetime = String(s).trim().split(' ');
-    if(datetime.length > 2) return BADNUM;
-
-    var p = datetime[0].split('-'); // date part
-
-    var CE = true; // common era, ie positive year
-    if(p[0] === '') {
-        // first part is blank: year starts with a minus sign
-        CE = false;
-        p.splice(0, 1);
+    var match = String(s).match(DATETIME_REGEXP);
+    if(!match) return BADNUM;
+    var y = match[1],
+        m = Number(match[3] || 1),
+        d = Number(match[5] || 1),
+        H = Number(match[7] || 0),
+        M = Number(match[9] || 0),
+        S = Number(match[11] || 0);
+    if(y.length === 2) {
+        y = (Number(y) + 2000 - YFIRST) % 100 + YFIRST;
     }
-
-    var plen = p.length;
-    if(plen > 3 || (plen !== 3 && datetime[1]) || !plen) return BADNUM;
-
-    // year
-    if(p[0].length === 4) y = Number(p[0]);
-    else if(p[0].length === 2) {
-        if(!CE) return BADNUM;
-        var yNow = new Date().getFullYear();
-        y = ((Number(p[0]) - yNow + 70) % 100 + 200) % 100 + yNow - 70;
-    }
-    else return BADNUM;
-    if(!isNumeric(y)) return BADNUM;
+    else y = Number(y);
 
     // javascript takes new Date(0..99,m,d) to mean 1900-1999, so
     // to support years 0-99 we need to use setFullYear explicitly
-    var baseDate = new Date(0, 0, 1);
-    baseDate.setFullYear(CE ? y : -y);
-    if(p.length > 1) {
-
-        // month - may be 1 or 2 digits
-        m = Number(p[1]) - 1; // new Date() uses zero-based months
-        if(p[1].length > 2 || !(m >= 0 && m <= 11)) return BADNUM;
-        baseDate.setMonth(m);
-
-        if(p.length > 2) {
-
-            // day - may be 1 or 2 digits
-            d = Number(p[2]);
-            if(p[2].length > 2 || !(d >= 1 && d <= 31)) return BADNUM;
-            baseDate.setDate(d);
-
-            // does that date exist in this month?
-            if(baseDate.getDate() !== d) return BADNUM;
-
-            if(datetime[1]) {
+    var date = new Date(Date.UTC(2000, m - 1, d, H, M));
+    date.setUTCFullYear(y);
 
-                p = datetime[1].split(':');
-                if(p.length > 3) return BADNUM;
+    if(date.getUTCDate() !== d) return BADNUM;
 
-                // hour - may be 1 or 2 digits
-                h = Number(p[0]);
-                if(p[0].length > 2 || !p[0].length || !(h >= 0 && h <= 23)) return BADNUM;
-                baseDate.setHours(h);
-
-                // does that hour exist in this day? (Daylight time!)
-                // (TODO: remove this check when we move to UTC)
-                if(baseDate.getHours() !== h) return BADNUM;
-
-                if(p.length > 1) {
-                    d = baseDate.getTime();
-
-                    // minute - must be 2 digits
-                    m = Number(p[1]);
-                    if(p[1].length !== 2 || !(m >= 0 && m <= 59)) return BADNUM;
-                    d += ONEMIN * m;
-                    if(p.length === 2) return d;
-
-                    // second (and milliseconds) - must have 2-digit seconds
-                    if(p[2].split('.')[0].length !== 2) return BADNUM;
-                    s = Number(p[2]);
-                    if(!(s >= 0 && s < 60)) return BADNUM;
-                    return d + s * ONESEC;
-                }
-            }
-        }
-    }
-    return baseDate.getTime();
+    return date.getTime() + S * ONESEC;
 };
 
 MIN_MS = exports.MIN_MS = exports.dateTime2ms('-9999');
@@ -191,16 +148,41 @@ exports.ms2DateTime = function(ms, r) {
 
     if(!r) r = 0;
 
-    var d = new Date(Math.floor(ms)),
-        dateStr = d3.time.format('%Y-%m-%d')(d),
+    var msecTenths = Math.round(((ms % 1) + 1) * 10) % 10,
+        d = new Date(Math.round(ms - msecTenths / 10)),
+        dateStr = d3.time.format.utc('%Y-%m-%d')(d),
         // <90 days: add hours and minutes - never *only* add hours
-        h = (r < NINETYDAYS) ? d.getHours() : 0,
-        m = (r < NINETYDAYS) ? d.getMinutes() : 0,
+        h = (r < NINETYDAYS) ? d.getUTCHours() : 0,
+        m = (r < NINETYDAYS) ? d.getUTCMinutes() : 0,
         // <3 hours: add seconds
-        s = (r < THREEHOURS) ? d.getSeconds() : 0,
+        s = (r < THREEHOURS) ? d.getUTCSeconds() : 0,
         // <5 minutes: add ms (plus one extra digit, this is msec*10)
-        msec10 = (r < FIVEMIN) ? Math.round((d.getMilliseconds() + (((ms % 1) + 1) % 1)) * 10) : 0;
+        msec10 = (r < FIVEMIN) ? d.getUTCMilliseconds() * 10 + msecTenths : 0;
 
+    return includeTime(dateStr, h, m, s, msec10);
+};
+
+// For converting old-style milliseconds to date strings,
+// we use the local timezone rather than UTC like we use
+// everywhere else, both for backward compatibility and
+// because that's how people mostly use javasript date objects.
+// Clip one extra day off our date range though so we can't get
+// thrown beyond the range by the timezone shift.
+exports.ms2DateTimeLocal = function(ms) {
+    if(!(ms >= MIN_MS + ONEDAY && ms <= MAX_MS - ONEDAY)) return BADNUM;
+
+    var msecTenths = Math.round(((ms % 1) + 1) * 10) % 10,
+        d = new Date(Math.round(ms - msecTenths / 10)),
+        dateStr = d3.time.format('%Y-%m-%d')(d),
+        h = d.getHours(),
+        m = d.getMinutes(),
+        s = d.getSeconds(),
+        msec10 = d.getUTCMilliseconds() * 10 + msecTenths;
+
+    return includeTime(dateStr, h, m, s, msec10);
+};
+
+function includeTime(dateStr, h, m, s, msec10) {
     // include each part that has nonzero data in or after it
     if(h || m || s || msec10) {
         dateStr += ' ' + lpad(h, 2) + ':' + lpad(m, 2);
@@ -217,7 +199,7 @@ exports.ms2DateTime = function(ms, r) {
         }
     }
     return dateStr;
-};
+}
 
 // normalize date format to date string, in case it starts as
 // a Date object or milliseconds
@@ -227,7 +209,7 @@ exports.cleanDate = function(v, dflt) {
         // NOTE: if someone puts in a year as a number rather than a string,
         // this will mistakenly convert it thinking it's milliseconds from 1970
         // that is: '2012' -> Jan. 1, 2012, but 2012 -> 2012 epoch milliseconds
-        v = exports.ms2DateTime(+v);
+        v = exports.ms2DateTimeLocal(+v);
         if(!v && dflt !== undefined) return dflt;
     }
     else if(!exports.isDateTime(v)) {

src/lib/index.js

@@ -28,6 +28,7 @@ var datesModule = require('./dates');
 lib.dateTime2ms = datesModule.dateTime2ms;
 lib.isDateTime = datesModule.isDateTime;
 lib.ms2DateTime = datesModule.ms2DateTime;
+lib.ms2DateTimeLocal = datesModule.ms2DateTimeLocal;
 lib.cleanDate = datesModule.cleanDate;
 lib.isJSDate = datesModule.isJSDate;
 lib.MIN_MS = datesModule.MIN_MS;

src/plots/cartesian/axes.js

@@ -931,9 +931,9 @@ axes.tickIncrement = function(x, dtick, axrev) {
     // Dates: months (or years)
     if(tType === 'M') {
         var y = new Date(x);
-        // is this browser consistent? setMonth edits a date but
+        // is this browser consistent? setUTCMonth edits a date but
         // returns that date's milliseconds
-        return y.setMonth(y.getMonth() + dtSigned);
+        return y.setUTCMonth(y.getUTCMonth() + dtSigned);
     }
 
     // Log scales: Linear, Digits
@@ -984,9 +984,9 @@ axes.tickFirst = function(ax) {
     if(tType === 'M') {
         t0 = new Date(tick0);
         r0 = new Date(r0);
-        mdif = (r0.getFullYear() - t0.getFullYear()) * 12 +
-            r0.getMonth() - t0.getMonth();
-        t1 = t0.setMonth(t0.getMonth() +
+        mdif = (r0.getUTCFullYear() - t0.getUTCFullYear()) * 12 +
+            r0.getUTCMonth() - t0.getUTCMonth();
+        t1 = t0.setUTCMonth(t0.getUTCMonth() +
             (Math.round(mdif / dtNum) + (axrev ? 1 : -1)) * dtNum);
 
         while(axrev ? t1 > r0 : t1 < r0) {
@@ -1010,12 +1010,13 @@ axes.tickFirst = function(ax) {
     else throw 'unrecognized dtick ' + String(dtick);
 };
 
-var yearFormat = d3.time.format('%Y'),
-    monthFormat = d3.time.format('%b %Y'),
-    dayFormat = d3.time.format('%b %-d'),
-    yearMonthDayFormat = d3.time.format('%b %-d, %Y'),
-    minuteFormat = d3.time.format('%H:%M'),
-    secondFormat = d3.time.format(':%S');
+var utcFormat = d3.time.format.utc,
+    yearFormat = utcFormat('%Y'),
+    monthFormat = utcFormat('%b %Y'),
+    dayFormat = utcFormat('%b %-d'),
+    yearMonthDayFormat = utcFormat('%b %-d, %Y'),
+    minuteFormat = utcFormat('%H:%M'),
+    secondFormat = utcFormat(':%S');
 
 // add one item to d3's vocabulary:
 // %{n}f where n is the max number of digits
@@ -1028,10 +1029,10 @@ function modDateFormat(fmt, x) {
         var digits = Math.min(+fm[1] || 6, 6),
             fracSecs = String((x / 1000 % 1) + 2.0000005)
                 .substr(2, digits).replace(/0+$/, '') || '0';
-        return d3.time.format(fmt.replace(fracMatch, fracSecs))(d);
+        return utcFormat(fmt.replace(fracMatch, fracSecs))(d);
     }
     else {
-        return d3.time.format(fmt)(d);
+        return utcFormat(fmt)(d);
     }
 }
 

src/plots/cartesian/dragbox.js

@@ -298,16 +298,18 @@ module.exports = function dragBox(gd, plotinfo, x, y, w, h, ns, ew) {
     function zoomAxRanges(axList, r0Fraction, r1Fraction) {
         var i,
             axi,
-            axRangeLinear;
+            axRangeLinear0,
+            axRangeLinearSpan;
 
         for(i = 0; i < axList.length; i++) {
             axi = axList[i];
             if(axi.fixedrange) continue;
 
-            axRangeLinear = axi.range.map(axi.r2l);
+            axRangeLinear0 = axi._rl[0];
+            axRangeLinearSpan = axi._rl[1] - axRangeLinear0;
             axi.range = [
-                axi.l2r(axRangeLinear[0] + (axRangeLinear[1] - axRangeLinear[0]) * r0Fraction),
-                axi.l2r(axRangeLinear[0] + (axRangeLinear[1] - axRangeLinear[0]) * r1Fraction)
+                axi.l2r(axRangeLinear0 + axRangeLinearSpan * r0Fraction),
+                axi.l2r(axRangeLinear0 + axRangeLinearSpan * r1Fraction)
             ];
         }
     }

src/plots/cartesian/set_convert.js

@@ -265,10 +265,10 @@ module.exports = function setConvert(ax) {
             // NOTE: Changed this behavior: previously we took any numeric value
             // to be a ms, even if it was a string that could be a bare year.
             // Now we convert it as a date if at all possible, and only try
-            // as ms if that fails.
+            // as (local) ms if that fails.
             var ms = Lib.dateTime2ms(v);
             if(ms === BADNUM) {
-                if(isNumeric(v)) ms = Number(v);
+                if(isNumeric(v)) ms = Lib.dateTime2ms(new Date(v));
                 else return BADNUM;
             }
             return Lib.constrain(ms, Lib.MIN_MS, Lib.MAX_MS);

src/plots/plots.js

@@ -1390,7 +1390,7 @@ plots.graphJson = function(gd, dataonly, mode, output, useDefaults) {
 
         // convert native dates to date strings...
         // mostly for external users exporting to plotly
-        if(Lib.isJSDate(d)) return Lib.ms2DateTime(+d);
+        if(Lib.isJSDate(d)) return Lib.ms2DateTimeLocal(+d);
 
         return d;
     }