Merge pull request #77 from bonitoo-io/fix/timestamp

fix: better default timestamp generating

Author: vlastahajek

CHANGELOG.md

@@ -1,8 +1,10 @@
 # Changelog
+
 ## Version 3.3.0 (in progress)
  - [NEW] Added possibility skip server certification validation (`setInsecure()` method)
- - [NEW] Added possibility to query flux on InfuxDB 1.8 using V1 approach
+ - [NEW] Added possibility to query flux on secured InfuxDB 1.8 using V1 approach
  - [NEW] `validateConnection()` can be used also for the [forward compatibility](https://docs.influxdata.com/influxdb/latest/tools/api/#influxdb-2-0-api-compatibility-endpoints) connection to InfluxDB 1.8
+ - [FIX] More precice default timestamp generating, up to microseconds
  - [FIX] Debug compilation error
  - [FIX] SecureBatchWrite compile error
 

README.md

@@ -153,7 +153,7 @@ InfluxDB client for Arduino can write data in batches. A batch is simply a set o
 If using batch writes, the timestamp should be employed. Timestamp specifies the time where data was gathered and it is used in the form of a number of seconds (milliseconds, etc) from epoch (1.1.1970) UTC.
 If points have no timestamp assigned, InfluxDB assigns timestamp at the time of writing, which could happen much later than the data has been obtained, because final batch write will happen when the batch is full (or when [flush buffer](#buffer-handling-and-retrying) is forced).
 
-InfuxDB allows sending timestamp in various precisions - nanoseconds, microseconds, milliseconds or seconds. The milliseconds precision is usually enough for using on Arduino.
+InfuxDB allows sending timestamp in various precisions - nanoseconds, microseconds, milliseconds or seconds. The milliseconds precision is usually enough for using on Arduino. Maximum avavailable precision is microseconds. Setting to nanosecond will just add zeroes for microseconds fraction.
 
 The client has to be configured with time precision. The default settings is not using the timestamp. The `setWriteOptions` functions allow setting various parameters and one of them is __write precision__:
 ``` cpp
@@ -164,8 +164,10 @@ When a write precision is configured, the client will automatically assign curre
 
 If you want to manage timestamp on your own, there are several ways how to set timestamp explicitly.
 - `setTime(WritePrecision writePrecision)` - Sets timestamp to actual time in desired precision
-- `setTime(unsigned long seconds)` -  Sets timestamp in seconds since epoch. Write precision must be set to `S` 
-- `setTime(String timestamp)` - Set custom timestamp in precision specified in InfluxDBClient. 
+- `setTime(unsigned long long timestamp)` -  Sets timestamp in an offset since epoch. Correct precision must be set InfluxDBClient::setWriteOptions.
+- `setTime(String timestamp)` - Sets timestamp in an offset since epoch. Correct precision must be set InfluxDBClient::setWriteOptions.
+
+The `getTime()` method allows copying timestamp between points.
 
 
 ### Configure Time

src/InfluxDbClient.cpp

@@ -112,32 +112,30 @@ String Point::toLineProtocol() const {
 }
 
 void  Point::setTime(WritePrecision precision) {
-    static char buff[10];
-    time_t now = time(nullptr);
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    
     switch(precision) {
         case WritePrecision::NS:
-             sprintf(buff, "%06d000",  micros()%1000000uL);
-            _timestamp = String(now) + buff;
+            setTime(getTimeStamp(&tv,9));
             break;
         case WritePrecision::US:
-             sprintf(buff, "%06d",  micros()%1000000uL);
-            _timestamp = String(now) + buff;
+            setTime(getTimeStamp(&tv,6));
             break;
-        case WritePrecision::MS:
-             sprintf(buff, "%03d",  millis()%1000u);
-            _timestamp = String(now) + buff;
+        case WritePrecision::MS: 
+            setTime(getTimeStamp(&tv,3));
+            break;
+        case WritePrecision::S:
+            setTime(getTimeStamp(&tv,0));
             break;
         case WritePrecision::NoTime:
             _timestamp = "";
             break;
-        case WritePrecision::S:
-             _timestamp = String(now);
-             break;
     }
 }
 
-void  Point::setTime(unsigned long timestamp) {
-    _timestamp = String(timestamp);
+void  Point::setTime(unsigned long long timestamp) {
+    _timestamp = timeStampToString(timestamp);
 }
 
 void  Point::setTime(String timestamp) {

src/InfluxDbClient.h

@@ -84,9 +84,9 @@ class Point {
     void addField(String name, const char *value);
     // Set timestamp to `now()` and store it in specified precision, nanoseconds by default. Date and time must be already set. See `configTime` in the device API
     void setTime(WritePrecision writePrecision = WritePrecision::NS);
-    // Set timestamp in seconds since epoch (1.1.1970). Precision should be set to `S` 
-    void setTime(unsigned long seconds);
-    // Set timestamp in desired precision (specified in InfluxDBClient) since epoch (1.1.1970 00:00:00)
+    // Set timestamp in offset since epoch (1.1.1970). Correct precision must be set InfluxDBClient::setWriteOptions.
+    void setTime(unsigned long long timestamp);
+    // Set timestamp in offset since epoch (1.1.1970 00:00:00). Correct precision must be set InfluxDBClient::setWriteOptions.
     void setTime(String timestamp);
     // Clear all fields. Usefull for reusing point  
     void clearFields();
@@ -96,10 +96,12 @@ class Point {
     bool hasFields() const { return _fields.length() > 0; }
     // True if a point contains at least one tag
     bool hasTags() const   { return _tags.length() > 0; }
-     // True if a point contains timestamp
+    // True if a point contains timestamp
     bool hasTime() const   { return _timestamp.length() > 0; }
     // Creates line protocol
     String toLineProtocol() const;
+    // returns current timestamp
+    String getTime() const { return _timestamp; } 
   protected:
     String _measurement;
     String _tags;

src/util/helpers.cpp

@@ -47,6 +47,33 @@ void timeSync(const char *tzInfo, const char* ntpServer1, const char* ntpServer2
   Serial.println(ctime(&tnow));
 }
 
+unsigned long long getTimeStamp(struct timeval *tv, int secFracDigits) {
+    unsigned long long tsVal = 0;
+    switch(secFracDigits) {
+        case 0: 
+            tsVal = tv->tv_sec;
+            break;
+        case 6: 
+            tsVal = tv->tv_sec * 1000000LL + tv->tv_usec;
+            break;
+        case 9:
+            tsVal = tv->tv_sec * 1000000000LL + tv->tv_usec * 1000LL;
+            break;
+        case 3: 
+        default:
+            tsVal = tv->tv_sec * 1000LL + tv->tv_usec / 1000LL;
+            break;
+
+    }
+    return tsVal;
+}
+
+String timeStampToString(unsigned long long timestamp) {
+    static char buff[50];
+    snprintf(buff, 50, "%llu", timestamp);
+    return String(buff);
+}
+
 String escapeKey(String key, bool escapeEqual) {
     String ret;
     ret.reserve(key.length()+5); //5 is estimate of  chars needs to escape,

src/util/helpers.h

@@ -28,14 +28,22 @@
 #define _INFLUXDB_CLIENT_HELPERS_H
 
 #include <Arduino.h>
+#include <sys/time.h>  
 
 // Synchronize time with NTP servers and waits for completition. Prints waiting progress and final synchronized time to the serial.
 // Accurate time is necessary for certificate validion and writing points in batch
 // For the fastest time sync find NTP servers in your area: https://www.pool.ntp.org/zone/
 void timeSync(const char *tzInfo, const char* ntpServer1, const char* ntpServer2 = nullptr, const char* ntpServer3 = nullptr);
 
+// Create timestamp in offset from epoch. secFracDigits specify resulution. 0 - seconds, 3 - milliseconds, etc. Maximum and default is 9 - nanoseconds.
+unsigned long long getTimeStamp(struct timeval *tv, int secFracDigits = 3);
+
+// Converts unsigned long long timestamp to String
+String timeStampToString(unsigned long long timestamp);
+
 // Escape invalid chars in measurement, tag key, tag value and field key
 String escapeKey(String key, bool escapeEqual = true);
+
 // Escape invalid chars in field value
 String escapeValue(const char *value);
 

test/TestSupport.h

@@ -1,10 +1,14 @@
 #ifndef _TEST_SUPPORT_H_
 #define _TEST_SUPPORT_H_
 
-#define TEST_INIT(name) int temp = failures; const char *testName = name; do { Serial.println(testName)
-#define TEST_END()  } while(0); Serial.printf("%s %s\n",testName,failures == temp?"SUCCEEDED":"FAILED")
-#define TEST_ASSERT(a) if(testAssert(__LINE__, (a))) break
-#define TEST_ASSERTM(a,m) if(testAssertm(__LINE__, (a),(m))) break
+#define TEST_INIT(name)  int temp = failures;\
+ const char *testName = name; \
+  do { \
+  Serial.println(testName)
+#define TEST_END()  } while(0); \
+ end: Serial.printf("%s %s\n",testName,failures == temp?"SUCCEEDED":"FAILED")
+#define TEST_ASSERT(a) if(testAssert(__LINE__, (a))) goto end
+#define TEST_ASSERTM(a,m) if(testAssertm(__LINE__, (a),(m))) goto end
 
 #include "query/FluxParser.h"
 

test/test.ino

@@ -55,7 +55,6 @@ void loop() {
     testPoint();
     testFluxTypes();
     testFluxParserEmpty();
-    
     testFluxParserSingleTable();
     testFluxParserNilValue();
     testFluxParserMultiTables(false);
@@ -145,15 +144,22 @@ void testPoint() {
 
     TEST_ASSERT(!p.hasTime());
     time_t now = time(nullptr);
+    String snow(now);
     p.setTime(now);
-    String testLineTime = testLine + " " + now;
+    String testLineTime = testLine + " " + snow;
+    line = p.toLineProtocol();
+    TEST_ASSERTM(line == testLineTime, line);
+    
+    unsigned long long ts = now*1000000000LL+123456789;
+    p.setTime(ts);
+    testLineTime = testLine + " " + snow + "123456789";
     line = p.toLineProtocol();
     TEST_ASSERTM(line == testLineTime, line);
 
     now += 10;
-    String nowStr(now);
-    p.setTime(nowStr);
-    testLineTime = testLine + " " + nowStr;
+    snow = now;
+    p.setTime(snow);
+    testLineTime = testLine + " " + snow;
     line = p.toLineProtocol();
     TEST_ASSERTM(line == testLineTime, line);
 
@@ -162,29 +168,29 @@ void testPoint() {
     int partsCount;
     String *parts = getParts(line, ' ', partsCount);
     TEST_ASSERTM(partsCount == 3, String("3 != ") + partsCount);
-    TEST_ASSERT(parts[2].length() == nowStr.length());
+    TEST_ASSERT(parts[2].length() == snow.length());
     delete[] parts;
 
     p.setTime(WritePrecision::MS);
     TEST_ASSERT(p.hasTime());
     line = p.toLineProtocol();
     parts = getParts(line, ' ', partsCount);
     TEST_ASSERT(partsCount == 3);
-    TEST_ASSERT(parts[2].length() == nowStr.length() + 3);
+    TEST_ASSERT(parts[2].length() == snow.length() + 3);
     delete[] parts;
 
     p.setTime(WritePrecision::US);
     line = p.toLineProtocol();
     parts = getParts(line, ' ', partsCount);
     TEST_ASSERT(partsCount == 3);
-    TEST_ASSERT(parts[2].length() == nowStr.length() + 6);
+    TEST_ASSERT(parts[2].length() == snow.length() + 6);
     delete[] parts;
 
     p.setTime(WritePrecision::NS);
     line = p.toLineProtocol();
     parts = getParts(line, ' ', partsCount);
     TEST_ASSERT(partsCount == 3);
-    TEST_ASSERT(parts[2].length() == nowStr.length() + 9);
+    TEST_ASSERT(parts[2].length() == snow.length() + 9);
     delete[] parts;
 
     p.clearFields();
@@ -749,6 +755,31 @@ void testFailedWrites() {
 
 void testTimestamp() {
     TEST_INIT("testTimestamp");
+
+    struct timeval tv;
+    tv.tv_usec = 1234;
+    tv.tv_sec = 5678;
+    unsigned long long ts = getTimeStamp(&tv, 0);
+    TEST_ASSERTM( ts == 5678, timeStampToString(ts));
+    ts = getTimeStamp(&tv, 3);
+    TEST_ASSERTM( ts == 5678001, timeStampToString(ts));
+    ts = getTimeStamp(&tv, 6);
+    TEST_ASSERTM( ts == 5678001234, timeStampToString(ts));
+    ts = getTimeStamp(&tv, 9);
+    TEST_ASSERTM( ts == 5678001234000, timeStampToString(ts));
+
+    // Test increasing timestamp
+    String prev = "";
+    for(int i = 0;i<2000;i++) {
+        Point p("test");
+        p.setTime(WritePrecision::US);
+        String act = p.getTime();
+        TEST_ASSERTM( i == 0 || prev < act, String(i) + ": " + prev + " vs " + act);
+        prev = act;
+        delayMicroseconds(100);
+    }
+
+
     serverLog(INFLUXDB_CLIENT_TESTING_URL, "testTimestamp");
     InfluxDBClient client(INFLUXDB_CLIENT_TESTING_URL, INFLUXDB_CLIENT_TESTING_ORG, INFLUXDB_CLIENT_TESTING_BUC, INFLUXDB_CLIENT_TESTING_TOK);
     client.setWriteOptions(WritePrecision::S, 1, 5);
@@ -1007,6 +1038,7 @@ bool testFluxDateTimeValue(FluxQueryResult flux, int columnIndex,  const char *c
         TEST_ASSERTM(dt.microseconds == us,  String(dt.microseconds) + " vs " + String(us));
         return true;
     } while(0);
+end:
     return false;
 }
 
@@ -1017,6 +1049,7 @@ bool testStringValue(FluxQueryResult flux, int columnIndex,  const char *columnN
         TEST_ASSERTM(flux.getValueByName(columnName).getRawValue() == rawValue, flux.getValueByName(columnName).getRawValue());
         return true;
     } while(0);
+end:
     return false;
 }
 
@@ -1027,12 +1060,12 @@ bool testStringVector(std::vector<String> vect, const char *values[], int size)
             if(vect[i] != values[i]) {
                 Serial.print("assert failure: ");
                 Serial.println(vect[i]);
-                goto fail;
+                goto end;
             }
         }
         return true;
     } while(0);
-fail:
+end:
     return false;
 }
 
@@ -1043,6 +1076,7 @@ bool testDoubleValue(FluxQueryResult flux, int columnIndex,  const char *columnN
         TEST_ASSERTM(flux.getValueByName(columnName).getRawValue() == rawValue, flux.getValueByName(columnName).getRawValue());
         return true;
     } while(0);
+end:
     return false;
 }
 
@@ -1053,6 +1087,7 @@ bool testLongValue(FluxQueryResult flux, int columnIndex,  const char *columnNam
         TEST_ASSERTM(flux.getValueByName(columnName).getRawValue() == rawValue, flux.getValueByName(columnName).getRawValue());
         return true;
     } while(0);
+end:
     return false;
 }
 
@@ -1063,6 +1098,7 @@ bool testUnsignedLongValue(FluxQueryResult flux, int columnIndex,  const char *c
         TEST_ASSERTM(flux.getValueByName(columnName).getRawValue() == rawValue, flux.getValueByName(columnName).getRawValue());
         return true;
     } while(0);
+end:
     return false;
 }
 
@@ -1076,6 +1112,7 @@ bool testTableColumns(FluxQueryResult flux,  const char *columns[], int columnsC
         TEST_ASSERTM(flux.getColumnIndex("x") == -1, "flux.getColumnIndex(\"x\")");
         return true;
     } while(0);
+end:
     return false;
 }
 
@@ -1500,7 +1537,7 @@ void initInet() {
         Serial.print("Ip: ");
         Serial.println(WiFi.localIP());
 
-        timeSync("CET-1CEST,M3.5.0,M10.5.0/3", "pool.ntp.org", "0.cz.pool.ntp.org", "1.cz.pool.ntp.org");
+        timeSync("CET-1CEST,M3.5.0,M10.5.0/3", "0.cz.pool.ntp.org", "1.cz.pool.ntp.org", "pool.ntp.org");
 
         deleteAll(INFLUXDB_CLIENT_TESTING_URL);
     }