CSHARP-4539: Improving Time Series Bucketing Scalability (#1228)

Author: adelinowona

specifications/collection-management/tests/timeseries-collection.json

@@ -250,6 +250,71 @@
           ]
         }
       ]
+    },
+    {
+      "description": "createCollection with bucketing options",
+      "runOnRequirements": [
+        {
+          "minServerVersion": "6.3"
+        }
+      ],
+      "operations": [
+        {
+          "name": "dropCollection",
+          "object": "database0",
+          "arguments": {
+            "collection": "test"
+          }
+        },
+        {
+          "name": "createCollection",
+          "object": "database0",
+          "arguments": {
+            "collection": "test",
+            "timeseries": {
+              "timeField": "time",
+              "bucketMaxSpanSeconds": 3600,
+              "bucketRoundingSeconds": 3600
+            }
+          }
+        },
+        {
+          "name": "assertCollectionExists",
+          "object": "testRunner",
+          "arguments": {
+            "databaseName": "ts-tests",
+            "collectionName": "test"
+          }
+        }
+      ],
+      "expectEvents": [
+        {
+          "client": "client0",
+          "events": [
+            {
+              "commandStartedEvent": {
+                "command": {
+                  "drop": "test"
+                },
+                "databaseName": "ts-tests"
+              }
+            },
+            {
+              "commandStartedEvent": {
+                "command": {
+                  "create": "test",
+                  "timeseries": {
+                    "timeField": "time",
+                    "bucketMaxSpanSeconds": 3600,
+                    "bucketRoundingSeconds": 3600
+                  }
+                },
+                "databaseName": "ts-tests"
+              }
+            }
+          ]
+        }
+      ]
     }
   ]
 }

specifications/collection-management/tests/timeseries-collection.yml

@@ -83,21 +83,21 @@ tests:
         arguments:
           documents: &docs
             - {
-                _id: 1,
-                time: {
-                  $date: {
-                    $numberLong: "1552949630482"
-                  }
+              _id: 1,
+              time: {
+                $date: {
+                  $numberLong: "1552949630482"
                 }
               }
+            }
             - {
-                _id: 1,
-                time: {
-                  $date: {
-                    $numberLong: "1552949630483"
-                  }
+              _id: 1,
+              time: {
+                $date: {
+                  $numberLong: "1552949630483"
                 }
               }
+            }
       - name: find
         object: *collection0
         arguments:
@@ -127,3 +127,38 @@ tests:
                 filter: {}
                 sort: { time: 1 }
               databaseName: *database0Name
+
+  - description: "createCollection with bucketing options"
+    runOnRequirements:
+      - minServerVersion: "6.3"
+    operations:
+      - name: dropCollection
+        object: *database0
+        arguments:
+          collection: *collection0Name
+      - name: createCollection
+        object: *database0
+        arguments:
+          collection: *collection0Name
+          timeseries: &timeseries1
+            timeField: "time"
+            bucketMaxSpanSeconds: 3600
+            bucketRoundingSeconds: 3600
+      - name: assertCollectionExists
+        object: testRunner
+        arguments:
+          databaseName: *database0Name
+          collectionName: *collection0Name
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                drop: *collection0Name
+              databaseName: *database0Name
+          - commandStartedEvent:
+              command:
+                create: *collection0Name
+                timeseries: *timeseries1
+              databaseName: *database0Name
+

src/MongoDB.Driver.Core/TimeSeriesOptions.cs

@@ -27,18 +27,24 @@ public class TimeSeriesOptions
         private readonly TimeSeriesGranularity? _granularity;
         private readonly string _metaField;
         private readonly string _timeField;
+        private readonly int? _bucketMaxSpanSeconds;
+        private readonly int? _bucketRoundingSeconds;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="TimeSeriesOptions"/> class.
         /// </summary>
         /// <param name="timeField">The name of the top-level field to be used for time.</param>
         /// <param name="metaField">The name of the top-level field describing the series upon which related data will be grouped.</param>
-        /// <param name="granularity">The <see cref="TimeSeriesGranularity"/> for the time series.</param>
-        public TimeSeriesOptions(string timeField, Optional<string> metaField = default, Optional<TimeSeriesGranularity?> granularity = default)
+        /// <param name="granularity">The <see cref="TimeSeriesGranularity"/> for the time series. Do not set if using bucketMaxSpanSeconds</param>
+        /// <param name="bucketMaxSpanSeconds">The maximum time between timestamps in the same bucket.</param>
+        /// <param name="bucketRoundingSeconds">The interval used to round down the first timestamp when opening a new bucket.</param>
+        public TimeSeriesOptions(string timeField, Optional<string> metaField = default, Optional<TimeSeriesGranularity?> granularity = default, Optional<int?> bucketMaxSpanSeconds = default, Optional<int?> bucketRoundingSeconds = default)
         {
             _timeField = Ensure.IsNotNullOrEmpty(timeField, nameof(timeField));
             _metaField = metaField.WithDefault(null);
             _granularity = granularity.WithDefault(null);
+            _bucketMaxSpanSeconds = bucketMaxSpanSeconds.WithDefault(null);
+            _bucketRoundingSeconds = bucketRoundingSeconds.WithDefault(null);
         }
 
         /// <summary>
@@ -56,6 +62,16 @@ public TimeSeriesOptions(string timeField, Optional<string> metaField = default,
         /// </summary>
         public string TimeField => _timeField;
 
+        /// <summary>
+        /// The maximum time between timestamps in the same bucket.
+        /// </summary>
+        public int? BucketMaxSpanSeconds => _bucketMaxSpanSeconds;
+
+        /// <summary>
+        /// The interval used to round down the first timestamp when opening a new bucket.
+        /// </summary>
+        public int? BucketRoundingSeconds => _bucketRoundingSeconds;
+
         /// <summary>
         /// The BSON representation of the time series options.
         /// </summary>
@@ -66,7 +82,9 @@ public BsonDocument ToBsonDocument()
             {
                 { "timeField", _timeField },
                 { "metaField", _metaField, _metaField != null },
-                { "granularity", () => _granularity.Value.ToString().ToLowerInvariant(), _granularity.HasValue }
+                { "granularity", () => _granularity.Value.ToString().ToLowerInvariant(), _granularity.HasValue },
+                { "bucketMaxSpanSeconds", _bucketMaxSpanSeconds, _bucketMaxSpanSeconds != null },
+                { "bucketRoundingSeconds", _bucketRoundingSeconds, _bucketRoundingSeconds != null }
             };
         }
     }

tests/MongoDB.Driver.Core.Tests/TimeSeriesOptionsTests.cs

@@ -54,12 +54,34 @@ public void constructor_with_all_parameters_should_initialize_instance()
             const string timeField = "time";
             const string metaField = "meta";
             const TimeSeriesGranularity granularity = TimeSeriesGranularity.Hours;
+            const int bucketMaxSpanSeconds = 30;
+            const int bucketRoundingSeconds = 30;
 
-            var result = new TimeSeriesOptions(timeField, metaField, granularity);
+            var result = new TimeSeriesOptions(timeField, metaField, granularity, bucketMaxSpanSeconds, bucketRoundingSeconds);
 
             result.TimeField.Should().Be(timeField);
             result.MetaField.Should().Be(metaField);
             result.Granularity.Should().Be(granularity);
+            result.BucketMaxSpanSeconds.Should().Be(bucketMaxSpanSeconds);
+            result.BucketRoundingSeconds.Should().Be(bucketRoundingSeconds);
+        }
+
+        [Theory]
+        [InlineData(true, "{ timeField: 'time' }")]
+        [InlineData(false, "{ timeField: 'time', metaField: 'meta', granularity: 'hours', bucketMaxSpanSeconds: 30, bucketRoundingSeconds: 30 }")]
+        public void ToBsonDocument_should_return_expected_result(bool withDefaults, string expected)
+        {
+            const string timeField = "time";
+            const string metaField = "meta";
+            const TimeSeriesGranularity granularity = TimeSeriesGranularity.Hours;
+            const int bucketMaxSpanSeconds = 30;
+            const int bucketRoundingSeconds = 30;
+
+            var result = withDefaults
+                ? new TimeSeriesOptions(timeField)
+                : new TimeSeriesOptions(timeField, metaField, granularity, bucketMaxSpanSeconds, bucketRoundingSeconds);
+
+            result.ToBsonDocument().Should().Be(expected);
         }
     }
 }

tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedCreateCollectionOperation.cs

@@ -215,7 +215,16 @@ public IUnifiedEntityTestOperation Build(string targetDatabaseId, BsonDocument a
                         {
                             granularity = (TimeSeriesGranularity)Enum.Parse(typeof(TimeSeriesGranularity), granularityValue.AsString, true);
                         }
-                        timeSeriesOptions = new TimeSeriesOptions(timeField, metaField, granularity);
+
+                        var bucketMaxSpanSeconds =
+                            timeseries.TryGetValue("bucketMaxSpanSeconds", out var bucketMaxSpanSecondsValue)
+                                ? bucketMaxSpanSecondsValue.AsInt32
+                                : (int?)null;
+                        var bucketRoundingSeconds =
+                            timeseries.TryGetValue("bucketRoundingSeconds", out var bucketRoundingSecondsValue)
+                                ? bucketRoundingSecondsValue.AsInt32
+                                : (int?)null;
+                        timeSeriesOptions = new TimeSeriesOptions(timeField, metaField, granularity, bucketMaxSpanSeconds, bucketRoundingSeconds);
                         break;
                     case "viewOn":
                         viewOn = argument.Value.AsString;