Merge pull request #3455 from elastic/6.4

Add implemented 6.4.0 features to 6.x

Author: russcam

.paket/Paket.Restore.targets

@@ -144,11 +144,12 @@
         <PackageName>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0])</PackageName>
         <PackageVersion>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1])</PackageVersion>
         <AllPrivateAssets>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4])</AllPrivateAssets>
+        <CopyLocal>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5])</CopyLocal>
       </PaketReferencesFileLinesInfo>
       <PackageReference Include="%(PaketReferencesFileLinesInfo.PackageName)">
         <Version>%(PaketReferencesFileLinesInfo.PackageVersion)</Version>
         <PrivateAssets Condition=" ('%(PaketReferencesFileLinesInfo.AllPrivateAssets)' == 'true') Or ('$(PackAsTool)' == 'true') ">All</PrivateAssets>
-        <ExcludeAssets Condition="%(PaketReferencesFileLinesInfo.AllPrivateAssets) == 'exclude'">runtime</ExcludeAssets>
+        <ExcludeAssets Condition="%(PaketReferencesFileLinesInfo.CopyLocal) == 'false'">runtime</ExcludeAssets>
         <Publish Condition=" '$(PackAsTool)' == 'true' ">true</Publish>
       </PackageReference>
     </ItemGroup>

build/scripts/Commandline.fsx

@@ -41,6 +41,7 @@ NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to for
 Execution hints can be provided anywhere on the command line
 - skiptests : skip running tests as part of the target chain
 - skipdocs : skip generating documentation
+- docs:<B> : the branch name B to use when generating documentation
 - seed:<N> : provide a seed to run the tests with.
 - random:<K><:B> : sets random K to bool B if if B is omitted will default to true
   K can be: sourceserializer, typedkeys or oldconnection (only valid on windows)

build/scripts/Targets.fsx

@@ -33,6 +33,7 @@ open Commandline
 open Differ
 open Differ.Differ
 open Fake.IO
+open Octokit
 
 Commandline.parse()
 
@@ -107,14 +108,22 @@ Target "Cluster" <| fun _ ->
     let clusterVersion = getBuildParam "clusterVersion"
     let testsProjectDirectory = Path.Combine(Path.GetFullPath(Paths.Output("Tests.ClusterLauncher")), "netcoreapp2.1")
     let tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+    
+    let sourceDir = Paths.Source("Tests/Tests.Configuration");
+    let defaultYaml = Path.Combine(sourceDir, "tests.default.yaml");
+    let userYaml = Path.Combine(sourceDir, "tests.yaml");
+    let e f = File.Exists f;
+    match ((e userYaml), (e defaultYaml)) with
+    | (true, _) -> setProcessEnvironVar "NEST_YAML_FILE" (Path.GetFullPath(userYaml))
+    | (_, true) -> setProcessEnvironVar "NEST_YAML_FILE" (Path.GetFullPath(defaultYaml))
+    | _ -> ignore()
+    
     Shell.copyDir tempDir testsProjectDirectory (fun s -> true)
-    trace testsProjectDirectory
-    trace tempDir
     let command = sprintf "%s %s" clusterName clusterVersion
     DotNetCli.RunCommand(fun p ->
         { p with
             WorkingDir = tempDir;
-            TimeOut = TimeSpan.FromMinutes(60.)
+            TimeOut = TimeSpan.FromMinutes(120.)
         }) (sprintf "%s %s" (Path.Combine(tempDir, "Tests.ClusterLauncher.dll")) command)
 
     Shell.deleteDir tempDir

docs/aggregations.asciidoc

@@ -4,7 +4,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/master/src/Tests/aggregations.asciidoc. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/master/src/Tests/Tests/aggregations.asciidoc. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////
@@ -62,6 +62,8 @@ The values are typically extracted from the fields of the document (using the fi
 
 * <<value-count-aggregation-usage,Value Count Aggregation Usage>>
 
+* <<weighted-average-aggregation-usage,Weighted Average Aggregation Usage>>
+
 See the Elasticsearch documentation on {ref_current}/search-aggregations-metrics.html[Metric aggregations] for more details.
 
 :includes-from-dirs: aggregations/metric
@@ -94,6 +96,8 @@ include::aggregations/metric/top-hits/top-hits-aggregation-usage.asciidoc[]
 
 include::aggregations/metric/value-count/value-count-aggregation-usage.asciidoc[]
 
+include::aggregations/metric/weighted-average/weighted-average-aggregation-usage.asciidoc[]
+
 [[bucket-aggregations]]
 == Bucket Aggregations
 
@@ -233,6 +237,8 @@ There are many different types of pipeline aggregation, each computing different
 
 * <<moving-average-simple-aggregation-usage,Moving Average Simple Aggregation Usage>>
 
+* <<moving-function-aggregation-usage,Moving Function Aggregation Usage>>
+
 * <<percentiles-bucket-aggregation-usage,Percentiles Bucket Aggregation Usage>>
 
 * <<serial-differencing-aggregation-usage,Serial Differencing Aggregation Usage>>
@@ -273,6 +279,8 @@ include::aggregations/pipeline/moving-average/moving-average-linear-aggregation-
 
 include::aggregations/pipeline/moving-average/moving-average-simple-aggregation-usage.asciidoc[]
 
+include::aggregations/pipeline/moving-function/moving-function-aggregation-usage.asciidoc[]
+
 include::aggregations/pipeline/percentiles-bucket/percentiles-bucket-aggregation-usage.asciidoc[]
 
 include::aggregations/pipeline/serial-differencing/serial-differencing-aggregation-usage.asciidoc[]

docs/aggregations/aggregation-meta-usage.asciidoc

@@ -7,7 +7,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Aggregations/AggregationMetaUsageTests.cs. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Tests/Aggregations/AggregationMetaUsageTests.cs. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////

docs/aggregations/bucket/adjacency-matrix/adjacency-matrix-usage.asciidoc

@@ -7,7 +7,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Aggregations/Bucket/AdjacencyMatrix/AdjacencyMatrixUsageTests.cs. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Tests/Aggregations/Bucket/AdjacencyMatrix/AdjacencyMatrixUsageTests.cs. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////

docs/aggregations/bucket/children/children-aggregation-usage.asciidoc

@@ -7,7 +7,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Aggregations/Bucket/Children/ChildrenAggregationUsageTests.cs. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Tests/Aggregations/Bucket/Children/ChildrenAggregationUsageTests.cs. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////

docs/aggregations/bucket/composite/composite-aggregation-usage.asciidoc

@@ -7,7 +7,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////
@@ -145,7 +145,7 @@ new CompositeAggregation("my_buckets")
 
 ==== Handling Responses
 
-Each Composite aggregation bucket key is an `CompositeKey`, a specialized
+Each Composite aggregation bucket key is a `CompositeKey` type, a specialized
 `IReadOnlyDictionary<string, object>` type with methods to convert values to supported types
 
 [source,csharp]
@@ -185,3 +185,133 @@ foreach (var item in composite.Buckets)
 }
 ----
 
+[float]
+=== Missing buckets
+
+By default documents without a value for a given source are ignored.
+It is possible to include them in the response by setting missing_bucket to `true` (defaults to `false`):
+
+NOTE: Only available in Elasticsearch 6.4.0+
+
+==== Fluent DSL example
+
+[source,csharp]
+----
+a => a
+.Composite("my_buckets", date => date
+    .Sources(s => s
+        .Terms("branches", t => t
+            .Field(f => f.Branches.Suffix("keyword"))
+            .MissingBucket()
+            .Order(SortOrder.Ascending)
+        )
+    )
+    .Aggregations(childAggs => childAggs
+        .Nested("project_tags", n => n
+            .Path(p => p.Tags)
+            .Aggregations(nestedAggs => nestedAggs
+                .Terms("tags", avg => avg.Field(p => p.Tags.First().Name))
+            )
+        )
+    )
+)
+----
+
+==== Object Initializer syntax example
+
+[source,csharp]
+----
+new CompositeAggregation("my_buckets")
+{
+    Sources = new List<ICompositeAggregationSource>
+    {
+        new TermsCompositeAggregationSource("branches")
+        {
+            Field = Infer.Field<Project>(f => f.Branches.Suffix("keyword")),
+            MissingBucket = true,
+            Order = SortOrder.Ascending
+        }
+    },
+    Aggregations = new NestedAggregation("project_tags")
+    {
+        Path = Field<Project>(p => p.Tags),
+        Aggregations = new TermsAggregation("tags")
+        {
+            Field = Field<Project>(p => p.Tags.First().Name)
+        }
+    }
+}
+----
+
+[source,javascript]
+.Example json output
+----
+{
+  "my_buckets": {
+    "composite": {
+      "sources": [
+        {
+          "branches": {
+            "terms": {
+              "field": "branches.keyword",
+              "order": "asc",
+              "missing_bucket": true
+            }
+          }
+        }
+      ]
+    },
+    "aggs": {
+      "project_tags": {
+        "nested": {
+          "path": "tags"
+        },
+        "aggs": {
+          "tags": {
+            "terms": {
+              "field": "tags.name"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+----
+
+==== Handling Responses
+
+Each Composite aggregation bucket key is an `CompositeKey`, a specialized
+`IReadOnlyDictionary<string, object>` type with methods to convert values to supported types
+
+[source,csharp]
+----
+response.ShouldBeValid();
+
+var composite = response.Aggregations.Composite("my_buckets");
+composite.Should().NotBeNull();
+composite.Buckets.Should().NotBeNullOrEmpty();
+composite.AfterKey.Should().NotBeNull();
+
+if (TestConfiguration.Instance.InRange(">=6.3.0"))
+    composite.AfterKey.Should().HaveCount(1).And.ContainKeys("branches");
+
+var i = 0;
+foreach (var item in composite.Buckets)
+{
+    var key = item.Key;
+    key.Should().NotBeNull();
+
+    key.TryGetValue("branches", out string branches).Should().BeTrue("expected to find 'branches' in composite bucket");
+    if (i == 0) branches.Should().BeNull("First key should be null as we expect to have some projects with no branches");
+    else branches.Should().NotBeNullOrEmpty();
+
+    var nested = item.Nested("project_tags");
+    nested.Should().NotBeNull();
+
+    var nestedTerms = nested.Terms("tags");
+    nestedTerms.Buckets.Count.Should().BeGreaterThan(0);
+    i++;
+}
+----
+

docs/aggregations/bucket/date-histogram/date-histogram-aggregation-usage.asciidoc

@@ -7,7 +7,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Aggregations/Bucket/DateHistogram/DateHistogramAggregationUsageTests.cs. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Tests/Aggregations/Bucket/DateHistogram/DateHistogramAggregationUsageTests.cs. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////
@@ -114,14 +114,12 @@ new DateHistogramAggregation("projects_started_per_month")
 }
 ----
 
-=== Handling responses
+==== Handling responses
 
 The `AggregateDictionary found on `.Aggregations` on `ISearchResponse<T>` has several helper methods
 so we can fetch our aggregation results easily in the correct type.
 <<handling-aggregate-response, Be sure to read more about these helper methods>>
 
-==== Handling Responses
-
 [source,csharp]
 ----
 response.ShouldBeValid();

docs/aggregations/bucket/date-range/date-range-aggregation-usage.asciidoc

@@ -7,7 +7,7 @@
 ////
 IMPORTANT NOTE
 ==============
-This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Aggregations/Bucket/DateRange/DateRangeAggregationUsageTests.cs. 
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/6.x/src/Tests/Tests/Aggregations/Bucket/DateRange/DateRangeAggregationUsageTests.cs. 
 If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
 please modify the original csharp file found at the link and submit the PR with that change. Thanks!
 ////
@@ -93,14 +93,12 @@ new DateRangeAggregation("projects_date_ranges")
 }
 ----
 
-=== Handling Responses
+==== Handling Responses
 
 The `AggregateDictionary found on `.Aggregations` on `ISearchResponse<T>` has several helper methods
 so we can fetch our aggregation results easily in the correct type.
 <<handling-aggregate-response, Be sure to read more about these helper methods>>
 
-==== Handling Responses
-
 [source,csharp]
 ----
 response.ShouldBeValid();