Add phrase_limit to highlighting

- Add Encoder and PhraseLimit to integration test
- Deprecate Encoder on HighlightField

Closes #2851

(cherry picked from commit 56d9166)

Author: russcam

src/Nest/Search/Search/Highlighting/HighlightField.cs

@@ -75,7 +75,8 @@ public interface IHighlightField
 		/// Define how highlighted text will be encoded.
 		/// It can be either default (no encoding) or html (will escape html, if you use html highlighting tags).
 		/// </summary>
-		[JsonProperty("encoder")]
+		[JsonIgnore]
+		[Obsolete("Encoder not valid on highlight field. Removed in NEST 6.x")]
 		string Encoder { get; set; }
 
 		/// <summary>
@@ -133,6 +134,7 @@ public interface IHighlightField
 		/// </summary>
 		[JsonProperty("fragmenter")]
 		HighlighterFragmenter? Fragmenter { get; set; }
+
 		/// <summary>
 		/// The type of highlighter to use. Can be a defined or custom highlighter
 		/// </summary>
@@ -159,6 +161,15 @@ public interface IHighlightField
 		/// </summary>
 		[JsonProperty("highlight_query")]
 		QueryContainer HighlightQuery { get; set; }
+
+		/// <summary>
+		/// Controls the number of matching phrases in a document that are considered. Prevents the
+		/// <see cref="HighlighterType.Fvh"/> highlighter from analyzing too many phrases and consuming too much memory.
+		/// When using matched_fields, <see cref="PhraseLimit"/> phrases per matched field are considered. Raising the limit increases query time
+		/// and consumes more memory. Only supported by the <see cref="HighlighterType.Fvh"/> highlighter. Defaults to 256.
+		/// </summary>
+		[JsonProperty("phrase_limit")]
+		int? PhraseLimit { get; set; }
 	}
 
 	public class HighlightField : IHighlightField
@@ -180,6 +191,7 @@ public class HighlightField : IHighlightField
 		/// <inheritdoc/>
 		public int? BoundaryMaxScan { get; set; }
 		/// <inheritdoc/>
+		[Obsolete("Encoder not valid on highlight field. Removed in NEST 6.x")]
 		public string Encoder { get; set; }
 		/// <inheritdoc/>
 		public string Order { get; set; }
@@ -205,6 +217,8 @@ public class HighlightField : IHighlightField
 		public Fields MatchedFields { get; set; }
 		/// <inheritdoc/>
 		public QueryContainer HighlightQuery { get; set; }
+		/// <inheritdoc/>
+		public int? PhraseLimit { get; set; }
 	}
 
 	public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescriptor<T>,IHighlightField>, IHighlightField
@@ -218,6 +232,7 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
 		int? IHighlightField.NumberOfFragments { get; set; }
 		int? IHighlightField.FragmentOffset { get; set; }
 		int? IHighlightField.BoundaryMaxScan { get; set; }
+		[Obsolete("Encoder not valid on highlight field. Removed in NEST 6.x")]
 		string IHighlightField.Encoder { get; set; }
 		string IHighlightField.Order { get; set; }
 		string IHighlightField.TagsSchema { get; set; }
@@ -231,6 +246,7 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
 		bool? IHighlightField.ForceSource { get; set; }
 		Fields IHighlightField.MatchedFields { get; set; }
 		QueryContainer IHighlightField.HighlightQuery { get; set; }
+		int? IHighlightField.PhraseLimit { get; set; }
 
 		/// <inheritdoc/>
 		public HighlightFieldDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
@@ -278,6 +294,7 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
 		public HighlightFieldDescriptor<T> FragmentOffset(int? fragmentOffset) => Assign(a => a.FragmentOffset = fragmentOffset);
 
 		/// <inheritdoc/>
+		[Obsolete("Encoder not valid on highlight field. Removed in NEST 6.x")]
 		public HighlightFieldDescriptor<T> Encoder(string encoder) => Assign(a => a.Encoder = encoder);
 
 		/// <inheritdoc/>
@@ -311,5 +328,8 @@ public HighlightFieldDescriptor<T> HighlightQuery(Func<QueryContainerDescriptor<
 
 		/// <inheritdoc/>
 		public HighlightFieldDescriptor<T> Fragmenter(HighlighterFragmenter? fragmenter) => Assign(a => a.Fragmenter = fragmenter);
+
+		/// <inheritdoc/>
+		public HighlightFieldDescriptor<T> PhraseLimit(int phraseLimit) => Assign(a => a.PhraseLimit = phraseLimit);
 	}
 }

src/Tests/Search/Request/HighlightingUsageTests.cs

@@ -29,9 +29,9 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 		{
 			query = new
 			{
-				match = new JObject
+				match = new Dictionary<string, object>
 				{
-					{ "name.standard", new JObject
+					{ "name.standard", new Dictionary<string, object>
 						{
 							{ "query", "Upton Sons Shield Rice Rowe Roberts" }
 						}
@@ -42,9 +42,10 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 			{
 				pre_tags = new[] { "<tag1>" },
 				post_tags = new[] { "</tag1>" },
-				fields = new JObject
+				encoder = "html",
+				fields = new Dictionary<string, object>
 				{
-					{ "name.standard", new JObject
+					{ "name.standard", new Dictionary<string, object>
 						{
 							{ "type", "plain" },
 							{ "force_source", true},
@@ -54,17 +55,18 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 							{ "no_match_size", 150 }
 						}
 					},
-					{ "leadDeveloper.firstName", new JObject
+					{ "leadDeveloper.firstName", new Dictionary<string, object>
 						{
 							{ "type", "fvh" },
+							{ "phrase_limit", 10 },
 							{ "boundary_max_scan", 50 },
-							{ "pre_tags", new JArray { "<name>" } },
-							{ "post_tags", new JArray { "</name>" } },
-							{ "highlight_query", new JObject
+							{ "pre_tags", new [] { "<name>" } },
+							{ "post_tags", new [] { "</name>" } },
+							{ "highlight_query", new Dictionary<string, object>
 								{
-									{ "match", new JObject
+									{ "match", new Dictionary<string, object>
 										{
-											{ "leadDeveloper.firstName", new JObject
+											{ "leadDeveloper.firstName", new Dictionary<string, object>
 												{
 													{ "query", "Kurt Edgardo Naomi Dariana Justice Felton" }
 												}
@@ -75,16 +77,20 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 							}
 						}
 					},
-					{ "state.offsets", new JObject
+					{ "leadDeveloper.lastName", new Dictionary<string, object>
 						{
-							{ "type", "postings" },
-							{ "pre_tags", new JArray { "<state>" } },
-							{ "post_tags", new JArray { "</state>" } },
-							{ "highlight_query", new JObject
+							{ "type", "unified" },
+							{ "pre_tags", new [] { "<name>" } },
+							{ "post_tags", new [] { "</name>" } },
+							{ "highlight_query", new Dictionary<string, object>
 								{
-									{ "terms", new JObject
+									{ "match", new Dictionary<string, object>
 										{
-											{ "state.offsets", new JArray { "stable" , "bellyup" } }
+											{ "leadDeveloper.lastName", new Dictionary<string, object>
+												{
+													{ "query", LastNameSearch }
+												}
+											}
 										}
 									}
 								}
@@ -105,6 +111,7 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 			.Highlight(h => h
 				.PreTags("<tag1>")
 				.PostTags("</tag1>")
+				.Encoder("html")
 				.Fields(
 					fs => fs
 						.Field(p => p.Name.Suffix("standard"))
@@ -120,24 +127,22 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 						.PreTags("<name>")
 						.PostTags("</name>")
 						.BoundaryMaxScan(50)
+						.PhraseLimit(10)
 						.HighlightQuery(q => q
 							.Match(m => m
 								.Field(p => p.LeadDeveloper.FirstName)
 								.Query("Kurt Edgardo Naomi Dariana Justice Felton")
 							)
 						),
 					fs => fs
-						.Field(p => p.State.Suffix("offsets"))
-						.Type(HighlighterType.Postings)
-						.PreTags("<state>")
-						.PostTags("</state>")
+						.Field(p => p.LeadDeveloper.LastName)
+						.Type(HighlighterType.Unified)
+						.PreTags("<name>")
+						.PostTags("</name>")
 						.HighlightQuery(q => q
-							.Terms(t => t
-								.Field(f => f.State.Suffix("offsets"))
-								.Terms(
-									StateOfBeing.Stable.ToString().ToLowerInvariant(),
-									StateOfBeing.BellyUp.ToString().ToLowerInvariant()
-								)
+							.Match(m => m
+								.Field(p => p.LeadDeveloper.LastName)
+								.Query(LastNameSearch)
 							)
 						)
 				)
@@ -155,6 +160,7 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 				{
 					PreTags = new[] { "<tag1>" },
 					PostTags = new[] { "</tag1>" },
+					Encoder = "html",
 					Fields = new Dictionary<Field, IHighlightField>
 					{
 						{ "name.standard", new HighlightField
@@ -170,6 +176,7 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 						{ "leadDeveloper.firstName", new HighlightField
 							{
 								Type = "fvh",
+								PhraseLimit = 10,
 								BoundaryMaxScan = 50,
 								PreTags = new[] { "<name>"},
 								PostTags = new[] { "</name>"},
@@ -180,15 +187,15 @@ public HighlightingUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : ba
 								}
 							}
 						},
-						{ "state.offsets", new HighlightField
+						{ "leadDeveloper.lastName", new HighlightField
 							{
-								Type = HighlighterType.Postings,
-								PreTags = new[] { "<state>"},
-								PostTags = new[] { "</state>"},
-								HighlightQuery = new TermsQuery
+								Type = HighlighterType.Unified,
+								PreTags = new[] { "<name>"},
+								PostTags = new[] { "</name>"},
+								HighlightQuery = new MatchQuery
 								{
-									Field = "state.offsets",
-									Terms = new [] { "stable", "bellyup" }
+									Field = "leadDeveloper.lastName",
+									Query = LastNameSearch
 								}
 							}
 						}