Include similarity in index settings (#2924)

Non binary breaking back port of 648c3cf

Similarity settings are nested under settings>index
Add ClassicSimilarity
Add DFISimilarity
Add similarity settings to create index integration tests
Add XML comments for each similarity
Refactor SimilarityJsonConverter to not look up the NEST ISimilarity type with Type.GetType() but to infer it from the type property value directly.
Add Similarity to the updatable index settings

Closes #2890

Author: russcam

src/Nest/CommonAbstractions/DictionaryLike/IsADictionaryBase.cs

@@ -10,7 +10,7 @@ public abstract class IsADictionaryBase<TKey, TValue> : IIsADictionary<TKey, TVa
 		protected Dictionary<TKey, TValue> BackingDictionary { get; }
 		private ICollection<KeyValuePair<TKey, TValue>> Self => BackingDictionary;
 
-		protected IsADictionaryBase() { this.BackingDictionary = new Dictionary<TKey, TValue>(); }
+		protected IsADictionaryBase() => this.BackingDictionary = new Dictionary<TKey, TValue>();
 
 		protected IsADictionaryBase(IDictionary<TKey, TValue> backingDictionary)
 		{
@@ -52,14 +52,14 @@ void ICollection<KeyValuePair<TKey, TValue>>.Add(KeyValuePair<TKey, TValue> item
 
 		TValue IDictionary<TKey, TValue>.this[TKey key]
 		{
-			get { return this.BackingDictionary[key]; }
-			set { this.BackingDictionary[ValidateKey(key)] = value; }
+			get => this.BackingDictionary[key];
+			set => this.BackingDictionary[ValidateKey(key)] = value;
 		}
 
 		public TValue this[TKey key]
 		{
-			get { return this.BackingDictionary[key]; }
-			set { this.BackingDictionary[ValidateKey(key)] = value; }
+			get => this.BackingDictionary[key];
+			set => this.BackingDictionary[ValidateKey(key)] = value;
 		}
 	}
 }

src/Nest/IndexModules/IndexSettings/IndexState.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System;
+using Newtonsoft.Json;
 
 namespace Nest
 {
@@ -14,7 +15,8 @@ public interface IIndexState
 		[JsonProperty("mappings")]
 		IMappings Mappings { get; set; }
 
-		[JsonProperty("similarity")]
+		[JsonIgnore]
+		[Obsolete("Use Similarity within Settings. Removed in NEST 6.x")]
 		ISimilarities Similarity { get; set; }
 	}
 
@@ -26,6 +28,7 @@ public class IndexState : IIndexState
 
 		public IAliases Aliases { get; set; }
 
+		[Obsolete("Use Similarity within Settings. Removed in NEST 6.x")]
 		public ISimilarities Similarity { get; set; }
 	}
 }

src/Nest/IndexModules/IndexSettings/Settings/DynamicIndexSettings.cs

@@ -91,6 +91,11 @@ public interface IDynamicIndexSettings : IIsADictionary<string, object>
 		/// Configure analysis
 		/// </summary>
 		IAnalysis Analysis { get; set; }
+
+		/// <summary>
+		/// Configure similarity
+		/// </summary>
+		ISimilarities Similarity { get; set; }
 	}
 
 	public class DynamicIndexSettings : IsADictionaryBase<string, object>, IDynamicIndexSettings
@@ -147,6 +152,9 @@ public DynamicIndexSettings(IDictionary<string, object> container) : base(contai
 		/// <inheritdoc/>
 		public IAnalysis Analysis { get; set; }
 
+		/// <inheritdoc/>
+		public ISimilarities Similarity { get; set; }
+
 		/// <summary>
 		/// Add any setting to the index
 		/// </summary>
@@ -227,7 +235,12 @@ public TDescriptor Translog(Func<TranslogSettingsDescriptor, ITranslogSettings>
 		public TDescriptor UnassignedNodeLeftDelayedTimeout(Time time) =>
 			Assign(a => a.UnassignedNodeLeftDelayedTimeout = time);
 
+		/// <inheritdoc/>
 		public TDescriptor Analysis(Func<AnalysisDescriptor, IAnalysis> selector) =>
 			Assign(a => a.Analysis = selector?.Invoke(new AnalysisDescriptor()));
+
+		/// <inheritdoc/>
+		public TDescriptor Similarity(Func<SimilaritiesDescriptor, IPromise<ISimilarities>> selector) =>
+			Assign(a => a.Similarity = selector?.Invoke(new SimilaritiesDescriptor())?.Value);
 	}
 }

src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsConverter.cs

@@ -75,40 +75,42 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
 			d[UpdatableIndexSettings.SlowlogIndexingSource] = indexing?.Source;
 
 			d[UpdatableIndexSettings.Analysis] = ds.Analysis;
+			d[UpdatableIndexSettings.Similarity] = ds.Similarity;
 
 			var indexSettings = value as IIndexSettings;
-            d[FixedIndexSettings.NumberOfShards] = indexSettings?.NumberOfShards;
+			d[FixedIndexSettings.NumberOfShards] = indexSettings?.NumberOfShards;
             d[FixedIndexSettings.RoutingPartitionSize] = indexSettings?.RoutingPartitionSize;
             d[UpdatableIndexSettings.StoreType] = indexSettings?.FileSystemStorageImplementation;
             d[UpdatableIndexSettings.QueriesCacheEnabled] = indexSettings?.Queries?.Cache?.Enabled;
 
 			base.WriteJson(writer, d, serializer);
 		}
 
+		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+		{
+			var s = new IndexSettings();
+			SetKnownIndexSettings(reader, serializer, s);
+			if (!typeof (IUpdateIndexSettingsRequest).IsAssignableFrom(objectType)) return s;
 
-		public JObject Flatten(JObject original, string prefix = "", JObject newObject = null)
+			var request = new UpdateIndexSettingsRequest() { IndexSettings =  s};
+			return request;
+		}
+
+		private static JObject Flatten(JObject original, string prefix = "", JObject newObject = null)
 		{
 			newObject = newObject ?? new JObject();
 			foreach (var property in original.Properties())
 			{
-				if (property.Value is JObject && property.Name != UpdatableIndexSettings.Analysis)
+				if (property.Value is JObject &&
+				    property.Name != UpdatableIndexSettings.Analysis &&
+				    property.Name != UpdatableIndexSettings.Similarity)
 					Flatten(property.Value.Value<JObject>(), prefix + property.Name + ".", newObject);
 				else newObject.Add(prefix + property.Name, property.Value);
 			}
 			return newObject;
 		}
 
-		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-		{
-			var s = new IndexSettings();
-			SetKnownIndexSettings(reader, serializer, s);
-			if (!typeof (IUpdateIndexSettingsRequest).IsAssignableFrom(objectType)) return s;
-
-			var request = new UpdateIndexSettingsRequest() { IndexSettings =  s};
-			return request;
-		}
-
-		private void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s)
+		private static void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s)
 		{
 			var settings = Flatten(JObject.Load(reader)).Properties().ToDictionary(kv => kv.Name);
 
@@ -194,6 +196,8 @@ private void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer,
 				var setting = kv.Value;
 				if (kv.Key == UpdatableIndexSettings.Analysis || kv.Key == "index.analysis")
 					s.Analysis = setting.Value.Value<JObject>().ToObject<Analysis>(serializer);
+				if (kv.Key == UpdatableIndexSettings.Similarity || kv.Key == "index.similarity")
+					s.Similarity = setting.Value.Value<JObject>().ToObject<Similarities>(serializer);
 				else
 				{
 					dict?.Add(kv.Key, serializer.Deserialize(kv.Value.Value.CreateReader()));

src/Nest/IndexModules/IndexSettings/Settings/UpdatableIndexSettings.cs

@@ -50,6 +50,8 @@ public static class UpdatableIndexSettings
 		public const string MergeSchedulerMaxThreadCount = "index.merge.scheduler.max_thread_count";
 		public const string MergeSchedulerAutoThrottle = "index.merge.scheduler.auto_throttle";
 
+		public const string Similarity = "similarity";
+
 		public const string SlowlogSearchThresholdQueryWarn = "index.search.slowlog.threshold.query.warn";
 		public const string SlowlogSearchThresholdQueryInfo = "index.search.slowlog.threshold.query.info";
 		public const string SlowlogSearchThresholdQueryDebug = "index.search.slowlog.threshold.query.debug";

src/Nest/IndexModules/Similarity/BM25Similarity.cs

@@ -3,8 +3,8 @@
 namespace Nest
 {
 	/// <summary>
-	/// BM25 Similarity. Introduced in Stephen E. Robertson, Steve Walker, Susan Jones, Micheline Hancock-Beaulieu, 
-	/// and Mike Gatford. Okapi at TREC-3. In Proceedings of the Third Text REtrieval Conference (TREC 1994). Gaithersburg, USA, November 1994.
+	/// BM25 Similarity. Introduced in Stephen E. Robertson, Steve Walker, Susan Jones, Micheline Hancock-Beaulieu,
+	/// and Mike Gatford. Okapi at TREC-3. In Proceedings of the Third Text Retrieval Conference (TREC 1994). Gaithersburg, USA, November 1994.
 	/// </summary>
 	public interface IBM25Similarity : ISimilarity
 	{
@@ -41,7 +41,7 @@ public class BM25Similarity : IBM25Similarity
 		public bool? DiscountOverlaps { get; set; }
 	}
 	/// <inheritdoc/>
-	public class BM25SimilarityDescriptor 
+	public class BM25SimilarityDescriptor
 		: DescriptorBase<BM25SimilarityDescriptor, IBM25Similarity>, IBM25Similarity
 	{
 		string ISimilarity.Type => "BM25";

src/Nest/IndexModules/Similarity/ClassicSimilarity.cs

@@ -0,0 +1,37 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// The classic similarity that is based on the TF/IDF model.
+	/// </summary>
+	public interface IClassicSimilarity : ISimilarity
+	{
+		/// <summary>
+		/// Determines whether overlap tokens (tokens with 0 position increment) are ignored when computing norm.
+		/// By default this is <c>true</c>, meaning overlap tokens do not count when computing norms.
+		/// </summary>
+		[JsonProperty("discount_overlaps")]
+		bool? DiscountOverlaps { get; set; }
+	}
+
+	/// <inheritdoc />
+	public class ClassicSimilarity : IClassicSimilarity
+	{
+		public string Type => "classic";
+
+		/// <inheritdoc />
+		public bool? DiscountOverlaps { get; set; }
+	}
+
+	/// <inheritdoc />
+	public class ClassicSimilarityDescriptor
+		: DescriptorBase<ClassicSimilarityDescriptor, IClassicSimilarity>, IClassicSimilarity
+	{
+		string ISimilarity.Type => "classic";
+		bool? IClassicSimilarity.DiscountOverlaps { get; set; }
+
+		/// <inheritdoc />
+		public ClassicSimilarityDescriptor DiscountOverlaps(bool discount = true) => Assign(a => a.DiscountOverlaps = discount);
+	}
+}

src/Nest/IndexModules/Similarity/CustomSimilarity.cs

@@ -4,33 +4,41 @@
 
 namespace Nest
 {
+	/// <summary>
+	/// A custom similarity
+	/// </summary>
 	public interface ICustomSimilarity : ISimilarity, IIsADictionary<string, object> { }
 
 	/// <inheritdoc/>
 	public class CustomSimilarity : IsADictionaryBase<string, object>, ICustomSimilarity
 	{
-		public string Type { get { return this["type"] as string; } set { this.Add("type", value); } }
+		public string Type
+		{
+			get => this["type"] as string;
+			set => this.Add("type", value);
+		}
 
 		public CustomSimilarity(string type)
 		{
 			if (!string.IsNullOrEmpty(type)) this.Type = type;
 		}
 
 		internal CustomSimilarity(IDictionary<string, object> container) : base(container) { }
+
 		internal CustomSimilarity(Dictionary<string, object> container)
-			: base(container.Select(kv => kv).ToDictionary(kv => kv.Key, kv => kv.Value))
-		{}
+			: base(container.Select(kv => kv).ToDictionary(kv => kv.Key, kv => kv.Value)) { }
 
 		public void Add(string key, object value) => BackingDictionary.Add(key, value);
 	}
+
 	/// <inheritdoc/>
 	public class CustomSimilarityDescriptor
 		: IsADictionaryDescriptorBase<CustomSimilarityDescriptor, ICustomSimilarity, string, object>
 	{
-
 		public CustomSimilarityDescriptor() : base(new CustomSimilarity(string.Empty)) { }
 
 		internal CustomSimilarityDescriptor Type(string type) => Assign("type", type);
+
 		public CustomSimilarityDescriptor Add(string key, object value) => Assign(key, value);
 	}
 

src/Nest/IndexModules/Similarity/DFI/DFIIndependenceMeasure.cs

@@ -0,0 +1,22 @@
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace Nest
+{
+	/// <summary>
+	/// <see cref="IDFISimilarity"/> independence measure
+	/// </summary>
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum DFIIndependenceMeasure
+	{
+		[EnumMember(Value = "standardized")]
+		Standardized,
+
+		[EnumMember(Value = "saturated")]
+		Saturated,
+
+		[EnumMember(Value = "chisquared")]
+		ChiSquared
+	}
+}

src/Nest/IndexModules/Similarity/DFI/DFISimilarity.cs

@@ -0,0 +1,37 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// Similarity that implements the divergence from independence model
+	/// </summary>
+	public interface IDFISimilarity : ISimilarity
+	{
+		/// <summary>
+		/// The independence measure
+		/// </summary>
+		[JsonProperty("independence_measure")]
+		DFIIndependenceMeasure? IndependenceMeasure { get; set; }
+	}
+
+	/// <inheritdoc/>
+	public class DFISimilarity : IDFISimilarity
+	{
+		public string Type => "DFI";
+
+		/// <inheritdoc/>
+		public DFIIndependenceMeasure? IndependenceMeasure { get; set; }
+	}
+
+	/// <inheritdoc/>
+	public class DFISimilarityDescriptor
+		: DescriptorBase<DFISimilarityDescriptor, IDFISimilarity>, IDFISimilarity
+	{
+		string ISimilarity.Type => "DFI";
+		DFIIndependenceMeasure? IDFISimilarity.IndependenceMeasure { get; set; }
+
+		/// <inheritdoc/>
+		public DFISimilarityDescriptor IndependenceMeasure(DFIIndependenceMeasure independenceMeasure) =>
+			Assign(a => a.IndependenceMeasure = independenceMeasure);
+	}
+}