Add DailyModelSnapshotRetentionAfterDays to ml jobs (#4812) (#4831)

Relates: elastic/elasticsearch#56125, #4803

This commit adds the DailyModelSnapshotRetentionAfterDays
to ML jobs. It also updates the XML comment for
ModelSnapshotRetentionDays to align with the new default in
Elasticsearch 7.8.

Co-authored-by: Russ Cam <russ.cam@elastic.co>

Author: github-actions[bot]

src/Nest/XPack/MachineLearning/Job/Config/Job.cs

@@ -90,11 +90,27 @@ public class Job
 
 		/// <summary>
 		/// The time in days that model snapshots are retained for the job.
-		/// Older snapshots are deleted. The default value is 1 day.
+		/// Older snapshots are deleted. The default value is 10 days in Elasticsearch 7.8.0+
+		/// and 1 day in older versions.
 		/// </summary>
 		[DataMember(Name = "model_snapshot_retention_days")]
 		public long? ModelSnapshotRetentionDays { get; set; }
 
+		/// <summary>
+		/// Specifies a number of days between 0 and the value of <see cref="ModelSnapshotRetentionDays"/>.
+		/// After this period of time, only the first model snapshot per day is retained for this job.
+		/// Age is calculated relative to the timestamp of the newest model snapshot. For new jobs, the default
+		/// value is <c>1</c>, which means that all snapshots are retained for one day. Older snapshots
+		/// are thinned out such that only one per day is retained. For jobs that were
+		/// created before this setting was available, the default value matches the
+		/// <see cref="ModelSnapshotRetentionDays"/> value, which preserves the original behavior
+		/// and no thinning out of model snapshots occurs.
+		/// <para />
+		/// Available in Elasticsearch 7.8.0+
+		/// </summary>
+		[DataMember(Name = "daily_model_snapshot_retention_after_days")]
+		public long? DailyModelSnapshotRetentionAfterDays { get; set; }
+
 		/// <summary>
 		/// Advanced configuration option. The period over which adjustments to the score are applied, as new data
 		/// is seen. The default value is the longer of 30 days or 100 bucket spans.

src/Nest/XPack/MachineLearning/MachineLearningInfo/MachineLearningInfoResponse.cs

@@ -39,6 +39,12 @@ public class AnomalyDetectors
 		[DataMember(Name = "model_snapshot_retention_days")]
 		public int ModelSnapshotRetentionDays { get; internal set; }
 
+		/// <summary>
+		/// Available in Elasticsearch 7.8.0+
+		/// </summary>
+		[DataMember(Name = "daily_model_snapshot_retention_after_days")]
+		public long DailyModelSnapshotRetentionAfterDays { get; internal set; }
+
 		[DataMember(Name = "categorization_analyzer")]
 		public CategorizationAnalyzer CategorizationAnalyzer { get; internal set; }
 	}

src/Nest/XPack/MachineLearning/PutJob/PutJobRequest.cs

@@ -48,11 +48,27 @@ public partial interface IPutJobRequest
 
 		/// <summary>
 		/// The time in days that model snapshots are retained for the job.
-		/// Older snapshots are deleted. The default value is 1 day.
+		/// Older snapshots are deleted. The default value is 10 days in Elasticsearch 7.8.0+
+		/// and 1 day in older versions.
 		/// </summary>
 		[DataMember(Name ="model_snapshot_retention_days")]
 		long? ModelSnapshotRetentionDays { get; set; }
 
+		/// <summary>
+		/// Specifies a number of days between 0 and the value of <see cref="ModelSnapshotRetentionDays"/>.
+		/// After this period of time, only the first model snapshot per day is retained for this job.
+		/// Age is calculated relative to the timestamp of the newest model snapshot. For new jobs, the default
+		/// value is <c>1</c>, which means that all snapshots are retained for one day. Older snapshots
+		/// are thinned out such that only one per day is retained. For jobs that were
+		/// created before this setting was available, the default value matches the
+		/// <see cref="ModelSnapshotRetentionDays"/> value, which preserves the original behavior
+		/// and no thinning out of model snapshots occurs.
+		/// <para />
+		/// Available in Elasticsearch 7.8.0+
+		/// </summary>
+		[DataMember(Name ="daily_model_snapshot_retention_after_days")]
+		long? DailyModelSnapshotRetentionAfterDays { get; set; }
+
 		/// <summary>
 		/// The name of the index in which to store the machine learning results.
 		/// The default value is shared, which corresponds to the index name .ml-anomalies-shared.
@@ -93,6 +109,9 @@ public partial class PutJobRequest
 		/// <inheritdoc />
 		public long? ModelSnapshotRetentionDays { get; set; }
 
+		/// <inheritdoc />
+		public long? DailyModelSnapshotRetentionAfterDays { get; set; }
+
 		/// <inheritdoc />
 		public IndexName ResultsIndexName { get; set; }
 
@@ -109,41 +128,46 @@ public partial class PutJobDescriptor<TDocument> where TDocument : class
 		string IPutJobRequest.Description { get; set; }
 		IModelPlotConfig IPutJobRequest.ModelPlotConfig { get; set; }
 		long? IPutJobRequest.ModelSnapshotRetentionDays { get; set; }
+		long? IPutJobRequest.DailyModelSnapshotRetentionAfterDays { get; set; }
 		IndexName IPutJobRequest.ResultsIndexName { get; set; }
 		bool? IPutJobRequest.AllowLazyOpen { get; set; }
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.AnalysisConfig"/>
 		public PutJobDescriptor<TDocument> AnalysisConfig(Func<AnalysisConfigDescriptor<TDocument>, IAnalysisConfig> selector) =>
 			Assign(selector, (a, v) => a.AnalysisConfig = v?.Invoke(new AnalysisConfigDescriptor<TDocument>()));
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.AnalysisLimits"/>
 		public PutJobDescriptor<TDocument> AnalysisLimits(Func<AnalysisLimitsDescriptor, IAnalysisLimits> selector) =>
 			Assign(selector, (a, v) => a.AnalysisLimits = v?.Invoke(new AnalysisLimitsDescriptor()));
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.DataDescription"/>
 		public PutJobDescriptor<TDocument> DataDescription(Func<DataDescriptionDescriptor<TDocument>, IDataDescription> selector) =>
 			Assign(selector.InvokeOrDefault(new DataDescriptionDescriptor<TDocument>()), (a, v) => a.DataDescription = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.Description"/>
 		public PutJobDescriptor<TDocument> Description(string description) => Assign(description, (a, v) => a.Description = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.ModelPlotConfig"/>
 		public PutJobDescriptor<TDocument> ModelPlot(Func<ModelPlotConfigDescriptor<TDocument>, IModelPlotConfig> selector) =>
 			Assign(selector, (a, v) => a.ModelPlotConfig = v?.Invoke(new ModelPlotConfigDescriptor<TDocument>()));
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.ModelSnapshotRetentionDays"/>
 		public PutJobDescriptor<TDocument> ModelSnapshotRetentionDays(long? modelSnapshotRetentionDays) =>
 			Assign(modelSnapshotRetentionDays, (a, v) => a.ModelSnapshotRetentionDays = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.DailyModelSnapshotRetentionAfterDays"/>
+		public PutJobDescriptor<TDocument> DailyModelSnapshotRetentionAfterDays(long? dailyModelSnapshotRetentionAfterDays) =>
+			Assign(dailyModelSnapshotRetentionAfterDays, (a, v) => a.DailyModelSnapshotRetentionAfterDays = v);
+
+		/// <inheritdoc cref="IPutJobRequest.ResultsIndexName"/>
 		public PutJobDescriptor<TDocument> ResultsIndexName(IndexName indexName) =>
 			Assign(indexName, (a, v) => a.ResultsIndexName = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.ResultsIndexName"/>
 		public PutJobDescriptor<TDocument> ResultsIndexName<TIndex>() =>
 			Assign(typeof(TIndex), (a, v) => a.ResultsIndexName = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IPutJobRequest.AllowLazyOpen"/>
 		public PutJobDescriptor<TDocument> AllowLazyOpen(bool? allowLazyOpen = true) =>
 			Assign(allowLazyOpen, (a, v) => a.AllowLazyOpen = v);
 	}

src/Nest/XPack/MachineLearning/UpdateJob/UpdateJobRequest.cs

@@ -52,11 +52,27 @@ public partial interface IUpdateJobRequest
 
 		/// <summary>
 		/// The time in days that model snapshots are retained for the job.
-		/// Older snapshots are deleted. The default value is 1 day.
+		/// Older snapshots are deleted. The default value is 10 days in Elasticsearch 7.8.0+
+		/// and 1 day in older versions.
 		/// </summary>
 		[DataMember(Name ="model_snapshot_retention_days")]
 		long? ModelSnapshotRetentionDays { get; set; }
 
+		/// <summary>
+		/// Specifies a number of days between 0 and the value of <see cref="ModelSnapshotRetentionDays"/>.
+		/// After this period of time, only the first model snapshot per day is retained for this job.
+		/// Age is calculated relative to the timestamp of the newest model snapshot. For new jobs, the default
+		/// value is <c>1</c>, which means that all snapshots are retained for one day. Older snapshots
+		/// are thinned out such that only one per day is retained. For jobs that were
+		/// created before this setting was available, the default value matches the
+		/// <see cref="ModelSnapshotRetentionDays"/> value, which preserves the original behavior
+		/// and no thinning out of model snapshots occurs.
+		/// <para />
+		/// Available in Elasticsearch 7.8.0+
+		/// </summary>
+		[DataMember(Name ="daily_model_snapshot_retention_after_days")]
+		long? DailyModelSnapshotRetentionAfterDays { get; set; }
+
 		/// <summary>
 		/// The period over which adjustments to the score are applied, as new data is seen.
 		/// </summary>
@@ -104,6 +120,9 @@ public partial class UpdateJobRequest
 		/// <inheritdoc />
 		public long? ModelSnapshotRetentionDays { get; set; }
 
+		/// <inheritdoc />
+		public long? DailyModelSnapshotRetentionAfterDays { get; set; }
+
 		/// <inheritdoc />
 		public long? RenormalizationWindowDays { get; set; }
 
@@ -123,43 +142,47 @@ public partial class UpdateJobDescriptor<TDocument> where TDocument : class
 		string IUpdateJobRequest.Description { get; set; }
 		IModelPlotConfigEnabled IUpdateJobRequest.ModelPlotConfig { get; set; }
 		long? IUpdateJobRequest.ModelSnapshotRetentionDays { get; set; }
+		long? IUpdateJobRequest.DailyModelSnapshotRetentionAfterDays { get; set; }
 		long? IUpdateJobRequest.RenormalizationWindowDays { get; set; }
 		long? IUpdateJobRequest.ResultsRetentionDays { get; set; }
-
 		bool? IUpdateJobRequest.AllowLazyOpen { get; set; }
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.AnalysisLimits" />
 		public UpdateJobDescriptor<TDocument> AnalysisLimits(Func<AnalysisMemoryLimitDescriptor, IAnalysisMemoryLimit> selector) =>
 			Assign(selector, (a, v) => a.AnalysisLimits = v?.Invoke(new AnalysisMemoryLimitDescriptor()));
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.BackgroundPersistInterval" />
 		public UpdateJobDescriptor<TDocument> BackgroundPersistInterval(Time backgroundPersistInterval) =>
 			Assign(backgroundPersistInterval, (a, v) => a.BackgroundPersistInterval = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.CustomSettings" />
 		public UpdateJobDescriptor<TDocument> CustomSettings(Func<FluentDictionary<string, object>, FluentDictionary<string, object>> customSettingsDictionary
 		) =>
 			Assign(customSettingsDictionary(new FluentDictionary<string, object>()), (a, v) => a.CustomSettings = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.Description" />
 		public UpdateJobDescriptor<TDocument> Description(string description) => Assign(description, (a, v) => a.Description = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.ModelPlotConfig" />
 		public UpdateJobDescriptor<TDocument> ModelPlot(Func<ModelPlotConfigEnabledDescriptor<TDocument>, IModelPlotConfigEnabled> selector) =>
 			Assign(selector, (a, v) => a.ModelPlotConfig = v?.Invoke(new ModelPlotConfigEnabledDescriptor<TDocument>()));
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.ModelSnapshotRetentionDays" />
 		public UpdateJobDescriptor<TDocument> ModelSnapshotRetentionDays(long? modelSnapshotRetentionDays) =>
 			Assign(modelSnapshotRetentionDays, (a, v) => a.ModelSnapshotRetentionDays = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.DailyModelSnapshotRetentionAfterDays" />
+		public UpdateJobDescriptor<TDocument> DailyModelSnapshotRetentionAfterDays(long? dailyModelSnapshotRetentionAfterDays) =>
+			Assign(dailyModelSnapshotRetentionAfterDays, (a, v) => a.DailyModelSnapshotRetentionAfterDays = v);
+
+		/// <inheritdoc cref="IUpdateJobRequest.RenormalizationWindowDays" />
 		public UpdateJobDescriptor<TDocument> RenormalizationWindowDays(long? renormalizationWindowDays) =>
 			Assign(renormalizationWindowDays, (a, v) => a.RenormalizationWindowDays = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.ResultsRetentionDays" />
 		public UpdateJobDescriptor<TDocument> ResultsRetentionDays(long? resultsRetentionDays) => Assign(resultsRetentionDays, (a, v) => a.ResultsRetentionDays = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IUpdateJobRequest.AllowLazyOpen" />
 		public UpdateJobDescriptor<TDocument> AllowLazyOpen(bool? allowLazyOpen = true) =>
 			Assign(allowLazyOpen, (a, v) => a.AllowLazyOpen = v);
 	}

tests/Tests/XPack/MachineLearning/MachineLearningInfo/MachineLearningInfoApiTests.cs

@@ -47,6 +47,9 @@ protected override void ExpectResponse(MachineLearningInfoResponse response)
 				? 10
 				: 1);
 
+			if (TestClient.Configuration.InRange(">=7.8.0"))
+				anomalyDetectors.DailyModelSnapshotRetentionAfterDays.Should().Be(1);
+
 			response.Defaults.Datafeeds.ScrollSize.Should().Be(1000);
 
 			if (Cluster.ClusterConfiguration.Version >= "7.6.0")