Continuing tweaking search relevance

config/synonyms.yml

@@ -1,6 +1,8 @@
 synonyms:
   - [ ".net", "c#", "csharp", "dotnet", "net" ]
-  - [ "esql", "es|ql" ]
+  - [ "esql", "es|ql => esql" ]
+  - [ "data-stream", "data stream", "datastream => data-streams"]
+  - [ "data-streams", "data streams", "datastreams"]
   - [ "motlp", "managed otlp" ]
   - [ "s3", "aws s3", "amazon s3" ]
   - [ "es", "elasticsearch" ]
@@ -27,7 +29,7 @@ synonyms:
   - [ "edot", "elastic distribution of opentelemetry" ]
   - [ "k8s", "kubernetes" ]
   - [ "ecs", "elastic common schema" ]
-  - [ "ml", "machine learning" ]
+  - [ "machine-learning", "machine learning", "ml => machine learning" ]
   - [ "eis", "elastic inference service" ]
   - [ "traffic filter", "network security" ]
   - [ "sso", "single sign-on" ]

src/Elastic.Documentation.Navigation/INavigationTraversable.cs

@@ -7,6 +7,42 @@
 
 namespace Elastic.Documentation.Navigation;
 
+public static class NavigationExtensions
+{
+	extension(INavigationItem navigationItem)
+	{
+		public INavigationItem[] GetParents()
+		{
+			var parents = new List<INavigationItem>();
+			var parent = navigationItem.Parent;
+			do
+			{
+				if (parent is null)
+					continue;
+				if (parents.All(i => i.Url != parent.Url))
+					parents.Add(parent);
+
+				parent = parent.Parent;
+			} while (parent != null);
+
+			return [.. parents];
+		}
+
+		public int NavigationDepth => navigationItem.GetParents().Length;
+
+		public string? NavigationSection
+		{
+			get
+			{
+				var parents = navigationItem.GetParents();
+				if (parents.Length <= 1)
+					return navigationItem.NavigationTitle.ToLowerInvariant();
+				return parents.Reverse().Skip(1).FirstOrDefault()?.NavigationTitle.ToLowerInvariant();
+			}
+		}
+	}
+}
+
 public interface INavigationTraversable
 {
 	ConditionalWeakTable<IDocumentationFile, INavigationItem> NavigationDocumentationFileLookup { get; }
@@ -71,22 +107,7 @@ INavigationItem GetNavigationFor(IDocumentationFile file) =>
 		NavigationDocumentationFileLookup.TryGetValue(file, out var navigation)
 			? navigation : throw new InvalidOperationException($"Could not find {file.NavigationTitle} in navigation");
 
-	INavigationItem[] GetParents(INavigationItem current)
-	{
-		var parents = new List<INavigationItem>();
-		var parent = current.Parent;
-		do
-		{
-			if (parent is null)
-				continue;
-			if (parents.All(i => i.Url != parent.Url))
-				parents.Add(parent);
-
-			parent = parent.Parent;
-		} while (parent != null);
-
-		return [.. parents];
-	}
+	INavigationItem[] GetParents(INavigationItem current) => current.GetParents();
 
 	INavigationItem[] GetParentsOfMarkdownFile(IDocumentationFile file) =>
 		NavigationDocumentationFileLookup.TryGetValue(file, out var navigation) ? GetParents(navigation) : [];

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -25,6 +25,15 @@ public record DocumentationDocument
 	[JsonPropertyName("url")]
 	public string Url { get; set; } = string.Empty;
 
+	[JsonPropertyName("navigation_depth")]
+	public int NavigationDepth { get; set; } = 50; //default to a high number so that omission gets penalized.
+
+	[JsonPropertyName("navigation_table_of_contents")]
+	public int NavigationTableOfContents { get; set; } = 50; //default to a high number so that omission gets penalized.
+
+	[JsonPropertyName("navigation_section")]
+	public string? NavigationSection { get; set; }
+
 	/// The date of the batch update this document was part of last.
 	/// This date could be higher than the date_last_updated.
 	[JsonPropertyName("batch_index_date")]

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -69,6 +69,7 @@ public DocumentationGenerator(
 		_logger = logFactory.CreateLogger(nameof(DocumentationGenerator));
 
 		DocumentationSet = docSet;
+		PositionalNavigation = positionalNavigation ?? docSet;
 		Context = docSet.Context;
 		var productVersionInferrer = new ProductVersionInferrerService(DocumentationSet.Context.ProductsConfiguration, DocumentationSet.Context.VersionsConfiguration);
 		HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem, new DescriptionGenerator(), positionalNavigation, navigationHtmlWriter, legacyUrlMapper, productVersionInferrer);
@@ -83,6 +84,8 @@ public DocumentationGenerator(
 		_logger.LogInformation("Output directory: {OutputPath} Exists: {OutputPathExists}", docSet.OutputDirectory, docSet.OutputDirectory.Exists);
 	}
 
+	private INavigationTraversable PositionalNavigation { get; }
+
 	public GenerationState? GetPreviousGenerationState()
 	{
 		var stateFile = DocumentationSet.OutputStateFile;
@@ -256,7 +259,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 				foreach (var exporter in _markdownExporters)
 				{
 					var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(DocumentationSet.TryFindDocumentByRelativePath, ctx);
-					var navigationItem = DocumentationSet.FindNavigationByMarkdown(markdown);
+					var navigationItem = PositionalNavigation.GetNavigationFor(markdown);
 					_ = await exporter.ExportAsync(new MarkdownExportFileContext
 					{
 						BuildContext = Context,
@@ -265,6 +268,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 						SourceFile = markdown,
 						DefaultOutputFile = outputFile,
 						DocumentationSet = DocumentationSet,
+						PositionaNavigation = PositionalNavigation,
 						NavigationItem = navigationItem
 					}, ctx);
 				}