Move flattened field to core. (elastic#70122)

This field mapper only lived in its own module so it could be licensed as x-pack
basic. Now it can be moved to core, which matches its status as a core type.

The PR also removes flattened fields from xpack usage stats now that they live
in core. The usage stats were already removed in on master, but this PR was not
backported to avoid a breaking change on 7.x. We have now clarified that usage
stats updates are not considered breaking, so it is safe to backport this
change.

Author: jtibshirani

docs/reference/mapping/types/flattened.asciidoc

@@ -1,6 +1,3 @@
-[role="xpack"]
-[testenv="basic"]
-
 [[flattened]]
 === Flattened field type
 ++++

docs/reference/rest-api/info.asciidoc

@@ -86,10 +86,6 @@ Example response:
           "available" : true,
           "enabled" : true
       },
-      "flattened" : {
-         "available" : true,
-         "enabled" : true
-      },
       "frozen_indices" : {
          "available" : true,
          "enabled" : true

docs/reference/rest-api/usage.asciidoc

@@ -284,11 +284,6 @@ GET /_xpack/usage
     "available" : true,
     "enabled" : true
   },
-  "flattened" : {
-    "available" : true,
-    "enabled" : true,
-    "field_count" : 0
-  },
   "vectors" : {
     "available" : true,
     "enabled" : true,

x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml renamed to rest-api-spec/src/main/resources/rest-api-spec/test/search/340_flattened.yml

@@ -1,9 +1,10 @@
----
-"Test exists query on flattened object field":
+setup:
   - skip:
-      version: " - 7.2.99"
-      reason: "Flattened fields were implemented in 7.3."
+      version: " - 7.12.99"
+      reason: "Flattened fields were moved from xpack to core in 7.13."
 
+---
+"Test exists query on flattened object field":
   - do:
       indices.create:
         index:  flattened_test
@@ -53,10 +54,6 @@
 
 ---
 "Test query string query on flattened object field":
-  - skip:
-      version: " - 7.2.99"
-      reason: "Flattened fields were implemented in 7.3."
-
   - do:
       indices.create:
         index:  test
@@ -113,10 +110,6 @@
 
 ---
 "Test fields option on flattened object field":
-  - skip:
-      version: " - 7.10.99"
-      reason: "Fields option on search request was added in 7.10"
-
   - do:
       indices.create:
         index:  test
@@ -159,9 +152,6 @@
 
 ---
 "Test fields option on flattened object field with include_unmapped":
-  -  skip:
-        version: ' - 7.10.99'
-        reason: support was introduced in 7.11
   - do:
       indices.create:
         index:  test

server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java

@@ -25,8 +25,7 @@
  * sure to passes an empty multi-fields list to help prevent conflicting sub-keys from being
  * registered.
  *
- * Note: we anticipate that 'flattened' fields will be the only implementation of this
- * interface. Flattened object fields live in the 'mapper-flattened' module.
+ * Note: currently 'flattened' fields are the only implementation of this interface.
  */
 public abstract class DynamicKeyFieldMapper extends FieldMapper {
 

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java renamed to server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -1,11 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.flattened.mapper;
+package org.elasticsearch.index.mapper.flattened;
 
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParser.java renamed to server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java

@@ -1,11 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.flattened.mapper;
+package org.elasticsearch.index.mapper.flattened;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldData.java renamed to server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java

@@ -1,11 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.flattened.mapper;
+package org.elasticsearch.index.mapper.flattened;
 
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.SortedSetDocValues;

server/src/main/java/org/elasticsearch/indices/IndicesModule.java

@@ -45,6 +45,7 @@
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncAction;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
@@ -123,18 +124,21 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
         }
         mappers.put(BooleanFieldMapper.CONTENT_TYPE, BooleanFieldMapper.PARSER);
         mappers.put(BinaryFieldMapper.CONTENT_TYPE, BinaryFieldMapper.PARSER);
+        mappers.put(CompletionFieldMapper.CONTENT_TYPE, CompletionFieldMapper.PARSER);
+
         DateFieldMapper.Resolution milliseconds = DateFieldMapper.Resolution.MILLISECONDS;
         mappers.put(milliseconds.type(), DateFieldMapper.MILLIS_PARSER);
         DateFieldMapper.Resolution nanoseconds = DateFieldMapper.Resolution.NANOSECONDS;
         mappers.put(nanoseconds.type(), DateFieldMapper.NANOS_PARSER);
+
+        mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
+        mappers.put(FlattenedFieldMapper.CONTENT_TYPE, FlattenedFieldMapper.PARSER);
+        mappers.put(GeoPointFieldMapper.CONTENT_TYPE, GeoPointFieldMapper.PARSER);
         mappers.put(IpFieldMapper.CONTENT_TYPE, IpFieldMapper.PARSER);
-        mappers.put(TextFieldMapper.CONTENT_TYPE, TextFieldMapper.PARSER);
         mappers.put(KeywordFieldMapper.CONTENT_TYPE, KeywordFieldMapper.PARSER);
         mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser());
         mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
-        mappers.put(CompletionFieldMapper.CONTENT_TYPE, CompletionFieldMapper.PARSER);
-        mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
-        mappers.put(GeoPointFieldMapper.CONTENT_TYPE, GeoPointFieldMapper.PARSER);
+        mappers.put(TextFieldMapper.CONTENT_TYPE, TextFieldMapper.PARSER);
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {

server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java

@@ -11,16 +11,22 @@
 import org.elasticsearch.common.collect.List;
 import org.elasticsearch.common.collect.Set;
 import org.elasticsearch.index.mapper.TypeFieldMapper.TypeFieldType;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
 
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
 
 public class FieldTypeLookupTests extends ESTestCase {
 
@@ -218,4 +224,106 @@ private static int size(Iterable<MappedFieldType> iterable) {
         }
         return count;
     }
+
+    public void testFlattenedLookup() {
+        String fieldName = "object1.object2.field";
+        FlattenedFieldMapper mapper = createFlattenedMapper(fieldName);
+
+        FieldTypeLookup lookup = new FieldTypeLookup("_doc", singletonList(mapper), emptyList(), emptyList());
+        assertEquals(mapper.fieldType(), lookup.get(fieldName));
+
+        String objectKey = "key1.key2";
+        String searchFieldName = fieldName + "." + objectKey;
+
+        MappedFieldType searchFieldType = lookup.get(searchFieldName);
+        assertEquals(mapper.keyedFieldName(), searchFieldType.name());
+        assertThat(searchFieldType, Matchers.instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class));
+
+        FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) searchFieldType;
+        assertEquals(objectKey, keyedFieldType.key());
+    }
+
+    public void testFlattenedLookupWithAlias() {
+        String fieldName = "object1.object2.field";
+        FlattenedFieldMapper mapper = createFlattenedMapper(fieldName);
+
+        String aliasName = "alias";
+        FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName);
+
+        FieldTypeLookup lookup = new FieldTypeLookup("_doc", singletonList(mapper), singletonList(alias), emptyList());
+        assertEquals(mapper.fieldType(), lookup.get(aliasName));
+
+        String objectKey = "key1.key2";
+        String searchFieldName = aliasName + "." + objectKey;
+
+        MappedFieldType searchFieldType = lookup.get(searchFieldName);
+        assertEquals(mapper.keyedFieldName(), searchFieldType.name());
+        assertThat(searchFieldType, Matchers.instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class));
+
+        FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) searchFieldType;
+        assertEquals(objectKey, keyedFieldType.key());
+    }
+
+    public void testFlattenedLookupWithMultipleFields() {
+        String field1 = "object1.object2.field";
+        String field2 = "object1.field";
+        String field3 = "object2.field";
+
+        FlattenedFieldMapper mapper1 = createFlattenedMapper(field1);
+        FlattenedFieldMapper mapper2 = createFlattenedMapper(field2);
+        FlattenedFieldMapper mapper3 = createFlattenedMapper(field3);
+
+        FieldTypeLookup lookup = new FieldTypeLookup("_doc", Arrays.asList(mapper1, mapper2), emptyList(), emptyList());
+        assertNotNull(lookup.get(field1 + ".some.key"));
+        assertNotNull(lookup.get(field2 + ".some.key"));
+
+        lookup = new FieldTypeLookup("_doc", Arrays.asList(mapper1, mapper2, mapper3), emptyList(), emptyList());
+        assertNotNull(lookup.get(field1 + ".some.key"));
+        assertNotNull(lookup.get(field2 + ".some.key"));
+        assertNotNull(lookup.get(field3 + ".some.key"));
+    }
+
+    public void testMaxDynamicKeyDepth() {
+        Map<String, DynamicKeyFieldMapper> mappers = new HashMap<>();
+        Map<String, String> aliases = new HashMap<>();
+        assertEquals(0, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));
+
+        // Add a flattened object field.
+        String name = "object1.object2.field";
+        FlattenedFieldMapper flattenedMapper = createFlattenedMapper(name);
+        mappers.put(name, flattenedMapper);
+        assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));
+
+        // Add a short alias to that field.
+        String aliasName = "alias";
+        aliases.put(aliasName, name);
+        assertEquals(3,  DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));
+
+        // Add a longer alias to that field.
+        String longAliasName = "object1.object2.object3.alias";
+        aliases.put(longAliasName, name);
+        assertEquals(4,  DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));
+
+        // Update the long alias to refer to a non-flattened object field.
+        String fieldName = "field";
+        aliases.put(longAliasName, fieldName);
+        assertEquals(3,  DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));
+    }
+
+    public void testFieldLookupIterator() {
+        MockFieldMapper mapper = new MockFieldMapper("foo");
+        FlattenedFieldMapper flattenedMapper = createFlattenedMapper("object1.object2.field");
+
+        FieldTypeLookup lookup = new FieldTypeLookup("_doc", Arrays.asList(mapper, flattenedMapper), emptyList(), emptyList());
+
+        Collection<String> fieldNames = new HashSet<String>();
+        lookup.filter(ft -> true).forEach(ft -> fieldNames.add(ft.name()));
+
+        assertThat(fieldNames, containsInAnyOrder(
+            mapper.name(), flattenedMapper.name(), flattenedMapper.keyedFieldName()));
+    }
+
+    private FlattenedFieldMapper createFlattenedMapper(String fieldName) {
+        return new FlattenedFieldMapper.Builder(fieldName).build(new ContentPath());
+    }
 }