TSDB: Add _tsid field to time_series indices

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml

@@ -178,3 +178,39 @@ can't shadow metrics:
           runtime_mappings:
             deep.deeper.deepest:
               type: keyword
+
+---
+# Test that _tsid field is not added if an index is not a time-series index
+no _tsid in standard indices:
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      field_caps:
+        index: test
+        fields: [metricset, _tsid]
+
+  - match: {fields.metricset.keyword.searchable:            true}
+  - match: {fields.metricset.keyword.aggregatable:          true}
+  - match: {fields.metricset.keyword.time_series_dimension: true}
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - is_false: fields._tsid  # _tsid metadata field must not exist in non-time-series indices

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml

@@ -316,7 +316,7 @@ field capabilities:
   - is_false: fields.metricset.keyword.indices
   - is_false: fields.metricset.keyword.non_searchable_indices
   - is_false: fields.metricset.keyword.non_aggregatable_indices
-  - match: { fields._tsid._tsid.metadata_field:               true}
+  - match: {fields._tsid._tsid.metadata_field:                true}
   - match: {fields._tsid._tsid.searchable:                    false}
   - match: {fields._tsid._tsid.aggregatable:                  true}
   - is_false: fields._tsid._tsid.indices

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -206,9 +206,14 @@ Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> getMetadataMapper
         if (existingMapper == null) {
             for (MetadataFieldMapper.TypeParser parser : metadataMapperParsers.values()) {
                 MetadataFieldMapper metadataFieldMapper = parser.getDefault(parserContext());
-                metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper);
+                // A MetadataFieldMapper may choose to not be added to the metadata mappers
+                // of an index (eg TimeSeriesIdFieldMapper is only added to time series indices)
+                // In this case its TypeParser will return null instead of the MetadataFieldMapper
+                // instance.
+                if (metadataFieldMapper != null) {
+                    metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper);
+                }
             }
-
         } else {
             metadataMappers.putAll(existingMapper.mapping().getMetadataMappersMap());
         }

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java

@@ -82,7 +82,22 @@ public TimeSeriesIdFieldMapper build() {
         }
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> new TimeSeriesIdFieldMapper(), c -> new Builder());
+    /**
+     * A no-op builder does not return an instance of the field mapper. Instead, it returns null
+     * This is going to be used for non-time-series indices that should have this
+     *
+     */
+    public static class NoopBuilder extends Builder {
+        @Override
+        public TimeSeriesIdFieldMapper build() {
+            return null;
+        }
+    }
+
+    public static final TypeParser PARSER = new ConfigurableTypeParser(
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? new TimeSeriesIdFieldMapper() : null,
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? new Builder() : new NoopBuilder()
+    );
 
     public static final class TimeSeriesIdFieldType extends MappedFieldType {
         private TimeSeriesIdFieldType() {
@@ -126,9 +141,6 @@ private TimeSeriesIdFieldMapper() {
 
     @Override
     public void postParse(DocumentParserContext context) throws IOException {
-        if (context.indexSettings().getMode() != IndexMode.TIME_SERIES) {
-            return;
-        }
         assert fieldType().isSearchable() == false;
 
         // SortedMap is expected to be sorted by key (field name)

server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java

@@ -75,17 +75,21 @@ public void testEnabledInTimeSeriesMode() throws Exception {
 
     public void testDisabledInStandardMode() throws Exception {
         DocumentMapper docMapper = createMapperService(
-            getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), "standard").build(),
+            getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(),
             mapping(b -> {})
         ).documentMapper();
-        ParsedDocument doc = docMapper.parse(source(b -> b.field("field", "value")));
+        assertThat(docMapper.metadataMapper(TimeSeriesIdFieldMapper.class), is(nullValue()));
 
+        ParsedDocument doc = docMapper.parse(source(b -> b.field("field", "value")));
         assertThat(doc.rootDoc().getBinaryValue("_tsid"), is(nullValue()));
         assertThat(doc.rootDoc().get("field"), equalTo("value"));
     }
 
     public void testIncludeInDocumentNotAllowed() throws Exception {
-        DocumentMapper docMapper = createDocumentMapper(mapping(b -> {}));
+        DocumentMapper docMapper = createDocumentMapper(
+            "a",
+            mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); })
+        );
         Exception e = expectThrows(MapperParsingException.class, () -> parseDocument(docMapper, b -> b.field("_tsid", "foo")));
 
         assertThat(e.getCause().getMessage(), containsString("Field [_tsid] is a metadata field and cannot be added inside a document"));