TSDB: Add _tsid field to time_series indices

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -225,7 +225,7 @@ public void testNewReplicas() throws Exception {
     }
 
     public void testSearchTimeSeriesMode() throws Exception {
-        assumeTrue("time series mode introduced in 8.0.0", getOldClusterVersion().onOrAfter(Version.V_8_0_0));
+        assumeTrue("time series index sort by _tsid introduced in 8.1.0", getOldClusterVersion().onOrAfter(Version.V_8_1_0));
         int numDocs;
         if (isRunningAgainstOldCluster()) {
             numDocs = createTimeSeriesModeIndex(1);
@@ -267,7 +267,7 @@ public void testSearchTimeSeriesMode() throws Exception {
     }
 
     public void testNewReplicasTimeSeriesMode() throws Exception {
-        assumeTrue("time series mode introduced in 8.0.0", getOldClusterVersion().onOrAfter(Version.V_8_0_0));
+        assumeTrue("time series index sort by _tsid introduced in 8.1.0", getOldClusterVersion().onOrAfter(Version.V_8_1_0));
         if (isRunningAgainstOldCluster()) {
             createTimeSeriesModeIndex(0);
         } else {

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -258,7 +258,7 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio
     }
 
     public void testTsdb() throws IOException {
-        assumeTrue("tsdb added in 8.0.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_0_0));
+        assumeTrue("sort by _tsid added in 8.1.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_1_0));
 
         StringBuilder bulk = new StringBuilder();
         switch (CLUSTER_TYPE) {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml

@@ -178,3 +178,39 @@ can't shadow metrics:
           runtime_mappings:
             deep.deeper.deepest:
               type: keyword
+
+---
+# Test that _tsid field is not added if an index is not a time-series index
+no _tsid in standard indices:
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      field_caps:
+        index: test
+        fields: [metricset, _tsid]
+
+  - match: {fields.metricset.keyword.searchable:            true}
+  - match: {fields.metricset.keyword.aggregatable:          true}
+  - match: {fields.metricset.keyword.time_series_dimension: true}
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - is_false: fields._tsid  # _tsid metadata field must not exist in non-time-series indices

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

@@ -186,6 +186,8 @@ set start_time and end_time:
                 end_time: 1632625792000
           mappings:
             properties:
+              "@timestamp":
+                type: date
               metricset:
                 type: keyword
                 time_series_dimension: true

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -204,6 +204,9 @@ runtime field matching routing path:
               properties:
                 "@timestamp":
                   type: date
+                dim_kw:
+                    type: "keyword"
+                    time_series_dimension: true
                 dim:
                   type: object
                   dynamic: runtime
@@ -214,7 +217,7 @@ runtime field matching routing path:
         index: test
         body:
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": "a"}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim_kw": "dim", "dim": {"foo": "a"}}'
   - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] and without the [script] parameter. [dim.foo] was a runtime [keyword]."}
 
 ---

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml

@@ -17,8 +17,8 @@ teardown:
 ---
 "Create a snapshot and then restore it":
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
       features: ["allowed_warnings"]
 
   # Create index
@@ -134,10 +134,13 @@ teardown:
       search:
         index: test_index
         body:
+          fields:
+            - field: _tsid
           query:
             query_string:
               query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
 
   - match: {hits.total.value: 1}
   - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507}
-  # TODO assert the _tsid once we generate it
+  - match: {hits.hits.0.fields._tsid: [ { k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod } ] }
+

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml

@@ -90,7 +90,20 @@ query a metric:
 
   - match: {hits.total.value: 1}
 
-# TODO add test showing that quering _tsid fails
+---
+"query tsid fails":
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      catch: /\[_tsid\] is not searchable/
+      search:
+        index: test
+        body:
+          query:
+            term:
+              _tsid: wont't work
 
 ---
 fetch a dimension:
@@ -151,7 +164,24 @@ fetch a tag:
   - match: {hits.hits.0.fields.k8s\.pod\.ip: ['10.10.55.2']}
   - is_false: hits.hits.0.fields._tsid   # tsid isn't fetched by default
 
-# TODO add test to fetch the tsid
+---
+"fetch the tsid":
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      search:
+        index: test
+        body:
+          fields:
+            - field: _tsid
+          query:
+            query_string:
+              query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
+
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
 
 ---
 aggregate a dimension:
@@ -229,23 +259,44 @@ aggregate a tag:
   - match: {aggregations.ips.buckets.2.key: 10.10.55.3}
   - match: {aggregations.ips.buckets.2.doc_count: 4}
 
+---
+"aggregate the tsid":
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
 
-# TODO add a test aggregating the _tsid
+  - match: {hits.total.value: 8}
+  - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}
 
 ---
 field capabilities:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
 
   - do:
       field_caps:
         index: test
-        fields: [k8s.pod.uid, k8s.pod.network.rx, k8s.pod.ip, _tsid]
+        fields: [k8s.pod.uid, k8s.pod.network.rx, k8s.pod.ip, metricset, _tsid]
 
-  # TODO assert time_series_metric and time_series_dimension
-  - match: {fields.k8s\.pod\.uid.keyword.searchable:          true}
-  - match: {fields.k8s\.pod\.uid.keyword.aggregatable:        true}
+  - match: {fields.k8s\.pod\.uid.keyword.searchable:            true}
+  - match: {fields.k8s\.pod\.uid.keyword.aggregatable:          true}
+  - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
   - is_false: fields.k8s\.pod\.uid.keyword.indices
   - is_false: fields.k8s\.pod\.uid.keyword.non_searchable_indices
   - is_false: fields.k8s\.pod\.uid.keyword.non_aggregatable_indices
@@ -259,4 +310,15 @@ field capabilities:
   - is_false: fields.k8s\.pod\.ip.ip.indices
   - is_false: fields.k8s\.pod\.ip.ip.non_searchable_indices
   - is_false: fields.k8s\.pod\.ip.ip.non_aggregatable_indices
-  # TODO assert tsid once we build it:
+  - match: {fields.metricset.keyword.searchable:            true}
+  - match: {fields.metricset.keyword.aggregatable:          true}
+  - match: {fields.metricset.keyword.time_series_dimension: true}
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - match: {fields._tsid._tsid.metadata_field:                true}
+  - match: {fields._tsid._tsid.searchable:                    false}
+  - match: {fields._tsid._tsid.aggregatable:                  true}
+  - is_false: fields._tsid._tsid.indices
+  - is_false: fields._tsid._tsid.non_searchable_indices
+  - is_false: fields._tsid._tsid.non_aggregatable_indices

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml

@@ -57,13 +57,40 @@ setup:
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
 
-# TODO search on _tsid in an alias
+---
+search an alias:
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      indices.put_alias:
+        index: test
+        name:  test_alias
+
+  - do:
+      search:
+        index: test_alias
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+
+  - match: {hits.total.value: 8}
+  - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}
 
 ---
 index into alias:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
 
   - do:
       indices.put_alias:
@@ -85,4 +112,23 @@ index into alias:
           - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39", "ip": "10.10.55.4", "network": {"tx": 1434595272, "rx": 530605511}}}}'
   - match: {errors: false}
 
-  # TODO search on tsid once we generate it
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+
+  - match: {hits.total.value: 12}
+  - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}
+  - match: {aggregations.tsids.buckets.2.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}}
+  - match: {aggregations.tsids.buckets.2.doc_count: 4}
+