Add source fallback support for date and date_nanos mapped types

docs/changelog/89440.yaml

@@ -0,0 +1,5 @@
+pr: 89440
+summary: Add source fallback support for date and `date_nanos` mapped types
+area: Mapping
+type: enhancement
+issues: []

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml

@@ -14,8 +14,14 @@ setup:
                             doc_values: false
                         date:
                             type: date
+                        date_no_doc_values:
+                            type: date
+                            doc_values: false
                         nanos:
                             type: date_nanos
+                        nanos_no_doc_values:
+                            type: date_nanos
+                            doc_values: false
                         geo_point:
                             type: geo_point
                         geo_point_no_doc_values:
@@ -93,7 +99,9 @@ setup:
                 boolean: true
                 boolean_no_doc_values: true
                 date: 2017-01-01T12:11:12
+                date_no_doc_values: 2017-01-01T12:11:12
                 nanos: 2015-01-01T12:10:30.123456789Z
+                nanos_no_doc_values: 2015-01-01T12:10:30.123456789Z
                 geo_point: 41.12,-71.34
                 geo_point_no_doc_values: 41.12,-71.34
                 ip: 192.168.0.19
@@ -136,7 +144,9 @@ setup:
               boolean_no_doc_values: [true, false, true]
               ip: ["10.1.2.3", "2001:db8::2:1"]
               date: [2017-01-01T12:11:12, 2018-01-01T12:11:12]
+              date_no_doc_values: [2017-01-01T12:11:12, 2018-01-01T12:11:12]
               nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z]
+              nanos_no_doc_values: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z]
               geo_point: [[-71.34,41.12],[60.32,21.25]]
               geo_point_no_doc_values: [[60.32,21.25],[-71.34,41.12]]
               keyword: ["one string", "another string"]
@@ -692,6 +702,244 @@ setup:
                   source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('nanos')) times.add(zdt); times"
     - match: { hits.hits.0.fields.field: ["2015-01-01T12:10:30.123456789Z", "2015-01-01T12:10:30.987654321Z"] }
 
+---
+"date_no_doc_values":
+  - skip:
+      features: "warnings"
+
+  - do:
+      catch: bad_request
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "doc.date_no_doc_values.get(0)"
+  - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" }
+
+  - do:
+      catch: bad_request
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "doc.date_no_doc_values.value"
+  - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "field('date_no_doc_values').get(null)"
+  - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('date_no_doc_values', null)"
+  - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: 1 } }
+          script_fields:
+            field:
+              script:
+                source: "field('date_no_doc_values').get(null).getMillis()"
+  - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: 1 } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('date_no_doc_values', null).getMillis()"
+  - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: 1 } }
+          script_fields:
+            field:
+              script:
+                source: "field('date_no_doc_values').get(null).millis"
+  - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: 1 } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('date_no_doc_values', null).millis"
+  - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "2" } }
+          script_fields:
+            field:
+              script:
+                source: "field('date_no_doc_values').get(ZonedDateTime.parse('2018-01-01T12:11:12.000Z'))"
+  - match: { hits.hits.0.fields.field.0: '2018-01-01T12:11:12.000Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "2" } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('date_no_doc_values', ZonedDateTime.parse('2018-01-01T12:11:12.000Z'))"
+  - match: { hits.hits.0.fields.field.0: '2018-01-01T12:11:12.000Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "field('nanos_no_doc_values').get(null)"
+  - match: { hits.hits.0.fields.field.0: '2015-01-01T12:10:30.123456789Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('nanos_no_doc_values', null)"
+  - match: { hits.hits.0.fields.field.0: '2015-01-01T12:10:30.123456789Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "2" } }
+          script_fields:
+            field:
+              script:
+                source: "field('nanos_no_doc_values').get(ZonedDateTime.parse('2016-01-01T12:10:30.123Z'))"
+  - match: { hits.hits.0.fields.field.0: '2016-01-01T12:10:30.123Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "2" } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('nanos_no_doc_values', ZonedDateTime.parse('2016-01-01T12:10:30.123Z'))"
+  - match: { hits.hits.0.fields.field.0: '2016-01-01T12:10:30.123Z' }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "field('nanos_no_doc_values').get(null).getNano()"
+  - match: { hits.hits.0.fields.field.0: 123456789 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "1" } }
+          script_fields:
+            field:
+              script:
+                source: "/* avoid yaml stash */ $('nanos_no_doc_values', null).getNano()"
+  - match: { hits.hits.0.fields.field.0: 123456789 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "2" } }
+          script_fields:
+            field:
+              script:
+                source: "field('nanos_no_doc_values').get(ZonedDateTime.parse('2016-01-01T12:10:30.123Z')).getNano()"
+  - match: { hits.hits.0.fields.field.0: 123000000 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "3" } }
+          script_fields:
+            field:
+              script:
+                source: "field('date_no_doc_values').get(1, null)"
+  - match: { hits.hits.0.fields.field.0: "2018-01-01T12:11:12.000Z" }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "3" } }
+          script_fields:
+            field:
+              script:
+                source: "field('nanos_no_doc_values').get(1, null)"
+  - match: { hits.hits.0.fields.field.0: "2015-01-01T12:10:30.987654321Z" }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "3" } }
+          script_fields:
+            field:
+              script:
+                source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('date_no_doc_values')) times.add(zdt); times"
+  - match: { hits.hits.0.fields.field: ["2017-01-01T12:11:12.000Z", "2018-01-01T12:11:12.000Z"] }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: "3" } }
+          script_fields:
+            field:
+              script:
+                source: "List times = new ArrayList(); for (ZonedDateTime zdt : field('nanos_no_doc_values')) times.add(zdt); times"
+  - match: { hits.hits.0.fields.field: ["2015-01-01T12:10:30.123456789Z", "2015-01-01T12:10:30.987654321Z"] }
+
 ---
 "geo_point":
     - do:

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.DateRangeIncludingNowQuery;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -64,6 +65,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -519,6 +521,17 @@ public String parseSourceValue(Object value) {
             };
         }
 
+        // returns a Long to support source fallback which emulates numeric doc values for dates
+        private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
+            return new SourceValueFetcher(sourcePaths, nullValue) {
+                @Override
+                public Long parseSourceValue(Object value) {
+                    String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString();
+                    return parse(date);
+                }
+            };
+        }
+
         private String format(long timestamp, DateFormatter formatter) {
             ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
             return formatter.format(dateTime);
@@ -750,8 +763,34 @@ public Function<byte[], Number> pointReaderIfPossible() {
 
         @Override
         public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
-            failIfNoDocValues();
-            return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType(), resolution.getDefaultToScriptFieldFactory());
+            FielddataOperation operation = fieldDataContext.fielddataOperation();
+
+            if (operation == FielddataOperation.SEARCH) {
+                failIfNoDocValues();
+            }
+
+            if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) {
+                return new SortedNumericIndexFieldData.Builder(
+                    name(),
+                    resolution.numericType(),
+                    resolution.getDefaultToScriptFieldFactory()
+                );
+            }
+
+            if (operation == FielddataOperation.SCRIPT) {
+                SearchLookup searchLookup = fieldDataContext.lookupSupplier().get();
+                Set<String> sourcePaths = fieldDataContext.sourcePathsLookup().apply(name());
+
+                return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
+                    name(),
+                    resolution.numericType().getValuesSourceType(),
+                    sourceValueFetcher(sourcePaths),
+                    searchLookup.source(),
+                    resolution.getDefaultToScriptFieldFactory()
+                );
+            }
+
+            throw new IllegalStateException("unknown field data operation [" + operation.name() + "]");
         }
 
         @Override