Add ignore_empty_value parameter in set ingest processor (#57030) (#5…

…8108)

docs/reference/ingest/processors/set.asciidoc

@@ -11,6 +11,7 @@ its value will be replaced with the provided one.
 | `field` | yes       | -        | The field to insert, upsert, or update. Supports <<accessing-template-fields,template snippets>>.
 | `value` | yes       | -        | The value to be set for the field. Supports <<accessing-template-fields,template snippets>>.
 | `override` | no        | true     | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched.
+| `ignore_empty_value` | no        | `false`  | If `true` and `value` is a <<accessing-template-fields,template snippet>> that evaluates to `null` or the empty string, the processor quietly exits without modifying the document
 include::common-options.asciidoc[]
 |======
 

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java

@@ -40,16 +40,18 @@ public final class SetProcessor extends AbstractProcessor {
     private final boolean overrideEnabled;
     private final TemplateScript.Factory field;
     private final ValueSource value;
+    private final boolean ignoreEmptyValue;
 
     SetProcessor(String tag, TemplateScript.Factory field, ValueSource value)  {
-        this(tag, field, value, true);
+        this(tag, field, value, true, false);
     }
 
-    SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled)  {
+    SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled, boolean ignoreEmptyValue)  {
         super(tag);
         this.overrideEnabled = overrideEnabled;
         this.field = field;
         this.value = value;
+        this.ignoreEmptyValue = ignoreEmptyValue;
     }
 
     public boolean isOverrideEnabled() {
@@ -64,10 +66,14 @@ public ValueSource getValue() {
         return value;
     }
 
+    public boolean isIgnoreEmptyValue() {
+        return ignoreEmptyValue;
+    }
+
     @Override
     public IngestDocument execute(IngestDocument document) {
         if (overrideEnabled || document.hasField(field) == false || document.getFieldValue(field, Object.class) == null) {
-            document.setFieldValue(field, value);
+            document.setFieldValue(field, value, ignoreEmptyValue);
         }
         return document;
     }
@@ -93,11 +99,13 @@ public SetProcessor create(Map<String, Processor.Factory> registry, String proce
             boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true);
             TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
                 "field", field, scriptService);
+            boolean ignoreEmptyValue = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_empty_value", false);
             return new SetProcessor(
                     processorTag,
                     compiledTemplate,
                     ValueSource.wrap(value, scriptService),
-                    overrideEnabled);
+                    overrideEnabled,
+                    ignoreEmptyValue);
         }
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java

@@ -38,7 +38,7 @@ public void testSetExistingFields() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
         Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
-        Processor processor = createSetProcessor(fieldName, fieldValue, true);
+        Processor processor = createSetProcessor(fieldName, fieldValue, true, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(true));
         assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@@ -50,7 +50,7 @@ public void testSetNewFields() throws Exception {
         IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), testIngestDocument, fieldValue);
-        Processor processor = createSetProcessor(fieldName, fieldValue, true);
+        Processor processor = createSetProcessor(fieldName, fieldValue, true, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(true));
         assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@@ -59,7 +59,7 @@ public void testSetNewFields() throws Exception {
     public void testSetFieldsTypeMismatch() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         ingestDocument.setFieldValue("field", "value");
-        Processor processor = createSetProcessor("field.inner", "value", true);
+        Processor processor = createSetProcessor("field.inner", "value", true, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -73,7 +73,7 @@ public void testSetNewFieldWithOverrideDisabled() throws Exception {
         IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
-        Processor processor = createSetProcessor(fieldName, fieldValue, false);
+        Processor processor = createSetProcessor(fieldName, fieldValue, false, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(true));
         assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@@ -83,7 +83,7 @@ public void testSetExistingFieldWithOverrideDisabled() throws Exception {
         IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
         Object fieldValue = "foo";
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = createSetProcessor(fieldName, "bar", false);
+        Processor processor = createSetProcessor(fieldName, "bar", false, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(true));
         assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@@ -94,54 +94,54 @@ public void testSetExistingNullFieldWithOverrideDisabled() throws Exception {
         Object fieldValue = null;
         Object newValue = "bar";
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = createSetProcessor(fieldName, newValue, false);
+        Processor processor = createSetProcessor(fieldName, newValue, false, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(true));
         assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(newValue));
     }
 
     public void testSetMetadataExceptVersion() throws Exception {
         Metadata randomMetadata = randomFrom(Metadata.INDEX, Metadata.TYPE, Metadata.ID, Metadata.ROUTING);
-        Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", true);
+        Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(randomMetadata.getFieldName(), String.class), Matchers.equalTo("_value"));
     }
 
     public void testSetMetadataVersion() throws Exception {
         long version = randomNonNegativeLong();
-        Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, true);
+        Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(Metadata.VERSION.getFieldName(), Long.class), Matchers.equalTo(version));
     }
 
     public void testSetMetadataVersionType() throws Exception {
         String versionType = randomFrom("internal", "external", "external_gte");
-        Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, true);
+        Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(Metadata.VERSION_TYPE.getFieldName(), String.class), Matchers.equalTo(versionType));
     }
 
     public void testSetMetadataIfSeqNo() throws Exception {
         long ifSeqNo = randomNonNegativeLong();
-        Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, true);
+        Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(Metadata.IF_SEQ_NO.getFieldName(), Long.class), Matchers.equalTo(ifSeqNo));
     }
 
     public void testSetMetadataIfPrimaryTerm() throws Exception {
         long ifPrimaryTerm = randomNonNegativeLong();
-        Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, true);
+        Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), Long.class), Matchers.equalTo(ifPrimaryTerm));
     }
 
-    private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled) {
+    private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled, boolean ignoreEmptyValue) {
         return new SetProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(fieldName),
-                ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled);
+                ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled, ignoreEmptyValue);
     }
 }

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml

@@ -0,0 +1,55 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "1"
+        ignore: 404
+
+---
+"Test set processor with template value":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "set" : {
+                  "field" : "foo",
+                  "value" : "{{bar}}",
+                  "ignore_empty_value" : true
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "1"
+        body: {
+          foo: "hello"
+        }
+  - do:
+      index:
+        index: test
+        id: 2
+        pipeline: "1"
+        body: {
+          foo: "hello",
+          bar: ""
+        }
+
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source.foo: "hello" }
+
+  - do:
+      get:
+        index: test
+        id: 2
+  - match: { _source.foo: "hello" }

server/src/main/java/org/elasticsearch/ingest/IngestDocument.java

@@ -427,6 +427,32 @@ public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource
         setFieldValue(fieldPathTemplate.newInstance(model).execute(), valueSource.copyAndResolve(model), false);
     }
 
+    /**
+     * Sets the provided value to the provided path in the document.
+     * Any non existing path element will be created. If the last element is a list,
+     * the value will replace the existing list.
+     * @param fieldPathTemplate Resolves to the path with dot-notation within the document
+     * @param valueSource The value source that will produce the value to put in for the path key
+     * @param ignoreEmptyValue The flag to determine whether to exit quietly when the value produced by TemplatedValue is null or empty
+     * @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the
+     * item identified by the provided path.
+     */
+    public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource, boolean ignoreEmptyValue) {
+        Map<String, Object> model = createTemplateModel();
+        Object value = valueSource.copyAndResolve(model);
+        if (ignoreEmptyValue && valueSource instanceof ValueSource.TemplatedValue) {
+            if (value == null) {
+                return;
+            }
+            String valueStr = (String) value;
+            if (valueStr.isEmpty()) {
+                return;
+            }
+        }
+
+        setFieldValue(fieldPathTemplate.newInstance(model).execute(), value, false);
+    }
+
     private void setFieldValue(String path, Object value, boolean append) {
         FieldPath fieldPath = new FieldPath(path);
         Object context = fieldPath.initialContext;