diff --git a/docs/reference/ingest/processors/set.asciidoc b/docs/reference/ingest/processors/set.asciidoc index f2c4895379d12..22a9fb4c5cd55 100644 --- a/docs/reference/ingest/processors/set.asciidoc +++ b/docs/reference/ingest/processors/set.asciidoc @@ -11,6 +11,7 @@ its value will be replaced with the provided one. | `field` | yes | - | The field to insert, upsert, or update. Supports <>. | `value` | yes | - | The value to be set for the field. Supports <>. | `override` | no | true | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched. +| `ignore_empty_value` | no | `false` | If `true` and `value` is a <> that evaluates to `null` or the empty string, the processor quietly exits without modifying the document include::common-options.asciidoc[] |====== diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java index 0af51e5b895e4..0b75283c11a78 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java @@ -40,16 +40,18 @@ public final class SetProcessor extends AbstractProcessor { private final boolean overrideEnabled; private final TemplateScript.Factory field; private final ValueSource value; + private final boolean ignoreEmptyValue; SetProcessor(String tag, TemplateScript.Factory field, ValueSource value) { - this(tag, field, value, true); + this(tag, field, value, true, false); } - SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled) { + SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled, boolean ignoreEmptyValue) { super(tag); this.overrideEnabled = overrideEnabled; this.field = field; this.value = value; + this.ignoreEmptyValue = ignoreEmptyValue; } public boolean isOverrideEnabled() { @@ -64,10 +66,14 @@ public ValueSource getValue() { return value; } + public boolean isIgnoreEmptyValue() { + return ignoreEmptyValue; + } + @Override public IngestDocument execute(IngestDocument document) { if (overrideEnabled || document.hasField(field) == false || document.getFieldValue(field, Object.class) == null) { - document.setFieldValue(field, value); + document.setFieldValue(field, value, ignoreEmptyValue); } return document; } @@ -93,11 +99,13 @@ public SetProcessor create(Map registry, String proce boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true); TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService); + boolean ignoreEmptyValue = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_empty_value", false); return new SetProcessor( processorTag, compiledTemplate, ValueSource.wrap(value, scriptService), - overrideEnabled); + overrideEnabled, + ignoreEmptyValue); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java index 09dfade8f5ca3..8cd0a03554b90 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java @@ -38,7 +38,7 @@ public void testSetExistingFields() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); Object fieldValue = RandomDocumentPicks.randomFieldValue(random()); - Processor processor = createSetProcessor(fieldName, fieldValue, true); + Processor processor = createSetProcessor(fieldName, fieldValue, true, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(true)); assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue)); @@ -50,7 +50,7 @@ public void testSetNewFields() throws Exception { IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); Object fieldValue = RandomDocumentPicks.randomFieldValue(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), testIngestDocument, fieldValue); - Processor processor = createSetProcessor(fieldName, fieldValue, true); + Processor processor = createSetProcessor(fieldName, fieldValue, true, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(true)); assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue)); @@ -59,7 +59,7 @@ public void testSetNewFields() throws Exception { public void testSetFieldsTypeMismatch() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); ingestDocument.setFieldValue("field", "value"); - Processor processor = createSetProcessor("field.inner", "value", true); + Processor processor = createSetProcessor("field.inner", "value", true, false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -73,7 +73,7 @@ public void testSetNewFieldWithOverrideDisabled() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); Object fieldValue = RandomDocumentPicks.randomFieldValue(random()); - Processor processor = createSetProcessor(fieldName, fieldValue, false); + Processor processor = createSetProcessor(fieldName, fieldValue, false, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(true)); assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue)); @@ -83,7 +83,7 @@ public void testSetExistingFieldWithOverrideDisabled() throws Exception { IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); Object fieldValue = "foo"; String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = createSetProcessor(fieldName, "bar", false); + Processor processor = createSetProcessor(fieldName, "bar", false, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(true)); assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue)); @@ -94,7 +94,7 @@ public void testSetExistingNullFieldWithOverrideDisabled() throws Exception { Object fieldValue = null; Object newValue = "bar"; String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = createSetProcessor(fieldName, newValue, false); + Processor processor = createSetProcessor(fieldName, newValue, false, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(true)); assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(newValue)); @@ -102,7 +102,7 @@ public void testSetExistingNullFieldWithOverrideDisabled() throws Exception { public void testSetMetadataExceptVersion() throws Exception { Metadata randomMetadata = randomFrom(Metadata.INDEX, Metadata.TYPE, Metadata.ID, Metadata.ROUTING); - Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", true); + Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", true, false); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(randomMetadata.getFieldName(), String.class), Matchers.equalTo("_value")); @@ -110,7 +110,7 @@ public void testSetMetadataExceptVersion() throws Exception { public void testSetMetadataVersion() throws Exception { long version = randomNonNegativeLong(); - Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, true); + Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, true, false); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(Metadata.VERSION.getFieldName(), Long.class), Matchers.equalTo(version)); @@ -118,7 +118,7 @@ public void testSetMetadataVersion() throws Exception { public void testSetMetadataVersionType() throws Exception { String versionType = randomFrom("internal", "external", "external_gte"); - Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, true); + Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, true, false); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(Metadata.VERSION_TYPE.getFieldName(), String.class), Matchers.equalTo(versionType)); @@ -126,7 +126,7 @@ public void testSetMetadataVersionType() throws Exception { public void testSetMetadataIfSeqNo() throws Exception { long ifSeqNo = randomNonNegativeLong(); - Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, true); + Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, true, false); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(Metadata.IF_SEQ_NO.getFieldName(), Long.class), Matchers.equalTo(ifSeqNo)); @@ -134,14 +134,14 @@ public void testSetMetadataIfSeqNo() throws Exception { public void testSetMetadataIfPrimaryTerm() throws Exception { long ifPrimaryTerm = randomNonNegativeLong(); - Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, true); + Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, true, false); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), Long.class), Matchers.equalTo(ifPrimaryTerm)); } - private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled) { + private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled, boolean ignoreEmptyValue) { return new SetProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(fieldName), - ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled); + ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled, ignoreEmptyValue); } } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml new file mode 100644 index 0000000000000..be60767aef6f6 --- /dev/null +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml @@ -0,0 +1,55 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "1" + ignore: 404 + +--- +"Test set processor with template value": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "processors": [ + { + "set" : { + "field" : "foo", + "value" : "{{bar}}", + "ignore_empty_value" : true + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "1" + body: { + foo: "hello" + } + - do: + index: + index: test + id: 2 + pipeline: "1" + body: { + foo: "hello", + bar: "" + } + + - do: + get: + index: test + id: 1 + - match: { _source.foo: "hello" } + + - do: + get: + index: test + id: 2 + - match: { _source.foo: "hello" } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index e924c097c469a..46a7488c019d0 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -427,6 +427,32 @@ public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource setFieldValue(fieldPathTemplate.newInstance(model).execute(), valueSource.copyAndResolve(model), false); } + /** + * Sets the provided value to the provided path in the document. + * Any non existing path element will be created. If the last element is a list, + * the value will replace the existing list. + * @param fieldPathTemplate Resolves to the path with dot-notation within the document + * @param valueSource The value source that will produce the value to put in for the path key + * @param ignoreEmptyValue The flag to determine whether to exit quietly when the value produced by TemplatedValue is null or empty + * @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the + * item identified by the provided path. + */ + public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource, boolean ignoreEmptyValue) { + Map model = createTemplateModel(); + Object value = valueSource.copyAndResolve(model); + if (ignoreEmptyValue && valueSource instanceof ValueSource.TemplatedValue) { + if (value == null) { + return; + } + String valueStr = (String) value; + if (valueStr.isEmpty()) { + return; + } + } + + setFieldValue(fieldPathTemplate.newInstance(model).execute(), value, false); + } + private void setFieldValue(String path, Object value, boolean append) { FieldPath fieldPath = new FieldPath(path); Object context = fieldPath.initialContext;