From 63680d9f2ff203c93d2ce1678bacccc8831c3feb Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 20 Feb 2025 11:46:46 +0200 Subject: [PATCH] Use min node version to guard injecting settings in logs provider --- .../LogsIndexModeRollingUpgradeIT.java | 2 +- .../xpack/logsdb/LogsDBPlugin.java | 1 + .../LogsdbIndexModeSettingsProvider.java | 10 +- .../LogsdbIndexModeSettingsProviderTests.java | 141 +++++++++++++----- ...dexSettingsProviderLegacyLicenseTests.java | 3 + 5 files changed, 115 insertions(+), 42 deletions(-) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java index 1eb7cbd3f70c2..a2446474b2967 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java @@ -40,7 +40,7 @@ public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCas .module("x-pack-aggregate-metric") .module("x-pack-stack") .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.license.self_generated.type", randomBoolean() ? "trial" : "basic") // We upgrade from standard to logsdb, so we need to start with logsdb disabled, // then later cluster.logsdb.enabled gets set to true and next rollover data stream is in logsdb mode. .setting("cluster.logsdb.enabled", "false") diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index 1fb46f74a09b2..655fcd0ab68db 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -80,6 +80,7 @@ public Collection getAdditionalIndexSettingProviders(Index IndexVersion.current(), parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion() ), + () -> parameters.clusterService().state().nodes().getMinNodeVersion(), DiscoveryNode.isStateless(settings) == false, DiscoveryNode.isStateless(settings) == false ); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 01f989e8062fc..0257f2c3da8be 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.UUIDs; @@ -52,6 +53,7 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); private final SetOnce> createdIndexVersion = new SetOnce<>(); + private final SetOnce> minNodeVersion = new SetOnce<>(); private final SetOnce supportFallbackToStoredSource = new SetOnce<>(); private final SetOnce supportFallbackLogsdbRouting = new SetOnce<>(); @@ -69,11 +71,13 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { void init( CheckedFunction factory, Supplier indexVersion, + Supplier minNodeVersion, boolean supportFallbackToStoredSource, boolean supportFallbackLogsdbRouting ) { this.mapperServiceFactory.set(factory); this.createdIndexVersion.set(indexVersion); + this.minNodeVersion.set(minNodeVersion); this.supportFallbackToStoredSource.set(supportFallbackToStoredSource); this.supportFallbackLogsdbRouting.set(supportFallbackLogsdbRouting); } @@ -111,7 +115,9 @@ && matchesLogsPattern(dataStreamName)) { MappingHints mappingHints = getMappingHints(indexName, templateIndexMode, settings, combinedTemplateMappings); // Inject stored source mode if synthetic source if not available per licence. - if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) { + if (mappingHints.hasSyntheticSourceUsage + && supportFallbackToStoredSource.get() + && minNodeVersion.get().get().onOrAfter(Version.V_8_17_0)) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed( @@ -128,7 +134,7 @@ && matchesLogsPattern(dataStreamName)) { } } - if (isLogsDB) { + if (isLogsDB && minNodeVersion.get().get().onOrAfter(Version.V_8_18_0)) { // Inject sorting on [host.name], in addition to [@timestamp]. if (mappingHints.sortOnHostName) { if (mappingHints.addHostNameField) { diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 09086b75f9384..0565d37d5d367 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.logsdb; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; import org.elasticsearch.cluster.metadata.DataStream; @@ -83,6 +84,10 @@ public void setup() throws Exception { } private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { + return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT); + } + + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled, Version version) { newMapperServiceCounter.set(0); var provider = new LogsdbIndexModeSettingsProvider( logsdbLicenseService, @@ -91,15 +96,28 @@ private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boole provider.init(im -> { newMapperServiceCounter.incrementAndGet(); return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); - }, IndexVersion::current, true, true); + }, IndexVersion::current, () -> version, true, true); + return provider; + } + + private LogsdbIndexModeSettingsProvider withoutMapperService(boolean enabled) { + var provider = new LogsdbIndexModeSettingsProvider( + logsdbLicenseService, + Settings.builder().put("cluster.logsdb.enabled", enabled).build() + ); + provider.init(im -> null, IndexVersion::current, () -> Version.CURRENT, true, true); return provider; } private Settings generateLogsdbSettings(Settings settings) throws IOException { - return generateLogsdbSettings(settings, null); + return generateLogsdbSettings(settings, null, Version.CURRENT); } private Settings generateLogsdbSettings(Settings settings, String mapping) throws IOException { + return generateLogsdbSettings(settings, mapping, Version.CURRENT); + } + + private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException { Metadata metadata = Metadata.EMPTY_METADATA; var provider = new LogsdbIndexModeSettingsProvider( logsdbLicenseService, @@ -108,7 +126,7 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw provider.init(im -> { newMapperServiceCounter.incrementAndGet(); return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); - }, IndexVersion::current, true, true); + }, IndexVersion::current, () -> version, true, true); var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0), DATA_STREAM_NAME, @@ -217,11 +235,7 @@ public void testNonLogsDataStream() throws IOException { } public void testWithoutLogsComponentTemplate() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -236,11 +250,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { } public void testWithLogsComponentTemplate() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -255,11 +265,7 @@ public void testWithLogsComponentTemplate() throws IOException { } public void testWithMultipleComponentTemplates() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -274,11 +280,7 @@ public void testWithMultipleComponentTemplates() throws IOException { } public void testWithCustomComponentTemplatesOnly() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -293,11 +295,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { } public void testNonMatchingTemplateIndexPattern() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -331,10 +329,7 @@ public void testCaseSensitivity() throws IOException { } public void testMultipleHyphensInDataStreamName() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, @@ -349,12 +344,8 @@ public void testMultipleHyphensInDataStreamName() throws IOException { assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); } - public void testBeforeAndAFterSettingUpdate() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", false).build() - ); - + public void testBeforeAndAfterSettingUpdate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(false); final Settings beforeSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -627,7 +618,7 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept } } - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException { + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { String dataStreamName = DATA_STREAM_NAME; Metadata.Builder mb = Metadata.builder( DataStreamTestHelper.getClusterStateWithDataStreams( @@ -698,6 +689,33 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws assertThat(newMapperServiceCounter.get(), equalTo(4)); } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() { + logsdbLicenseService.setSyntheticSourceFallback(true); + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true, Version.V_8_16_0); + Metadata.Builder mb = Metadata.builder( + DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple(DATA_STREAM_NAME, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ).getMetadata() + ); + Metadata metadata = mb.build(); + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .build(); + var result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 2), + DATA_STREAM_NAME, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertTrue(result.isEmpty()); + } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException { logsdbLicenseService.setSyntheticSourceFallback(true); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true); @@ -773,6 +791,15 @@ public void testRoutingPathOnSortFields() throws Exception { assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message")); } + public void testRoutingPathOnSortFieldsDisabledInOldNode() throws Exception { + var settings = Settings.builder() + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message") + .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) + .build(); + Settings result = generateLogsdbSettings(settings, null, Version.V_8_17_0); + assertTrue(result.isEmpty()); + } + public void testRoutingPathOnSortFieldsFilterTimestamp() throws Exception { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp") @@ -868,6 +895,42 @@ public void testSortAndHostNameWithCustomSortConfig() throws Exception { assertEquals(0, newMapperServiceCounter.get()); } + public void testSortAndHostNoHost() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNoHostOldNode() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); + assertTrue(result.isEmpty()); + } + public void testSortAndHostNameKeyword() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 540f5ad162a41..e3faf3bdb5c1b 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.logsdb; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; @@ -53,6 +54,7 @@ public void setup() throws Exception { provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), IndexVersion::current, + () -> Version.CURRENT, true, true ); @@ -116,6 +118,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), IndexVersion::current, + () -> Version.CURRENT, true, true );