Skip to content

Commit

Permalink
Use min node version to guard injecting settings in logs provider
Browse files Browse the repository at this point in the history
  • Loading branch information
@kkrik-es
kkrik-es committed Feb 20, 2025
1 parent 18df4d0 commit 63680d9
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 42 deletions.
2 changes: 1 addition & 1 deletion ...grade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCas
.module("x-pack-aggregate-metric")
.module("x-pack-stack")
.setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial")
.setting("xpack.license.self_generated.type", randomBoolean() ? "trial" : "basic")
// We upgrade from standard to logsdb, so we need to start with logsdb disabled,
// then later cluster.logsdb.enabled gets set to true and next rollover data stream is in logsdb mode.
.setting("cluster.logsdb.enabled", "false")
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
IndexVersion.current(),
parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion()
),
() -> parameters.clusterService().state().nodes().getMinNodeVersion(),
DiscoveryNode.isStateless(settings) == false,
DiscoveryNode.isStateless(settings) == false
);
Expand Down
10 changes: 8 additions & 2 deletions .../logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.UUIDs;
Expand Down Expand Up @@ -52,6 +53,7 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
private final LogsdbLicenseService licenseService;
private final SetOnce<CheckedFunction<IndexMetadata, MapperService, IOException>> mapperServiceFactory = new SetOnce<>();
private final SetOnce<Supplier<IndexVersion>> createdIndexVersion = new SetOnce<>();
private final SetOnce<Supplier<Version>> minNodeVersion = new SetOnce<>();
private final SetOnce<Boolean> supportFallbackToStoredSource = new SetOnce<>();
private final SetOnce<Boolean> supportFallbackLogsdbRouting = new SetOnce<>();

Expand All @@ -69,11 +71,13 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) {
void init(
CheckedFunction<IndexMetadata, MapperService, IOException> factory,
Supplier<IndexVersion> indexVersion,
Supplier<Version> minNodeVersion,
boolean supportFallbackToStoredSource,
boolean supportFallbackLogsdbRouting
) {
this.mapperServiceFactory.set(factory);
this.createdIndexVersion.set(indexVersion);
this.minNodeVersion.set(minNodeVersion);
this.supportFallbackToStoredSource.set(supportFallbackToStoredSource);
this.supportFallbackLogsdbRouting.set(supportFallbackLogsdbRouting);
}
Expand Down Expand Up @@ -111,7 +115,9 @@ && matchesLogsPattern(dataStreamName)) {
MappingHints mappingHints = getMappingHints(indexName, templateIndexMode, settings, combinedTemplateMappings);

// Inject stored source mode if synthetic source if not available per licence.
if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) {
if (mappingHints.hasSyntheticSourceUsage
&& supportFallbackToStoredSource.get()
&& minNodeVersion.get().get().onOrAfter(Version.V_8_17_0)) {
// This index name is used when validating component and index templates, we should skip this check in that case.
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
Expand All @@ -128,7 +134,7 @@ && matchesLogsPattern(dataStreamName)) {
}
}

if (isLogsDB) {
if (isLogsDB && minNodeVersion.get().get().onOrAfter(Version.V_8_18_0)) {
// Inject sorting on [host.name], in addition to [@timestamp].
if (mappingHints.sortOnHostName) {
if (mappingHints.addHostNameField) {
Expand Down
141 changes: 102 additions & 39 deletions ...db/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.xpack.logsdb;

import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.DataStream;
Expand Down Expand Up @@ -83,6 +84,10 @@ public void setup() throws Exception {
}

private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) {
return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT);
}

private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled, Version version) {
newMapperServiceCounter.set(0);
var provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Expand All @@ -91,15 +96,28 @@ private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boole
provider.init(im -> {
newMapperServiceCounter.incrementAndGet();
return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
}, IndexVersion::current, true, true);
}, IndexVersion::current, () -> version, true, true);
return provider;
}

private LogsdbIndexModeSettingsProvider withoutMapperService(boolean enabled) {
var provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", enabled).build()
);
provider.init(im -> null, IndexVersion::current, () -> Version.CURRENT, true, true);
return provider;
}

private Settings generateLogsdbSettings(Settings settings) throws IOException {
return generateLogsdbSettings(settings, null);
return generateLogsdbSettings(settings, null, Version.CURRENT);
}

private Settings generateLogsdbSettings(Settings settings, String mapping) throws IOException {
return generateLogsdbSettings(settings, mapping, Version.CURRENT);
}

private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException {
Metadata metadata = Metadata.EMPTY_METADATA;
var provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Expand All @@ -108,7 +126,7 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw
provider.init(im -> {
newMapperServiceCounter.incrementAndGet();
return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
}, IndexVersion::current, true, true);
}, IndexVersion::current, () -> version, true, true);
var result = provider.getAdditionalIndexSettings(
DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0),
DATA_STREAM_NAME,
Expand Down Expand Up @@ -217,11 +235,7 @@ public void testNonLogsDataStream() throws IOException {
}

public void testWithoutLogsComponentTemplate() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);

final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true);
final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
null,
"logs-apache-production",
Expand All @@ -236,11 +250,7 @@ public void testWithoutLogsComponentTemplate() throws IOException {
}

public void testWithLogsComponentTemplate() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);

final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true);
final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
null,
"logs-apache-production",
Expand All @@ -255,11 +265,7 @@ public void testWithLogsComponentTemplate() throws IOException {
}

public void testWithMultipleComponentTemplates() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);

final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true);
final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
null,
"logs-apache-production",
Expand All @@ -274,11 +280,7 @@ public void testWithMultipleComponentTemplates() throws IOException {
}

public void testWithCustomComponentTemplatesOnly() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);

final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true);
final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
null,
"logs-apache-production",
Expand All @@ -293,11 +295,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException {
}

public void testNonMatchingTemplateIndexPattern() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);

final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true);
final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
null,
"logs-apache-production",
Expand Down Expand Up @@ -331,10 +329,7 @@ public void testCaseSensitivity() throws IOException {
}

public void testMultipleHyphensInDataStreamName() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);
final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true);

final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
null,
Expand All @@ -349,12 +344,8 @@ public void testMultipleHyphensInDataStreamName() throws IOException {
assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName());
}

public void testBeforeAndAFterSettingUpdate() throws IOException {
final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", false).build()
);

public void testBeforeAndAfterSettingUpdate() throws IOException {
final LogsdbIndexModeSettingsProvider provider = withoutMapperService(false);
final Settings beforeSettings = provider.getAdditionalIndexSettings(
null,
"logs-apache-production",
Expand Down Expand Up @@ -627,7 +618,7 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept
}
}

public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException {
public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() {
String dataStreamName = DATA_STREAM_NAME;
Metadata.Builder mb = Metadata.builder(
DataStreamTestHelper.getClusterStateWithDataStreams(
Expand Down Expand Up @@ -698,6 +689,33 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
assertThat(newMapperServiceCounter.get(), equalTo(4));
}

public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() {
logsdbLicenseService.setSyntheticSourceFallback(true);
LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true, Version.V_8_16_0);
Metadata.Builder mb = Metadata.builder(
DataStreamTestHelper.getClusterStateWithDataStreams(
List.of(Tuple.tuple(DATA_STREAM_NAME, 1)),
List.of(),
Instant.now().toEpochMilli(),
builder().build(),
1
).getMetadata()
);
Metadata metadata = mb.build();
Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
.build();
var result = provider.getAdditionalIndexSettings(
DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 2),
DATA_STREAM_NAME,
null,
metadata,
Instant.ofEpochMilli(1L),
settings,
List.of()
);
assertTrue(result.isEmpty());
}

public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException {
logsdbLicenseService.setSyntheticSourceFallback(true);
LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true);
Expand Down Expand Up @@ -773,6 +791,15 @@ public void testRoutingPathOnSortFields() throws Exception {
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message"));
}

public void testRoutingPathOnSortFieldsDisabledInOldNode() throws Exception {
var settings = Settings.builder()
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message")
.put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
.build();
Settings result = generateLogsdbSettings(settings, null, Version.V_8_17_0);
assertTrue(result.isEmpty());
}

public void testRoutingPathOnSortFieldsFilterTimestamp() throws Exception {
var settings = Settings.builder()
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp")
Expand Down Expand Up @@ -868,6 +895,42 @@ public void testSortAndHostNameWithCustomSortConfig() throws Exception {
assertEquals(0, newMapperServiceCounter.get());
}

public void testSortAndHostNoHost() throws Exception {
var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
var mappings = """
{
"_doc": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
}
""";
Settings result = generateLogsdbSettings(settings, mappings);
assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
assertEquals(1, newMapperServiceCounter.get());
}

public void testSortAndHostNoHostOldNode() throws Exception {
var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
var mappings = """
{
"_doc": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
}
""";
Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0);
assertTrue(result.isEmpty());
}

public void testSortAndHostNameKeyword() throws Exception {
var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
var mappings = """
Expand Down
3 changes: 3 additions & 0 deletions ...st/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.xpack.logsdb;

import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexMode;
Expand Down Expand Up @@ -53,6 +54,7 @@ public void setup() throws Exception {
provider.init(
im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
IndexVersion::current,
() -> Version.CURRENT,
true,
true
);
Expand Down Expand Up @@ -116,6 +118,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception
provider.init(
im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
IndexVersion::current,
() -> Version.CURRENT,
true,
true
);
Expand Down

0 comments on commit 63680d9

Please sign in to comment.