Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TSDB: Add _tsid field to time_series indices #80276

Merged
merged 47 commits into from
Nov 29, 2021
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
2119b6d
Initial commit
csoulios Nov 3, 2021
9461331
Checkstyle
csoulios Nov 3, 2021
2ab1ff5
Merge branch 'master' into tsid-gen
csoulios Nov 3, 2021
fff4d7e
Removed TimeSeriesIdGeneratorService class
csoulios Nov 3, 2021
ca6c86e
Cleanup
csoulios Nov 3, 2021
12d89cf
Merge branch 'master' into tsid-gen
csoulios Nov 3, 2021
681831f
Revert
csoulios Nov 3, 2021
8267f74
Remove check for dynamic mapping update
csoulios Nov 3, 2021
7fbedc8
Return a shared instance of the tsid field mapper
csoulios Nov 3, 2021
1b5ef49
Remove tsid generator from Mapping class
csoulios Nov 4, 2021
95278e4
Remove TimeSeriesIdGenerator class
csoulios Nov 4, 2021
505d42b
Merge branch 'master' into tsid-gen
csoulios Nov 8, 2021
3ecaa8e
Merge branch 'master' into tsid-gen
csoulios Nov 9, 2021
e62254d
Delegate tsid generation to TsidFieldMapper
csoulios Nov 9, 2021
2975b48
Simplified dimension fields
csoulios Nov 9, 2021
7587518
More cleanup
csoulios Nov 9, 2021
b6f3f94
More cleanup
csoulios Nov 9, 2021
68e1526
Cleanup exception handling when parsing _tsid
csoulios Nov 9, 2021
a125aab
Fix broken tests
csoulios Nov 10, 2021
07ed667
Merge branch 'master' into tsid-gen
csoulios Nov 10, 2021
bc43ba0
Added more tests
csoulios Nov 10, 2021
5302edb
More tests
csoulios Nov 10, 2021
6860f38
More tests
csoulios Nov 10, 2021
05ed0d7
Cleanup
csoulios Nov 10, 2021
1a7e308
Merge branch 'master' into tsid-gen
csoulios Nov 11, 2021
8272182
Added rest tests for _tsid
csoulios Nov 11, 2021
05fc921
Merge branch 'master' into tsid-gen
csoulios Nov 11, 2021
15334e0
Sort by _tsid, @timestamp
csoulios Nov 11, 2021
5aaf469
Catch IOException at tsid field mapper level
csoulios Nov 11, 2021
9474774
spotless apply
csoulios Nov 11, 2021
359a22c
Fix test
csoulios Nov 13, 2021
02c2091
Fix test
csoulios Nov 13, 2021
14c23fe
Fix test
csoulios Nov 13, 2021
287a4c8
Merge branch 'master' into tsid-gen
csoulios Nov 13, 2021
7b87a71
Fix tests
csoulios Nov 15, 2021
271c66a
Merge branch 'master' into tsid-gen
csoulios Nov 15, 2021
1c1f0f8
Fix tests
csoulios Nov 15, 2021
713041d
Add _tsid field mapper only to time_series indices
csoulios Nov 16, 2021
c1cf213
Merge branch 'master' into tsid-gen
csoulios Nov 23, 2021
6a8dca6
Fix conflicts
csoulios Nov 23, 2021
5106a6e
Delegated building of tsid field mapper creation
csoulios Nov 23, 2021
0e10639
Fix brojen test
csoulios Nov 23, 2021
6d7d4ae
Merge branch 'master' into tsid-gen
csoulios Nov 23, 2021
91c91db
Fix broken tests
csoulios Nov 23, 2021
709700b
Addressed reviewer comments
csoulios Nov 25, 2021
1453e63
Merge branch 'master' into tsid-gen
csoulios Nov 25, 2021
12e5105
Merge branch 'master' into tsid-gen
csoulios Nov 29, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
import org.elasticsearch.core.Nullable;
import org.elasticsearch.gateway.MetadataStateFormat;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.IndexLongFieldRange;
Expand Down Expand Up @@ -476,6 +478,8 @@ public static APIBlock readFrom(StreamInput input) throws IOException {

private final IndexLongFieldRange timestampRange;

private final IndexMode mode;

private final int priority;

private final long creationDate;
Expand Down Expand Up @@ -516,7 +520,8 @@ private IndexMetadata(
final int priority,
final long creationDate,
final boolean ignoreDiskWatermarks,
@Nullable final List<String> tierPreference
@Nullable final List<String> tierPreference,
final IndexMode mode
) {

this.index = index;
Expand Down Expand Up @@ -557,6 +562,9 @@ private IndexMetadata(
this.creationDate = creationDate;
this.ignoreDiskWatermarks = ignoreDiskWatermarks;
this.tierPreference = tierPreference;
this.mode = mode;
assert false == (mode.organizeIntoTimeSeries() && isRoutingPartitionedIndex())
: "time series indices incompatible with routing partitioned indices";
assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards;
}

Expand Down Expand Up @@ -1455,6 +1463,8 @@ public IndexMetadata build() {

final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);

final IndexMode mode = IndexSettings.MODE.get(settings);

List<String> tierPreference;
try {
tierPreference = DataTier.parseTierList(DataTier.TIER_PREFERENCE_SETTING.get(settings));
Expand Down Expand Up @@ -1497,7 +1507,8 @@ public IndexMetadata build() {
IndexMetadata.INDEX_PRIORITY_SETTING.get(settings),
settings.getAsLong(SETTING_CREATION_DATE, -1L),
DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS.get(settings),
tierPreference
tierPreference,
mode
);
}

Expand Down Expand Up @@ -2050,6 +2061,10 @@ public static int getRoutingFactor(int sourceNumberOfShards, int targetNumberOfS
return factor;
}

public IndexMode mode() {
return mode;
}

/**
* Parses the number from the rolled over index name. It also supports the date-math format (ie. index name is wrapped in &lt; and &gt;)
* E.g.
Expand Down
9 changes: 9 additions & 0 deletions server/src/main/java/org/elasticsearch/index/IndexMode.java
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ private void validateTimeStampField(Object timestampFieldValue) {
);
}
}

@Override
public boolean organizeIntoTimeSeries() {
csoulios marked this conversation as resolved.
Show resolved Hide resolved
return true;
}
};

private static final List<Setting<?>> TIME_SERIES_UNSUPPORTED = List.of(
Expand Down Expand Up @@ -171,4 +176,8 @@ private void validateTimeStampField(Object timestampFieldValue) {
* Validate and/or modify the mappings after after they've been parsed.
*/
public abstract void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder);

public boolean organizeIntoTimeSeries() {
return false;
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -650,7 +650,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
nodeName = Node.NODE_NAME_SETTING.get(settings);
this.indexMetadata = indexMetadata;
numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null);
mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD;
this.mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD;
csoulios marked this conversation as resolved.
Show resolved Hide resolved

this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
Expand Down
Loading