Skip to content

Commit

Permalink
WIP make kibana reporting a system data stream
Browse files Browse the repository at this point in the history
  • Loading branch information
@dakrone
dakrone committed Jan 9, 2025
1 parent f2e428a commit edce9fb
Show file tree
Hide file tree
Showing 5 changed files with 212 additions and 70 deletions.
1 change: 1 addition & 0 deletions modules/kibana/src/main/java/module-info.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@

module org.elasticsearch.kibana {
requires org.elasticsearch.server;
requires org.elasticsearch.xcontent;
}
211 changes: 211 additions & 0 deletions modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,24 @@

package org.elasticsearch.kibana;

import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.SystemDataStreamDescriptor;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.SystemIndexDescriptor.Type;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SystemIndexPlugin;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import static org.elasticsearch.indices.ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS;

public class KibanaPlugin extends Plugin implements SystemIndexPlugin {

Expand All @@ -31,6 +41,202 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
.setAllowsTemplates()
.build();

public static final SystemDataStreamDescriptor KIBANA_REPORTING_DS_DESCRIPTOR;

static {
try {
KIBANA_REPORTING_DS_DESCRIPTOR = new SystemDataStreamDescriptor(
".kibana-reporting",
"system data stream for reporting",
SystemDataStreamDescriptor.Type.EXTERNAL,
ComposableIndexTemplate.builder()
.indexPatterns(List.of(".kibana-reporting"))
.priority(200L)
.version(15L)
.allowAutoCreate(true)
.deprecated(false)
.ignoreMissingComponentTemplates(List.of("kibana-reporting@custom"))
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(true, false))
.metadata(Map.of("managed", "true", "description", "default kibana reporting template installed by elasticsearch"))
.componentTemplates(List.of("kibana-reporting@settings", "kibana-reporting@custom"))
.template(Template.builder().mappings(CompressedXContent.fromJSON("""
{
"properties" : {
"kibana_name" : {
"type" : "keyword"
},
"created_at" : {
"type" : "date"
},
"priority" : {
"type" : "byte"
},
"jobtype" : {
"type" : "keyword"
},
"created_by" : {
"type" : "keyword"
},
"migration_version" : {
"type" : "keyword"
},
"timeout" : {
"type" : "long"
},
"kibana_id" : {
"type" : "keyword"
},
"output" : {
"type" : "object",
"properties" : {
"content_type" : {
"type" : "keyword"
},
"size" : {
"type" : "long"
},
"csv_contains_formulas" : {
"type" : "boolean"
},
"warnings" : {
"type" : "text"
},
"chunk" : {
"type" : "long"
},
"error_code" : {
"type" : "keyword"
},
"max_size_reached" : {
"type" : "boolean"
},
"content" : {
"type" : "object",
"enabled" : false
}
}
},
"process_expiration" : {
"type" : "date"
},
"completed_at" : {
"type" : "date"
},
"payload" : {
"type" : "object",
"enabled" : false
},
"meta" : {
"properties" : {
"layout" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"isDeprecated" : {
"type" : "boolean"
},
"objectType" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
}
}
},
"parent_id" : {
"type" : "keyword"
},
"max_attempts" : {
"type" : "short"
},
"started_at" : {
"type" : "date"
},
"metrics" : {
"type" : "object",
"properties" : {
"pdf" : {
"type" : "object",
"properties" : {
"pages" : {
"type" : "long"
},
"memory" : {
"type" : "long"
},
"cpuInPercentage" : {
"type" : "double"
},
"cpu" : {
"type" : "double"
},
"memoryInMegabytes" : {
"type" : "double"
}
}
},
"csv" : {
"type" : "object",
"properties" : {
"rows" : {
"type" : "long"
}
}
},
"png" : {
"type" : "object",
"properties" : {
"memory" : {
"type" : "long"
},
"cpuInPercentage" : {
"type" : "double"
},
"cpu" : {
"type" : "double"
},
"memoryInMegabytes" : {
"type" : "double"
}
}
}
}
},
"attempts" : {
"type" : "short"
},
"status" : {
"type" : "keyword"
}
}
}""")).lifecycle(DataStreamLifecycle.newBuilder().enabled(true).build()))
.build(),
Map.of(
"kibana-reporting@settings",
new ComponentTemplate(
Template.builder()
.settings(Settings.builder().put("index.number_of_shards", 1).put("index.auto_expand_replicas", "0-1"))
.build(),
null,
null
)
),
KIBANA_PRODUCT_ORIGIN,
DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
);
} catch (IOException e) {
throw new RuntimeException("unable to read kibana reporting template JSON", e);
}
}

public static final SystemIndexDescriptor REPORTING_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
.setIndexPattern(".reporting-*")
.setDescription("system index for reporting")
Expand All @@ -52,6 +258,11 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
.setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
.build();

@Override
public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
return List.of(KIBANA_REPORTING_DS_DESCRIPTOR);
}

@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
return List.of(
Expand Down
45 changes: 0 additions & 45 deletions x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,6 @@ setup:
indices.get_index_template:
name: synthetics

- do:
indices.get_index_template:
name: .kibana-reporting

---
"Test logs index auto creation":
- do:
Expand Down Expand Up @@ -221,47 +217,6 @@ setup:
indices.delete_data_stream:
name: synthetics-foo-bar

---
"Test kibana reporting index auto creation":
- requires:
test_runner_features: ["headers"]

- do:
headers: { X-elastic-product-origin: kibana }
index:
index: .kibana-reporting-foo
body:
"@timestamp": "2020-01-01"
jobtype: "thing"

- do:
indices.get_data_stream:
name: .kibana-reporting-foo

- match: { data_streams.0.name: .kibana-reporting-foo }
- match: { data_streams.0.hidden: true }
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
- match: { data_streams.0.generation: 1 }
- length: { data_streams.0.indices: 1 }
- match: { data_streams.0.lifecycle.enabled: true }
- match: { data_streams.0.indices.0.index_name: '/\.ds-.kibana-reporting-foo-(\d{4}\.\d{2}\.\d{2}-)?000001/' }

- set: { data_streams.0.indices.0.index_name: idx0name }

- do:
indices.get:
index: $idx0name

- is_true: .$idx0name.settings
- is_true: .$idx0name.mappings
- match: { .$idx0name.mappings.properties.meta.properties.objectType.type: "text" }
- match: { .$idx0name.mappings.properties.meta.properties.layout.type: "text" }
- match: { .$idx0name.data_stream: ".kibana-reporting-foo" }

- do:
indices.delete_data_stream:
name: .kibana-reporting-foo

---
"Test wrong data_stream type":

Expand Down
20 changes: 0 additions & 20 deletions x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
public static final String SYNTHETICS_ILM_POLICY_NAME = "synthetics@lifecycle";
public static final String SYNTHETICS_INDEX_TEMPLATE_NAME = "synthetics";

///////////////////////////////////
// Kibana reporting template
///////////////////////////////////
public static final String KIBANA_REPORTING_INDEX_TEMPLATE_NAME = ".kibana-reporting";
public static final String KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME = "kibana-reporting@settings";

public StackTemplateRegistry(
Settings nodeSettings,
ClusterService clusterService,
Expand Down Expand Up @@ -198,13 +192,6 @@ private Map<String, ComponentTemplate> loadComponentTemplateConfigs() {
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE,
ADDITIONAL_TEMPLATE_VARIABLES
),
new IndexTemplateConfig(
KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
"/kibana-reporting@settings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE,
ADDITIONAL_TEMPLATE_VARIABLES
)
)) {
try {
Expand Down Expand Up @@ -286,13 +273,6 @@ protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE,
ADDITIONAL_TEMPLATE_VARIABLES
),
new IndexTemplateConfig(
KIBANA_REPORTING_INDEX_TEMPLATE_NAME,
"/kibana-reporting@template.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE,
ADDITIONAL_TEMPLATE_VARIABLES
)
);

Expand Down
5 changes: 0 additions & 5 deletions .../plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes);
Expand Down Expand Up @@ -472,10 +471,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
Expand Down

0 comments on commit edce9fb

Please sign in to comment.