Skip to content
This repository has been archived by the owner on Aug 9, 2022. It is now read-only.

Reporting backend metrics #282

Merged
merged 28 commits into from
Jan 6, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
036b82c
Add new stats endpoint and dummy response
abbashus Dec 21, 2020
665b0df
Add metric classes
abbashus Dec 21, 2020
a742391
Add json dependencies and more metric classes
abbashus Dec 21, 2020
7a8d7c4
Collect metrics to json
abbashus Dec 21, 2020
6739842
Add metrics json to stats response
abbashus Dec 21, 2020
a5611d7
Add json unflattener
abbashus Dec 21, 2020
43444a5
Add Json unflattening logic
abbashus Dec 21, 2020
fddeb3e
Add metric names for reporting backend
abbashus Dec 23, 2020
39b9d33
Merge branch 'dev' into report-metrics
abbashus Dec 23, 2020
75a92fc
Add new test endpoint and url params
abbashus Dec 28, 2020
7b8e2e0
Place metric counts for some report defintion APIs
abbashus Dec 28, 2020
c5b7319
Add a metric and some tests
abbashus Dec 29, 2020
6c6a264
Merge branch 'dev' into report-metrics
abbashus Jan 4, 2021
86117b6
Place more metrics
abbashus Jan 4, 2021
96f172a
Merge branch 'dev' into report-metrics
abbashus Jan 4, 2021
36a3c48
Refactor
abbashus Jan 4, 2021
15b9321
Refactor and place more metrics
abbashus Jan 4, 2021
acf8db3
Remove commented code
abbashus Jan 4, 2021
b3aeaca
Fx metric naming
abbashus Jan 4, 2021
534a460
remove test code
abbashus Jan 4, 2021
884ee9f
Fix copyright year
abbashus Jan 4, 2021
fa90f50
Address comments
abbashus Jan 4, 2021
22d9e07
Address comments
abbashus Jan 5, 2021
766ffe5
Address more comments
abbashus Jan 6, 2021
6f47ecb
Address comments
abbashus Jan 6, 2021
2da9ca5
Address comments
abbashus Jan 6, 2021
083f7f5
Fix metric name
abbashus Jan 6, 2021
b68ad64
Add javadoc to Metrics enum
abbashus Jan 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add metric classes
  • Loading branch information
abbashus committed Dec 21, 2020
commit 665b0dfb4a0780d0e22d1403e32f92aba6a82777
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

import java.util.concurrent.atomic.LongAdder;

public class BasicCounter implements Counter<Long> {
private LongAdder count = new LongAdder();

@Override
public void increment() {
count.increment();
}

@Override
public void add(long n) {
count.add(n);
}

@Override
public Long getValue() {
return count.longValue();
}

@Override
public void reset() {
count.reset();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

public interface Counter<T> {
void increment();

void add(long n);

T getValue();

void reset();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

import java.util.function.Supplier;

/**
* Gauge metric, an instant value like cpu usage, state and so on
*/
public class GaugeMetric<T> extends Metric<T> {
private Supplier<T> loadValue;

public GaugeMetric(String name, Supplier<T> supplier) {
super(name);
this.loadValue = supplier;
}

public String getName() {
return super.getName();
}

public T getValue() {
return loadValue.get();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

public abstract class Metric<T> {

private String name;

public Metric(String name) {
this.name = name;
}

public String getName() {
return name;
}

public abstract T getValue();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

import java.util.concurrent.ConcurrentHashMap;

public class Metrics {

private static Metrics metrics = new Metrics();
private ConcurrentHashMap<String, Metric<?>> registeredMetricsByName = new ConcurrentHashMap<>();

public static Metrics getInstance() {
return metrics;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

public class NumericMetric<T> extends Metric<T> {
private Counter<T> counter;

public NumericMetric(String name, Counter<T> counter) {
super(name);
this.counter = counter;
}

public String getName() {
return super.getName();
}

public Counter<T> getCounter() {
return counter;
}

public void increment() {
counter.increment();
}

public void increment(long n) {
counter.add(n);
}

public T getValue() {
return counter.getValue();
}

public void clear() {
counter.reset();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics;

import java.time.Clock;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.LongAdder;

/**
* Rolling counter. The count is refreshed every interval. In every interval the count is cumulative.
*/
public class RollingCounter implements Counter<Long> {
private static final long METRICS_ROLLING_WINDOW_VALUE = 3600L;
private static final long METRICS_ROLLING_INTERVAL_VALUE = 60L;

private final long capacity;
private final long window;
private final long interval;
private final Clock clock;
private final ConcurrentSkipListMap<Long, Long> time2CountWin;
private final LongAdder count;

public RollingCounter() {
this(METRICS_ROLLING_WINDOW_VALUE, METRICS_ROLLING_INTERVAL_VALUE);
}

public RollingCounter(long window, long interval, Clock clock) {
this.window = window;
this.interval = interval;
this.clock = clock;
time2CountWin = new ConcurrentSkipListMap<>();
count = new LongAdder();
capacity = window / interval * 2;
}

public RollingCounter(long window, long interval) {
this(window, interval, Clock.systemDefaultZone());
}

@Override
public void increment() {
add(1L);
}

@Override
public void add(long n) {
trim();
time2CountWin.compute(getKey(clock.millis()), (k, v) -> (v == null) ? n : v + n);
}

@Override
public Long getValue() {
return getValue(getPreKey(clock.millis()));
}

public long getValue(long key) {
Long res = time2CountWin.get(key);
if (res == null) {
return 0;
}
return res;
}

public long getSum() {
return count.longValue();
}

private void trim() {
if (time2CountWin.size() > capacity) {
time2CountWin.headMap(getKey(clock.millis() - window * 1000)).clear();
}
}

private long getKey(long millis) {
return millis / 1000 / this.interval;
}

private long getPreKey(long millis) {
return getKey(millis) - 1;
}

public int size() {
return time2CountWin.size();
}

public void reset() {
time2CountWin.clear();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.reportsscheduler.metrics

import com.amazon.opendistroforelasticsearch.reportsscheduler.resthandler.ReportStatsRestHandler.Companion.COUNT

enum class MetricName(val metricName: String, val value: Int, val numerical: Boolean) {

REPORT_DEFINITION_CREATE_COUNT("report_definition_create_count", COUNT, true),
REPORT_DEFINITION_CREATE_USER_ERROR("report_definition_create_user_error", COUNT, true),
REPORT_DEFINITION_CREATE_SYSTEM_ERROR("report_definition_create_system_error", COUNT, true),
REPORT_DEFINITION_UPDATE_COUNT("report_definition_update_count", COUNT, true),
REPORT_DEFINITION_UPDATE_USER_ERROR("report_definition_update_user_error", COUNT, true),
REPORT_DEFINITION_UPDATE_SYSTEM_ERROR("report_definition_update_system_error", COUNT, true),
REPORT_DEFINITION_DELETE_COUNT("report_definition_delete_count", COUNT, true),
REPORT_DEFINITION_DELETE_USER_ERROR("report_definition_delete_user_error", COUNT, true),
REPORT_DEFINITION_DELETE_SYSTEM_ERROR("report_definition_delete_system_error", COUNT, true),
REPORT_DEFINITION_LIST_COUNT("report_definition_list_count", COUNT, true),
REPORT_DEFINITION_LIST_USER_ERROR("report_definition_list_user_error", COUNT, true),
REPORT_DEFINITION_LIST_SYSTEM_ERROR("report_definition_list_system_error", COUNT, true),

REPORT_INSTANCE_LIST_COUNT("report_instance_list_count", COUNT, true),
REPORT_INSTANCE_LIST_USER_ERROR("report_instance_list_user_error", COUNT, true),
REPORT_INSTANCE_LIST_SYSTEM_ERROR("report_instance_list_system_error", COUNT, true)
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,24 @@
package com.amazon.opendistroforelasticsearch.reportsscheduler.resthandler

import com.amazon.opendistroforelasticsearch.reportsscheduler.ReportsSchedulerPlugin.Companion.BASE_REPORTS_URI
import com.amazon.opendistroforelasticsearch.reportsscheduler.action.CreateReportDefinitionAction
import com.amazon.opendistroforelasticsearch.reportsscheduler.action.DeleteReportDefinitionAction
import com.amazon.opendistroforelasticsearch.reportsscheduler.action.GetReportDefinitionAction
// import com.amazon.opendistroforelasticsearch.reportsscheduler.action.CreateReportDefinitionAction
// import com.amazon.opendistroforelasticsearch.reportsscheduler.action.DeleteReportDefinitionAction
// import com.amazon.opendistroforelasticsearch.reportsscheduler.action.GetReportDefinitionAction
import com.amazon.opendistroforelasticsearch.reportsscheduler.action.ReportDefinitionActions
import com.amazon.opendistroforelasticsearch.reportsscheduler.action.UpdateReportDefinitionAction
import com.amazon.opendistroforelasticsearch.reportsscheduler.model.CreateReportDefinitionRequest
import com.amazon.opendistroforelasticsearch.reportsscheduler.model.DeleteReportDefinitionRequest
import com.amazon.opendistroforelasticsearch.reportsscheduler.model.GetReportDefinitionRequest
import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.REPORT_DEFINITION_ID_FIELD
import com.amazon.opendistroforelasticsearch.reportsscheduler.model.UpdateReportDefinitionRequest
import com.amazon.opendistroforelasticsearch.reportsscheduler.util.contentParserNextToken
// import com.amazon.opendistroforelasticsearch.reportsscheduler.action.UpdateReportDefinitionAction
// import com.amazon.opendistroforelasticsearch.reportsscheduler.model.CreateReportDefinitionRequest
// import com.amazon.opendistroforelasticsearch.reportsscheduler.model.DeleteReportDefinitionRequest
// import com.amazon.opendistroforelasticsearch.reportsscheduler.model.GetReportDefinitionRequest
// import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.REPORT_DEFINITION_ID_FIELD
// import com.amazon.opendistroforelasticsearch.reportsscheduler.model.UpdateReportDefinitionRequest
// import com.amazon.opendistroforelasticsearch.reportsscheduler.util.contentParserNextToken
import org.elasticsearch.client.node.NodeClient
import org.elasticsearch.rest.BaseRestHandler
import org.elasticsearch.rest.BaseRestHandler.RestChannelConsumer
import org.elasticsearch.rest.BytesRestResponse
import org.elasticsearch.rest.RestHandler.Route
import org.elasticsearch.rest.RestRequest
import org.elasticsearch.rest.RestRequest.Method.DELETE
import org.elasticsearch.rest.RestRequest.Method.GET
import org.elasticsearch.rest.RestRequest.Method.POST
import org.elasticsearch.rest.RestRequest.Method.PUT
import org.elasticsearch.rest.RestStatus

/**
Expand Down