Update Elastic Package Registry integration to use new indexer metric…

…s (get operations) (#3863) Update field mappings, ingest pipeline and dashboard accordingly with the changes done in elastic/package-registry#853 Specially, get operations related to indexers are now reporting the values per each indexer (using labels).
elastic · Jul 28, 2022 · a1664e9 · a1664e9
1 parent 711f246
commit a1664e9
Show file tree

Hide file tree

Showing 11 changed files with 314 additions and 241 deletions.
diff --git a/packages/elastic_package_registry/_dev/deploy/docker/Dockerfile b/packages/elastic_package_registry/_dev/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/package-registry/package-registry:v1.10.0
+FROM docker.elastic.co/package-registry/package-registry:v1.11.0
 
 COPY packages /packages
 COPY config.yml /package-registry/config.yml
diff --git a/packages/elastic_package_registry/changelog.yml b/packages/elastic_package_registry/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.0.3"
+  changes:
+    - description: Update indexer metrics related to get operations
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3863/
 - version: "0.0.2"
   changes:
     - description: Remove release experimental field

diff --git a/packages/elastic_package_registry/data_stream/metrics/_dev/test/pipeline/test-default.json b/packages/elastic_package_registry/data_stream/metrics/_dev/test/pipeline/test-default.json
@@ -117,7 +117,7 @@
                         ]
                     }
                 },
-                "epr_storage_indexer_get_duration_seconds": {
+                "epr_indexer_get_duration_seconds": {
                     "histogram": {
                         "counts": [
                             0,

diff --git a/...c_package_registry/data_stream/metrics/_dev/test/pipeline/test-default.json-expected.json b/...c_package_registry/data_stream/metrics/_dev/test/pipeline/test-default.json-expected.json
@@ -5,25 +5,10 @@
             "ecs": {
                 "version": "8.3.1"
             },
-            "service": {
-                "address": "http://elastic-package-service_elastic_package_registry_1:9110/metrics",
-                "type": "elastic_package_registry"
-            },
             "event": {
                 "kind": "metric"
             },
             "package_registry": {
-                "labels": {
-                    "code": "200",
-                    "instance": "elastic-package-service_elastic_package_registry_1:9110",
-                    "job": "prometheus",
-                    "version": "1.10.1",
-                    "method": "get",
-                    "path": "/"
-                },
-                "http_requests_total": {
-                    "counter": 20
-                },
                 "http": {
                     "request_duration_seconds": {
                         "histogram": {
@@ -114,11 +99,11 @@
                         }
                     }
                 },
+                "http_requests_total": {
+                    "counter": 20
+                },
                 "in_flight_requests": 0,
-                "number_indexed_packages": 1600,
-                "start_time_seconds": 1.65831636059E9,
-                "start_time": "2022-07-20T11:26:00.590Z",
-                "storage_indexer": {
+                "indexer": {
                     "get_duration_seconds": {
                         "histogram": {
                             "counts": [
@@ -150,10 +135,20 @@
                                 15
                             ]
                         }
-                    },
-                    "update_index_success_total": {
-                        "counter": 0
-                    },
+                    }
+                },
+                "labels": {
+                    "code": "200",
+                    "instance": "elastic-package-service_elastic_package_registry_1:9110",
+                    "job": "prometheus",
+                    "method": "get",
+                    "path": "/",
+                    "version": "1.10.1"
+                },
+                "number_indexed_packages": 1600,
+                "start_time": "2022-07-20T11:26:00.590Z",
+                "start_time_seconds": 1.65831636059E9,
+                "storage_indexer": {
                     "update_index_duration_seconds": {
                         "histogram": {
                             "counts": [
@@ -188,12 +183,19 @@
                     },
                     "update_index_error_total": {
                         "counter": 0
+                    },
+                    "update_index_success_total": {
+                        "counter": 0
                     }
                 },
                 "storage_requests_total": {
                     "counter": 0
                 },
                 "uptime": 114
+            },
+            "service": {
+                "address": "http://elastic-package-service_elastic_package_registry_1:9110/metrics",
+                "type": "elastic_package_registry"
             }
         }
     ]

diff --git a/...es/elastic_package_registry/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml b/...es/elastic_package_registry/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml
@@ -19,8 +19,8 @@ processors:
       target_field: package_registry.labels
       ignore_missing: true
   - rename:
-      field: tmp.epr_storage_indexer_get_duration_seconds
-      target_field: package_registry.storage_indexer.get_duration_seconds
+      field: tmp.epr_indexer_get_duration_seconds
+      target_field: package_registry.indexer.get_duration_seconds
       ignore_missing: true
   - rename:
       field: tmp.epr_storage_indexer_update_index_error_total

diff --git a/packages/elastic_package_registry/data_stream/metrics/fields/fields.yml b/packages/elastic_package_registry/data_stream/metrics/fields/fields.yml
@@ -9,6 +9,10 @@
       type: keyword
       description: Component type of the storage (statics, artifacts, signature...)
       dimension: true
+    - name: indexer
+      type: keyword
+      description: Indexer type
+      dimension: true
     - name: instance
       type: keyword
       description: Elastic Package Registry instance
@@ -104,7 +108,7 @@
   description: >
     Number of indexed packages
 
-- name: package_registry.storage_indexer
+- name: package_registry.indexer
   type: group
   fields:
     - name: get_duration_seconds.histogram
@@ -122,30 +126,33 @@
       description: >
         Bucket values of the histogram of latencies for get processes run by the indexer
 
+- name: package_registry.storage_indexer
+  type: group
+  fields:
     - name: update_index_duration_seconds.histogram
       type: histogram
       description: >
-        A histogram of latencies for update index processes run by the indexer
+        A histogram of latencies for update index processes run by the storage indexer
 
     - name: update_index_duration_seconds.histogram.counts
       type: histogram
       description: >
-        Counters of the histogram of latencies for update index processes run by the indexer
+        Counters of the histogram of latencies for update index processes run by the storage indexer
 
     - name: update_index_duration_seconds.histogram.values
       type: histogram
       description: >
-        Bucket values of the histogram of latencies for update index processes run by the indexer
+        Bucket values of the histogram of latencies for update index processes run by the storage indexer
 
     - name: update_index_error_total.counter
       type: long
       description: >
-        A counter for all the update index processes that finished with error
+        A counter for all the update index processes that finished with error in the storage indexer
 
     - name: update_index_success_total.counter
       type: long
       description: >
-        A counter for all the update index processes that finished with error
+        A counter for all the update index processes that finished with success in the storage indexer
 
 - name: package_registry.start_time_seconds
   type: double

diff --git a/packages/elastic_package_registry/data_stream/metrics/sample_event.json b/packages/elastic_package_registry/data_stream/metrics/sample_event.json
@@ -1,11 +1,11 @@
 {
-    "@timestamp": "2022-07-20T12:31:14.439Z",
+    "@timestamp": "2022-07-28T09:54:47.993Z",
     "agent": {
-        "ephemeral_id": "2e34ed1d-357c-482c-9640-d0ab91f43e0b",
-        "id": "2e305fba-81f6-49b1-be45-f7198c6aaa53",
+        "ephemeral_id": "fb0962b4-3f3f-48d6-81d6-3df63366f744",
+        "id": "97cba3e2-ea7d-4d80-aa69-75752faa1576",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
-        "version": "8.3.2"
+        "version": "8.0.0"
     },
     "data_stream": {
         "dataset": "elastic_package_registry.metrics",
@@ -16,15 +16,15 @@
         "version": "8.3.1"
     },
     "elastic_agent": {
-        "id": "2e305fba-81f6-49b1-be45-f7198c6aaa53",
+        "id": "97cba3e2-ea7d-4d80-aa69-75752faa1576",
         "snapshot": false,
-        "version": "8.3.2"
+        "version": "8.0.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "elastic_package_registry.metrics",
-        "duration": 27108917,
-        "ingested": "2022-07-20T12:31:15Z",
+        "duration": 7602509,
+        "ingested": "2022-07-28T09:54:51Z",
         "kind": "metric",
         "module": "prometheus"
     },
@@ -33,10 +33,10 @@
         "containerized": false,
         "hostname": "docker-fleet-agent",
         "ip": [
-            "192.168.112.7"
+            "172.31.0.7"
         ],
         "mac": [
-            "02:42:c0:a8:70:07"
+            "02:42:ac:1f:00:07"
         ],
         "name": "docker-fleet-agent",
         "os": {
@@ -46,24 +46,16 @@
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
-            "version": "20.04.4 LTS (Focal Fossa)"
+            "version": "20.04.3 LTS (Focal Fossa)"
         }
     },
     "metricset": {
         "name": "collector",
         "period": 30000
     },
     "package_registry": {
-        "in_flight_requests": 0,
-        "labels": {
-            "instance": "elastic-package-service_elastic_package_registry_1:9110",
-            "job": "prometheus"
-        },
-        "number_indexed_packages": 1434,
-        "start_time": "2022-07-20T12:30:46.030Z",
-        "start_time_seconds": 1658320246.03,
-        "storage_indexer": {
-            "get_duration_seconds": {
+        "http": {
+            "request_duration_seconds": {
                 "histogram": {
                     "counts": [
                         0,
@@ -95,11 +87,37 @@
                     ]
                 }
             },
-            "update_index_duration_seconds": {
+            "request_size_bytes": {
                 "histogram": {
                     "counts": [
                         0,
                         0,
+                        0,
+                        0,
+                        0,
+                        0,
+                        0,
+                        0,
+                        0,
+                        0
+                    ],
+                    "values": [
+                        8,
+                        24,
+                        48,
+                        96,
+                        192,
+                        384,
+                        768,
+                        33280,
+                        163840,
+                        458752
+                    ]
+                }
+            },
+            "response_size_bytes": {
+                "histogram": {
+                    "counts": [
                         0,
                         0,
                         0,
@@ -112,29 +130,30 @@
                         0
                     ],
                     "values": [
-                        0.0025,
-                        0.0075,
-                        0.0175,
-                        0.037500000000000006,
-                        0.07500000000000001,
-                        0.175,
-                        0.375,
-                        0.75,
-                        1.75,
-                        3.75,
-                        7.5,
-                        15
+                        8,
+                        24,
+                        48,
+                        96,
+                        192,
+                        384,
+                        768,
+                        33280,
+                        163840,
+                        458752
                     ]
                 }
-            },
-            "update_index_error_total": {
-                "counter": 0
-            },
-            "update_index_success_total": {
-                "counter": 0
             }
         },
-        "uptime": 28
+        "http_requests_total": {
+            "counter": 20
+        },
+        "labels": {
+            "code": "200",
+            "instance": "elastic-package-service_elastic_package_registry_1:9110",
+            "job": "prometheus",
+            "method": "get",
+            "path": "/"
+        }
     },
     "service": {
         "address": "http://elastic-package-service_elastic_package_registry_1:9110/metrics",