Add profiling plugin

docs/changelog/91640.yaml

@@ -0,0 +1,5 @@
+pr: 91640
+summary: Add profiling plugin
+area: Search
+type: enhancement
+issues: []

x-pack/plugin/profiler/build.gradle

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-es-plugin'
+
+esplugin {
+  name 'x-pack-profiling'
+  description 'The profiler plugin adds support for retrieving data from Universal Profiler.'
+  classname 'org.elasticsearch.xpack.profiler.ProfilingPlugin'
+}

x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/GetProfilingActionIT.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
+public class GetProfilingActionIT extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(ProfilingPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(ProfilingPlugin.PROFILING_ENABLED.getKey(), true)
+            .build();
+    }
+
+    private byte[] read(String resource) throws IOException {
+        return GetProfilingAction.class.getClassLoader().getResourceAsStream(resource).readAllBytes();
+    }
+
+    private void createIndex(String name, String bodyFileName) throws Exception {
+        client().admin().indices().prepareCreate(name).setSource(read(bodyFileName), XContentType.JSON).execute().get();
+    }
+
+    private void indexDoc(String index, String id, Map<String, Object> source) {
+        IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).get();
+        assertEquals(RestStatus.CREATED, indexResponse.status());
+    }
+
+    @Before
+    public void setupData() throws Exception {
+
+        for (String idx : EventsIndex.indexNames()) {
+            createIndex(idx, "events.json");
+        }
+        createIndex("profiling-stackframes", "stackframes.json");
+        createIndex("profiling-stacktraces", "stacktraces.json");
+        createIndex("profiling-executables", "executables.json");
+        ensureGreen();
+
+        // ensure that we have this in every index, so we find an event
+        for (String idx : EventsIndex.indexNames()) {
+            indexDoc(
+                idx,
+                "QjoLteG7HX3VUUXr-J4kHQ",
+                Map.of("@timestamp", 1668761065, "Stacktrace.id", "QjoLteG7HX3VUUXr-J4kHQ", "Stacktrace.count", 1)
+            );
+        }
+
+        indexDoc(
+            "profiling-stacktraces",
+            "QjoLteG7HX3VUUXr-J4kHQ",
+            Map.of("Stacktrace.frame.ids", "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf", "Stacktrace.frame.types", "AQI")
+        );
+        indexDoc(
+            "profiling-stackframes",
+            "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf",
+            Map.of("Stackframe.function.name", "_raw_spin_unlock_irqrestore")
+        );
+        indexDoc("profiling-executables", "QCCDqjSg3bMK1C4YRK6Tiw", Map.of("Executable.file.name", "libc.so.6"));
+
+        refresh();
+    }
+
+    public void testGetProfilingDataUnfiltered() throws Exception {
+        GetProfilingRequest request = new GetProfilingRequest(1, null);
+        GetProfilingResponse response = client().execute(GetProfilingAction.INSTANCE, request).get();
+        assertEquals(RestStatus.OK, response.status());
+        assertEquals(1, response.getTotalFrames());
+        assertNotNull(response.getStackTraces());
+        StackTrace stackTrace = response.getStackTraces().get("QjoLteG7HX3VUUXr-J4kHQ");
+        assertArrayEquals(new int[] { 1083999 }, stackTrace.addressOrLines);
+        assertArrayEquals(new String[] { "QCCDqjSg3bMK1C4YRK6Tiw" }, stackTrace.fileIds);
+        assertArrayEquals(new String[] { "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf" }, stackTrace.frameIds);
+        assertArrayEquals(new int[] { 2 }, stackTrace.typeIds);
+
+        assertNotNull(response.getStackFrames());
+        StackFrame stackFrame = response.getStackFrames().get("QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf");
+        assertEquals("_raw_spin_unlock_irqrestore", stackFrame.functionName);
+        assertNotNull(response.getStackTraceEvents());
+        assertEquals(1, (int) response.getStackTraceEvents().get("QjoLteG7HX3VUUXr-J4kHQ"));
+
+        assertNotNull(response.getExecutables());
+        assertNotNull("libc.so.6", response.getExecutables().get("QCCDqjSg3bMK1C4YRK6Tiw"));
+    }
+}

x-pack/plugin/profiler/src/internalClusterTest/resources/events.json

@@ -0,0 +1,78 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "4",
+      "max_result_window": 150000,
+      "refresh_interval": "10s",
+      "sort": {
+        "field": [
+          "service.name",
+          "@timestamp",
+          "orchestrator.resource.name",
+          "container.name",
+          "process.thread.name",
+          "host.id"
+        ]
+      }
+    },
+    "codec": "best_compression"
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "enabled": false
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "service.name": {
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        },
+        "host.id": {
+          "type": "keyword"
+        },
+        "Stacktrace.id": {
+          "type": "keyword",
+          "index": false
+        },
+        "orchestrator.resource.name": {
+          "type": "keyword"
+        },
+        "container.name": {
+          "type": "keyword"
+        },
+        "process.thread.name": {
+          "type": "keyword"
+        },
+        "Stacktrace.count": {
+          "type": "short",
+          "index": false
+        },
+        "agent.version": {
+          "type": "keyword"
+        },
+        "host.ip": {
+          "type": "ip"
+        },
+        "host.ipstring": {
+          "type": "keyword"
+        },
+        "host.name": {
+          "type": "keyword"
+        },
+        "os.kernel": {
+          "type": "keyword"
+        },
+        "tags": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json

@@ -0,0 +1,32 @@
+{
+  "settings": {
+    "index": {
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "Executable.build.id": {
+          "type": "keyword",
+          "index": true
+        },
+        "Executable.file.name": {
+          "type": "keyword",
+          "index": true
+        },
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json

@@ -0,0 +1,41 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "16",
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "Stackframe.line.number": {
+          "type": "integer",
+          "index": false
+        },
+        "Stackframe.file.name": {
+          "type": "keyword",
+          "index": false
+        },
+        "Stackframe.source.type": {
+          "type": "short",
+          "index": false
+        },
+        "Stackframe.function.name": {
+          "type": "keyword",
+          "index": false
+        },
+        "Stackframe.function.offset": {
+          "type": "integer",
+          "index": false
+        }
+      }
+    }
+  }
+}

x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json

@@ -0,0 +1,38 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "16",
+      "refresh_interval": "10s",
+      "sort": {
+        "field": [
+          "Stacktrace.frame.ids"
+        ]
+      }
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "Stacktrace.frame.ids": {
+          "type": "keyword",
+          "index": false
+        },
+        "Stacktrace.frame.types": {
+          "type": "keyword",
+          "index": false
+        },
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        }
+      }
+    }
+  }
+}