From db2c91e0d1e38ff7d69d146d3fc7ffec4615179a Mon Sep 17 00:00:00 2001 From: Chris Laverdiere Date: Tue, 25 Jul 2023 14:40:54 -0400 Subject: [PATCH] Update openai overview and metadata for web integration (#15350) * Update overview and metadata for web integration * Remove newline * Add API prefix * Update metadata * Update metadata * Update openai/README.md Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com> * Add api usage overview dashboard * Update openai/manifest.json Co-authored-by: Florent Clarret --------- Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com> Co-authored-by: Florent Clarret --- openai/README.md | 6 +- .../dashboards/usage_overview_dashboard.json | 555 ++++++++++++++++++ openai/manifest.json | 3 +- openai/metadata.csv | 5 + 4 files changed, 567 insertions(+), 2 deletions(-) create mode 100644 openai/assets/dashboards/usage_overview_dashboard.json diff --git a/openai/README.md b/openai/README.md index 389e75229c85c..a67f36de7088e 100644 --- a/openai/README.md +++ b/openai/README.md @@ -2,7 +2,7 @@ ## Overview -Get cost estimation, prompt and completion sampling, error tracking, performance metrics, and more out of [OpenAI][1] Python and Node.js library requests using Datadog metrics, APM, and logs. +Get cost estimation, prompt and completion sampling, error tracking, performance metrics, and more out of [OpenAI][1] account-level, Python, and Node.js library requests using Datadog metrics, APM, and logs. ## Setup @@ -12,6 +12,10 @@ Get cost estimation, prompt and completion sampling, error tracking, performance ### Installation + + + + 1. Enable APM and StatsD in your Datadog Agent. For example, in Docker: ```shell diff --git a/openai/assets/dashboards/usage_overview_dashboard.json b/openai/assets/dashboards/usage_overview_dashboard.json new file mode 100644 index 0000000000000..e0b997de990b5 --- /dev/null +++ b/openai/assets/dashboards/usage_overview_dashboard.json @@ -0,0 +1,555 @@ +{ + "title": "OpenAI Usage Overview", + "description": "Monitor OpenAI Usage metrics.", + "widgets": [ + { + "id": 6756498291092174, + "definition": { + "title": "", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1590802316495700, + "definition": { + "type": "image", + "url": "/static/images/logos/openai_large.svg", + "url_dark_theme": "/static/images/logos/openai_reversed_large.svg", + "sizing": "cover", + "has_background": true, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 2 + } + }, + { + "id": 7992422381542136, + "definition": { + "type": "note", + "content": "Track OpenAI API usage. You can use this dashboard to monitor your API requests and token usage at an account level.", + "background_color": "transparent", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 2, + "width": 6, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 5 + } + }, + { + "id": 1569477937421566, + "definition": { + "title": "Usage Overview", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 3104121683554248, + "definition": { + "title": "Total OpenAI requests", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "sum", + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_requests{$model,$operation}.as_count()" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "yaxis": {}, + "type": "bars" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 1848203546593340, + "definition": { + "title": "Total Response Tokens", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_generated_tokens_total{$model,$operation}" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "custom_unit": "tokens", + "precision": 2, + "timeseries_background": { + "yaxis": {}, + "type": "bars" + } + }, + "layout": { + "x": 2, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 4830582785535768, + "definition": { + "title": "Total Context Tokens", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_context_tokens_total{$model,$operation}" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "custom_unit": "tokens", + "precision": 2, + "timeseries_background": { + "yaxis": {}, + "type": "bars" + } + }, + "layout": { + "x": 4, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 1815867005998886, + "definition": { + "title": "Avg Context Tokens per request", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "avg", + "data_source": "metrics", + "name": "query1", + "query": "avg:openai.api.usage.n_context{$model,$operation}.as_count()" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "custom_unit": "tokens/req", + "precision": 2, + "timeseries_background": { + "yaxis": {}, + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 2, + "width": 3, + "height": 2 + } + }, + { + "id": 7545620141129682, + "definition": { + "title": "Avg Response Tokens per request", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "avg", + "data_source": "metrics", + "name": "query1", + "query": "avg:openai.api.usage.n_generated{$model,$operation}.as_count()" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "custom_unit": "tokens/req", + "precision": 2, + "timeseries_background": { + "yaxis": {}, + "type": "area" + } + }, + "layout": { + "x": 3, + "y": 2, + "width": 3, + "height": 2 + } + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 5 + } + }, + { + "id": 2219734989484006, + "definition": { + "title": "Request Usage", + "background_color": "vivid_pink", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7282385674326408, + "definition": { + "title": "Total OpenAI requests breakdown by model/operation", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "sum", + "data_source": "metrics", + "name": "query1", + "query": "avg:openai.api.usage.n_requests{$model,$operation} by {model,operation}" + } + ], + "response_format": "scalar" + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 5, + "width": 12, + "height": 4 + } + }, + { + "id": 3080457545117630, + "definition": { + "title": "Token Usage (Context)", + "background_color": "blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7581406721040970, + "definition": { + "title": "Context tokens breakdown by model/operation", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "sum", + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_context{$model,$operation} by {model,operation}.as_count()" + } + ], + "response_format": "scalar" + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 7215038769331264, + "definition": { + "title": "Total Context Token Usage by Model/Operation", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Prompt Token Count", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_context_tokens_total{$model,$operation} by {model,operation}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 6, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 7 + } + }, + { + "id": 3410216988240622, + "definition": { + "title": "Token Usage (Response)", + "background_color": "vivid_orange", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 5703892962287386, + "definition": { + "title": "Response tokens breakdown by model/operation", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "sum", + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_generated{$model,$operation} by {model,operation}.as_count()" + } + ], + "response_format": "scalar" + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 6479108410363704, + "definition": { + "title": "Total Response Token Usage by Model/Operation", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Prompt Token Count", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_generated_tokens_total{$model,$operation} by {model,operation}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 6, + "height": 3 + } + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 7 + } + } + ], + "template_variables": [ + { + "name": "model", + "prefix": "model", + "available_values": [], + "default": "*" + }, + { + "name": "operation", + "prefix": "operation", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} diff --git a/openai/manifest.json b/openai/manifest.json index bf68b9eb712b5..a6e3c30ac3c18 100644 --- a/openai/manifest.json +++ b/openai/manifest.json @@ -37,7 +37,8 @@ } }, "dashboards": { - "OpenAI Overview Dashboard": "assets/dashboards/overview_dashboard.json" + "OpenAI Overview Dashboard": "assets/dashboards/overview_dashboard.json", + "OpenAI Usage Overview": "assets/dashboards/usage_overview_dashboard.json" }, "monitors": { "Request Limits": "assets/recommended_monitors/request_limits.json", diff --git a/openai/metadata.csv b/openai/metadata.csv index 226762644faef..f1379da509460 100644 --- a/openai/metadata.csv +++ b/openai/metadata.csv @@ -8,3 +8,8 @@ openai.organization.ratelimit.requests.remaining,gauge,10,request,,Number of req openai.organization.ratelimit.tokens.remaining,gauge,10,token,,Number of tokens remaining in the rate limit.,0,openai,openai, openai.ratelimit.requests,gauge,10,request,,Number of requests in the rate limit.,0,openai,openai, openai.ratelimit.tokens,gauge,10,token,,Number of tokens in the rate limit.,0,openai,openai, +openai.api.usage.n_requests,count,,request,,Total number of requests,0,openai,n requests, +openai.api.usage.n_context,count,,token,,Number of context tokens used,0,openai,n context, +openai.api.usage.n_context_tokens_total,gauge,,token,,Total number of context tokens used (all-time),0,openai,n context tokens total, +openai.api.usage.n_generated,count,,token,,Number of generated response tokens,0,openai,n generated, +openai.api.usage.n_generated_tokens_total,gauge,,token,,Total number of generated response tokens (all-time),0,openai,n generated tokens total,