From dfda7c36c0aefcd1aef74b3ce29e50d8e061c131 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Mon, 31 Jul 2023 12:20:03 -0700 Subject: [PATCH] Fix send to gdrive Signed-off-by: Merwane Hamadi --- agbenchmark/conftest.py | 2 - agbenchmark/generate_test.py | 3 ++ agbenchmark/utils/get_data_from_helicone.py | 1 + get_data_from_helicone.py | 45 --------------------- send_to_googledrive.py | 42 ++++++++++--------- 5 files changed, 26 insertions(+), 67 deletions(-) delete mode 100644 get_data_from_helicone.py diff --git a/agbenchmark/conftest.py b/agbenchmark/conftest.py index 3e04007b379..a9ebba7ea4c 100644 --- a/agbenchmark/conftest.py +++ b/agbenchmark/conftest.py @@ -8,7 +8,6 @@ from typing import Any, Dict, Generator import pytest -from helicone.lock import HeliconeLockManager from agbenchmark.reports.reports import ( finalize_reports, @@ -169,7 +168,6 @@ def pytest_runtest_makereport(item: Any, call: Any) -> None: test_name = challenge_data["name"] generate_single_call_report(item, call, challenge_data) # else: it's a same_task=false suite (tests aren't combined) - HeliconeLockManager.write_custom_property("challenge", test_name) if call.when == "teardown": finalize_reports(item, challenge_data) diff --git a/agbenchmark/generate_test.py b/agbenchmark/generate_test.py index a347599bb25..928751837be 100644 --- a/agbenchmark/generate_test.py +++ b/agbenchmark/generate_test.py @@ -56,6 +56,9 @@ def create_single_test( # Define test method within the dynamically created class def test_method(self, config: Dict[str, Any], request) -> None: # type: ignore + from helicone.lock import HeliconeLockManager + HeliconeLockManager.write_custom_property("challenge", self.data.name) + cutoff = self.data.cutoff or 60 self.setup_challenge(config, cutoff) diff --git a/agbenchmark/utils/get_data_from_helicone.py b/agbenchmark/utils/get_data_from_helicone.py index 754a13418a4..b7ac78a052e 100644 --- a/agbenchmark/utils/get_data_from_helicone.py +++ b/agbenchmark/utils/get_data_from_helicone.py @@ -79,3 +79,4 @@ def get_data_from_helicone(challenge: str) -> Optional[float]: ) except Exception as err: print(f"Error occurred: {err}") + raise diff --git a/get_data_from_helicone.py b/get_data_from_helicone.py deleted file mode 100644 index c99da6576ff..00000000000 --- a/get_data_from_helicone.py +++ /dev/null @@ -1,45 +0,0 @@ -import json - -import requests - -# Define the endpoint of your GraphQL server -url = "https://www.helicone.ai/api/graphql" - -# Set the headers, usually you'd need to set the content type and possibly an authorization token -headers = {"authorization": "Bearer sk-"} - -# Define the query, variables, and operation name -query = """ -query ExampleQuery($limit: Int, $filters: [HeliconeRequestFilter!]) { - user { - id - } - heliconeRequest(limit: $limit,filters: $filters) { - responseBody - } -} -""" - -variables = { - "limit": 100, - "filters": [{"property": {"value": {"equals": "beebot"}, "name": "agent"}}], -} - -operation_name = "ExampleQuery" - -# Make the request -response = requests.post( - url, - headers=headers, - json={"query": query, "variables": variables, "operationName": operation_name}, -) -data = response.json() -total_tokens_sum = 0 - -for item in data["data"]["heliconeRequest"]: - total_tokens_sum += item["responseBody"]["usage"]["total_tokens"] - -# Extract the data from the response (consider adding error checks) - -print(json.dumps(data, indent=4, ensure_ascii=False)) -print(total_tokens_sum) diff --git a/send_to_googledrive.py b/send_to_googledrive.py index 0b0b98b69ba..798bcf50cc9 100644 --- a/send_to_googledrive.py +++ b/send_to_googledrive.py @@ -71,26 +71,28 @@ def process_test( # Loop over each directory in the base directory -for sub_dir in os.listdir(base_dir): - # Define the subdirectory path - sub_dir_path = os.path.join(base_dir, sub_dir) - - # Ensure the sub_dir_path is a directory - if os.path.isdir(sub_dir_path): - # Loop over each file in the subdirectory - for filename in os.listdir(sub_dir_path): - # Check if the file is a JSON file - if filename.endswith(".json"): - # Define the file path - file_path = os.path.join(sub_dir_path, filename) - - # Load the JSON data from the file - with open(file_path, "r") as f: - data = json.load(f) - - # Loop through each test - for test_name, test_info in data["tests"].items(): - process_test(test_name, test_info, sub_dir, data) +for agent_dir in os.listdir(base_dir): + agent_dir_path = os.path.join(base_dir, agent_dir) + + # Ensure the agent_dir_path is a directory + if os.path.isdir(agent_dir_path): + # Loop over each sub-directory in the agent directory (e.g., "folder49_07-28-03-53") + for report_folder in os.listdir(agent_dir_path): + report_folder_path = os.path.join(agent_dir_path, report_folder) + + # Ensure the report_folder_path is a directory + if os.path.isdir(report_folder_path): + # Check for a file named "report.json" in the sub-directory + report_path = os.path.join(report_folder_path, "report.json") + + if os.path.exists(report_path): + # Load the JSON data from the file + with open(report_path, "r") as f: + data = json.load(f) + + # Loop through each test + for test_name, test_info in data["tests"].items(): + process_test(test_name, test_info, agent_dir, data) # Convert the list of rows into a DataFrame df = pd.DataFrame(rows)