Add static linters ci (#45)

.flake8

@@ -0,0 +1,13 @@
+[flake8]
+max-line-length = 88
+select = "E303, W293, W291, W292, E305, E231, E302"
+exclude =
+    .tox,
+    __pycache__,
+    *.pyc,
+    .env
+    venv*/*,
+    .venv/*,
+    reports/*,
+    dist/*,
+    agent/*

.github/workflows/autogpt.yml

@@ -46,14 +46,14 @@ jobs:
 
       - name: Set up venv and install Python dependencies
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          poetry install
+          poetry install --only main
+          poetry build
+          
 
-      - name: Build project
+      - name: Run regression tests
         run: |
+          python -m venv venv
           source venv/bin/activate
-          poetry build
           cd agent/Auto-GPT
           pip install -r requirements.txt
           pip install ../../dist/agbenchmark-0.1.0-py3-none-any.whl

.github/workflows/ci.yml

@@ -0,0 +1,68 @@
+name: Python CI
+
+on:
+  push:
+    branches: [ master, ci-test* ]
+  pull_request:
+    branches: [ stable, master, release-* ]
+
+jobs:
+  lint:
+
+    runs-on: ubuntu-latest
+    env:
+      min-python-version: "3.10"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Set up Python ${{ env.min-python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.min-python-version }}
+
+      - id: get_date
+        name: Get date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+
+      - name: Set up Poetry cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pypoetry
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/poetry.lock') }}-${{ steps.get_date.outputs.date }}
+
+      - name: Install dependencies
+        run: |
+          poetry install
+
+      - name: Lint with flake8
+        run: poetry run flake8
+
+      - name: Check black formatting
+        run: poetry run  black . --check
+        if: success() || failure()
+
+      - name: Check isort formatting
+        run: poetry run  isort . --check
+        if: success() || failure()
+
+      - name: Check mypy formatting
+        run: poetry run mypy --ignore-missing-imports .
+        if: success() || failure()
+
+      - name: Check for unused imports and pass statements
+        run: |
+          cmd="poetry run autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring agbenchmark"
+          $cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1)
+        if: success() || failure()

.gitmodules

@@ -1,4 +1,4 @@
-[submodule "Auto-GPT"]
+[submodule "agent/Auto-GPT"]
 	path = agent/Auto-GPT
 	url = https://github.com/Significant-Gravitas/Auto-GPT.git
 	branch = benchmark-integration

.python-version

@@ -0,0 +1 @@
+3.10.10

agbenchmark/agent_interface.py

@@ -1,18 +1,22 @@
 import importlib
-
-from agbenchmark.mocks.MockManager import MockManager
 import os
-import sys
 import subprocess
+import sys
 import time
+from typing import Any, Dict, Optional
+
 from dotenv import load_dotenv
 
+from agbenchmark.mocks.mock_manager import MockManager
+
 load_dotenv()
 
 MOCK_FLAG = os.getenv("MOCK_TEST")
 
 
-def run_agent(task, mock_func, config):
+def run_agent(
+    task: Optional[str], mock_func: Optional[str], config: Dict[str, Any]
+) -> None:
     """Calling to get a response"""
 
     if mock_func == None and MOCK_FLAG == "True":
@@ -33,18 +37,24 @@ def run_agent(task, mock_func, config):
         # Add current directory to Python's import path
         sys.path.append(cwd)
 
-
         module_name = config["func_path"].replace("/", ".").rstrip(".py")
         module = importlib.import_module(module_name)
 
-
         command = [sys.executable, "benchmarks.py", str(task)]
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=cwd)
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            cwd=cwd,
+        )
 
         start_time = time.time()
         timeout = config["cutoff"]
 
         while True:
+            if process.stdout is None:
+                continue
             output = process.stdout.readline()
             print(output.strip())
 
@@ -55,7 +65,9 @@ def run_agent(task, mock_func, config):
 
             # Check if process has exceeded timeout
             if time.time() - start_time > timeout:
-                print("The Python function has exceeded the time limit and was terminated.")
+                print(
+                    "The Python function has exceeded the time limit and was terminated."
+                )
                 process.terminate()
                 break
 
@@ -66,5 +78,4 @@ def run_agent(task, mock_func, config):
         process.wait()
 
 
-
 ENVIRONMENT = os.getenv("ENVIRONMENT") or "production"

agbenchmark/Challenge.py → agbenchmark/challenge.py

@@ -1,11 +1,13 @@
-import os
 import glob
-import pytest
+import os
 from abc import ABC, abstractmethod
-from agbenchmark.challenges.define_task_types import Ground
-from agbenchmark.challenges.define_task_types import ChallengeData
+from typing import Any, Dict, List, Optional
+
+import pytest
 from dotenv import load_dotenv
 
+from agbenchmark.challenges.define_task_types import ChallengeData, Ground
+
 load_dotenv()
 
 mock_test_str = os.getenv("MOCK_TEST")
@@ -27,11 +29,11 @@ def data(self) -> ChallengeData:
         return ChallengeData.deserialize(self.get_file_path())
 
     @property
-    def mock(self):
+    def mock(self) -> Optional[str]:
         return self.data.mock.mock_func if self.data.mock else None
 
     @property
-    def task(self):
+    def task(self) -> Optional[str]:
         return (
             self.data.mock.mock_task if self.data.mock and MOCK_TEST else self.data.task
         )
@@ -40,7 +42,7 @@ def task(self):
     def dependencies(self) -> list:
         return self.data.dependencies
 
-    def setup_challenge(self, config):
+    def setup_challenge(self, config: Dict[str, Any]) -> None:
         from agbenchmark.agent_interface import run_agent
 
         run_agent(self.task, self.mock, config)
@@ -54,18 +56,18 @@ def name(self) -> str:
         [data],
         indirect=True,
     )
-    def test_method(self, config):
+    def test_method(self, config: Dict[str, Any]) -> None:
         raise NotImplementedError
 
     @staticmethod
-    def open_file(workspace: str, filename: str):
+    def open_file(workspace: str, filename: str) -> str:
         script_dir = os.path.abspath(workspace)
         workspace_dir = os.path.join(script_dir, filename)
         with open(workspace_dir, "r") as f:
             return f.read()
 
     @staticmethod
-    def open_files(workspace: str, file_patterns: list):
+    def open_files(workspace: str, file_patterns: list) -> List[str]:
         script_dir = os.path.abspath(workspace)
         files_contents = []
 
@@ -85,7 +87,7 @@ def open_files(workspace: str, file_patterns: list):
         return files_contents
 
     @staticmethod
-    def write_to_file(workspace: str, filename: str, content: str):
+    def write_to_file(workspace: str, filename: str, content: str) -> None:
         script_dir = os.path.abspath(workspace)
         print("Writing file at", script_dir)
         workspace_dir = os.path.join(script_dir, filename)
@@ -95,14 +97,14 @@ def write_to_file(workspace: str, filename: str, content: str):
             # Write the content to the file.
             f.write(content)
 
-    def get_filenames_in_workspace(self, workspace: str):
+    def get_filenames_in_workspace(self, workspace: str) -> List[str]:
         return [
             filename
             for filename in os.listdir(workspace)
             if os.path.isfile(os.path.join(workspace, filename))
         ]
 
-    def scoring(self, content: str, ground: Ground):
+    def scoring(self, content: str, ground: Ground) -> float:
         if ground.should_contain:
             for should_contain_word in ground.should_contain:
                 if should_contain_word not in content:

agbenchmark/challenges/define_task_types.py

@@ -1,7 +1,7 @@
-from pydantic import BaseModel
-from typing import List, Optional
 import json
-import os
+from typing import List, Optional
+
+from pydantic import BaseModel
 
 
 class Mock(BaseModel):

agbenchmark/challenges/retrieval/Retrieval.py

@@ -1,9 +1,8 @@
-from agbenchmark.Challenge import Challenge
 import pytest
 
+from agbenchmark.challenge import Challenge
+
 
 @pytest.mark.retrieval
 class RetrievalChallenge(Challenge):
     """Challenge for information-retrieval"""
-
-    pass

agbenchmark/challenges/retrieval/r1/r1_test.py

@@ -1,5 +1,7 @@
-from agbenchmark.challenges.retrieval.Retrieval import RetrievalChallenge
 import os
+from typing import Any, Dict
+
+from agbenchmark.challenges.retrieval.retrieval import RetrievalChallenge
 
 
 class TestRetrieval1(RetrievalChallenge):
@@ -8,7 +10,7 @@ class TestRetrieval1(RetrievalChallenge):
     def get_file_path(self) -> str:  # all tests must implement this method
         return os.path.join(os.path.dirname(__file__), "r1_data.json")
 
-    def test_method(self, config):
+    def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)
         files_contents = self.open_files(config["workspace"], self.data.ground.files)