Skip to content

Commit

Permalink
Rework plugin config to be file-based (Significant-Gravitas#4673)
Browse files Browse the repository at this point in the history
  • Loading branch information
erik-megarad authored Jun 14, 2023
1 parent 0c8f2cf commit 49d1a5a
Show file tree
Hide file tree
Showing 10 changed files with 262 additions and 176 deletions.
14 changes: 3 additions & 11 deletions .env.template
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ OPENAI_API_KEY=your-openai-api-key
## AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml)
# AI_SETTINGS_FILE=ai_settings.yaml

## PLUGINS_CONFIG_FILE - The path to the plugins_config.yaml file (Default plugins_config.yaml)
# PLUGINS_CONFIG_FILE=plugins_config.yaml

## PROMPT_SETTINGS_FILE - Specifies which Prompt Settings file to use (defaults to prompt_settings.yaml)
# PROMPT_SETTINGS_FILE=prompt_settings.yaml

Expand All @@ -38,7 +41,6 @@ OPENAI_API_KEY=your-openai-api-key
## DISABLED_COMMAND_CATEGORIES - The list of categories of commands that are disabled (Default: None)
# DISABLED_COMMAND_CATEGORIES=


################################################################################
### LLM PROVIDER
################################################################################
Expand Down Expand Up @@ -194,16 +196,6 @@ OPENAI_API_KEY=your-openai-api-key
## ELEVENLABS_VOICE_ID - Eleven Labs voice ID (Example: None)
# ELEVENLABS_VOICE_ID=

################################################################################
### ALLOWLISTED PLUGINS
################################################################################

## ALLOWLISTED_PLUGINS - Sets the listed plugins that are allowed (Default: None)
# ALLOWLISTED_PLUGINS=

## DENYLISTED_PLUGINS - Sets the listed plugins that are not allowed (Default: None)
# DENYLISTED_PLUGINS=

################################################################################
### CHAT MESSAGES
################################################################################
Expand Down
18 changes: 18 additions & 0 deletions autogpt/config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from auto_gpt_plugin_template import AutoGPTPluginTemplate
from colorama import Fore

import autogpt
from autogpt.singleton import Singleton


Expand Down Expand Up @@ -156,20 +157,37 @@ def __init__(self) -> None:
self.plugins: List[AutoGPTPluginTemplate] = []
self.plugins_openai = []

# Deprecated. Kept for backwards-compatibility. Will remove in a future version.
plugins_allowlist = os.getenv("ALLOWLISTED_PLUGINS")
if plugins_allowlist:
self.plugins_allowlist = plugins_allowlist.split(",")
else:
self.plugins_allowlist = []

# Deprecated. Kept for backwards-compatibility. Will remove in a future version.
plugins_denylist = os.getenv("DENYLISTED_PLUGINS")
if plugins_denylist:
self.plugins_denylist = plugins_denylist.split(",")
else:
self.plugins_denylist = []

# Avoid circular imports
from autogpt.plugins import DEFAULT_PLUGINS_CONFIG_FILE

self.plugins_config_file = os.getenv(
"PLUGINS_CONFIG_FILE", DEFAULT_PLUGINS_CONFIG_FILE
)
self.load_plugins_config()

self.chat_messages_enabled = os.getenv("CHAT_MESSAGES_ENABLED") == "True"

def load_plugins_config(self) -> "autogpt.plugins.PluginsConfig":
# Avoid circular import
from autogpt.plugins.plugins_config import PluginsConfig

self.plugins_config = PluginsConfig.load_config(global_config=self)
return self.plugins_config

def get_azure_deployment_id_for_model(self, model: str) -> str:
"""
Returns the relevant deployment id for the model specified.
Expand Down
59 changes: 22 additions & 37 deletions autogpt/plugins.py → autogpt/plugins/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,14 @@
from auto_gpt_plugin_template import AutoGPTPluginTemplate
from openapi_python_client.config import Config as OpenAPIConfig

from autogpt.config import Config
from autogpt.config.config import Config
from autogpt.logs import logger
from autogpt.models.base_open_ai_plugin import BaseOpenAIPlugin

DEFAULT_PLUGINS_CONFIG_FILE = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "..", "plugins_config.yaml"
)


def inspect_zip_for_modules(zip_path: str, debug: bool = False) -> list[str]:
"""
Expand Down Expand Up @@ -215,9 +219,7 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
loaded_plugins = []
# Generic plugins
plugins_path_path = Path(cfg.plugins_dir)

logger.debug(f"Allowlisted Plugins: {cfg.plugins_allowlist}")
logger.debug(f"Denylisted Plugins: {cfg.plugins_denylist}")
plugins_config = cfg.plugins_config

# Directory-based plugins
for plugin_path in [f.path for f in os.scandir(cfg.plugins_dir) if f.is_dir()]:
Expand All @@ -232,11 +234,14 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
__import__(qualified_module_name)
plugin = sys.modules[qualified_module_name]

if not plugins_config.is_enabled(plugin_module_name):
logger.warn(f"Plugin {plugin_module_name} found but not configured")
continue

for _, class_obj in inspect.getmembers(plugin):
if (
hasattr(class_obj, "_abc_impl")
and AutoGPTPluginTemplate in class_obj.__bases__
and denylist_allowlist_check(plugin_module_name, cfg)
):
loaded_plugins.append(class_obj())

Expand All @@ -249,6 +254,12 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
logger.debug(f"Plugin: {plugin} Module: {module}")
zipped_package = zipimporter(str(plugin))
zipped_module = zipped_package.load_module(str(module.parent))
plugin_module_name = zipped_module.__name__.split(os.path.sep)[-1]

if not plugins_config.is_enabled(plugin_module_name):
logger.warn(f"Plugin {plugin_module_name} found but not configured")
continue

for key in dir(zipped_module):
if key.startswith("__"):
continue
Expand All @@ -257,7 +268,6 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
if (
"_abc_impl" in a_keys
and a_module.__name__ != "AutoGPTPluginTemplate"
and denylist_allowlist_check(a_module.__name__, cfg)
):
loaded_plugins.append(a_module())

Expand All @@ -269,40 +279,15 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
manifests_specs, cfg, debug
)
for url, openai_plugin_meta in manifests_specs_clients.items():
if denylist_allowlist_check(url, cfg):
plugin = BaseOpenAIPlugin(openai_plugin_meta)
loaded_plugins.append(plugin)
if not plugins_config.is_enabled(url):
logger.warn(f"Plugin {plugin_module_name} found but not configured")
continue

plugin = BaseOpenAIPlugin(openai_plugin_meta)
loaded_plugins.append(plugin)

if loaded_plugins:
logger.info(f"\nPlugins found: {len(loaded_plugins)}\n" "--------------------")
for plugin in loaded_plugins:
logger.info(f"{plugin._name}: {plugin._version} - {plugin._description}")
return loaded_plugins


def denylist_allowlist_check(plugin_name: str, cfg: Config) -> bool:
"""Check if the plugin is in the allowlist or denylist.
Args:
plugin_name (str): Name of the plugin.
cfg (Config): Config object.
Returns:
True or False
"""
logger.debug(f"Checking if plugin {plugin_name} should be loaded")
if (
plugin_name in cfg.plugins_denylist
or "all" in cfg.plugins_denylist
or "none" in cfg.plugins_allowlist
):
logger.debug(f"Not loading plugin {plugin_name} as it was in the denylist.")
return False
if plugin_name in cfg.plugins_allowlist or "all" in cfg.plugins_allowlist:
logger.debug(f"Loading plugin {plugin_name} as it was in the allowlist.")
return True
ack = input(
f"WARNING: Plugin {plugin_name} found. But not in the"
f" allowlist... Load? ({cfg.authorise_key}/{cfg.exit_key}): "
)
return ack.lower() == cfg.authorise_key
14 changes: 14 additions & 0 deletions autogpt/plugins/plugin_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
from typing import Any


class PluginConfig:
"""Class for holding configuration of a single plugin"""

def __init__(self, name: str, enabled: bool = False, config: dict[str, Any] = None):
self.name = name
self.enabled = enabled
# Arbitray config options for this plugin. API keys or plugin-specific options live here.
self.config = config or {}

def __repr__(self):
return f"PluginConfig('{self.name}', {self.enabled}, {str(self.config)}"
81 changes: 81 additions & 0 deletions autogpt/plugins/plugins_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import os
from typing import Any, Union

import yaml

from autogpt.config.config import Config
from autogpt.logs import logger
from autogpt.plugins.plugin_config import PluginConfig


class PluginsConfig:
"""Class for holding configuration of all plugins"""

def __init__(self, plugins_config: dict[str, Any]):
self.plugins = {}
for name, plugin in plugins_config.items():
if type(plugin) == dict:
self.plugins[name] = PluginConfig(
name,
plugin.get("enabled", False),
plugin.get("config", {}),
)
elif type(plugin) == PluginConfig:
self.plugins[name] = plugin
else:
raise ValueError(f"Invalid plugin config data type: {type(plugin)}")

def __repr__(self):
return f"PluginsConfig({self.plugins})"

def get(self, name: str) -> Union[PluginConfig, None]:
return self.plugins.get(name)

def is_enabled(self, name) -> bool:
plugin_config = self.plugins.get(name)
return plugin_config and plugin_config.enabled

@classmethod
def load_config(cls, global_config: Config) -> "PluginsConfig":
empty_config = cls({})

try:
config_data = cls.deserialize_config_file(global_config=global_config)
if type(config_data) != dict:
logger.error(
f"Expected plugins config to be a dict, got {type(config_data)}, continuing without plugins"
)
return empty_config
return cls(config_data)

except BaseException as e:
logger.error(
f"Plugin config is invalid, continuing without plugins. Error: {e}"
)
return empty_config

@classmethod
def deserialize_config_file(cls, global_config: Config) -> dict[str, Any]:
plugins_config_path = global_config.plugins_config_file
if not os.path.exists(plugins_config_path):
logger.warn("plugins_config.yaml does not exist, creating base config.")
cls.create_empty_plugins_config(global_config=global_config)

with open(plugins_config_path, "r") as f:
return yaml.load(f, Loader=yaml.FullLoader)

@staticmethod
def create_empty_plugins_config(global_config: Config):
"""Create an empty plugins_config.yaml file. Fill it with values from old env variables."""
base_config = {}

# Backwards-compatibility shim
for plugin_name in global_config.plugins_denylist:
base_config[plugin_name] = {"enabled": False, "config": {}}

for plugin_name in global_config.plugins_allowlist:
base_config[plugin_name] = {"enabled": True, "config": {}}

with open(global_config.plugins_config_file, "w+") as f:
f.write(yaml.dump(base_config))
return base_config
3 changes: 1 addition & 2 deletions docs/configuration/options.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@ Configuration is controlled through the `Config` object. You can set configurati
## Environment Variables

- `AI_SETTINGS_FILE`: Location of AI Settings file. Default: ai_settings.yaml
- `ALLOWLISTED_PLUGINS`: List of plugins allowed. Optional.
- `AUDIO_TO_TEXT_PROVIDER`: Audio To Text Provider. Only option currently is `huggingface`. Default: huggingface
- `AUTHORISE_COMMAND_KEY`: Key response accepted when authorising commands. Default: y
- `BROWSE_CHUNK_MAX_LENGTH`: When browsing website, define the length of chunks to summarize. Default: 3000
- `BROWSE_SPACY_LANGUAGE_MODEL`: [spaCy language model](https://spacy.io/usage/models) to use when creating chunks. Default: en_core_web_sm
- `CHAT_MESSAGES_ENABLED`: Enable chat messages. Optional
- `DENYLISTED_PLUGINS`: List of plugins not allowed. Optional.
- `DISABLED_COMMAND_CATEGORIES`: Command categories to disable. Command categories are Python module names, e.g. autogpt.commands.analyze_code. See the directory `autogpt/commands` in the source for all command modules. Default: None
- `ELEVENLABS_API_KEY`: ElevenLabs API Key. Optional.
- `ELEVENLABS_VOICE_ID`: ElevenLabs Voice ID. Optional.
Expand All @@ -34,6 +32,7 @@ Configuration is controlled through the `Config` object. You can set configurati
- `OPENAI_API_KEY`: *REQUIRED*- Your [OpenAI API Key](https://platform.openai.com/account/api-keys).
- `OPENAI_ORGANIZATION`: Organization ID in OpenAI. Optional.
- `PLAIN_OUTPUT`: Plain output, which disables the spinner. Default: False
- `PLUGINS_CONFIG_FILE`: Path of plugins_config.yaml file. Default: plugins_config.yaml
- `PROMPT_SETTINGS_FILE`: Location of Prompt Settings file. Default: prompt_settings.yaml
- `REDIS_HOST`: Redis Host. Default: localhost
- `REDIS_PASSWORD`: Redis Password. Optional. Default:
Expand Down
12 changes: 12 additions & 0 deletions docs/plugins.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@

⚠️💀 **WARNING** 💀⚠️: Review the code of any plugin you use thoroughly, as plugins can execute any Python code, potentially leading to malicious activities, such as stealing your API keys.

To configure plugins, you can create or edit the `plugins_config.yaml` file in the root directory of Auto-GPT. This file allows you to enable or disable plugins as desired. For specific configuration instructions, please refer to the documentation provided for each plugin. The file should be formatted in YAML. Here is an example for your reference:

```yaml
plugin_a:
config:
api_key: my-api-key
enabled: false
plugin_b:
config: {}
enabled: true
```
See our [Plugins Repo](https://github.com/Significant-Gravitas/Auto-GPT-Plugins) for more info on how to install all the amazing plugins the community has built!
Alternatively, developers can use the [Auto-GPT Plugin Template](https://github.com/Significant-Gravitas/Auto-GPT-Plugin-Template) as a starting point for creating your own plugins.
Expand Down
20 changes: 19 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import os
from pathlib import Path
from tempfile import TemporaryDirectory

import pytest
import yaml
from pytest_mock import MockerFixture

from autogpt.agent.agent import Agent
Expand Down Expand Up @@ -32,9 +34,25 @@ def workspace(workspace_root: Path) -> Workspace:
return Workspace(workspace_root, restrict_to_workspace=True)


@pytest.fixture
def temp_plugins_config_file():
"""Create a plugins_config.yaml file in a temp directory so that it doesn't mess with existing ones"""
config_directory = TemporaryDirectory()
config_file = os.path.join(config_directory.name, "plugins_config.yaml")
with open(config_file, "w+") as f:
f.write(yaml.dump({}))

yield config_file


@pytest.fixture()
def config(mocker: MockerFixture, workspace: Workspace) -> Config:
def config(
temp_plugins_config_file: str, mocker: MockerFixture, workspace: Workspace
) -> Config:
config = Config()
config.plugins_dir = "tests/unit/data/test_plugins"
config.plugins_config_file = temp_plugins_config_file
config.load_plugins_config()

# Do a little setup and teardown since the config object is a singleton
mocker.patch.multiple(
Expand Down
Loading

0 comments on commit 49d1a5a

Please sign in to comment.