diff --git a/.ci/integration.cloudbuild.yaml b/.ci/integration.cloudbuild.yaml index d4c561396..a953f3307 100644 --- a/.ci/integration.cloudbuild.yaml +++ b/.ci/integration.cloudbuild.yaml @@ -15,89 +15,109 @@ steps: - id: "install-dependencies" name: golang:1 - waitFor: ['-'] + waitFor: ["-"] env: - - 'GOPATH=/gopath' + - "GOPATH=/gopath" volumes: - - name: 'go' - path: '/gopath' + - name: "go" + path: "/gopath" script: | - go get -d ./... + go get -d ./... - id: "cloud-sql-pg" name: golang:1 - waitFor: ['install-dependencies'] + waitFor: ["install-dependencies"] entrypoint: /bin/bash env: - - 'GOPATH=/gopath' + - "GOPATH=/gopath" - "CLOUD_SQL_POSTGRES_PROJECT=$PROJECT_ID" - "CLOUD_SQL_POSTGRES_INSTANCE=$_CLOUD_SQL_POSTGRES_INSTANCE" - "CLOUD_SQL_POSTGRES_DATABASE=$_DATABASE_NAME" - "CLOUD_SQL_POSTGRES_REGION=$_REGION" - secretEnv: ["CLOUD_SQL_POSTGRES_USER", "CLOUD_SQL_POSTGRES_PASS"] + - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" + secretEnv: + ["CLOUD_SQL_POSTGRES_USER", "CLOUD_SQL_POSTGRES_PASS", "CLIENT_ID"] volumes: - - name: 'go' - path: '/gopath' - args: + - name: "go" + path: "/gopath" + args: - -c - | go test -race -v -tags=integration,cloudsql ./tests - id: "alloydb-pg" name: golang:1 - waitFor: ['install-dependencies'] + waitFor: ["install-dependencies"] entrypoint: /bin/bash env: - - 'GOPATH=/gopath' + - "GOPATH=/gopath" - "ALLOYDB_POSTGRES_PROJECT=$PROJECT_ID" - "ALLOYDB_POSTGRES_CLUSTER=$_ALLOYDB_POSTGRES_CLUSTER" - "ALLOYDB_POSTGRES_INSTANCE=$_ALLOYDB_POSTGRES_INSTANCE" - "ALLOYDB_POSTGRES_DATABASE=$_DATABASE_NAME" - "ALLOYDB_POSTGRES_REGION=$_REGION" - secretEnv: ["ALLOYDB_POSTGRES_USER", "ALLOYDB_POSTGRES_PASS"] + - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" + secretEnv: ["ALLOYDB_POSTGRES_USER", "ALLOYDB_POSTGRES_PASS", "CLIENT_ID"] volumes: - - name: 'go' - path: '/gopath' - args: + - name: "go" + path: "/gopath" + args: - -c - | go test -race -v -tags=integration,alloydb ./tests - id: "postgres" name: golang:1 - waitFor: ['install-dependencies'] + waitFor: ["install-dependencies"] entrypoint: /bin/bash env: - - 'GOPATH=/gopath' + - "GOPATH=/gopath" - "POSTGRES_DATABASE=$_DATABASE_NAME" - "POSTGRES_HOST=$_POSTGRES_HOST" - "POSTGRES_PORT=$_POSTGRES_PORT" secretEnv: ["POSTGRES_USER", "POSTGRES_PASS"] volumes: - - name: 'go' - path: '/gopath' - args: + - name: "go" + path: "/gopath" + args: - -c - | go test -race -v -tags=integration,postgres ./tests - id: "spanner" name: golang:1 - waitFor: ['install-dependencies'] + waitFor: ["install-dependencies"] entrypoint: /bin/bash env: - - 'GOPATH=/gopath' + - "GOPATH=/gopath" - "SPANNER_PROJECT=$PROJECT_ID" - "SPANNER_DATABASE=$_DATABASE_NAME" - "SPANNER_INSTANCE=$_SPANNER_INSTANCE" volumes: - - name: 'go' - path: '/gopath' - args: + - name: "go" + path: "/gopath" + args: - -c - | go test -race -v -tags=integration,spanner ./tests + - id: "neo4j" + name: golang:1 + waitFor: ["install-dependencies"] + entrypoint: /bin/bash + env: + - "GOPATH=/gopath" + - "NEO4J_DATABASE=$_NEO4J_DATABASE" + - "NEO4J_URI=$_NEO4J_URI" + secretEnv: ["NEO4J_USER", "NEO4J_PASS"] + volumes: + - name: "go" + path: "/gopath" + args: + - -c + - | + go test -race -v -tags=integration,neo4j ./tests + availableSecrets: secretManager: - versionName: projects/$PROJECT_ID/secrets/cloud_sql_pg_user/versions/latest @@ -112,11 +132,17 @@ availableSecrets: env: POSTGRES_USER - versionName: projects/$PROJECT_ID/secrets/postgres_pass/versions/latest env: POSTGRES_PASS + - versionName: projects/$PROJECT_ID/secrets/client_id/versions/latest + env: CLIENT_ID + - versionName: projects/$PROJECT_ID/secrets/neo4j_user/versions/latest + env: NEO4J_USER + - versionName: projects/$PROJECT_ID/secrets/neo4j_pass/versions/latest + env: NEO4J_PASS options: logging: CLOUD_LOGGING_ONLY automapSubstitutions: true - substitutionOption: 'ALLOW_LOOSE' + substitutionOption: "ALLOW_LOOSE" dynamicSubstitutions: true pool: name: projects/$PROJECT_ID/locations/us-central1/workerPools/integration-testing # Necessary for VPC network connection @@ -130,3 +156,4 @@ substitutions: _POSTGRES_HOST: 127.0.0.1 _POSTGRES_PORT: "5432" _SPANNER_INSTANCE: "spanner-testing" + _NEO4J_DATABASE: "neo4j" diff --git a/.github/workflows/docs_deploy.yaml b/.github/workflows/docs_deploy.yaml new file mode 100644 index 000000000..e1050e85b --- /dev/null +++ b/.github/workflows/docs_deploy.yaml @@ -0,0 +1,81 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "docs" + +permissions: + contents: write + +on: + push: + branches: + - main + paths: + - 'docs/**' + - 'github/workflows/docs**' + - '.hugo' + + # Allow triggering manually. + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: .hugo + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod + + - name: Setup Hugo + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: "latest" + extended: true + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - run: npm ci + - run: hugo --minify + env: + # HUGO_BASEURL: https://${{ github.repository_owner }}.github.io/${{ github.repository }} + # While private, GitHub uses an obfuscated url instead: + HUGO_BASEURL: https://vigilant-guacamole-plnwrm9.pages.github.io/ + HUGO_RELATIVEURLS: false + + - name: Deploy + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: .hugo/public + # Do not delete previews on each production deploy. + # CSS or JS changes will require manual clean-up. + keep_files: true + commit_message: "deploy: ${{ github.event.head_commit.message }}" \ No newline at end of file diff --git a/.github/workflows/docs_preview_clean.yaml b/.github/workflows/docs_preview_clean.yaml new file mode 100644 index 000000000..b4ec45a95 --- /dev/null +++ b/.github/workflows/docs_preview_clean.yaml @@ -0,0 +1,59 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "docs" + +permissions: + contents: write + pull-requests: write + +# This Workflow depends on 'github.event.number', +# not compatible with branch or manual triggers. +on: + pull_request: + types: + - closed + +jobs: + clean: + if: ${{ !github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-24.04 + concurrency: + # Shared concurrency group wih preview staging. + group: "preview-${{ github.event.number }}" + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + with: + ref: gh-pages + + - name: Remove Preview + run: | + rm -Rf ./previews/PR-${{ github.event.number }} + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git add -u previews/PR-${{ github.event.number }} + git commit --message "cleanup: previews/PR-${{ github.event.number }}" + git push + + - name: Comment + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.payload.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: "๐Ÿงจ Preview deployments removed." + }) \ No newline at end of file diff --git a/.github/workflows/docs_preview_deploy.yaml b/.github/workflows/docs_preview_deploy.yaml new file mode 100644 index 000000000..fa30be3d0 --- /dev/null +++ b/.github/workflows/docs_preview_deploy.yaml @@ -0,0 +1,95 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "docs" + +permissions: + contents: write + pull-requests: write + +# This Workflow depends on 'github.event.number', +# not compatible with branch or manual triggers. +on: + pull_request: + # Sync with github_actions_preview_fallback.yml on.pull_request.paths-ignore + paths: + - 'docs/**' + - 'github/workflows/docs**' + - '.hugo' + +jobs: + preview: + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: .hugo + concurrency: + # Shared concurrency group wih preview cleanup. + group: "preview-${{ github.event.number }}" + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod + + - name: Setup Hugo + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: "latest" + extended: true + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - run: npm ci + - run: hugo --minify + env: + # HUGO_BASEURL: https://${{ github.repository_owner }}.github.io/${{ github.repository }}/previews/PR-${{ github.event.number }}/ + # While private, GitHub uses an obfuscated url instead: + HUGO_BASEURL: https://vigilant-guacamole-plnwrm9.pages.github.io/previews/PR-${{ github.event.number }}/ + HUGO_ENVIRONMENT: preview + HUGO_RELATIVEURLS: false + + - name: Deploy + # If run from a fork, GitHub write operations will fail. + if: ${{ !github.event.pull_request.head.repo.fork }} + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: .hugo/public + destination_dir: ./previews/PR-${{ github.event.number }} + commit_message: "stage: PR-${{ github.event.number }}: ${{ github.event.head_commit.message }}" + + - name: Comment + # If run from a fork, GitHub write operations will fail. + if: ${{ !github.event.pull_request.head.repo.fork }} + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.payload.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: "๐Ÿ”Ž Preview at https://vigilant-guacamole-plnwrm9.pages.github.io/previews/PR-${{ github.event.number }}/" + }) \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5c2b98ff5..7d975d5b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,13 @@ +# direnv +.envrc + # vscode .vscode/ + +# npm +node_modules + +# hugo +.hugo/public/ +.hugo/resources/_gen +.hugo_build.lock \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..c430b6512 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docs2/themes/godocs"] + path = docs2/themes/godocs + url = https://github.com/themefisher/godocs.git diff --git a/.hugo/archetypes/default.md b/.hugo/archetypes/default.md new file mode 100644 index 000000000..25b67521d --- /dev/null +++ b/.hugo/archetypes/default.md @@ -0,0 +1,5 @@ ++++ +date = '{{ .Date }}' +draft = true +title = '{{ replace .File.ContentBaseName "-" " " | title }}' ++++ diff --git a/.hugo/assets/icons/logo.svg b/.hugo/assets/icons/logo.svg new file mode 100644 index 000000000..13b80b8c9 --- /dev/null +++ b/.hugo/assets/icons/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/.hugo/assets/scss/_variables_project.scss b/.hugo/assets/scss/_variables_project.scss new file mode 100644 index 000000000..9cd721c47 --- /dev/null +++ b/.hugo/assets/scss/_variables_project.scss @@ -0,0 +1,2 @@ +$primary: #D84040; +$secondary: #8E1616; diff --git a/.hugo/go.mod b/.hugo/go.mod new file mode 100644 index 000000000..649f7f63f --- /dev/null +++ b/.hugo/go.mod @@ -0,0 +1,5 @@ +module github.com/googleapis/genai-toolbox + +go 1.23.2 + +require github.com/google/docsy v0.11.0 // indirect diff --git a/.hugo/go.sum b/.hugo/go.sum new file mode 100644 index 000000000..558b7c83e --- /dev/null +++ b/.hugo/go.sum @@ -0,0 +1,4 @@ +github.com/FortAwesome/Font-Awesome v0.0.0-20240716171331-37eff7fa00de/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo= +github.com/google/docsy v0.11.0 h1:QnV40cc28QwS++kP9qINtrIv4hlASruhC/K3FqkHAmM= +github.com/google/docsy v0.11.0/go.mod h1:hGGW0OjNuG5ZbH5JRtALY3yvN8ybbEP/v2iaK4bwOUI= +github.com/twbs/bootstrap v5.3.3+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0= diff --git a/.hugo/hugo.toml b/.hugo/hugo.toml new file mode 100644 index 000000000..f55468471 --- /dev/null +++ b/.hugo/hugo.toml @@ -0,0 +1,43 @@ +title = 'Gen AI Toolbox' +relativeURLs = true + +languageCode = 'en-us' +contentDir = "../docs/en" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = false + +enableGitInfo = true +enableRobotsTXT = true + +[languages] + [languages.en] + languageName ="English" + weight = 1 + +[module] + proxy = "direct" + [module.hugoVersion] + extended = true + min = "0.73.0" + [[module.imports]] + path = "github.com/google/docsy" + disable = false + +[params] + copyright = "Google LLC" + github_repo = "https://github.com/googleapis/genai-toolbox" + github_project_repo = "https://github.com/googleapis/genai-toolbox" + github_subdir = "docs" + offlineSearch = true + [params.ui] + ul_show = 100 + showLightDarkModeMenu = true + breadcrumb_disable = true + sidebar_menu_foldable = true + sidebar_menu_compact = false + +[[menu.main]] + name = "GitHub" + weight = 50 + url = "https://github.com/googleapis/genai-toolbox" + pre = "" \ No newline at end of file diff --git a/.hugo/layouts/_default/_markup/render-heading.html b/.hugo/layouts/_default/_markup/render-heading.html new file mode 100644 index 000000000..7f8e97424 --- /dev/null +++ b/.hugo/layouts/_default/_markup/render-heading.html @@ -0,0 +1 @@ +{{ template "_default/_markup/td-render-heading.html" . }} diff --git a/.hugo/layouts/robot.txt b/.hugo/layouts/robot.txt new file mode 100644 index 000000000..8d85a8645 --- /dev/null +++ b/.hugo/layouts/robot.txt @@ -0,0 +1,4 @@ +User-agent: * +{{ if eq hugo.Environment "preview" }} +Disallow: /* +{{ end }} \ No newline at end of file diff --git a/.hugo/package-lock.json b/.hugo/package-lock.json new file mode 100644 index 000000000..60ebef6a5 --- /dev/null +++ b/.hugo/package-lock.json @@ -0,0 +1,1097 @@ +{ + "name": "docs2", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "devDependencies": { + "autoprefixer": "^10.4.20", + "postcss": "^8.4.49", + "postcss-cli": "^11.0.0" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz", + "integrity": "sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/autoprefixer": { + "version": "10.4.20", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz", + "integrity": "sha512-XY25y5xSv/wEoqzDyXXME4AFfkZI0P23z6Fs3YgymDnKJkCGOnkL0iTxCa85UTqaSgfcqyf3UA6+c7wUvx/16g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.3", + "caniuse-lite": "^1.0.30001646", + "fraction.js": "^4.3.7", + "normalize-range": "^0.1.2", + "picocolors": "^1.0.1", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.4.tgz", + "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001692", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001692.tgz", + "integrity": "sha512-A95VKan0kdtrsnMubMKxEKUKImOPSuCpYgxSQBo036P5YYgVIcOYJEgt/txJWqObiRQeISNCfef9nvlQ0vbV7A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/dependency-graph": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", + "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.80", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.80.tgz", + "integrity": "sha512-LTrKpW0AqIuHwmlVNV+cjFYTnXtM9K37OGhpe0ZI10ScPSxqVSryZHIY3WnCS5NSYbBODRTZyhRMS2h5FAEqAw==", + "dev": true, + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.18.0.tgz", + "integrity": "sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fraction.js": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", + "integrity": "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stdin": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", + "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-14.0.2.tgz", + "integrity": "sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^2.1.0", + "fast-glob": "^3.3.2", + "ignore": "^5.2.4", + "path-type": "^5.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-type": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-5.0.0.tgz", + "integrity": "sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss": { + "version": "8.4.49", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", + "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-cli": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-11.0.0.tgz", + "integrity": "sha512-xMITAI7M0u1yolVcXJ9XTZiO9aO49mcoKQy6pCDFdMh9kGqhzLVpWxeD/32M/QBmkhcGypZFFOLNLmIW4Pg4RA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.3.0", + "dependency-graph": "^0.11.0", + "fs-extra": "^11.0.0", + "get-stdin": "^9.0.0", + "globby": "^14.0.0", + "picocolors": "^1.0.0", + "postcss-load-config": "^5.0.0", + "postcss-reporter": "^7.0.0", + "pretty-hrtime": "^1.0.3", + "read-cache": "^1.0.0", + "slash": "^5.0.0", + "yargs": "^17.0.0" + }, + "bin": { + "postcss": "index.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-load-config": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-5.1.0.tgz", + "integrity": "sha512-G5AJ+IX0aD0dygOE0yFZQ/huFFMSNneyfp0e3/bT05a8OfPC5FUoZRPfGijUdGOJNMewJiwzcHJXFafFzeKFVA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "lilconfig": "^3.1.1", + "yaml": "^2.4.2" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + } + } + }, + "node_modules/postcss-reporter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.1.0.tgz", + "integrity": "sha512-/eoEylGWyy6/DOiMP5lmFRdmDKThqgn7D6hP2dXKJI/0rJSO1ADFNngZfDzxL0YAxFvws+Rtpuji1YIHj4mySA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "picocolors": "^1.0.0", + "thenby": "^1.3.4" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pretty-hrtime": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", + "integrity": "sha512-66hKPCr+72mlfiSjlEB1+45IjXSqvVAIy6mocupoww4tBFE9R9IhwwUGoI4G++Tc9Aq+2rxOt0RFU6gPcrte0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/slash": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/thenby": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", + "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/unicorn-magic": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.1.0.tgz", + "integrity": "sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.2.tgz", + "integrity": "sha512-PPypAm5qvlD7XMZC3BujecnaOxwhrtoFR+Dqkk5Aa/6DssiH0ibKoketaj9w8LP7Bont1rYeoV5plxD7RTEPRg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", + "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/.hugo/package.json b/.hugo/package.json new file mode 100644 index 000000000..e80587730 --- /dev/null +++ b/.hugo/package.json @@ -0,0 +1,7 @@ +{ + "devDependencies": { + "autoprefixer": "^10.4.20", + "postcss": "^8.4.49", + "postcss-cli": "^11.0.0" + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 629a9d715..5e7bcc87f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## [0.0.5](https://github.com/googleapis/genai-toolbox/compare/v0.0.4...v0.0.5) (2025-01-14) + + +### โš  BREAKING CHANGES + +* replace Source field `ip_type` with `ipType` for consistency ([#197](https://github.com/googleapis/genai-toolbox/issues/197)) +* **toolbox-sdk:** deprecate 'add_auth_headers' in favor of 'add_auth_tokens' ([#170](https://github.com/googleapis/genai-toolbox/issues/170)) + +### Features + +* Add support for OpenTelemetry ([#205](https://github.com/googleapis/genai-toolbox/issues/205)) ([1fcc20a](https://github.com/googleapis/genai-toolbox/commit/1fcc20a8469794ed8e6846cded44196d26c306be)) +* Added Neo4j Source and Tool ([#189](https://github.com/googleapis/genai-toolbox/issues/189)) ([8a1224b](https://github.com/googleapis/genai-toolbox/commit/8a1224b9e0145c4e214d42f14f5308b508ea27ce)) +* **llamaindex-sdk:** Implement OAuth support for LlamaIndex. ([#159](https://github.com/googleapis/genai-toolbox/issues/159)) ([003ce51](https://github.com/googleapis/genai-toolbox/commit/003ce510a1fb37a23e4c64fdf21376e0e32ec8ab)) +* Replace Source field `ip_type` with `ipType` for consistency ([#197](https://github.com/googleapis/genai-toolbox/issues/197)) ([e069520](https://github.com/googleapis/genai-toolbox/commit/e069520bb79d086dbdd37ebc3ad9bb39b31c8fac)) +* Update log with given context ([#147](https://github.com/googleapis/genai-toolbox/issues/147)) ([809e547](https://github.com/googleapis/genai-toolbox/commit/809e547a481bd4af351bbaa2dcfd203b086bb51d)) + + +### Bug Fixes + +* Correct parsing of floats/ints from json ([#180](https://github.com/googleapis/genai-toolbox/issues/180)) ([387a5b5](https://github.com/googleapis/genai-toolbox/commit/387a5b56b53ccfe0637a0f44c0ddbec8e991cc39)) +* **doc:** Update example `clientId` field ([#198](https://github.com/googleapis/genai-toolbox/issues/198)) ([0c86e89](https://github.com/googleapis/genai-toolbox/commit/0c86e895066ee3dee9ab9bc20fe00934066b67ac)) +* Fix config name in auth doc samples ([#186](https://github.com/googleapis/genai-toolbox/issues/186)) ([bb03457](https://github.com/googleapis/genai-toolbox/commit/bb0345767e0550fcda975958f450086e44f6a913)) +* Handle shutdown gracefully ([#178](https://github.com/googleapis/genai-toolbox/issues/178)) ([66ab70f](https://github.com/googleapis/genai-toolbox/commit/66ab70f702d7178c61c8d90399483b6125ba01c8)) +* Improve return error for parameters ([#206](https://github.com/googleapis/genai-toolbox/issues/206)) ([346c57d](https://github.com/googleapis/genai-toolbox/commit/346c57da2394e398ee8cc527b84973aa2bcde642)) +* **toolbox-sdk:** Deprecate 'add_auth_headers' in favor of 'add_auth_tokens' ([#170](https://github.com/googleapis/genai-toolbox/issues/170)) ([b56fa68](https://github.com/googleapis/genai-toolbox/commit/b56fa685e379c3515025ed76d9abe61f93365a65)) + + +### Miscellaneous Chores + +* Release 0.0.5 ([#210](https://github.com/googleapis/genai-toolbox/issues/210)) ([bd407c0](https://github.com/googleapis/genai-toolbox/commit/bd407c0ab749c9a72523122a2212652f9d97ab03)) + ## [0.0.4](https://github.com/googleapis/genai-toolbox/compare/v0.0.3...v0.0.4) (2024-12-18) diff --git a/README.md b/README.md index e2226f258..f7162d967 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ You can also install Toolbox as a container: ```sh # see releases page for other versions +export VERSION=0.0.1 docker pull us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:$VERSION ``` @@ -180,6 +181,18 @@ sources: database: my_db ``` +Example for Neo4j + +```yaml +sources: + my-neo4j-source: + kind: neo4j + uri: neo4j+s://my-neo4j-host:7687 + user: neo4j + password: my-password + database: my_db +``` + For more details on configuring different types of sources, see the [Source documentation.](docs/sources/README.md) @@ -202,6 +215,22 @@ tools: description: 'id' represents the unique ID for each flight. ``` +Neo4j-Cypher Example + +```yaml +tools: + get_movies_in_year: + kind: neo4j-cypher + source: my-neo4j-instance + description: > + Use this tool to list all movies titles in a given year. + statement: "MATCH (m:Movie) WHERE m.year = $year RETURN m.title" + parameters: + - name: "year" + type: integer + description: 'year' represents a 4 digit year since 1900 up to the current year +``` + For more details on configuring different types of tools, see the [Tool documentation.](docs/tools/README.md) diff --git a/cmd/root.go b/cmd/root.go index d6592b479..2926463b9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,6 +27,7 @@ import ( "github.com/googleapis/genai-toolbox/internal/log" "github.com/googleapis/genai-toolbox/internal/server" + "github.com/googleapis/genai-toolbox/internal/telemetry" "github.com/spf13/cobra" "gopkg.in/yaml.v3" ) @@ -106,6 +107,9 @@ func NewCommand(opts ...Option) *Command { flags.StringVar(&cmd.tools_file, "tools_file", "tools.yaml", "File path specifying the tool configuration.") flags.Var(&cmd.cfg.LogLevel, "log-level", "Specify the minimum level logged. Allowed: 'DEBUG', 'INFO', 'WARN', 'ERROR'.") flags.Var(&cmd.cfg.LoggingFormat, "logging-format", "Specify logging format to use. Allowed: 'standard' or 'JSON'.") + flags.BoolVar(&cmd.cfg.TelemetryGCP, "telemetry-gcp", false, "Enable exporting directly to Google Cloud Monitoring.") + flags.StringVar(&cmd.cfg.TelemetryOTLP, "telemetry-otlp", "", "Enable exporting using OpenTelemetry Protocol (OTLP) to the specified endpoint (e.g. 'http://127.0.0.1:4318')") + flags.StringVar(&cmd.cfg.TelemetryServiceName, "telemetry-service-name", "toolbox", "Sets the value of the service.name resource attribute for telemetry data.") // wrap RunE command so that we have access to original Command object cmd.RunE = func(*cobra.Command, []string) error { return run(cmd) } @@ -173,6 +177,21 @@ func run(cmd *Command) error { return fmt.Errorf("logging format invalid.") } + // Set up OpenTelemetry + otelShutdown, err := telemetry.SetupOTel(ctx, cmd.Command.Version, cmd.cfg.TelemetryOTLP, cmd.cfg.TelemetryGCP, cmd.cfg.TelemetryServiceName) + if err != nil { + errMsg := fmt.Errorf("error setting up OpenTelemetry: %w", err) + cmd.logger.ErrorContext(ctx, errMsg.Error()) + return errMsg + } + defer func() { + err := otelShutdown(ctx) + if err != nil { + errMsg := fmt.Errorf("error shutting down OpenTelemetry: %w", err) + cmd.logger.ErrorContext(ctx, errMsg.Error()) + } + }() + // Read tool file contents buf, err := os.ReadFile(cmd.tools_file) if err != nil { diff --git a/cmd/root_test.go b/cmd/root_test.go index a3ab09d55..6a03bc9ce 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -41,6 +41,9 @@ func withDefaults(c server.ServerConfig) server.ServerConfig { if c.Port == 0 { c.Port = 5000 } + if c.TelemetryServiceName == "" { + c.TelemetryServiceName = "toolbox" + } return c } @@ -137,6 +140,27 @@ func TestServerConfigFlags(t *testing.T) { LogLevel: "WARN", }), }, + { + desc: "telemetry gcp", + args: []string{"--telemetry-gcp"}, + want: withDefaults(server.ServerConfig{ + TelemetryGCP: true, + }), + }, + { + desc: "telemetry otlp", + args: []string{"--telemetry-otlp", "http://127.0.0.1:4553"}, + want: withDefaults(server.ServerConfig{ + TelemetryOTLP: "http://127.0.0.1:4553", + }), + }, + { + desc: "telemetry service name", + args: []string{"--telemetry-service-name", "toolbox-custom"}, + want: withDefaults(server.ServerConfig{ + TelemetryServiceName: "toolbox-custom", + }), + }, } for _, tc := range tcs { t.Run(tc.desc, func(t *testing.T) { diff --git a/cmd/version.txt b/cmd/version.txt index 81340c7e7..bbdeab622 100644 --- a/cmd/version.txt +++ b/cmd/version.txt @@ -1 +1 @@ -0.0.4 +0.0.5 diff --git a/docs/authSources/README.md b/docs/authSources/README.md index b4fffc3bf..636b0d293 100644 --- a/docs/authSources/README.md +++ b/docs/authSources/README.md @@ -29,10 +29,10 @@ authSources: > authSources: > my_auth_app_1: > kind: google -> client_id: YOUR_CLIENT_ID_1 +> clientId: YOUR_CLIENT_ID_1 > my_auth_app_2: > kind: google -> client_id: YOUR_CLIENT_ID_2 +> clientId: YOUR_CLIENT_ID_2 > > tools: > my_tool: diff --git a/docs/en/_index.md b/docs/en/_index.md new file mode 100644 index 000000000..4ea04cd7f --- /dev/null +++ b/docs/en/_index.md @@ -0,0 +1,10 @@ +--- +title: "Documentation" +type: docs +notoc: false +weight: 1 +description: > + All of Toolbox's documentation. +--- + +Placeholder for top-level directory. \ No newline at end of file diff --git a/docs/en/concepts/_index.md b/docs/en/concepts/_index.md new file mode 100644 index 000000000..83a707309 --- /dev/null +++ b/docs/en/concepts/_index.md @@ -0,0 +1,6 @@ +--- +title: "Concepts" +type: docs +weight: 2 +description: Some core concepts in Toolbox +--- diff --git a/docs/en/concepts/overview.md b/docs/en/concepts/overview.md new file mode 100644 index 000000000..eeb54ac17 --- /dev/null +++ b/docs/en/concepts/overview.md @@ -0,0 +1,6 @@ +--- +title: "Architecture" +type: docs +weight: 2 +description: An overview of Toolbox's architecture +--- diff --git a/docs/en/development/_index.md b/docs/en/development/_index.md new file mode 100644 index 000000000..df195ffce --- /dev/null +++ b/docs/en/development/_index.md @@ -0,0 +1,6 @@ +--- +title: "Development" +type: docs +weight: 4 +description: A list of resources related to development of Toolbox +--- \ No newline at end of file diff --git a/docs/en/development/contributing.md b/docs/en/development/contributing.md new file mode 100644 index 000000000..9cdf67392 --- /dev/null +++ b/docs/en/development/contributing.md @@ -0,0 +1,7 @@ +--- +title: "Contributing" +type: docs +weight: 2 +description: How to participate in Toolbox Development +--- + diff --git a/docs/en/getting-started/_index.md b/docs/en/getting-started/_index.md new file mode 100644 index 000000000..6d861abcf --- /dev/null +++ b/docs/en/getting-started/_index.md @@ -0,0 +1,7 @@ +--- +title: "Getting Started" +type: docs +weight: 1 +description: > + How to get started with Toolbox +--- diff --git a/docs/en/getting-started/introduction.md b/docs/en/getting-started/introduction.md new file mode 100644 index 000000000..d0336eec1 --- /dev/null +++ b/docs/en/getting-started/introduction.md @@ -0,0 +1,10 @@ +--- +title: "Introduction" +type: docs +weight: 1 +description: An introduction to Toolbox +--- + +# Introduction + +This is a placeholder for the introduction. \ No newline at end of file diff --git a/docs/en/resources/_index.md b/docs/en/resources/_index.md new file mode 100644 index 000000000..f0cc9c52d --- /dev/null +++ b/docs/en/resources/_index.md @@ -0,0 +1,6 @@ +--- +title: "Resources" +type: docs +weight: 3 +description: List of reference documentation for resources in Toolbox +--- diff --git a/docs/sources/README.md b/docs/sources/README.md index 26f715b72..206f025f2 100644 --- a/docs/sources/README.md +++ b/docs/sources/README.md @@ -30,3 +30,4 @@ We currently support the following types of kinds of sources: PostgreSQL instance. * [postgres](./postgres.md) - Connect to any PostgreSQL compatible database. * [spanner](./spanner.md) - Connect to a Spanner database. +* [neo4j](./neo4j.md) - Connect to a Neo4j instance. \ No newline at end of file diff --git a/docs/sources/alloydb-pg.md b/docs/sources/alloydb-pg.md index 06e11fc34..eeee15576 100644 --- a/docs/sources/alloydb-pg.md +++ b/docs/sources/alloydb-pg.md @@ -32,11 +32,12 @@ IAM identity has been given the following IAM permissions: ### Network Path -Currently, this source only supports [connecting over Private -IP][private-ip]. Most notably, this means -you need to connect from a VPC that AlloyDB has been connected to. +Currently, AlloyDB supports connection over both [private IP][private-ip] and +[public IP][public-ip]. Set the `ipType` parameter in your source +configuration to `public` or `private`. [private-ip]: https://cloud.google.com/alloydb/docs/private-ip +[public-ip]: https://cloud.google.com/alloydb/docs/connect-public-ip ### Database User @@ -69,9 +70,7 @@ sources: | region | string | true | Name of the GCP region that the cluster was created in (e.g. "us-central1"). | | cluster | string | true | Name of the AlloyDB cluster (e.g. "my-cluster"). | | instance | string | true | Name of the AlloyDB instance within the cluser (e.g. "my-instance"). | -| ip_type | string | true | IP Type of the AlloyDB instance, must be either `public` or `private`. Default: `public`. | +| ipType | string | true | IP Type of the AlloyDB instance, must be either `public` or `private`. Default: `public`. | | database | string | true | Name of the Postgres database to connect to (e.g. "my_db"). | | user | string | true | Name of the Postgres user to connect as (e.g. "my-pg-user"). | | password | string | true | Password of the Postgres user (e.g. "my-password"). | - - diff --git a/docs/sources/cloud-sql-pg.md b/docs/sources/cloud-sql-pg.md index 873afe66f..66ae19cb5 100644 --- a/docs/sources/cloud-sql-pg.md +++ b/docs/sources/cloud-sql-pg.md @@ -30,16 +30,17 @@ IAM identity has been given the following IAM roles: ### Network Path -Currently, this source only supports [connecting over Public IP][public-ip]. -Because it uses the Go connector, is uses rotating client certificates to -establish a secure mTLS connection with the instance. +Currently, Cloud SQL supports connection over both [private IP][private-ip] and +[public IP][public-ip]. Set the `ipType` parameter in your source +configuration to `public` or `private`. +[private-ip]: https://cloud.google.com/sql/docs/postgres/configure-private-ip [public-ip]: https://cloud.google.com/sql/docs/postgres/configure-ip ### Database User Current, this source only uses standard authentication. You will need to [create a -PostreSQL user][cloud-sql-users] to login to the database with. +PostreSQL user][cloud-sql-users] to login to the database with. [cloud-sql-users]: https://cloud.google.com/sql/docs/postgres/create-manage-users @@ -65,7 +66,7 @@ sources: | project | string | true | Id of the GCP project that the cluster was created in (e.g. "my-project-id"). | | region | string | true | Name of the GCP region that the cluster was created in (e.g. "us-central1"). | | instance | string | true | Name of the Cloud SQL instance within the cluser (e.g. "my-instance"). | -| ip_type | string | true | IP Type of the Cloud SQL instance, must be either `public` or `private`. Default: `public`. | +| ipType | string | true | IP Type of the Cloud SQL instance, must be either `public` or `private`. Default: `public`. | | database | string | true | Name of the Postgres database to connect to (e.g. "my_db"). | | user | string | true | Name of the Postgres user to connect as (e.g. "my-pg-user"). | | password | string | true | Password of the Postgres user (e.g. "my-password"). | diff --git a/docs/sources/neo4j.md b/docs/sources/neo4j.md new file mode 100644 index 000000000..f90e3800c --- /dev/null +++ b/docs/sources/neo4j.md @@ -0,0 +1,40 @@ +# Neo4j Source + +[Neo4j][neo4j-docs] is a powerful, open source graph database +system with over 15 years of active development that has earned it a strong +reputation for reliability, feature robustness, and performance. + +[neo4j-docs]: https://neo4j.com/docs + +## Requirements + +### Database User + +This source only uses standard authentication. You will need to [create a +Neo4j user][neo4j-users] to log in to the database with, or use the default `neo4j` user if available. + +[neo4j-users]: https://neo4j.com/docs/operations-manual/current/authentication-authorization/manage-users/ + +## Example + +```yaml +sources: + my-neo4j-source: + kind: "neo4j" + uri: "neo4j+s://xxxx.databases.neo4j.io:7687" + user: "neo4j" + password: "my-password" + database: "neo4j" +``` + +## Reference + +| **field** | **type** | **required** | **description** | +|-----------|:--------:|:------------:|---------------------------------------------------------------------| +| kind | string | true | Must be "neo4j". | +| uri | string | true | Connect URI ("bolt://localhost", "neo4j+s://xxx.databases.neo4j.io") | +| user | string | true | Name of the Neo4j user to connect as (e.g. "neo4j"). | +| password | string | true | Password of the Neo4j user (e.g. "my-password"). | +| database | string | true | Name of the Neo4j database to connect to (e.g. "neo4j"). | + + diff --git a/docs/telemetry/guide_collector.md b/docs/telemetry/guide_collector.md new file mode 100644 index 000000000..aeb00d75c --- /dev/null +++ b/docs/telemetry/guide_collector.md @@ -0,0 +1,96 @@ +# Use collector to export telemetry (trace and metric) data +Collector receives telemetry data, processes the telemetry, and exports it to a wide variety of observability backends using its components. + +## Collector +The OpenTelemetry Collector removes the need to run, operate, and maintain multiple +agents/collector. This works well with scalability and supports open source +observability data formats senidng to one or more open source or commercial +backends. In addition, collector also provide other benefits such as allowing +your service to offload data quickly while it take care of additional handling +like retries, batching, encryption, or even sensitive data filtering. + +To run a collector, you will have to provide a configuration file. The +configuration file consists of four classes of pipeline component that access +telemetry data. +- `Receivers` +- `Processors` +- `Exporters` +- `Connectors` + +Example of setting up the classes of pipeline components (in this example, we +don't use connectors): + +```yaml +receivers: + otlp: + protocols: + http: + endpoint: "127.0.0.1:4553" + +exporters: + googlecloud: + project: + +processors: + batch: + send_batch_size: 200 +``` + +After each pipeline component is configured, you will enable it within the +`service` section of the configuration file. + +```yaml +service: + pipelines: + traces: + receivers: ["otlp"] + processors: ["batch"] + exporters: ["googlecloud"] +``` + +For a conceptual overview of the Collector, see [Collector][collector]. + +[collector]: https://opentelemetry.io/docs/collector/ + +## Using a Collector +There are a couple of steps to run and use a Collector. + +1. Obtain a Collector binary. Pull a binary or Docker image for the + OpenTelemetry contrib collector. + +1. Set up credentials for telemetry backend. + +1. Set up the Collector config. + Below are some examples for setting up the Collector config: + - [Google Cloud Exporter][google-cloud-exporter] + - [Google Managed Service for Prometheus Exporter][google-prometheus-exporter] + +1. Run the Collector with the configuration file. + + ```bash + ./otelcol-contrib --config=collector-config.yaml + ``` + +1. Run toolbox with the `--telemetry-otlp` flag. Configure it to send them to + `http://127.0.0.1:4553` (for HTTP) or the Collector's URL. + + ```bash + ./toolbox --telemetry-otlp=http://127.0.0.1:4553 + ``` + +1. Once telemetry datas are collected, you can view them in your telemetry + backend. If you are using GCP exporters, telemetry will be visible in GCP + dashboard at [Metrics Explorer][metrics-explorer] and [Trace + Explorer][trace-explorer]. + +> [!NOTE] +> If you are exporting to Google Cloud monitoring, we recommend that you use +> the Google Cloud Exporter for traces and the Google Managed Service for +> Prometheus Exporter for metrics. + +[google-cloud-exporter]: + https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/googlecloudexporter +[google-prometheus-exporter]: + https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/googlemanagedprometheusexporter#example-configuration +[metrics-explorer]: https://console.cloud.google.com/monitoring/metrics-explorer +[trace-explorer]: https://console.cloud.google.com/traces diff --git a/docs/telemetry/telemetry.md b/docs/telemetry/telemetry.md new file mode 100644 index 000000000..97c55a865 --- /dev/null +++ b/docs/telemetry/telemetry.md @@ -0,0 +1,183 @@ +# Telemetry for Toolbox + +Telemetry data such as logs, metrics, and traces will help developers understand +the internal state of the system. + +Toolbox exports telemetry data of logs via standard out/err, and traces/metrics +through OpenTelemetry. Additional flags can be passed to Toolbox to enable +different logging behavior, or to export metrics through a specific +[exporter](#exporter). + + +## Logging + +### Logging format +Toolbox supports both text and structured logging format. + +The text logging (also the default logging format) outputs log as string: +``` +2024-11-12T15:08:11.451377-08:00 INFO "Initialized 0 sources.\n" +``` + +The structured logging outputs log as JSON: +``` +{ + "timestamp":"2024-11-04T16:45:11.987299-08:00", + "severity":"ERROR", + "logging.googleapis.com/sourceLocation":{...}, + "message":"unable to parse tool file at \"tools.yaml\": \"cloud-sql-postgres1\" is not a valid kind of data source" +} +``` +> [!NOTE] +> `logging.googleapis.com/sourceLocation` shows the source code location +> information associated with the log entry, if any. + +### Log level +Toolbox supports four log levels, including `Debug`, `Info`, `Warn`, +and `Error`. Toolbox will only output logs that are equal or more severe to the +level that it is set. Below are the log levels that Toolbox supports in the +order of severity. + +| **Log level** | **Description** | +|---------------|-----------------| +| Debug | Debug logs typically contain information that is only useful during the debugging phase and may be of little value during production. | +| Info | Info logs include information about successful operations within the application, such as a successful start, pause, or exit of the application. | +| Warn | Warning logs are slightly less severe than error conditions. While it does not cause an error, it indicates that an operation might fail in the future if action is not taken now. | +| Error | Error log is assigned to event logs that contain an application error message. | + +### Logging Configurations +The following flags can be used to customize Toolbox logging: + +| **Flag** | **Description** | +|----------|-----------------| +| `--log-level` | Preferred log level, allowed values: `debug`, `info`, `warn`, `error`. Default: `info`. | +| `--logging-format` | Preferred logging format, allowed values: `standard`, `json`. Default: `standard`. | + +#### Example: + +```bash +./toolbox --tools_file "tools.yaml" --log-level warn --logging-format json +``` + +## Telemetry +### Metrics +A metric is a measurement of a service captured at runtime. The collected data +can be used to provide important insights into the service. +Toolbox provides the following custom metrics: + +| **Metric Name** | **Description** | +|-----------------|-----------------| +| `toolbox.server.toolset.get.count` | Counts the number of toolset manifest requests served | +| `toolbox.server.tool.get.count` | Counts the number of tool manifest requests served | +| `toolbox.server.tool.get.invoke` | Counts the number of tool invocation requests served | + +All custom metrics have the following attributes/labels: + +| **Metric Attributes** | **Description** | +|-----------------|-----------------| +| `toolbox.name` | Name of the toolset or tool, if applicable. | +| `toolbox.status` | Operation status code, for example: `success`, `failure`. | + +### Traces +Trace is a tree of spans that shows the path that a request makes through an +application. + +Spans generated by Toolbox server is prefixed with `toolbox/server/`. For +example, when user run Toolbox, it will generate spans for the following, with +`toolbox/server/init` as the root span: + +![traces](traces.png) + +### Exporter +Exporter is responsible for processing and exporting telemetry data. Toolbox +generates telemetry data within the OpenTelemetry Protocol (OTLP), and user can +choose to use exporters that are designed to support the OpenTelemetry +Protocol. Within Toolbox, we provide two types of exporter implementation to +choose from, either the Google Cloud Exporter that will send data directly to +the backend, or the OTLP Exporter along with a Collector that will act as a +proxy to collect and export data to the telemetry backend of user's choice. + +![telemetry_flow](telemetry_flow.png) + +#### Google Cloud Exporter +The Google Cloud Exporter directly exports telemetry to Google Cloud Monitoring. +It utilizes the [GCP Metric Exporter][gcp-metric-exporter] and [GCP Trace +Exporter][gcp-trace-exporter]. + +[gcp-metric-exporter]: + https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/tree/main/exporter/metric +[gcp-trace-exporter]: + https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/tree/main/exporter/trace + +> [!NOTE] +> If you're using Google Cloud Monitoring, the following APIs will need to be +enabled. For instructions on how to enable APIs, see [this +guide](https://cloud.google.com/endpoints/docs/openapi/enable-api): +> +> - logging.googleapis.com +> - monitoring.googleapis.com +> - cloudtrace.googleapis.com + +#### OTLP Exporter +This implementation uses the default OTLP Exporter over HTTP for +[metrics][otlp-metric-exporter] and [traces][otlp-trace-exporter]. You can use +this exporter if you choose to export your telemetry data to a Collector. + +[otlp-metric-exporter]: https://opentelemetry.io/docs/languages/go/exporters/#otlp-traces-over-http +[otlp-trace-exporter]: https://opentelemetry.io/docs/languages/go/exporters/#otlp-traces-over-http + +### Collector +A collector acts as a proxy between the application and the telemetry backend. It +receives telemetry data, transforms it, and then exports data to backends that +can store it permanently. Toolbox provide an option to export telemetry data to user's choice of +backend(s) that are compatible with the Open Telemetry Protocol (OTLP). If you +would like to use a collector, please refer to this +[guide](./guide_collector.md). + +### Telemetry Configurations +The following flags are used to determine Toolbox's telemetry configuration: + +| **flag** | **type** | **description** | +|-------------------------------|----------|-----------------| +| `--telemetry-gcp` | bool | Enable exporting directly to Google Cloud Monitoring. Default is `false`. | +| `--telemetry-otlp` | string | Enable exporting using OpenTelemetry Protocol (OTLP) to the specified endpoint (e.g. 'http://127.0.0.1:4318'). | +| `--telemetry-service-name` | string | Sets the value of the `service.name` resource attribute. Default is `toolbox`. | + +In addition to the flags noted above, you can also make additional configuration +for OpenTelemetry via the [General SDK Configuration][sdk-configuration] through +environmental variables. + +[sdk-configuration]: + https://opentelemetry.io/docs/languages/sdk-configuration/general/ + +#### Example usage + +To enable Google Cloud Exporter: +```bash +./toolbox --telemetry-gcp +``` + +To enable OTLP Exporter, provide Collector endpoint: +```bash +./toolbox --telemetry-otlp=http://127.0.0.1:4553 +``` + +#### Resource Attribute +All metrics and traces generated within Toolbox will be associated with a +unified [resource][resource]. The list of resource attributes included are: + +| **Resource Name** | **Description** | +|-------------------|-----------------| +| [TelemetrySDK](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/resource#WithTelemetrySDK) | TelemetrySDK version info. | +| [OS](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/resource#WithOS) | OS attributes including OS description and OS type. | +| [Container](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/resource#WithContainer) | Container attributes including container ID, if applicable. | +| [Host](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/resource#WithHost) | Host attributes including host name. | +| [SchemaURL](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/resource#WithSchemaURL) | Sets the schema URL for the configured resource. | +| `service.name` | Open telemetry service name. Defaulted to `toolbox`. User can set the service name via flag mentioned above to distinguish between different toolbox service. | +| `service.version` | The version of Toolbox used. | + + +[resource]: https://opentelemetry.io/docs/languages/go/resources/ + + + diff --git a/docs/telemetry/telemetry_flow.png b/docs/telemetry/telemetry_flow.png new file mode 100644 index 000000000..bfaf8a4f2 Binary files /dev/null and b/docs/telemetry/telemetry_flow.png differ diff --git a/docs/telemetry/traces.png b/docs/telemetry/traces.png new file mode 100644 index 000000000..4d970b30c Binary files /dev/null and b/docs/telemetry/traces.png differ diff --git a/docs/tools/README.md b/docs/tools/README.md index 890fd1fe3..5e3c1148b 100644 --- a/docs/tools/README.md +++ b/docs/tools/README.md @@ -50,6 +50,9 @@ We currently support the following types of kinds of tools: PostgreSQL-compatible database. * [spanner](./spanner.md) - Run a Spanner (either googlesql or postgresql) statement againts Spanner database. +* [neo4j-cypher](./neo4j-cypher.md) - Run a Cypher statement against a + Neo4j database. + ## Specifying Parameters @@ -96,9 +99,9 @@ requires another Parameter to be specified under the `items` field: type: array description: A list of airline, ordered by preference. items: - - name: name - type: string - description: Name of the airline. + name: name + type: string + description: Name of the airline. ``` | **field** | **type** | **required** | **description** | diff --git a/docs/tools/neo4j-cypher.md b/docs/tools/neo4j-cypher.md new file mode 100644 index 000000000..dab63c187 --- /dev/null +++ b/docs/tools/neo4j-cypher.md @@ -0,0 +1,59 @@ +# Neo4j Cypher Tool + +A "neo4j-cypher" tool executes a pre-defined Cypher statement against a Neo4j database. It's compatible with any of the following sources: +- [neo4j](../sources/neo4j.md) + +The specified Cypher statement is executed as a [parameterized statement][neo4j-parameters], +and specified parameters will be used according to their name: e.g. `$id`. + +[neo4j-parameters]: https://neo4j.com/docs/cypher-manual/current/syntax/parameters/ + +## Example + +```yaml +tools: + search_movies_by_actor: + kind: neo4j-cypher + source: my-neo4j-movies-instance + statement: | + MATCH (m:Movie)<-[:ACTED_IN]-(p:Person) + WHERE p.name = $name AND m.year > $year + RETURN m.title, m.year + LIMIT 10 + description: | + Use this tool to get a list of movies for a specific actor and a given minium release year. + Takes an full actor name, e.g. "Tom Hanks" and a year e.g 1993 and returns a list of movie titles and release years. + Do NOT use this tool with a movie title. Do NOT guess an actor name, Do NOT guess a year. + A actor name is a fully qualified name with first and last name separated by a space. + For example, if given "Hanks, Tom" the actor name is "Tom Hanks". + If the tool returns more than one option choose the most recent movies. + Example: + {{ + "name": "Meg Ryan", + "year": 1993 + }} + Example: + {{ + "name": "Clint Eastwood", + "year": 2000 + }} + parameters: + - name: name + type: string + description: Full actor name, "firstname lastname" + - name: year + type: integer + description: 4 digit number starting in 1900 up to the current year +``` + +## Reference + +| **field** | **type** | **required** | **description** | +|-------------|----------:|:------------:|----------------------------------------------------------------------------------------------------| +| kind | string | true | Must be "neo4j-cypher". | +| source | string | true | Name of the source the Cypher query should execute on. | +| description | string | true | Description of the tool | +| statement | string | true | Cypher statement to execute | +| parameters | parameter | true | List of [parameters](README.md#specifying-parameters) that will be used with the Cypher statement. | + + diff --git a/go.mod b/go.mod index 7f1d94c0e..1e96a1cf6 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,23 @@ require ( cloud.google.com/go/alloydbconn v1.13.2 cloud.google.com/go/cloudsqlconn v1.13.2 cloud.google.com/go/spanner v1.73.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.25.0 github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/httplog/v2 v2.1.1 github.com/go-chi/render v1.0.3 github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.7.1 + github.com/neo4j/neo4j-go-driver/v5 v5.26.0 github.com/spf13/cobra v1.8.1 + go.opentelemetry.io/contrib/propagators/autoprop v0.58.0 + go.opentelemetry.io/otel v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 + go.opentelemetry.io/otel/metric v1.33.0 + go.opentelemetry.io/otel/sdk v1.33.0 + go.opentelemetry.io/otel/sdk/metric v1.33.0 go.opentelemetry.io/otel/trace v1.33.0 google.golang.org/api v0.211.0 gopkg.in/yaml.v3 v3.0.1 @@ -28,9 +39,12 @@ require ( cloud.google.com/go/compute/metadata v0.5.2 // indirect cloud.google.com/go/longrunning v0.6.3 // indirect cloud.google.com/go/monitoring v1.22.0 // indirect + cloud.google.com/go/trace v1.11.2 // indirect github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect github.com/ajg/form v1.5.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect @@ -41,9 +55,9 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/s2a-go v0.1.8 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -55,10 +69,13 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.33.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.33.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 // indirect - go.opentelemetry.io/otel/sdk v1.33.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.33.0 // indirect + go.opentelemetry.io/contrib/propagators/aws v1.33.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.33.0 // indirect + go.opentelemetry.io/contrib/propagators/jaeger v1.33.0 // indirect + go.opentelemetry.io/contrib/propagators/ot v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect diff --git a/go.sum b/go.sum index 981fda3e1..e7612b7ee 100644 --- a/go.sum +++ b/go.sum @@ -356,6 +356,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6 cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= +cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= @@ -570,6 +572,8 @@ cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= +cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= @@ -627,6 +631,14 @@ github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2 h1:DBjmt6/otSdULyJdVg2 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 h1:o90wcURuxekmXrtxmYWTyNla0+ZEHhud6DI1ZTxd1vI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0/go.mod h1:6fTWu4m3jocfUZLYF5KsZC1TUfRvEjs7lM4crme/irw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.25.0 h1:4PoDbd/9/06IpwLGxSfvfNoEr9urvfkrN6mmJangGCg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.25.0/go.mod h1:EycllQ1gupHbjqbcmfCr/H6FKSGSmEUONJ2ivb86qeY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0 h1:jJKWl98inONJAr/IZrdFQUWcwUO95DLY1XMD1ZIut+g= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0/go.mod h1:l2fIqmwB+FKSfvn3bAD/0i+AXAxhIZjTK2svT/mgUXs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 h1:GYUJLfvd++4DMuMhCFLgLXvFwofIxh/qOwoGuS/LTew= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0/go.mod h1:wRbFgBQUVm1YXrvWKofAEmq9HNJTDphbAaJSSX01KUI= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= @@ -642,6 +654,8 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= @@ -844,6 +858,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -902,6 +918,8 @@ github.com/microsoft/go-mssqldb v1.8.0 h1:7cyZ/AT7ycDsEoWPIXibd+aVKFtteUNhDGf3ao github.com/microsoft/go-mssqldb v1.8.0/go.mod h1:6znkekS3T2vp0waiMhen4GPU1BiAsrP+iXHcE7a7rFo= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/neo4j/neo4j-go-driver/v5 v5.26.0 h1:GB3o4VtIGsvU+RmfgvF7L6nt1IpbPZaGtPMtPSOKmvc= +github.com/neo4j/neo4j-go-driver/v5 v5.26.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= @@ -977,8 +995,24 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/propagators/autoprop v0.58.0 h1:pL1MMoBcG/ol6fVsjE1bbOO9A8GMQiN+T73hnmaXDoU= +go.opentelemetry.io/contrib/propagators/autoprop v0.58.0/go.mod h1:EU5uMoCqafsagp4hzFqzu1Eyg/8L23JS5Y1hChoHf7s= +go.opentelemetry.io/contrib/propagators/aws v1.33.0 h1:MefPfPIut0IxEiQRK1qVv5AFADBOwizl189+m7QhpFg= +go.opentelemetry.io/contrib/propagators/aws v1.33.0/go.mod h1:VB6xPo12uW/PezOqtA/cY2/DiAGYshnhID606wC9NEY= +go.opentelemetry.io/contrib/propagators/b3 v1.33.0 h1:ig/IsHyyoQ1F1d6FUDIIW5oYpsuTVtN16AyGOgdjAHQ= +go.opentelemetry.io/contrib/propagators/b3 v1.33.0/go.mod h1:EsVYoNy+Eol5znb6wwN3XQTILyjl040gUpEnUSNZfsk= +go.opentelemetry.io/contrib/propagators/jaeger v1.33.0 h1:Jok/dG8kfp+yod29XKYV/blWgYPlMuRUoRHljrXMF5E= +go.opentelemetry.io/contrib/propagators/jaeger v1.33.0/go.mod h1:ku/EpGk44S5lyVMbtJRK2KFOnXEehxf6SDnhu1eZmjA= +go.opentelemetry.io/contrib/propagators/ot v1.33.0 h1:xj/pQFKo4ROsx0v129KpLgFwaYMgFTu3dAMEEih97cY= +go.opentelemetry.io/contrib/propagators/ot v1.33.0/go.mod h1:/xxHCLhTmaypEFwMViRGROj2qgrGiFrkxIlATt0rddc= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0 h1:bSjzTvsXZbLSWU8hnZXcKmEVaJjjnandxD0PxThhVU8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0/go.mod h1:aj2rilHL8WjXY1I5V+ra+z8FELtk681deydgYT8ikxU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= @@ -990,6 +1024,10 @@ go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37Cb go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/internal/server/api.go b/internal/server/api.go index bb1b63714..f24d25fe4 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -25,6 +25,9 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/render" "github.com/googleapis/genai-toolbox/internal/tools" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" ) // apiRouter creates a router that represents the routes under /api @@ -48,10 +51,33 @@ func apiRouter(s *Server) (chi.Router, error) { // toolsetHandler handles the request for information about a Toolset. func toolsetHandler(s *Server, w http.ResponseWriter, r *http.Request) { + ctx, span := s.instrumentation.Tracer.Start(r.Context(), "toolbox/server/toolset/get") + r = r.WithContext(ctx) + toolsetName := chi.URLParam(r, "toolsetName") + span.SetAttributes(attribute.String("toolset_name", toolsetName)) + var err error + defer func() { + if err != nil { + span.SetStatus(codes.Error, err.Error()) + } + span.End() + + status := "success" + if err != nil { + status = "error" + } + s.instrumentation.ToolsetGet.Add( + r.Context(), + 1, + metric.WithAttributes(attribute.String("toolbox.name", toolsetName)), + metric.WithAttributes(attribute.String("toolbox.operation.status", status)), + ) + }() + toolset, ok := s.toolsets[toolsetName] if !ok { - err := fmt.Errorf("Toolset %q does not exist", toolsetName) + err = fmt.Errorf("Toolset %q does not exist", toolsetName) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusNotFound)) return @@ -61,10 +87,32 @@ func toolsetHandler(s *Server, w http.ResponseWriter, r *http.Request) { // toolGetHandler handles requests for a single Tool. func toolGetHandler(s *Server, w http.ResponseWriter, r *http.Request) { + ctx, span := s.instrumentation.Tracer.Start(r.Context(), "toolbox/server/tool/get") + r = r.WithContext(ctx) + toolName := chi.URLParam(r, "toolName") + span.SetAttributes(attribute.String("tool_name", toolName)) + var err error + defer func() { + if err != nil { + span.SetStatus(codes.Error, err.Error()) + } + span.End() + + status := "success" + if err != nil { + status = "error" + } + s.instrumentation.ToolGet.Add( + r.Context(), + 1, + metric.WithAttributes(attribute.String("toolbox.name", toolName)), + metric.WithAttributes(attribute.String("toolbox.operation.status", status)), + ) + }() tool, ok := s.tools[toolName] if !ok { - err := fmt.Errorf("invalid tool name: tool with name %q does not exist", toolName) + err = fmt.Errorf("invalid tool name: tool with name %q does not exist", toolName) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusNotFound)) return @@ -82,10 +130,33 @@ func toolGetHandler(s *Server, w http.ResponseWriter, r *http.Request) { // toolInvokeHandler handles the API request to invoke a specific Tool. func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) { + ctx, span := s.instrumentation.Tracer.Start(r.Context(), "toolbox/server/tool/invoke") + r = r.WithContext(ctx) + toolName := chi.URLParam(r, "toolName") + span.SetAttributes(attribute.String("tool_name", toolName)) + var err error + defer func() { + if err != nil { + span.SetStatus(codes.Error, err.Error()) + } + span.End() + + status := "success" + if err != nil { + status = "error" + } + s.instrumentation.ToolInvoke.Add( + r.Context(), + 1, + metric.WithAttributes(attribute.String("toolbox.name", toolName)), + metric.WithAttributes(attribute.String("toolbox.operation.status", status)), + ) + }() + tool, ok := s.tools[toolName] if !ok { - err := fmt.Errorf("invalid tool name: tool with name %q does not exist", toolName) + err = fmt.Errorf("invalid tool name: tool with name %q does not exist", toolName) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusNotFound)) return @@ -97,7 +168,7 @@ func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) { for _, aS := range s.authSources { claims, err := aS.GetClaimsFromHeader(r.Header) if err != nil { - err := fmt.Errorf("failure getting claims from header: %w", err) + err = fmt.Errorf("failure getting claims from header: %w", err) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusBadRequest)) return @@ -119,16 +190,16 @@ func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) { // Check if any of the specified auth sources is verified isAuthorized := tool.Authorized(verifiedAuthSources) if !isAuthorized { - err := fmt.Errorf("tool invocation not authorized. Please make sure your specify correct auth headers") + err = fmt.Errorf("tool invocation not authorized. Please make sure your specify correct auth headers") s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusUnauthorized)) return } var data map[string]any - if err := decodeJSON(r.Body, &data); err != nil { + if err = decodeJSON(r.Body, &data); err != nil { render.Status(r, http.StatusBadRequest) - err := fmt.Errorf("request body was invalid JSON: %w", err) + err = fmt.Errorf("request body was invalid JSON: %w", err) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusBadRequest)) return @@ -136,7 +207,7 @@ func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) { params, err := tool.ParseParams(data, claimsFromAuth) if err != nil { - err := fmt.Errorf("provided parameters were invalid: %w", err) + err = fmt.Errorf("provided parameters were invalid: %w", err) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusBadRequest)) return @@ -144,7 +215,7 @@ func toolInvokeHandler(s *Server, w http.ResponseWriter, r *http.Request) { res, err := tool.Invoke(params) if err != nil { - err := fmt.Errorf("error while invoking tool: %w", err) + err = fmt.Errorf("error while invoking tool: %w", err) s.logger.DebugContext(context.Background(), err.Error()) _ = render.Render(w, r, newErrResponse(err, http.StatusInternalServerError)) return diff --git a/internal/server/api_test.go b/internal/server/api_test.go index 62eb22100..1a52b6faf 100644 --- a/internal/server/api_test.go +++ b/internal/server/api_test.go @@ -15,6 +15,7 @@ package server import ( + "context" "encoding/json" "fmt" "io" @@ -24,11 +25,14 @@ import ( "testing" "github.com/googleapis/genai-toolbox/internal/log" + "github.com/googleapis/genai-toolbox/internal/telemetry" "github.com/googleapis/genai-toolbox/internal/tools" ) var _ tools.Tool = &MockTool{} +const fakeVersionString = "0.0.0" + type MockTool struct { Name string Description string @@ -57,6 +61,9 @@ func (t MockTool) Authorized(verifiedAuthSources []string) bool { } func TestToolsetEndpoint(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Set up resources to test against tool1 := MockTool{ Name: "no_params", @@ -78,7 +85,7 @@ func TestToolsetEndpoint(t *testing.T) { "tool2_only": {tool2.Name}, } { tc := tools.ToolsetConfig{Name: name, ToolNames: l} - m, err := tc.Initialize("0.0.0", toolsMap) + m, err := tc.Initialize(fakeVersionString, toolsMap) if err != nil { t.Fatalf("unable to initialize toolset %q: %s", name, err) } @@ -87,9 +94,25 @@ func TestToolsetEndpoint(t *testing.T) { testLogger, err := log.NewStdLogger(os.Stdout, os.Stderr, "info") if err != nil { - t.Fatalf("unexpected error: %s", err) + t.Fatalf("unable to initialize logger: %s", err) + } + + otelShutdown, err := telemetry.SetupOTel(ctx, fakeVersionString, "", false, "toolbox") + if err != nil { + t.Fatalf("unable to setup otel: %s", err) } - server := Server{logger: testLogger, tools: toolsMap, toolsets: toolsets} + defer func() { + err := otelShutdown(ctx) + if err != nil { + t.Fatalf("error shutting down OpenTelemetry: %s", err) + } + }() + instrumentation, err := CreateTelemetryInstrumentation(fakeVersionString) + if err != nil { + t.Fatalf("unable to create custom metrics: %s", err) + } + + server := Server{logger: testLogger, instrumentation: instrumentation, tools: toolsMap, toolsets: toolsets} r, err := apiRouter(&server) if err != nil { t.Fatalf("unable to initialize router: %s", err) @@ -115,7 +138,7 @@ func TestToolsetEndpoint(t *testing.T) { toolsetName: "", want: wantResponse{ statusCode: http.StatusOK, - version: "0.0.0", + version: fakeVersionString, tools: []string{tool1.Name, tool2.Name}, }, }, @@ -132,7 +155,7 @@ func TestToolsetEndpoint(t *testing.T) { toolsetName: "tool1_only", want: wantResponse{ statusCode: http.StatusOK, - version: "0.0.0", + version: fakeVersionString, tools: []string{tool1.Name}, }, }, @@ -141,7 +164,7 @@ func TestToolsetEndpoint(t *testing.T) { toolsetName: "tool2_only", want: wantResponse{ statusCode: http.StatusOK, - version: "0.0.0", + version: fakeVersionString, tools: []string{tool2.Name}, }, }, @@ -186,6 +209,9 @@ func TestToolsetEndpoint(t *testing.T) { } } func TestToolGetEndpoint(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Set up resources to test against tool1 := MockTool{ Name: "no_params", @@ -202,9 +228,25 @@ func TestToolGetEndpoint(t *testing.T) { testLogger, err := log.NewStdLogger(os.Stdout, os.Stderr, "info") if err != nil { - t.Fatalf("unexpected error: %s", err) + t.Fatalf("unable to initialize logger: %s", err) } - server := Server{version: "0.0.0", logger: testLogger, tools: toolsMap} + + otelShutdown, err := telemetry.SetupOTel(ctx, fakeVersionString, "", false, "toolbox") + if err != nil { + t.Fatalf("unable to setup otel: %s", err) + } + defer func() { + err := otelShutdown(ctx) + if err != nil { + t.Fatalf("error shutting down OpenTelemetry: %s", err) + } + }() + instrumentation, err := CreateTelemetryInstrumentation(fakeVersionString) + if err != nil { + t.Fatalf("unable to create custom metrics: %s", err) + } + + server := Server{version: fakeVersionString, logger: testLogger, instrumentation: instrumentation, tools: toolsMap} r, err := apiRouter(&server) if err != nil { t.Fatalf("unable to initialize router: %s", err) @@ -230,7 +272,7 @@ func TestToolGetEndpoint(t *testing.T) { toolName: tool1.Name, want: wantResponse{ statusCode: http.StatusOK, - version: "0.0.0", + version: fakeVersionString, tools: []string{tool1.Name}, }, }, @@ -239,7 +281,7 @@ func TestToolGetEndpoint(t *testing.T) { toolName: tool2.Name, want: wantResponse{ statusCode: http.StatusOK, - version: "0.0.0", + version: fakeVersionString, tools: []string{tool2.Name}, }, }, diff --git a/internal/server/config.go b/internal/server/config.go index 64bf8f948..e9081fa00 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -22,9 +22,11 @@ import ( "github.com/googleapis/genai-toolbox/internal/sources" alloydbpgsrc "github.com/googleapis/genai-toolbox/internal/sources/alloydbpg" cloudsqlpgsrc "github.com/googleapis/genai-toolbox/internal/sources/cloudsqlpg" + neo4jrc "github.com/googleapis/genai-toolbox/internal/sources/neo4j" postgressrc "github.com/googleapis/genai-toolbox/internal/sources/postgres" spannersrc "github.com/googleapis/genai-toolbox/internal/sources/spanner" "github.com/googleapis/genai-toolbox/internal/tools" + neo4jtool "github.com/googleapis/genai-toolbox/internal/tools/neo4j" "github.com/googleapis/genai-toolbox/internal/tools/postgressql" "github.com/googleapis/genai-toolbox/internal/tools/spanner" "gopkg.in/yaml.v3" @@ -47,8 +49,14 @@ type ServerConfig struct { ToolsetConfigs ToolsetConfigs // LoggingFormat defines whether structured loggings are used. LoggingFormat logFormat - // LogLevel defines the levels to log + // LogLevel defines the levels to log. LogLevel StringLevel + // TelemetryGCP defines whether GCP exporter is used. + TelemetryGCP bool + // TelemetryOTLP defines OTLP collector url for telemetry exports. + TelemetryOTLP string + // TelemetryServiceName defines the value of service.name resource attribute. + TelemetryServiceName string } type logFormat string @@ -150,6 +158,12 @@ func (c *SourceConfigs) UnmarshalYAML(node *yaml.Node) error { return fmt.Errorf("unable to parse as %q: %w", k.Kind, err) } (*c)[name] = actual + case neo4jrc.SourceKind: + actual := neo4jrc.Config{Name: name} + if err := n.Decode(&actual); err != nil { + return fmt.Errorf("unable to parse as %q: %w", k.Kind, err) + } + (*c)[name] = actual default: return fmt.Errorf("%q is not a valid kind of data source", k.Kind) } @@ -162,7 +176,7 @@ func (c *SourceConfigs) UnmarshalYAML(node *yaml.Node) error { type AuthSourceConfigs map[string]auth.AuthSourceConfig // validate interface -var _ yaml.Unmarshaler = &SourceConfigs{} +var _ yaml.Unmarshaler = &AuthSourceConfigs{} func (c *AuthSourceConfigs) UnmarshalYAML(node *yaml.Node) error { *c = make(AuthSourceConfigs) @@ -229,6 +243,12 @@ func (c *ToolConfigs) UnmarshalYAML(node *yaml.Node) error { return fmt.Errorf("unable to parse as %q: %w", k.Kind, err) } (*c)[name] = actual + case neo4jtool.ToolKind: + actual := neo4jtool.Config{Name: name} + if err := n.Decode(&actual); err != nil { + return fmt.Errorf("unable to parse as %q: %w", k.Kind, err) + } + (*c)[name] = actual default: return fmt.Errorf("%q is not a valid kind of tool", k.Kind) } diff --git a/internal/server/instrumentation.go b/internal/server/instrumentation.go new file mode 100644 index 000000000..3974aa311 --- /dev/null +++ b/internal/server/instrumentation.go @@ -0,0 +1,85 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +const ( + TracerName = "github.com/googleapis/genai-toolbox/internal/opentel" + MetricName = "github.com/googleapis/genai-toolbox/internal/opentel" + + toolsetGetCountName = "toolbox.server.toolset.get.count" + toolGetCountName = "toolbox.server.tool.get.count" + toolInvokeCountName = "toolbox.server.tool.invoke.count" +) + +// Instrumentation defines the telemetry instrumentation for toolbox +type Instrumentation struct { + Tracer trace.Tracer + meter metric.Meter + ToolsetGet metric.Int64Counter + ToolGet metric.Int64Counter + ToolInvoke metric.Int64Counter +} + +func CreateTelemetryInstrumentation(versionString string) (*Instrumentation, error) { + tracer := otel.Tracer( + TracerName, + trace.WithInstrumentationVersion(versionString), + ) + + meter := otel.Meter(MetricName, metric.WithInstrumentationVersion(versionString)) + toolsetGet, err := meter.Int64Counter( + toolsetGetCountName, + metric.WithDescription("Number of toolset GET API calls."), + metric.WithUnit("{call}"), + ) + if err != nil { + return nil, fmt.Errorf("unable to create %s metric: %w", toolsetGetCountName, err) + } + + toolGet, err := meter.Int64Counter( + toolGetCountName, + metric.WithDescription("Number of tool GET API calls."), + metric.WithUnit("{call}"), + ) + if err != nil { + return nil, fmt.Errorf("unable to create %s metric: %w", toolGetCountName, err) + } + + toolInvoke, err := meter.Int64Counter( + toolInvokeCountName, + metric.WithDescription("Number of tool Invoke API calls."), + metric.WithUnit("{call}"), + ) + if err != nil { + return nil, fmt.Errorf("unable to create %s metric: %w", toolInvokeCountName, err) + } + + instrumentation := &Instrumentation{ + Tracer: tracer, + meter: meter, + ToolsetGet: toolsetGet, + ToolGet: toolGet, + ToolInvoke: toolInvoke, + } + return instrumentation, nil +} diff --git a/internal/server/server.go b/internal/server/server.go index 398c8c88b..f0a4b6021 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -29,15 +29,18 @@ import ( "github.com/googleapis/genai-toolbox/internal/log" "github.com/googleapis/genai-toolbox/internal/sources" "github.com/googleapis/genai-toolbox/internal/tools" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Server contains info for running an instance of Toolbox. Should be instantiated with NewServer(). type Server struct { - version string - srv *http.Server - listener net.Listener - root chi.Router - logger log.Logger + version string + srv *http.Server + listener net.Listener + root chi.Router + logger log.Logger + instrumentation *Instrumentation sources map[string]sources.Source authSources map[string]auth.AuthSource @@ -47,6 +50,14 @@ type Server struct { // NewServer returns a Server object based on provided Config. func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, error) { + instrumentation, err := CreateTelemetryInstrumentation(cfg.Version) + if err != nil { + return nil, fmt.Errorf("unable to create telemetry instrumentation: %w", err) + } + + parentCtx, span := instrumentation.Tracer.Start(context.Background(), "toolbox/server/init") + defer span.End() + // set up http serving r := chi.NewRouter() r.Use(middleware.Recoverer) @@ -84,9 +95,22 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er // initialize and validate the sources from configs sourcesMap := make(map[string]sources.Source) for name, sc := range cfg.SourceConfigs { - s, err := sc.Initialize() + s, err := func() (sources.Source, error) { + ctx, span := instrumentation.Tracer.Start( + parentCtx, + "toolbox/server/source/init", + trace.WithAttributes(attribute.String("source_kind", sc.SourceConfigKind())), + trace.WithAttributes(attribute.String("source_name", name)), + ) + defer span.End() + s, err := sc.Initialize(ctx, instrumentation.Tracer) + if err != nil { + return nil, fmt.Errorf("unable to initialize source %q: %w", name, err) + } + return s, nil + }() if err != nil { - return nil, fmt.Errorf("unable to initialize source %q: %w", name, err) + return nil, err } sourcesMap[name] = s } @@ -95,9 +119,22 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er // initialize and validate the auth sources from configs authSourcesMap := make(map[string]auth.AuthSource) for name, sc := range cfg.AuthSourceConfigs { - a, err := sc.Initialize() + a, err := func() (auth.AuthSource, error) { + _, span := instrumentation.Tracer.Start( + parentCtx, + "toolbox/server/auth/init", + trace.WithAttributes(attribute.String("auth_kind", sc.AuthSourceConfigKind())), + trace.WithAttributes(attribute.String("auth_name", name)), + ) + defer span.End() + a, err := sc.Initialize() + if err != nil { + return nil, fmt.Errorf("unable to initialize auth source %q: %w", name, err) + } + return a, nil + }() if err != nil { - return nil, fmt.Errorf("unable to initialize auth source %q: %w", name, err) + return nil, err } authSourcesMap[name] = a } @@ -106,9 +143,22 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er // initialize and validate the tools from configs toolsMap := make(map[string]tools.Tool) for name, tc := range cfg.ToolConfigs { - t, err := tc.Initialize(sourcesMap) + t, err := func() (tools.Tool, error) { + _, span := instrumentation.Tracer.Start( + parentCtx, + "toolbox/server/tool/init", + trace.WithAttributes(attribute.String("tool_kind", tc.ToolConfigKind())), + trace.WithAttributes(attribute.String("tool_name", name)), + ) + defer span.End() + t, err := tc.Initialize(sourcesMap) + if err != nil { + return nil, fmt.Errorf("unable to initialize tool %q: %w", name, err) + } + return t, nil + }() if err != nil { - return nil, fmt.Errorf("unable to initialize tool %q: %w", name, err) + return nil, err } toolsMap[name] = t } @@ -127,9 +177,21 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er // initialize and validate the toolsets from configs toolsetsMap := make(map[string]tools.Toolset) for name, tc := range cfg.ToolsetConfigs { - t, err := tc.Initialize(cfg.Version, toolsMap) + t, err := func() (tools.Toolset, error) { + _, span := instrumentation.Tracer.Start( + parentCtx, + "toolbox/server/toolset/init", + trace.WithAttributes(attribute.String("toolset_name", name)), + ) + defer span.End() + t, err := tc.Initialize(cfg.Version, toolsMap) + if err != nil { + return tools.Toolset{}, fmt.Errorf("unable to initialize toolset %q: %w", name, err) + } + return t, err + }() if err != nil { - return nil, fmt.Errorf("unable to initialize toolset %q: %w", name, err) + return nil, err } toolsetsMap[name] = t } @@ -139,10 +201,12 @@ func NewServer(ctx context.Context, cfg ServerConfig, l log.Logger) (*Server, er srv := &http.Server{Addr: addr, Handler: r} s := &Server{ - version: cfg.Version, - srv: srv, - root: r, - logger: l, + version: cfg.Version, + srv: srv, + root: r, + logger: l, + instrumentation: instrumentation, + sources: sourcesMap, authSources: authSourcesMap, tools: toolsMap, diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 712b8925a..ca998173e 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -25,6 +25,7 @@ import ( "github.com/googleapis/genai-toolbox/internal/log" "github.com/googleapis/genai-toolbox/internal/server" + "github.com/googleapis/genai-toolbox/internal/telemetry" ) func TestServe(t *testing.T) { @@ -38,6 +39,17 @@ func TestServe(t *testing.T) { Port: port, } + otelShutdown, err := telemetry.SetupOTel(ctx, "0.0.0", "", false, "toolbox") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + defer func() { + err := otelShutdown(ctx) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + }() + testLogger, err := log.NewStdLogger(os.Stdout, os.Stderr, "info") if err != nil { t.Fatalf("unexpected error: %s", err) diff --git a/internal/sources/alloydbpg/alloydb_pg.go b/internal/sources/alloydbpg/alloydb_pg.go index f5b939630..64cb9fa0f 100644 --- a/internal/sources/alloydbpg/alloydb_pg.go +++ b/internal/sources/alloydbpg/alloydb_pg.go @@ -23,6 +23,7 @@ import ( "cloud.google.com/go/alloydbconn" "github.com/googleapis/genai-toolbox/internal/sources" "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/trace" ) const SourceKind string = "alloydb-postgres" @@ -37,7 +38,7 @@ type Config struct { Region string `yaml:"region"` Cluster string `yaml:"cluster"` Instance string `yaml:"instance"` - IPType sources.IPType `yaml:"ip_type"` + IPType sources.IPType `yaml:"ipType"` User string `yaml:"user"` Password string `yaml:"password"` Database string `yaml:"database"` @@ -47,8 +48,8 @@ func (r Config) SourceConfigKind() string { return SourceKind } -func (r Config) Initialize() (sources.Source, error) { - pool, err := initAlloyDBPgConnectionPool(r.Project, r.Region, r.Cluster, r.Instance, r.IPType.String(), r.User, r.Password, r.Database) +func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error) { + pool, err := initAlloyDBPgConnectionPool(ctx, tracer, r.Name, r.Project, r.Region, r.Cluster, r.Instance, r.IPType.String(), r.User, r.Password, r.Database) if err != nil { return nil, fmt.Errorf("unable to create pool: %w", err) } @@ -82,18 +83,22 @@ func (s *Source) PostgresPool() *pgxpool.Pool { return s.Pool } -func getDialOpts(ip_type string) ([]alloydbconn.DialOption, error) { - switch strings.ToLower(ip_type) { +func getDialOpts(ipType string) ([]alloydbconn.DialOption, error) { + switch strings.ToLower(ipType) { case "private": return []alloydbconn.DialOption{alloydbconn.WithPrivateIP()}, nil case "public": return []alloydbconn.DialOption{alloydbconn.WithPublicIP()}, nil default: - return nil, fmt.Errorf("invalid ip_type %s", ip_type) + return nil, fmt.Errorf("invalid ipType %s", ipType) } } -func initAlloyDBPgConnectionPool(project, region, cluster, instance, ip_type, user, pass, dbname string) (*pgxpool.Pool, error) { +func initAlloyDBPgConnectionPool(ctx context.Context, tracer trace.Tracer, name, project, region, cluster, instance, ipType, user, pass, dbname string) (*pgxpool.Pool, error) { + //nolint:all // Reassigned ctx + ctx, span := sources.InitConnectionSpan(ctx, tracer, SourceKind, name) + defer span.End() + // Configure the driver to connect to the database dsn := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, dbname) config, err := pgxpool.ParseConfig(dsn) @@ -102,7 +107,7 @@ func initAlloyDBPgConnectionPool(project, region, cluster, instance, ip_type, us } // Create a new dialer with options - dialOpts, err := getDialOpts(ip_type) + dialOpts, err := getDialOpts(ipType) if err != nil { return nil, err } diff --git a/internal/sources/alloydbpg/alloydb_pg_test.go b/internal/sources/alloydbpg/alloydb_pg_test.go index c1ce7f675..cccd3397a 100644 --- a/internal/sources/alloydbpg/alloydb_pg_test.go +++ b/internal/sources/alloydbpg/alloydb_pg_test.go @@ -57,7 +57,7 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) { }, }, { - desc: "public ip_type", + desc: "public ipType", in: ` sources: my-pg-instance: @@ -66,7 +66,7 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) { region: my-region cluster: my-cluster instance: my-instance - ip_type: Public + ipType: Public database: my_db `, want: map[string]sources.SourceConfig{ @@ -83,7 +83,7 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) { }, }, { - desc: "private ip_type", + desc: "private ipType", in: ` sources: my-pg-instance: @@ -92,7 +92,7 @@ func TestParseFromYamlAlloyDBPg(t *testing.T) { region: my-region cluster: my-cluster instance: my-instance - ip_type: private + ipType: private database: my_db `, want: map[string]sources.SourceConfig{ @@ -132,7 +132,7 @@ func FailParseFromYamlAlloyDBPg(t *testing.T) { in string }{ { - desc: "invalid ip_type", + desc: "invalid ipType", in: ` sources: my-pg-instance: @@ -141,7 +141,7 @@ func FailParseFromYamlAlloyDBPg(t *testing.T) { region: my-region cluster: my-cluster instance: my-instance - ip_type: fail + ipType: fail database: my_db `, }, diff --git a/internal/sources/cloudsqlpg/cloud_sql_pg.go b/internal/sources/cloudsqlpg/cloud_sql_pg.go index 8e887565b..aeae9079a 100644 --- a/internal/sources/cloudsqlpg/cloud_sql_pg.go +++ b/internal/sources/cloudsqlpg/cloud_sql_pg.go @@ -23,6 +23,7 @@ import ( "cloud.google.com/go/cloudsqlconn" "github.com/googleapis/genai-toolbox/internal/sources" "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/trace" ) const SourceKind string = "cloud-sql-postgres" @@ -36,7 +37,7 @@ type Config struct { Project string `yaml:"project"` Region string `yaml:"region"` Instance string `yaml:"instance"` - IPType sources.IPType `yaml:"ip_type"` + IPType sources.IPType `yaml:"ipType"` User string `yaml:"user"` Password string `yaml:"password"` Database string `yaml:"database"` @@ -46,8 +47,8 @@ func (r Config) SourceConfigKind() string { return SourceKind } -func (r Config) Initialize() (sources.Source, error) { - pool, err := initCloudSQLPgConnectionPool(r.Project, r.Region, r.Instance, r.IPType.String(), r.User, r.Password, r.Database) +func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error) { + pool, err := initCloudSQLPgConnectionPool(ctx, tracer, r.Name, r.Project, r.Region, r.Instance, r.IPType.String(), r.User, r.Password, r.Database) if err != nil { return nil, fmt.Errorf("unable to create pool: %w", err) } @@ -81,18 +82,22 @@ func (s *Source) PostgresPool() *pgxpool.Pool { return s.Pool } -func getDialOpts(ip_type string) ([]cloudsqlconn.DialOption, error) { - switch strings.ToLower(ip_type) { +func getDialOpts(ipType string) ([]cloudsqlconn.DialOption, error) { + switch strings.ToLower(ipType) { case "private": return []cloudsqlconn.DialOption{cloudsqlconn.WithPrivateIP()}, nil case "public": return []cloudsqlconn.DialOption{cloudsqlconn.WithPublicIP()}, nil default: - return nil, fmt.Errorf("invalid ip_type %s", ip_type) + return nil, fmt.Errorf("invalid ipType %s", ipType) } } -func initCloudSQLPgConnectionPool(project, region, instance, ip_type, user, pass, dbname string) (*pgxpool.Pool, error) { +func initCloudSQLPgConnectionPool(ctx context.Context, tracer trace.Tracer, name, project, region, instance, ipType, user, pass, dbname string) (*pgxpool.Pool, error) { + //nolint:all // Reassigned ctx + ctx, span := sources.InitConnectionSpan(ctx, tracer, SourceKind, name) + defer span.End() + // Configure the driver to connect to the database dsn := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, dbname) config, err := pgxpool.ParseConfig(dsn) @@ -101,7 +106,7 @@ func initCloudSQLPgConnectionPool(project, region, instance, ip_type, user, pass } // Create a new dialer with options - dialOpts, err := getDialOpts(ip_type) + dialOpts, err := getDialOpts(ipType) if err != nil { return nil, err } diff --git a/internal/sources/cloudsqlpg/cloud_sql_pg_test.go b/internal/sources/cloudsqlpg/cloud_sql_pg_test.go index 21cc3b504..622ee169f 100644 --- a/internal/sources/cloudsqlpg/cloud_sql_pg_test.go +++ b/internal/sources/cloudsqlpg/cloud_sql_pg_test.go @@ -54,7 +54,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) { }, }, { - desc: "public ip_type", + desc: "public ipType", in: ` sources: my-pg-instance: @@ -62,7 +62,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) { project: my-project region: my-region instance: my-instance - ip_type: Public + ipType: Public database: my_db `, want: server.SourceConfigs{ @@ -78,7 +78,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) { }, }, { - desc: "private ip_type", + desc: "private ipType", in: ` sources: my-pg-instance: @@ -86,7 +86,7 @@ func TestParseFromYamlCloudSQLPg(t *testing.T) { project: my-project region: my-region instance: my-instance - ip_type: private + ipType: private database: my_db `, want: server.SourceConfigs{ @@ -126,7 +126,7 @@ func FailParseFromYamlCloudSQLPg(t *testing.T) { in string }{ { - desc: "invalid ip_type", + desc: "invalid ipType", in: ` sources: my-pg-instance: @@ -134,7 +134,7 @@ func FailParseFromYamlCloudSQLPg(t *testing.T) { project: my-project region: my-region instance: my-instance - ip_type: fail + ipType: fail database: my_db `, }, diff --git a/internal/sources/ip_type.go b/internal/sources/ip_type.go index 38f6b0766..9589da528 100644 --- a/internal/sources/ip_type.go +++ b/internal/sources/ip_type.go @@ -31,15 +31,15 @@ func (i *IPType) String() string { } func (i *IPType) UnmarshalYAML(node *yaml.Node) error { - var ip_type string - if err := node.Decode(&ip_type); err != nil { + var ipType string + if err := node.Decode(&ipType); err != nil { return err } - switch strings.ToLower(ip_type) { + switch strings.ToLower(ipType) { case "private", "public": - *i = IPType(strings.ToLower(ip_type)) + *i = IPType(strings.ToLower(ipType)) return nil default: - return fmt.Errorf(`ip_type invalid: must be one of "public", or "private"`) + return fmt.Errorf(`ipType invalid: must be one of "public", or "private"`) } } diff --git a/internal/sources/neo4j/neo4j.go b/internal/sources/neo4j/neo4j.go new file mode 100644 index 000000000..5f3de6e9c --- /dev/null +++ b/internal/sources/neo4j/neo4j.go @@ -0,0 +1,99 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package neo4j + +import ( + "context" + "fmt" + + "github.com/googleapis/genai-toolbox/internal/sources" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "go.opentelemetry.io/otel/trace" +) + +const SourceKind string = "neo4j" + +// validate interface +var _ sources.SourceConfig = Config{} + +type Config struct { + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Uri string `yaml:"uri"` + User string `yaml:"user"` + Password string `yaml:"password"` + Database string `yaml:"database"` +} + +func (r Config) SourceConfigKind() string { + return SourceKind +} + +func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error) { + driver, err := initNeo4jDriver(ctx, tracer, r.Uri, r.User, r.Password, r.Name) + if err != nil { + return nil, fmt.Errorf("Unable to create driver: %w", err) + } + + err = driver.VerifyConnectivity(context.Background()) + if err != nil { + return nil, fmt.Errorf("Unable to connect successfully: %w", err) + } + + if r.Database == "" { + r.Database = "neo4j" + } + s := &Source{ + Name: r.Name, + Kind: SourceKind, + Database: r.Database, + Driver: driver, + } + return s, nil +} + +var _ sources.Source = &Source{} + +type Source struct { + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Database string `yaml:"database"` + Driver neo4j.DriverWithContext +} + +func (s *Source) SourceKind() string { + return SourceKind +} + +func (s *Source) Neo4jDriver() neo4j.DriverWithContext { + return s.Driver +} + +func (s *Source) Neo4jDatabase() string { + return s.Database +} + +func initNeo4jDriver(ctx context.Context, tracer trace.Tracer, uri, user, password, name string) (neo4j.DriverWithContext, error) { + //nolint:all // Reassigned ctx + ctx, span := sources.InitConnectionSpan(ctx, tracer, SourceKind, name) + defer span.End() + + auth := neo4j.BasicAuth(user, password, "") + driver, err := neo4j.NewDriverWithContext(uri, auth) + if err != nil { + return nil, fmt.Errorf("unable to create connection driver: %w", err) + } + return driver, nil +} diff --git a/internal/sources/neo4j/neo4j_test.go b/internal/sources/neo4j/neo4j_test.go new file mode 100644 index 000000000..a9c165ad2 --- /dev/null +++ b/internal/sources/neo4j/neo4j_test.go @@ -0,0 +1,68 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package neo4j_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/googleapis/genai-toolbox/internal/server" + "github.com/googleapis/genai-toolbox/internal/sources/neo4j" + "github.com/googleapis/genai-toolbox/internal/testutils" + "gopkg.in/yaml.v3" +) + +func TestParseFromYamlNeo4j(t *testing.T) { + tcs := []struct { + desc string + in string + want server.SourceConfigs + }{ + { + desc: "basic example", + in: ` + sources: + my-neo4j-instance: + kind: neo4j + uri: neo4j+s://my-host:7687 + database: my_db + `, + want: server.SourceConfigs{ + "my-neo4j-instance": neo4j.Config{ + Name: "my-neo4j-instance", + Kind: neo4j.SourceKind, + Uri: "neo4j+s://my-host:7687", + Database: "my_db", + }, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.desc, func(t *testing.T) { + got := struct { + Sources server.SourceConfigs `yaml:"sources"` + }{} + // Parse contents + err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got) + if err != nil { + t.Fatalf("unable to unmarshal: %s", err) + } + if !cmp.Equal(tc.want, got.Sources) { + t.Fatalf("incorrect parse: want %v, got %v", tc.want, got.Sources) + } + }) + } + +} diff --git a/internal/sources/postgres/postgres.go b/internal/sources/postgres/postgres.go index 217336fb3..9694ca67b 100644 --- a/internal/sources/postgres/postgres.go +++ b/internal/sources/postgres/postgres.go @@ -20,6 +20,7 @@ import ( "github.com/googleapis/genai-toolbox/internal/sources" "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/trace" ) const SourceKind string = "postgres" @@ -41,8 +42,8 @@ func (r Config) SourceConfigKind() string { return SourceKind } -func (r Config) Initialize() (sources.Source, error) { - pool, err := initPostgresConnectionPool(r.Host, r.Port, r.User, r.Password, r.Database) +func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error) { + pool, err := initPostgresConnectionPool(ctx, tracer, r.Name, r.Host, r.Port, r.User, r.Password, r.Database) if err != nil { return nil, fmt.Errorf("Unable to create pool: %w", err) } @@ -76,7 +77,10 @@ func (s *Source) PostgresPool() *pgxpool.Pool { return s.Pool } -func initPostgresConnectionPool(host, port, user, pass, dbname string) (*pgxpool.Pool, error) { +func initPostgresConnectionPool(ctx context.Context, tracer trace.Tracer, name, host, port, user, pass, dbname string) (*pgxpool.Pool, error) { + //nolint:all // Reassigned ctx + ctx, span := sources.InitConnectionSpan(ctx, tracer, SourceKind, name) + defer span.End() // urlExample := "postgres:dd//username:password@localhost:5432/database_name" i := fmt.Sprintf("postgres://%s:%s@%s:%s/%s", user, pass, host, port, dbname) pool, err := pgxpool.New(context.Background(), i) diff --git a/internal/sources/postgres/postgres_test.go b/internal/sources/postgres/postgres_test.go index c06d69681..a43208dde 100644 --- a/internal/sources/postgres/postgres_test.go +++ b/internal/sources/postgres/postgres_test.go @@ -37,7 +37,7 @@ func TestParseFromYamlPostgres(t *testing.T) { my-pg-instance: kind: postgres host: my-host - port: 0000 + port: 0.0.0.0 database: my_db `, want: server.SourceConfigs{ @@ -45,7 +45,7 @@ func TestParseFromYamlPostgres(t *testing.T) { Name: "my-pg-instance", Kind: postgres.SourceKind, Host: "my-host", - Port: "0000", + Port: "0.0.0.0", Database: "my_db", }, }, diff --git a/internal/sources/sources.go b/internal/sources/sources.go index 1c9fa9352..78c3363f5 100644 --- a/internal/sources/sources.go +++ b/internal/sources/sources.go @@ -14,13 +14,31 @@ package sources +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + // SourceConfig is the interface for configuring a source. type SourceConfig interface { SourceConfigKind() string - Initialize() (Source, error) + Initialize(ctx context.Context, tracer trace.Tracer) (Source, error) } // Source is the interface for the source itself. type Source interface { SourceKind() string } + +// InitConnectionSpan adds a span for database pool connection initialization +func InitConnectionSpan(ctx context.Context, tracer trace.Tracer, sourceKind, sourceName string) (context.Context, trace.Span) { + ctx, span := tracer.Start( + ctx, + "toolbox/server/source/connect", + trace.WithAttributes(attribute.String("source_kind", sourceKind)), + trace.WithAttributes(attribute.String("source_name", sourceName)), + ) + return ctx, span +} diff --git a/internal/sources/spanner/spanner.go b/internal/sources/spanner/spanner.go index b43dc5103..de8cfa166 100644 --- a/internal/sources/spanner/spanner.go +++ b/internal/sources/spanner/spanner.go @@ -20,6 +20,7 @@ import ( "cloud.google.com/go/spanner" "github.com/googleapis/genai-toolbox/internal/sources" + "go.opentelemetry.io/otel/trace" ) const SourceKind string = "spanner" @@ -40,8 +41,8 @@ func (r Config) SourceConfigKind() string { return SourceKind } -func (r Config) Initialize() (sources.Source, error) { - client, err := initSpannerClient(r.Project, r.Instance, r.Database) +func (r Config) Initialize(ctx context.Context, tracer trace.Tracer) (sources.Source, error) { + client, err := initSpannerClient(ctx, tracer, r.Name, r.Project, r.Instance, r.Database) if err != nil { return nil, fmt.Errorf("unable to create client: %w", err) } @@ -76,7 +77,11 @@ func (s *Source) DatabaseDialect() string { return s.Dialect } -func initSpannerClient(project, instance, dbname string) (*spanner.Client, error) { +func initSpannerClient(ctx context.Context, tracer trace.Tracer, name, project, instance, dbname string) (*spanner.Client, error) { + //nolint:all // Reassigned ctx + ctx, span := sources.InitConnectionSpan(ctx, tracer, SourceKind, name) + defer span.End() + // Configure the connection to the database db := fmt.Sprintf("projects/%s/instances/%s/databases/%s", project, instance, dbname) @@ -89,8 +94,7 @@ func initSpannerClient(project, instance, dbname string) (*spanner.Client, error } // Create spanner client - ctx := context.Background() - client, err := spanner.NewClientWithConfig(ctx, db, spanner.ClientConfig{SessionPoolConfig: sessionPoolConfig}) + client, err := spanner.NewClientWithConfig(context.Background(), db, spanner.ClientConfig{SessionPoolConfig: sessionPoolConfig}) if err != nil { return nil, fmt.Errorf("unable to create new client: %w", err) } diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 000000000..81b0d7ca8 --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,159 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + "errors" + "fmt" + + mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" + texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" + "go.opentelemetry.io/contrib/propagators/autoprop" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func SetupOTel(ctx context.Context, versionString, telemetryOTLP string, telemetryGCP bool, telemetryServiceName string) (shutdown func(context.Context) error, err error) { + var shutdownFuncs []func(context.Context) error + + // shutdown calls cleanup functions registered via shutdownFuncs. + // The errors from the calls are joined. + // Each registered cleanup will be invoked once. + shutdown = func(ctx context.Context) error { + var err error + for _, fn := range shutdownFuncs { + err = errors.Join(err, fn(ctx)) + } + shutdownFuncs = nil + return err + } + + // handleErr calls shutdown for cleanup and makes sure that all errors are returned. + handleErr := func(inErr error) { + err = errors.Join(inErr, shutdown(ctx)) + } + + // Configure Context Propagation to use the default W3C traceparent format. + otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) + + res, err := newResource(ctx, versionString, telemetryServiceName) + if err != nil { + errMsg := fmt.Errorf("unable to set up resource: %w", err) + handleErr(errMsg) + return + } + + tracerProvider, err := newTracerProvider(ctx, res, telemetryOTLP, telemetryGCP) + if err != nil { + errMsg := fmt.Errorf("unable to set up trace provider: %w", err) + handleErr(errMsg) + return + } + shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) + otel.SetTracerProvider(tracerProvider) + + meterProvider, err := newMeterProvider(ctx, res, telemetryOTLP, telemetryGCP) + if err != nil { + errMsg := fmt.Errorf("unable to set up meter provider: %w", err) + handleErr(errMsg) + return + } + shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) + otel.SetMeterProvider(meterProvider) + + return shutdown, nil +} + +// newResource create default resources for telemetry data. +// Resource represents the entity producing telemetry. +func newResource(ctx context.Context, versionString string, telemetryServiceName string) (*resource.Resource, error) { + // Ensure default SDK resources and the required service name are set. + r, err := resource.New( + ctx, + resource.WithFromEnv(), // Discover and provide attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables. + resource.WithTelemetrySDK(), // Discover and provide information about the OTel SDK used. + resource.WithOS(), // Discover and provide OS information. + resource.WithContainer(), // Discover and provide container information. + resource.WithHost(), //Discover and provide host information. + resource.WithSchemaURL(semconv.SchemaURL), // Set the schema url. + resource.WithAttributes( // Add other custom resource attributes. + semconv.ServiceName(telemetryServiceName), + semconv.ServiceVersion(versionString), + ), + ) + if err != nil { + return nil, fmt.Errorf("trace provider fail to set up resource: %w", err) + } + return r, nil +} + +// newTracerProvider creates TracerProvider. +// TracerProvider is a factory for Tracers and is responsible for creating spans. +func newTracerProvider(ctx context.Context, r *resource.Resource, telemetryOTLP string, telemetryGCP bool) (*tracesdk.TracerProvider, error) { + traceOpts := []tracesdk.TracerProviderOption{} + if telemetryOTLP != "" { + // otlptracehttp provides an OTLP span exporter using HTTP with protobuf payloads. + // By default, the telemetry is sent to https://localhost:4318/v1/traces. + otlpExporter, err := otlptracehttp.New(ctx, otlptracehttp.WithEndpoint(telemetryOTLP)) + if err != nil { + return nil, err + } + traceOpts = append(traceOpts, tracesdk.WithBatcher(otlpExporter)) + } + if telemetryGCP { + gcpExporter, err := texporter.New() + if err != nil { + return nil, err + } + traceOpts = append(traceOpts, tracesdk.WithBatcher(gcpExporter)) + } + traceOpts = append(traceOpts, tracesdk.WithResource(r)) + + traceProvider := tracesdk.NewTracerProvider(traceOpts...) + return traceProvider, nil +} + +// newMeterProvider creates MeterProvider. +// MeterProvider is a factory for Meters, and is responsible for creating metrics. +func newMeterProvider(ctx context.Context, r *resource.Resource, telemetryOTLP string, telemetryGCP bool) (*metric.MeterProvider, error) { + metricOpts := []metric.Option{} + if telemetryOTLP != "" { + // otlpmetrichttp provides an OTLP metrics exporter using HTTP with protobuf payloads. + // By default, the telemetry is sent to https://localhost:4318/v1/metrics. + otlpExporter, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithEndpoint(telemetryOTLP)) + if err != nil { + return nil, err + } + metricOpts = append(metricOpts, metric.WithReader(metric.NewPeriodicReader(otlpExporter))) + } + if telemetryGCP { + gcpExporter, err := mexporter.New() + if err != nil { + return nil, err + } + metricOpts = append(metricOpts, metric.WithReader(metric.NewPeriodicReader(gcpExporter))) + } + + meterProvider := metric.NewMeterProvider(metricOpts...) + return meterProvider, nil +} diff --git a/internal/tools/neo4j/neo4j.go b/internal/tools/neo4j/neo4j.go new file mode 100644 index 000000000..216545658 --- /dev/null +++ b/internal/tools/neo4j/neo4j.go @@ -0,0 +1,132 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package neo4j + +import ( + "context" + "fmt" + "strings" + + neo4jsc "github.com/googleapis/genai-toolbox/internal/sources/neo4j" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + + "github.com/googleapis/genai-toolbox/internal/sources" + "github.com/googleapis/genai-toolbox/internal/tools" +) + +const ToolKind string = "neo4j-cypher" + +type compatibleSource interface { + Neo4jDriver() neo4j.DriverWithContext + Neo4jDatabase() string +} + +// validate compatible sources are still compatible +var _ compatibleSource = &neo4jsc.Source{} + +var compatibleSources = [...]string{neo4jsc.SourceKind} + +type Config struct { + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Source string `yaml:"source"` + Description string `yaml:"description"` + Statement string `yaml:"statement"` + Parameters tools.Parameters `yaml:"parameters"` +} + +// validate interface +var _ tools.ToolConfig = Config{} + +func (cfg Config) ToolConfigKind() string { + return ToolKind +} + +func (cfg Config) Initialize(srcs map[string]sources.Source) (tools.Tool, error) { + // verify source exists + rawS, ok := srcs[cfg.Source] + if !ok { + return nil, fmt.Errorf("no source named %q configured", cfg.Source) + } + + // verify the source is compatible + s, ok := rawS.(compatibleSource) + if !ok { + return nil, fmt.Errorf("invalid source for %q tool: source kind must be one of %q", ToolKind, compatibleSources) + } + + // finish tool setup + t := Tool{ + Name: cfg.Name, + Kind: ToolKind, + Parameters: cfg.Parameters, + Statement: cfg.Statement, + Driver: s.Neo4jDriver(), + Database: s.Neo4jDatabase(), + manifest: tools.Manifest{Description: cfg.Description, Parameters: cfg.Parameters.Manifest()}, + } + return t, nil +} + +// validate interface +var _ tools.Tool = Tool{} + +type Tool struct { + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Parameters tools.Parameters `yaml:"parameters"` + AuthRequired []string `yaml:"authRequired"` + + Driver neo4j.DriverWithContext + Database string + Statement string + manifest tools.Manifest +} + +func (t Tool) Invoke(params tools.ParamValues) (string, error) { + paramsMap := params.AsMap() + + fmt.Printf("Invoked tool %s\n", t.Name) + ctx := context.Background() + config := neo4j.ExecuteQueryWithDatabase(t.Database) + results, err := neo4j.ExecuteQuery[*neo4j.EagerResult](ctx, t.Driver, t.Statement, paramsMap, + neo4j.EagerResultTransformer, config) + if err != nil { + return "", fmt.Errorf("unable to execute query: %w", err) + } + + var out strings.Builder + keys := results.Keys + records := results.Records + for _, record := range records { + out.WriteString("\n") // fmt.Sprintf("Row: %d\n", row)) + for col, value := range record.Values { + out.WriteString(fmt.Sprintf("\t%s: %s\n", keys[col], value)) + } + } + return fmt.Sprintf("Stub tool call for %q! Parameters parsed: %q \n Output: %s", t.Name, paramsMap, out.String()), nil +} + +func (t Tool) ParseParams(data map[string]any, claimsMap map[string]map[string]any) (tools.ParamValues, error) { + return tools.ParseParams(t.Parameters, data, claimsMap) +} + +func (t Tool) Manifest() tools.Manifest { + return t.manifest +} + +func (t Tool) Authorized(verifiedAuthSources []string) bool { + return tools.IsAuthorized(t.AuthRequired, verifiedAuthSources) +} diff --git a/internal/tools/neo4j/neo4j_test.go b/internal/tools/neo4j/neo4j_test.go new file mode 100644 index 000000000..cfb3d1900 --- /dev/null +++ b/internal/tools/neo4j/neo4j_test.go @@ -0,0 +1,79 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package neo4j_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/googleapis/genai-toolbox/internal/server" + "github.com/googleapis/genai-toolbox/internal/testutils" + "github.com/googleapis/genai-toolbox/internal/tools" + "github.com/googleapis/genai-toolbox/internal/tools/neo4j" + "gopkg.in/yaml.v3" +) + +func TestParseFromYamlNeo4j(t *testing.T) { + tcs := []struct { + desc string + in string + want server.ToolConfigs + }{ + { + desc: "basic example", + in: ` + tools: + example_tool: + kind: neo4j-cypher + source: my-neo4j-instance + description: some tool description + statement: | + MATCH (c:Country) WHERE c.name = $country RETURN c.id as id; + parameters: + - name: country + type: string + description: country parameter description + `, + want: server.ToolConfigs{ + "example_tool": neo4j.Config{ + Name: "example_tool", + Kind: neo4j.ToolKind, + Source: "my-neo4j-instance", + Description: "some tool description", + Statement: "MATCH (c:Country) WHERE c.name = $country RETURN c.id as id;\n", + Parameters: []tools.Parameter{ + tools.NewStringParameter("country", "country parameter description"), + }, + }, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.desc, func(t *testing.T) { + got := struct { + Tools server.ToolConfigs `yaml:"tools"` + }{} + // Parse contents + err := yaml.Unmarshal(testutils.FormatYaml(tc.in), &got) + if err != nil { + t.Fatalf("unable to unmarshal: %s", err) + } + if diff := cmp.Diff(tc.want, got.Tools); diff != "" { + t.Fatalf("incorrect parse: diff %v", diff) + } + }) + } + +} diff --git a/internal/tools/parameters.go b/internal/tools/parameters.go index 24eb32cef..319cb6158 100644 --- a/internal/tools/parameters.go +++ b/internal/tools/parameters.go @@ -154,7 +154,7 @@ func parseParamFromNode(node *yaml.Node) (Parameter, error) { var p CommonParameter err := node.Decode(&p) if err != nil { - return nil, fmt.Errorf("parameter missing required fields") + return nil, fmt.Errorf("parameter missing required fields: %w", err) } switch p.Type { case typeString: diff --git a/internal/tools/spanner/spanner.go b/internal/tools/spanner/spanner.go index bbb77655e..55e1204ed 100644 --- a/internal/tools/spanner/spanner.go +++ b/internal/tools/spanner/spanner.go @@ -130,8 +130,7 @@ func (t Tool) Invoke(params tools.ParamValues) (string, error) { fmt.Printf("Invoked tool %s\n", t.Name) var out strings.Builder - ctx := context.Background() - _, err = t.Client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + _, err = t.Client.ReadWriteTransaction(context.Background(), func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { stmt := spanner.Statement{ SQL: t.Statement, Params: mapParams, diff --git a/sdks/langchain/integration.cloudbuild.yaml b/sdks/langchain/integration.cloudbuild.yaml index 74d3be6bb..70b30a57b 100644 --- a/sdks/langchain/integration.cloudbuild.yaml +++ b/sdks/langchain/integration.cloudbuild.yaml @@ -43,4 +43,4 @@ options: logging: CLOUD_LOGGING_ONLY substitutions: _VERSION: '3.13' - _TOOLBOX_VERSION: '0.0.4' \ No newline at end of file + _TOOLBOX_VERSION: '0.0.5' diff --git a/sdks/langchain/tests/test_e2e.py b/sdks/langchain/tests/test_e2e.py index d9d192865..e5272e282 100644 --- a/sdks/langchain/tests/test_e2e.py +++ b/sdks/langchain/tests/test_e2e.py @@ -87,6 +87,7 @@ async def test_load_toolset_all(self, toolbox): ##### Auth tests @pytest.mark.asyncio + @pytest.mark.skip(reason="b/389574566") async def test_run_tool_unauth_with_auth(self, toolbox, auth_token2): """Tests running a tool that doesn't require auth, with auth provided.""" tool = await toolbox.load_tool( @@ -105,14 +106,14 @@ async def test_run_tool_no_auth(self, toolbox): await tool.arun({"id": "2"}) @pytest.mark.asyncio - @pytest.mark.skip(reason="b/388259742") async def test_run_tool_wrong_auth(self, toolbox, auth_token2): """Tests running a tool with incorrect auth.""" toolbox.add_auth_token("my-test-auth", lambda: auth_token2) tool = await toolbox.load_tool( "get-row-by-id-auth", ) - with pytest.raises(ClientResponseError, match="401, message='Unauthorized'"): + # TODO: Fix error message (b/389577313) + with pytest.raises(ClientResponseError, match="400, message='Bad Request'"): await tool.arun({"id": "2"}) @pytest.mark.asyncio diff --git a/sdks/llamaindex/integration.cloudbuild.yaml b/sdks/llamaindex/integration.cloudbuild.yaml index 0669527d2..dc59b4c2e 100644 --- a/sdks/llamaindex/integration.cloudbuild.yaml +++ b/sdks/llamaindex/integration.cloudbuild.yaml @@ -32,6 +32,8 @@ steps: name: 'python:${_VERSION}' env: - TOOLBOX_URL=$_TOOLBOX_URL + - TOOLBOX_VERSION=$_TOOLBOX_VERSION + - GOOGLE_CLOUD_PROJECT=$PROJECT_ID args: - '-c' - >- @@ -40,4 +42,5 @@ steps: options: logging: CLOUD_LOGGING_ONLY substitutions: - _VERSION: '3.13' \ No newline at end of file + _VERSION: '3.13' + _TOOLBOX_VERSION: '0.0.5' diff --git a/sdks/llamaindex/pyproject.toml b/sdks/llamaindex/pyproject.toml index ee822c46c..f7632bf4f 100644 --- a/sdks/llamaindex/pyproject.toml +++ b/sdks/llamaindex/pyproject.toml @@ -39,7 +39,9 @@ test = [ "pytest-asyncio==0.24.0", "pytest==8.3.3", "pytest-cov==6.0.0", - "Pillow==10.4.0" + "Pillow==10.4.0", + "google-cloud-secret-manager==2.22.0", + "google-cloud-storage==2.19.0", ] [build-system] diff --git a/sdks/llamaindex/tests/conftest.py b/sdks/llamaindex/tests/conftest.py new file mode 100644 index 000000000..80c79272f --- /dev/null +++ b/sdks/llamaindex/tests/conftest.py @@ -0,0 +1,166 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contains pytest fixtures that are accessible from all +files present in the same directory.""" + +from __future__ import annotations + +import os +import platform +import subprocess +import tempfile +import time +from typing import Generator + +import google +import pytest_asyncio +from google.auth import compute_engine +from google.cloud import secretmanager, storage + + +#### Define Utility Functions +def get_env_var(key: str) -> str: + """Gets environment variables.""" + value = os.environ.get(key) + if value is None: + raise ValueError(f"Must set env var {key}") + return value + + +def access_secret_version( + project_id: str, secret_id: str, version_id: str = "latest" +) -> str: + """Accesses the payload of a given secret version from Secret Manager.""" + client = secretmanager.SecretManagerServiceClient() + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + response = client.access_secret_version(request={"name": name}) + return response.payload.data.decode("UTF-8") + + +def create_tmpfile(content: str) -> str: + """Creates a temporary file with the given content.""" + with tempfile.NamedTemporaryFile(delete=False, mode="w") as tmpfile: + tmpfile.write(content) + return tmpfile.name + + +def download_blob( + bucket_name: str, source_blob_name: str, destination_file_name: str +) -> None: + """Downloads a blob from a GCS bucket.""" + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + + print(f"Blob {source_blob_name} downloaded to {destination_file_name}.") + + +def get_toolbox_binary_url(toolbox_version: str) -> str: + """Constructs the GCS path to the toolbox binary.""" + os_system = platform.system().lower() + arch = ( + "arm64" if os_system == "darwin" and platform.machine() == "arm64" else "amd64" + ) + return f"v{toolbox_version}/{os_system}/{arch}/toolbox" + + +def get_auth_token(client_id: str) -> str: + """Retrieves an authentication token""" + request = google.auth.transport.requests.Request() + credentials = compute_engine.IDTokenCredentials( + request=request, + target_audience=client_id, + use_metadata_identity_endpoint=True, + ) + if not credentials.valid: + credentials.refresh(request) + return credentials.token + + +#### Define Fixtures +@pytest_asyncio.fixture(scope="session") +def project_id() -> str: + return get_env_var("GOOGLE_CLOUD_PROJECT") + + +@pytest_asyncio.fixture(scope="session") +def toolbox_version() -> str: + return get_env_var("TOOLBOX_VERSION") + + +@pytest_asyncio.fixture(scope="session") +def tools_file_path(project_id: str) -> Generator[str]: + """Provides a temporary file path containing the tools manifest.""" + tools_manifest = access_secret_version( + project_id=project_id, secret_id="sdk_testing_tools" + ) + tools_file_path = create_tmpfile(tools_manifest) + yield tools_file_path + os.remove(tools_file_path) + + +@pytest_asyncio.fixture(scope="session") +def auth_token1(project_id: str) -> str: + client_id = access_secret_version( + project_id=project_id, secret_id="sdk_testing_client1" + ) + return get_auth_token(client_id) + + +@pytest_asyncio.fixture(scope="session") +def auth_token2(project_id: str) -> str: + client_id = access_secret_version( + project_id=project_id, secret_id="sdk_testing_client2" + ) + return get_auth_token(client_id) + + +@pytest_asyncio.fixture(scope="session") +def toolbox_server(toolbox_version: str, tools_file_path: str) -> Generator[None]: + """Starts the toolbox server as a subprocess.""" + print("Downloading toolbox binary from gcs bucket...") + source_blob_name = get_toolbox_binary_url(toolbox_version) + download_blob("genai-toolbox", source_blob_name, "toolbox") + print("Toolbox binary downloaded successfully.") + try: + print("Opening toolbox server process...") + # Make toolbox executable + os.chmod("toolbox", 0o700) + # Run toolbox binary + toolbox_server = subprocess.Popen( + ["./toolbox", "--tools_file", tools_file_path] + ) + + # Wait for server to start + # Retry logic with a timeout + for _ in range(5): # retries + time.sleep(4) + print("Checking if toolbox is successfully started...") + if toolbox_server.poll() is None: + print("Toolbox server started successfully.") + break + else: + raise RuntimeError("Toolbox server failed to start after 5 retries.") + except subprocess.CalledProcessError as e: + print(e.stderr.decode("utf-8")) + print(e.stdout.decode("utf-8")) + raise RuntimeError(f"{e}\n\n{e.stderr.decode('utf-8')}") from e + yield + + # Clean up toolbox server + toolbox_server.terminate() + toolbox_server.wait() diff --git a/sdks/llamaindex/tests/test_e2e.py b/sdks/llamaindex/tests/test_e2e.py new file mode 100644 index 000000000..caae5547f --- /dev/null +++ b/sdks/llamaindex/tests/test_e2e.py @@ -0,0 +1,155 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end tests for the toolbox SDK interacting with the toolbox server. + +This file covers the following use cases: + +1. Loading a tool. +2. Loading a specific toolset. +3. Loading the default toolset (contains all tools). +4. Running a tool with no required auth, with auth provided. +5. Running a tool with required auth: + a. No auth provided. + b. Wrong auth provided: The tool requires a different authentication + than the one provided. + c. Correct auth provided. +6. Running a tool with a parameter that requires auth: + a. No auth provided. + b. Correct auth provided. + c. Auth provided does not contain the required claim. +""" + +import pytest +import pytest_asyncio +from aiohttp import ClientResponseError + +from toolbox_llamaindex_sdk.client import ToolboxClient + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("toolbox_server") +class TestE2EClient: + @pytest_asyncio.fixture(scope="function") + async def toolbox(self): + """Provides a ToolboxClient instance for each test.""" + toolbox = ToolboxClient("http://localhost:5000") + yield toolbox + await toolbox.close() + + #### Basic e2e tests + @pytest.mark.asyncio + async def test_load_tool(self, toolbox): + tool = await toolbox.load_tool("get-n-rows") + response = await tool.acall(num_rows="2") + result = response.raw_output["result"] + + assert "row1" in result + assert "row2" in result + assert "row3" not in result + + @pytest.mark.asyncio + async def test_load_toolset_specific(self, toolbox): + toolset = await toolbox.load_toolset("my-toolset") + assert len(toolset) == 1 + assert toolset[0].metadata.name == "get-row-by-id" + + toolset = await toolbox.load_toolset("my-toolset-2") + assert len(toolset) == 2 + tool_names = ["get-n-rows", "get-row-by-id"] + assert toolset[0].metadata.name in tool_names + assert toolset[1].metadata.name in tool_names + + @pytest.mark.asyncio + async def test_load_toolset_all(self, toolbox): + toolset = await toolbox.load_toolset() + assert len(toolset) == 5 + tool_names = [ + "get-n-rows", + "get-row-by-id", + "get-row-by-id-auth", + "get-row-by-email-auth", + "get-row-by-content-auth", + ] + for tool in toolset: + assert tool.metadata.name in tool_names + + ##### Auth tests + @pytest.mark.asyncio + @pytest.mark.skip(reason="b/389574566") + async def test_run_tool_unauth_with_auth(self, toolbox, auth_token2): + """Tests running a tool that doesn't require auth, with auth provided.""" + tool = await toolbox.load_tool( + "get-row-by-id", auth_tokens={"my-test-auth": lambda: auth_token2} + ) + response = await tool.acall(id="2") + assert "row2" in response.raw_output["result"] + + @pytest.mark.asyncio + async def test_run_tool_no_auth(self, toolbox): + """Tests running a tool requiring auth without providing auth.""" + tool = await toolbox.load_tool( + "get-row-by-id-auth", + ) + with pytest.raises(ClientResponseError, match="401, message='Unauthorized'"): + await tool.acall(id="2") + + @pytest.mark.asyncio + async def test_run_tool_wrong_auth(self, toolbox, auth_token2): + """Tests running a tool with incorrect auth.""" + toolbox.add_auth_token("my-test-auth", lambda: auth_token2) + tool = await toolbox.load_tool( + "get-row-by-id-auth", + ) + # TODO: Fix error message (b/389577313) + with pytest.raises(ClientResponseError, match="400, message='Bad Request'"): + await tool.acall(id="2") + + @pytest.mark.asyncio + async def test_run_tool_auth(self, toolbox, auth_token1): + """Tests running a tool with correct auth.""" + toolbox.add_auth_token("my-test-auth", lambda: auth_token1) + tool = await toolbox.load_tool( + "get-row-by-id-auth", + ) + response = await tool.acall(id="2") + assert "row2" in response.raw_output["result"] + + @pytest.mark.asyncio + async def test_run_tool_param_auth_no_auth(self, toolbox): + """Tests running a tool with a param requiring auth, without auth.""" + tool = await toolbox.load_tool("get-row-by-email-auth") + with pytest.raises(PermissionError, match="Login required"): + await tool.acall() + + @pytest.mark.asyncio + async def test_run_tool_param_auth(self, toolbox, auth_token1): + """Tests running a tool with a param requiring auth, with correct auth.""" + tool = await toolbox.load_tool( + "get-row-by-email-auth", auth_tokens={"my-test-auth": lambda: auth_token1} + ) + response = await tool.acall() + result = response.raw_output["result"] + assert "row4" in result + assert "row5" in result + assert "row6" in result + + @pytest.mark.asyncio + async def test_run_tool_param_auth_no_field(self, toolbox, auth_token1): + """Tests running a tool with a param requiring auth, with insufficient auth.""" + tool = await toolbox.load_tool( + "get-row-by-content-auth", auth_tokens={"my-test-auth": lambda: auth_token1} + ) + with pytest.raises(ClientResponseError, match="400, message='Bad Request'"): + await tool.acall() diff --git a/tests/alloydb_pg_integration_test.go b/tests/alloydb_pg_integration_test.go index a430b9dc4..046acdee3 100644 --- a/tests/alloydb_pg_integration_test.go +++ b/tests/alloydb_pg_integration_test.go @@ -20,13 +20,20 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" + "net" "net/http" "os" "reflect" "regexp" + "strings" "testing" "time" + + "cloud.google.com/go/alloydbconn" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" ) var ( @@ -39,7 +46,7 @@ var ( ALLOYDB_POSTGRES_PASS = os.Getenv("ALLOYDB_POSTGRES_PASS") ) -func requireAlloyDBPgVars(t *testing.T) { +func requireAlloyDBPgVars(t *testing.T) map[string]any { switch "" { case ALLOYDB_POSTGRES_PROJECT: t.Fatal("'ALLOYDB_POSTGRES_PROJECT' not set") @@ -56,10 +63,65 @@ func requireAlloyDBPgVars(t *testing.T) { case ALLOYDB_POSTGRES_PASS: t.Fatal("'ALLOYDB_POSTGRES_PASS' not set") } + return map[string]any{ + "kind": "alloydb-postgres", + "project": ALLOYDB_POSTGRES_PROJECT, + "cluster": ALLOYDB_POSTGRES_CLUSTER, + "instance": ALLOYDB_POSTGRES_INSTANCE, + "region": ALLOYDB_POSTGRES_REGION, + "database": ALLOYDB_POSTGRES_DATABASE, + "user": ALLOYDB_POSTGRES_USER, + "password": ALLOYDB_POSTGRES_PASS, + } +} + +// Copied over from alloydb_pg.go +func getAlloyDBDialOpts(ip_type string) ([]alloydbconn.DialOption, error) { + switch strings.ToLower(ip_type) { + case "private": + return []alloydbconn.DialOption{alloydbconn.WithPrivateIP()}, nil + case "public": + return []alloydbconn.DialOption{alloydbconn.WithPublicIP()}, nil + default: + return nil, fmt.Errorf("invalid ip_type %s", ip_type) + } +} + +// Copied over from alloydb_pg.go +func initAlloyDBPgConnectionPool(project, region, cluster, instance, ip_type, user, pass, dbname string) (*pgxpool.Pool, error) { + // Configure the driver to connect to the database + dsn := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, dbname) + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("unable to parse connection uri: %w", err) + } + + // Create a new dialer with options + dialOpts, err := getAlloyDBDialOpts(ip_type) + if err != nil { + return nil, err + } + d, err := alloydbconn.NewDialer(context.Background(), alloydbconn.WithDefaultDialOptions(dialOpts...)) + if err != nil { + return nil, fmt.Errorf("unable to parse connection uri: %w", err) + } + + // Tell the driver to use the AlloyDB Go Connector to create connections + i := fmt.Sprintf("projects/%s/locations/%s/clusters/%s/instances/%s", project, region, cluster, instance) + config.ConnConfig.DialFunc = func(ctx context.Context, _ string, instance string) (net.Conn, error) { + return d.Dial(ctx, i) + } + + // Interact with the driver directly as you normally would + pool, err := pgxpool.NewWithConfig(context.Background(), config) + if err != nil { + return nil, err + } + return pool, nil } -func TestAlloyDBPostgres(t *testing.T) { - requireAlloyDBPgVars(t) +func TestAlloyDBSimpleToolEndpoints(t *testing.T) { + sourceConfig := requireAlloyDBPgVars(t) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -68,16 +130,7 @@ func TestAlloyDBPostgres(t *testing.T) { // Write config into a file and pass it to command toolsFile := map[string]any{ "sources": map[string]any{ - "my-pg-instance": map[string]any{ - "kind": "alloydb-postgres", - "project": ALLOYDB_POSTGRES_PROJECT, - "instance": ALLOYDB_POSTGRES_INSTANCE, - "cluster": ALLOYDB_POSTGRES_CLUSTER, - "region": ALLOYDB_POSTGRES_REGION, - "database": ALLOYDB_POSTGRES_DATABASE, - "user": ALLOYDB_POSTGRES_USER, - "password": ALLOYDB_POSTGRES_PASS, - }, + "my-pg-instance": sourceConfig, }, "tools": map[string]any{ "my-simple-tool": map[string]any{ @@ -187,3 +240,152 @@ func TestAlloyDBPostgres(t *testing.T) { }) } } + +func TestPublicIpConnection(t *testing.T) { + // Test connecting to an AlloyDB source's public IP + sourceConfig := requireAlloyDBPgVars(t) + sourceConfig["ipType"] = "public" + RunSourceConnectionTest(t, sourceConfig, "postgres-sql") +} + +func TestPrivateIpConnection(t *testing.T) { + // Test connecting to an AlloyDB source's private IP + sourceConfig := requireAlloyDBPgVars(t) + sourceConfig["ipType"] = "private" + RunSourceConnectionTest(t, sourceConfig, "postgres-sql") +} + +// Set up tool calling with parameters test table +func setupParamTest(t *testing.T, ctx context.Context, tableName string) func(*testing.T) { + // Set up Tool invocation with parameters test + pool, err := initAlloyDBPgConnectionPool(ALLOYDB_POSTGRES_PROJECT, ALLOYDB_POSTGRES_REGION, ALLOYDB_POSTGRES_CLUSTER, ALLOYDB_POSTGRES_INSTANCE, "public", ALLOYDB_POSTGRES_USER, ALLOYDB_POSTGRES_PASS, ALLOYDB_POSTGRES_DATABASE) + if err != nil { + t.Fatalf("unable to create AlloyDB connection pool: %s", err) + } + + err = pool.Ping(ctx) + if err != nil { + t.Fatalf("unable to connect to test database: %s", err) + } + + _, err = pool.Query(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id SERIAL PRIMARY KEY, + name TEXT + ); + `, tableName)) + if err != nil { + t.Fatalf("unable to create test table %s: %s", tableName, err) + } + + // Insert test data + statement := fmt.Sprintf(` + INSERT INTO %s (name) + VALUES ($1), ($2), ($3); + `, tableName) + + params := []any{"Alice", "Jane", "Sid"} + _, err = pool.Query(ctx, statement, params...) + if err != nil { + t.Fatalf("unable to insert test data: %s", err) + } + + return func(t *testing.T) { + // tear down test + _, err = pool.Exec(ctx, fmt.Sprintf("DROP TABLE %s;", tableName)) + if err != nil { + t.Errorf("Teardown failed: %s", err) + } + } +} + +func TestToolInvocationWithParams(t *testing.T) { + // create test configs + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // create source config + sourceConfig := requireAlloyDBPgVars(t) + + // create table name with UUID + tableName := "param_test_table_" + strings.Replace(uuid.New().String(), "-", "", -1) + + // test setup function reterns teardown function + teardownTest := setupParamTest(t, ctx, tableName) + defer teardownTest(t) + + // call generic invocation test helper + RunToolInvocationWithParamsTest(t, sourceConfig, "postgres-sql", tableName) +} + +// Set up auth test database table +func setupAlloyDBAuthTest(t *testing.T, ctx context.Context, tableName string) func(*testing.T) { + // set up db connection pool + pool, err := initAlloyDBPgConnectionPool(ALLOYDB_POSTGRES_PROJECT, ALLOYDB_POSTGRES_REGION, ALLOYDB_POSTGRES_CLUSTER, ALLOYDB_POSTGRES_INSTANCE, "public", ALLOYDB_POSTGRES_USER, ALLOYDB_POSTGRES_PASS, ALLOYDB_POSTGRES_DATABASE) + if err != nil { + t.Fatalf("unable to create AlloyDB connection pool: %s", err) + } + + err = pool.Ping(ctx) + if err != nil { + t.Fatalf("unable to connect to test database: %s", err) + } + + _, err = pool.Query(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id SERIAL PRIMARY KEY, + name TEXT, + email TEXT + ); + `, tableName)) + if err != nil { + t.Fatalf("unable to create test table %s: %s", tableName, err) + } + + // Insert test data + statement := fmt.Sprintf(` + INSERT INTO %s (name, email) + VALUES ($1, $2), ($3, $4) + `, tableName) + params := []any{"Alice", SERVICE_ACCOUNT_EMAIL, "Jane", "janedoe@gmail.com"} + _, err = pool.Query(ctx, statement, params...) + if err != nil { + t.Fatalf("unable to insert test data: %s", err) + } + + return func(t *testing.T) { + // tear down test + _, err = pool.Exec(ctx, fmt.Sprintf("DROP TABLE %s;", tableName)) + if err != nil { + t.Errorf("Teardown failed: %s", err) + } + } +} + +func TestAlloyDBGoogleAuthenticatedParameter(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // create test configs + sourceConfig := requireAlloyDBPgVars(t) + + // create table name with UUID + tableName := "auth_table_" + strings.Replace(uuid.New().String(), "-", "", -1) + + // test setup function returns teardown funtion + teardownTest := setupAlloyDBAuthTest(t, ctx, tableName) + defer teardownTest(t) + + // call generic auth test helper + RunGoogleAuthenticatedParameterTest(t, sourceConfig, "postgres-sql", tableName) + +} + +func TestAlloyDBAuthRequiredToolInvocation(t *testing.T) { + // create test configs + sourceConfig := requireAlloyDBPgVars(t) + + // call generic auth test helper + RunAuthRequiredToolInvocationTest(t, sourceConfig, "postgres-sql") + +} diff --git a/tests/auth_test.go b/tests/auth_test.go index b1fa2e16a..c7a4616b7 100644 --- a/tests/auth_test.go +++ b/tests/auth_test.go @@ -17,94 +17,315 @@ package tests import ( + "bytes" + "context" + "encoding/json" + "fmt" "io" "net/http" + "os" "os/exec" + "regexp" + "strings" + "time" "testing" - "github.com/googleapis/genai-toolbox/internal/auth" - "github.com/googleapis/genai-toolbox/internal/auth/google" + "google.golang.org/api/idtoken" ) +var SERVICE_ACCOUNT_EMAIL = os.Getenv("SERVICE_ACCOUNT_EMAIL") +var clientId = os.Getenv("CLIENT_ID") + // Get a Google ID token func getGoogleIdToken(audience string) (string, error) { - // For local testing + // For local testing - use gcloud command to print personal ID token cmd := exec.Command("gcloud", "auth", "print-identity-token") output, err := cmd.Output() if err == nil { - return string(output), nil - } else { - // Cloud Build testing - url := "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity?audience=" + audience - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return "", err - } - req.Header.Set("Metadata-Flavor", "Google") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - return string(body), nil + return strings.TrimSpace(string(output)), nil + } + // For Cloud Build testing - retrieve ID token from GCE metadata server + ts, err := idtoken.NewTokenSource(context.Background(), clientId) + if err != nil { + return "", err } + token, err := ts.Token() + if err != nil { + return "", err + } + return token.AccessToken, nil } -func TestGoogleAuthVerification(t *testing.T) { - clientId := "32555940559.apps.googleusercontent.com" - tcs := []struct { - authSource auth.AuthSource - isErr bool +func RunGoogleAuthenticatedParameterTest(t *testing.T, sourceConfig map[string]any, toolKind string, tableName string) { + // create query statement + var statement string + switch { + case strings.EqualFold(toolKind, "postgres-sql"): + statement = fmt.Sprintf("SELECT * FROM %s WHERE email = $1;", tableName) + default: + t.Fatalf("invalid tool kind: %s", toolKind) + } + + // Write config into a file and pass it to command + toolsFile := map[string]any{ + "sources": map[string]any{ + "my-instance": sourceConfig, + }, + "authSources": map[string]any{ + "my-google-auth": map[string]any{ + "kind": "google", + "clientId": clientId, + }, + }, + "tools": map[string]any{ + "my-auth-tool": map[string]any{ + "kind": toolKind, + "source": "my-instance", + "description": "Tool to test authenticated parameters.", + // statement to auto-fill authenticated parameter + "statement": statement, + "parameters": []map[string]any{ + { + "name": "email", + "type": "string", + "description": "user email", + "authSources": []map[string]string{ + { + "name": "my-google-auth", + "field": "email", + }, + }, + }, + }, + }, + }, + } + // Initialize a test command + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var args []string + + cmd, cleanup, err := StartCmd(ctx, toolsFile, args...) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`)) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } + + // Get ID token + idToken, err := getGoogleIdToken(clientId) + if err != nil { + t.Fatalf("error getting Google ID token: %s", err) + } + + // Create wanted string + wantResult := fmt.Sprintf("Stub tool call for \"my-auth-tool\"! Parameters parsed: [{\"email\" \"%s\"}] \n Output: [%%!s(int32=1) Alice %s]", SERVICE_ACCOUNT_EMAIL, SERVICE_ACCOUNT_EMAIL) + + // Test tool invocation with authenticated parameters + invokeTcs := []struct { + name string + api string + requestHeader map[string]string + requestBody io.Reader + want string + isErr bool }{ { - authSource: google.AuthSource{ - Name: "my-google-auth", - Kind: google.AuthSourceKind, - ClientID: clientId, - }, - isErr: false, + name: "Invoke my-auth-tool with auth token", + api: "http://127.0.0.1:5000/api/tool/my-auth-tool/invoke", + requestHeader: map[string]string{"my-google-auth_token": idToken}, + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: false, + want: wantResult, + }, + { + name: "Invoke my-auth-tool with invalid auth token", + api: "http://127.0.0.1:5000/api/tool/my-auth-tool/invoke", + requestHeader: map[string]string{"my-google-auth_token": "INVALID_TOKEN"}, + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: true, }, { - authSource: google.AuthSource{ - Name: "err-google-auth", - Kind: google.AuthSourceKind, - ClientID: "random-client-id", + name: "Invoke my-auth-tool without auth token", + api: "http://127.0.0.1:5000/api/tool/my-auth-tool/invoke", + requestHeader: map[string]string{}, + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: true, + }, + } + for _, tc := range invokeTcs { + t.Run(tc.name, func(t *testing.T) { + // Send Tool invocation request with ID token + req, err := http.NewRequest(http.MethodPost, tc.api, tc.requestBody) + if err != nil { + t.Fatalf("unable to create request: %s", err) + } + req.Header.Add("Content-type", "application/json") + for k, v := range tc.requestHeader { + req.Header.Add(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("unable to send request: %s", err) + } + + if resp.StatusCode != http.StatusOK { + if tc.isErr == true { + return + } + bodyBytes, _ := io.ReadAll(resp.Body) + t.Fatalf("response status code is not 200, got %d: %s", resp.StatusCode, string(bodyBytes)) + } + + // Check response body + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") + } + got, ok := body["result"].(string) + if !ok { + t.Fatalf("unable to find result in response body") + } + + if got != tc.want { + t.Fatalf("unexpected value: got %q, want %q", got, tc.want) + } + }) + } +} + +func RunAuthRequiredToolInvocationTest(t *testing.T, sourceConfig map[string]any, toolKind string) { + // Write config into a file and pass it to command + toolsFile := map[string]any{ + "sources": map[string]any{ + "my-instance": sourceConfig, + }, + "authSources": map[string]any{ + "my-google-auth": map[string]any{ + "kind": "google", + "clientId": clientId, + }, + }, + "tools": map[string]any{ + "my-auth-tool": map[string]any{ + "kind": toolKind, + "source": "my-instance", + "description": "Tool to test auth required invocation.", + "statement": "SELECT 1;", + "authRequired": []string{ + "my-google-auth", + }, }, - isErr: true, }, } - for _, tc := range tcs { - token, err := getGoogleIdToken(clientId) + // Initialize a test command + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var args []string + + cmd, cleanup, err := StartCmd(ctx, toolsFile, args...) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + + out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`)) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } + + // Get ID token + idToken, err := getGoogleIdToken(clientId) + if err != nil { + t.Fatalf("error getting Google ID token: %s", err) + } + + // Test auth-required Tool invocation + invokeTcs := []struct { + name string + api string + requestHeader map[string]string + requestBody io.Reader + want string + isErr bool + }{ + { + name: "Invoke my-auth-tool with auth token", + api: "http://127.0.0.1:5000/api/tool/my-auth-tool/invoke", + requestHeader: map[string]string{"my-google-auth_token": idToken}, + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: false, + want: "Stub tool call for \"my-auth-tool\"! Parameters parsed: [] \n Output: [%!s(int32=1)]", + }, + { + name: "Invoke my-auth-tool with invalid auth token", + api: "http://127.0.0.1:5000/api/tool/my-auth-tool/invoke", + requestHeader: map[string]string{"my-google-auth_token": "INVALID_TOKEN"}, + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: true, + }, + { + name: "Invoke my-auth-tool without auth token", + api: "http://127.0.0.1:5000/api/tool/my-auth-tool/invoke", + requestHeader: map[string]string{}, + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: true, + }, + } + for _, tc := range invokeTcs { + t.Run(tc.name, func(t *testing.T) { + // Send Tool invocation request with ID token + req, err := http.NewRequest(http.MethodPost, tc.api, tc.requestBody) + if err != nil { + t.Fatalf("unable to create request: %s", err) + } + req.Header.Add("Content-type", "application/json") + for k, v := range tc.requestHeader { + req.Header.Add(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("unable to send request: %s", err) + } - if err != nil { - t.Fatalf("ID token generation error: %s", err) - } - headers := http.Header{} - headers.Add("my-google-auth_token", token) - claims, err := tc.authSource.GetClaimsFromHeader(headers) + if resp.StatusCode != http.StatusOK { + if tc.isErr == true { + return + } + bodyBytes, _ := io.ReadAll(resp.Body) + t.Fatalf("response status code is not 200, got %d: %s", resp.StatusCode, string(bodyBytes)) + } - if err != nil { - if tc.isErr { - return - } else { - t.Fatalf("Error getting claims from token: %s", err) + // Check response body + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") } - } - - _, ok := claims["sub"] - if !ok { - if tc.isErr { - return - } else { - t.Fatalf("Invalid claims.") + got, ok := body["result"].(string) + if !ok { + t.Fatalf("unable to find result in response body") + } + + if got != tc.want { + t.Fatalf("unexpected value: got %q, want %q", got, tc.want) } - } + }) } } diff --git a/tests/cloud_sql_pg_integration_test.go b/tests/cloud_sql_pg_integration_test.go index ff5045f01..2d5f955e6 100644 --- a/tests/cloud_sql_pg_integration_test.go +++ b/tests/cloud_sql_pg_integration_test.go @@ -21,13 +21,20 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" + "net" "net/http" "os" "reflect" "regexp" + "strings" "testing" "time" + + "cloud.google.com/go/cloudsqlconn" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" ) var ( @@ -39,7 +46,7 @@ var ( CLOUD_SQL_POSTGRES_PASS = os.Getenv("CLOUD_SQL_POSTGRES_PASS") ) -func requireCloudSQLPgVars(t *testing.T) { +func requireCloudSQLPgVars(t *testing.T) map[string]any { switch "" { case CLOUD_SQL_POSTGRES_PROJECT: t.Fatal("'CLOUD_SQL_POSTGRES_PROJECT' not set") @@ -54,10 +61,65 @@ func requireCloudSQLPgVars(t *testing.T) { case CLOUD_SQL_POSTGRES_PASS: t.Fatal("'CLOUD_SQL_POSTGRES_PASS' not set") } + + return map[string]any{ + "kind": "cloud-sql-postgres", + "project": CLOUD_SQL_POSTGRES_PROJECT, + "instance": CLOUD_SQL_POSTGRES_INSTANCE, + "region": CLOUD_SQL_POSTGRES_REGION, + "database": CLOUD_SQL_POSTGRES_DATABASE, + "user": CLOUD_SQL_POSTGRES_USER, + "password": CLOUD_SQL_POSTGRES_PASS, + } +} + +// Copied over from cloud_sql_pg.go +func getCloudSQLDialOpts(ip_type string) ([]cloudsqlconn.DialOption, error) { + switch strings.ToLower(ip_type) { + case "private": + return []cloudsqlconn.DialOption{cloudsqlconn.WithPrivateIP()}, nil + case "public": + return []cloudsqlconn.DialOption{cloudsqlconn.WithPublicIP()}, nil + default: + return nil, fmt.Errorf("invalid ip_type %s", ip_type) + } +} + +// Copied over from cloud_sql_pg.go +func initCloudSQLPgConnectionPool(project, region, instance, ip_type, user, pass, dbname string) (*pgxpool.Pool, error) { + // Configure the driver to connect to the database + dsn := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, dbname) + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("unable to parse connection uri: %w", err) + } + + // Create a new dialer with options + dialOpts, err := getCloudSQLDialOpts(ip_type) + if err != nil { + return nil, err + } + d, err := cloudsqlconn.NewDialer(context.Background(), cloudsqlconn.WithDefaultDialOptions(dialOpts...)) + if err != nil { + return nil, fmt.Errorf("unable to parse connection uri: %w", err) + } + + // Tell the driver to use the Cloud SQL Go Connector to create connections + i := fmt.Sprintf("%s:%s:%s", project, region, instance) + config.ConnConfig.DialFunc = func(ctx context.Context, _ string, instance string) (net.Conn, error) { + return d.Dial(ctx, i) + } + + // Interact with the driver directly as you normally would + pool, err := pgxpool.NewWithConfig(context.Background(), config) + if err != nil { + return nil, err + } + return pool, nil } func TestCloudSQLPostgres(t *testing.T) { - requireCloudSQLPgVars(t) + sourceConfig := requireCloudSQLPgVars(t) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -66,15 +128,7 @@ func TestCloudSQLPostgres(t *testing.T) { // Write config into a file and pass it to command toolsFile := map[string]any{ "sources": map[string]any{ - "my-pg-instance": map[string]any{ - "kind": "cloud-sql-postgres", - "project": CLOUD_SQL_POSTGRES_PROJECT, - "instance": CLOUD_SQL_POSTGRES_INSTANCE, - "region": CLOUD_SQL_POSTGRES_REGION, - "database": CLOUD_SQL_POSTGRES_DATABASE, - "user": CLOUD_SQL_POSTGRES_USER, - "password": CLOUD_SQL_POSTGRES_PASS, - }, + "my-pg-instance": sourceConfig, }, "tools": map[string]any{ "my-simple-tool": map[string]any{ @@ -123,8 +177,9 @@ func TestCloudSQLPostgres(t *testing.T) { t.Fatalf("error when sending a request: %s", err) } defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("response status code is not 200") + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + t.Fatalf("response status code is not 200, got %d: %s", resp.StatusCode, string(bodyBytes)) } var body map[string]interface{} @@ -164,8 +219,9 @@ func TestCloudSQLPostgres(t *testing.T) { t.Fatalf("error when sending a request: %s", err) } defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("response status code is not 200") + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + t.Fatalf("response status code is not 200, got %d: %s", resp.StatusCode, string(bodyBytes)) } var body map[string]interface{} @@ -184,3 +240,75 @@ func TestCloudSQLPostgres(t *testing.T) { }) } } + +// Set up auth test database table +func setupAuthTest(t *testing.T, ctx context.Context, tableName string) func(*testing.T) { + // set up testt + pool, err := initCloudSQLPgConnectionPool(CLOUD_SQL_POSTGRES_PROJECT, CLOUD_SQL_POSTGRES_REGION, CLOUD_SQL_POSTGRES_INSTANCE, "public", CLOUD_SQL_POSTGRES_USER, CLOUD_SQL_POSTGRES_PASS, CLOUD_SQL_POSTGRES_DATABASE) + if err != nil { + t.Fatalf("unable to create Cloud SQL connection pool: %s", err) + } + + err = pool.Ping(ctx) + if err != nil { + t.Fatalf("unable to connect to test database: %s", err) + } + + _, err = pool.Query(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id SERIAL PRIMARY KEY, + name TEXT, + email TEXT + ); + `, tableName)) + if err != nil { + t.Fatalf("unable to create test table: %s", err) + } + + // Insert test data + statement := fmt.Sprintf(` + INSERT INTO %s (name, email) + VALUES ($1, $2), ($3, $4) + `, tableName) + params := []any{"Alice", SERVICE_ACCOUNT_EMAIL, "Jane", "janedoe@gmail.com"} + _, err = pool.Query(ctx, statement, params...) + if err != nil { + t.Fatalf("unable to insert test data: %s", err) + } + + return func(t *testing.T) { + // tear down test + _, err := pool.Exec(ctx, fmt.Sprintf(`DROP TABLE %s;`, tableName)) + if err != nil { + t.Errorf("Teardown failed: %s", err) + } + } +} + +func TestCloudSQLGoogleAuthenticatedParameter(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // create test configs + sourceConfig := requireCloudSQLPgVars(t) + + // create table name with UUID + tableName := "auth_table_" + strings.Replace(uuid.New().String(), "-", "", -1) + + // test setup function reterns teardown function + teardownTest := setupAuthTest(t, ctx, tableName) + defer teardownTest(t) + + // call generic auth test helper + RunGoogleAuthenticatedParameterTest(t, sourceConfig, "postgres-sql", tableName) + +} + +func TestCloudSQLAuthRequiredToolInvocation(t *testing.T) { + // create test configs + sourceConfig := requireCloudSQLPgVars(t) + + // call generic auth test helper + RunAuthRequiredToolInvocationTest(t, sourceConfig, "postgres-sql") + +} diff --git a/tests/common_test.go b/tests/common_test.go index 3fad89897..9d175f0da 100644 --- a/tests/common_test.go +++ b/tests/common_test.go @@ -22,12 +22,17 @@ package tests import ( "bufio" + "bytes" "context" + "encoding/json" "fmt" "io" + "net/http" "os" "regexp" "strings" + "testing" + "time" "gopkg.in/yaml.v3" @@ -203,3 +208,163 @@ func (c *CmdExec) WaitForString(ctx context.Context, re *regexp.Regexp) (string, } } } + +func RunToolInvocationWithParamsTest(t *testing.T, sourceConfig map[string]any, toolKind string, tableName string) { + // Write config into a file and pass it to command + var statement string + switch toolKind { + case "postgres-sql": + statement = fmt.Sprintf("SELECT * FROM %s WHERE id = $1 OR name = $2;", tableName) + default: + t.Fatalf("invalid tool kind: %s", toolKind) + } + + toolsFile := map[string]any{ + "sources": map[string]any{ + "my-instance": sourceConfig, + }, + "tools": map[string]any{ + "my-tool": map[string]any{ + "kind": toolKind, + "source": "my-instance", + "description": "Tool to test invocation with params.", + "statement": statement, + "parameters": []any{ + map[string]any{ + "name": "id", + "type": "integer", + "description": "user ID", + }, + map[string]any{ + "name": "name", + "type": "string", + "description": "user name", + }, + }, + }, + }, + } + + // Initialize a test command + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var args []string + + cmd, cleanup, err := StartCmd(ctx, toolsFile, args...) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + + out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`)) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } + + // Test Tool invocation with parameters + invokeTcs := []struct { + name string + api string + + requestBody io.Reader + want string + isErr bool + }{ + { + name: "Invoke my-tool with parameters", + api: "http://127.0.0.1:5000/api/tool/my-tool/invoke", + requestBody: bytes.NewBuffer([]byte(`{"id": 3, "name": "Alice"}`)), + isErr: false, + want: "Stub tool call for \"my-tool\"! Parameters parsed: [{\"id\" '\\x03'} {\"name\" \"Alice\"}] \n Output: [%!s(int32=1) Alice][%!s(int32=3) Sid]", + }, + { + name: "Invoke my-tool without parameters", + api: "http://127.0.0.1:5000/api/tool/my-tool/invoke", + requestBody: bytes.NewBuffer([]byte(`{}`)), + isErr: true, + }, + { + name: "Invoke my-tool without insufficient parameters", + api: "http://127.0.0.1:5000/api/tool/my-tool/invoke", + requestBody: bytes.NewBuffer([]byte(`{"id": 1}`)), + isErr: true, + }, + } + for _, tc := range invokeTcs { + t.Run(tc.name, func(t *testing.T) { + // Send Tool invocation request with parameters + req, err := http.NewRequest(http.MethodPost, tc.api, tc.requestBody) + if err != nil { + t.Fatalf("unable to create request: %s", err) + } + req.Header.Add("Content-type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("unable to send request: %s", err) + } + + if resp.StatusCode != http.StatusOK { + if tc.isErr == true { + return + } + bodyBytes, _ := io.ReadAll(resp.Body) + t.Fatalf("response status code is not 200, got %d: %s", resp.StatusCode, string(bodyBytes)) + } + + // Check response body + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") + } + got, ok := body["result"].(string) + if !ok { + t.Fatalf("unable to find result in response body") + } + + if got != tc.want { + t.Fatalf("unexpected value: got %q, want %q", got, tc.want) + } + }) + } +} + +func RunSourceConnectionTest(t *testing.T, sourceConfig map[string]any, toolKind string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var args []string + + // Write config into a file and pass it to command + toolsFile := map[string]any{ + "sources": map[string]any{ + "my-instance": sourceConfig, + }, + "tools": map[string]any{ + "my-simple-tool": map[string]any{ + "kind": toolKind, + "source": "my-instance", + "description": "Simple tool to test end to end functionality.", + "statement": "SELECT 1;", + }, + }, + } + cmd, cleanup, err := StartCmd(ctx, toolsFile, args...) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`)) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } +} diff --git a/tests/neo4j_integration_test.go b/tests/neo4j_integration_test.go new file mode 100644 index 000000000..47fc5d151 --- /dev/null +++ b/tests/neo4j_integration_test.go @@ -0,0 +1,178 @@ +//go:build integration && neo4j + +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "reflect" + "regexp" + "testing" + "time" +) + +var ( + NEO4J_DATABASE = os.Getenv("NEO4J_DATABASE") + NEO4J_URI = os.Getenv("NEO4J_URI") + NEO4J_USER = os.Getenv("NEO4J_USER") + NEO4J_PASS = os.Getenv("NEO4J_PASS") +) + +func requireNeo4jVars(t *testing.T) { + switch "" { + case NEO4J_DATABASE: + t.Fatal("'NEO4J_DATABASE' not set") + case NEO4J_URI: + t.Fatal("'NEO4J_URI' not set") + case NEO4J_USER: + t.Fatal("'NEO4J_USER' not set") + case NEO4J_PASS: + t.Fatal("'NEO4J_PASS' not set") + } +} + +func TestNeo4j(t *testing.T) { + requireNeo4jVars(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var args []string + + // Write config into a file and pass it to command + toolsFile := map[string]any{ + "sources": map[string]any{ + "my-neo4j-instance": map[string]any{ + "kind": "neo4j", + "uri": NEO4J_URI, + "database": NEO4J_DATABASE, + "user": NEO4J_USER, + "password": NEO4J_PASS, + }, + }, + "tools": map[string]any{ + "my-simple-cypher-tool": map[string]any{ + "kind": "neo4j-cypher", + "source": "my-neo4j-instance", + "description": "Simple tool to test end to end functionality.", + "statement": "RETURN 1 as a;", + }, + }, + } + cmd, cleanup, err := StartCmd(ctx, toolsFile, args...) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := cmd.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`)) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } + + // Test tool get endpoint + tcs := []struct { + name string + api string + want map[string]any + }{ + { + name: "get my-simple-tool", + api: "http://127.0.0.1:5000/api/tool/my-simple-cypher-tool/", + want: map[string]any{ + "my-simple-cypher-tool": map[string]any{ + "description": "Simple tool to test end to end functionality.", + "parameters": []any{}, + }, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + resp, err := http.Get(tc.api) + if err != nil { + t.Fatalf("error when sending a request: %s", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("response status code is not 200") + } + + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") + } + + got, ok := body["tools"] + if !ok { + t.Fatalf("unable to find tools in response body") + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } + + // Test tool invoke endpoint + invokeTcs := []struct { + name string + api string + requestBody io.Reader + want string + }{ + { + name: "invoke my-simple-cypher-tool", + api: "http://127.0.0.1:5000/api/tool/my-simple-cypher-tool/invoke", + requestBody: bytes.NewBuffer([]byte(`{}`)), + want: "Stub tool call for \"my-simple-cypher-tool\"! Parameters parsed: map[] \n Output: \n\ta: %!s(int64=1)\n", + }, + } + for _, tc := range invokeTcs { + t.Run(tc.name, func(t *testing.T) { + resp, err := http.Post(tc.api, "application/json", tc.requestBody) + if err != nil { + t.Fatalf("error when sending a request: %s", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + t.Fatalf("response status code is not 200, got %d: %s", resp.StatusCode, string(bodyBytes)) + } + + var body map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&body) + if err != nil { + t.Fatalf("error parsing response body") + } + got, ok := body["result"].(string) + if !ok { + t.Fatalf("unable to find result in response body") + } + + if got != tc.want { + t.Fatalf("unexpected value: got %q, want %q", got, tc.want) + } + }) + } +}