diff --git a/.github/workflows/build_bff_sdk.yaml b/.github/workflows/build_bff_sdk.yaml new file mode 100644 index 000000000..0c6e19113 --- /dev/null +++ b/.github/workflows/build_bff_sdk.yaml @@ -0,0 +1,59 @@ +name: BFF SDK Tests/Push + +on: + push: + branches: + - 'main' + pull_request: + branches: [main] + workflow_dispatch: + inputs: + OnlyTestGeneratedSdk: + description: 'only test bff-sdk-generator, but not push' + required: true + default: 'true' + type: boolean + +env: + GOPROXY: https://proxy.golang.org/,direct + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +jobs: + build-bff-sdk: + env: + GRL_SDK_TEST_ONLY: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - uses: dorny/paths-filter@v2 + id: gql-changes + with: + filters: | + gql: + - 'apiserver/graph/schema/*.gql' + - name: setting token to npmrc + if: (( github.event_name == 'push' && steps.gql-changes.outputs.gql == 'true') || github.event_name == 'workflow_dispatch') + run: | + echo '//registry.npmjs.org/:_authToken="${{ secrets.NPM_TOKEN }}"' >> ~/.npmrc + - name: Set variable on Merge pull request + if: github.event_name == 'push' + run: | + echo "GRL_SDK_TEST_ONLY=false" >> $GITHUB_ENV + - name: Set variable on workflow_dispatch + if: github.event_name == 'workflow_dispatch' + run: | + echo "GRL_SDK_TEST_ONLY=$TEST_ONLY" >> $GITHUB_ENV + env: + TEST_ONLY: ${{ inputs.OnlyTestGeneratedSdk }} + - name: make sdk + if: (steps.gql-changes.outputs.gql == 'true' || github.event_name == 'workflow_dispatch') + run: tests/build-bff-sdk.sh + env: + GRL_SDK_TEST_ONLY: ${{ env.GRL_SDK_TEST_ONLY }} \ No newline at end of file diff --git a/.github/workflows/example_test.yaml b/.github/workflows/example_test.yaml index 8d134761b..9500ad795 100644 --- a/.github/workflows/example_test.yaml +++ b/.github/workflows/example_test.yaml @@ -7,12 +7,6 @@ on: pull_request: branches: [main] workflow_dispatch: - inputs: - OnlyTestGeneratedSdk: - description: 'only test bff-sdk-generator, but not push' - required: true - default: 'true' - type: boolean env: LOG_DIR: "/tmp/kubeagi-example-test/logs" @@ -59,8 +53,6 @@ jobs: path: /tmp/operator.image.tar example-test: - env: - GRL_SDK_TEST_ONLY: true needs: - build-image runs-on: ubuntu-latest @@ -114,33 +106,3 @@ jobs: with: name: ${{ github.sha }}-${{ matrix.no }}.logs path: ${{ env.LOG_DIR }} - - uses: dorny/paths-filter@v2 - if: matrix.no == 1 - id: gql-changes - with: - filters: | - gql: - - 'apiserver/graph/schema/*.gql' - - name: setting token to npmrc - if: (( github.event_name == 'push' && steps.gql-changes.outputs.gql == 'true') || github.event_name == 'workflow_dispatch') && matrix.no == 1 - run: | - echo '//dev-npm.k8s.com.cn/:_authToken="${{ secrets.TENX_NPM_TOKEN }}"' >> ~/.npmrc - echo '//registry.npmjs.org/:_authToken="${{ secrets.NPM_TOKEN }}"' >> ~/.npmrc - - name: Set variable on Merge pull request - if: github.event_name == 'push' && matrix.no == 1 - run: | - echo "GRL_SDK_TEST_ONLY=false" >> $GITHUB_ENV - - name: Set variable on workflow_dispatch - if: github.event_name == 'workflow_dispatch' - run: | - echo "GRL_SDK_TEST_ONLY=$TEST_ONLY" >> $GITHUB_ENV - env: - TEST_ONLY: ${{ inputs.OnlyTestGeneratedSdk }} - - name: make sdk - if: (steps.gql-changes.outputs.gql == 'true' || github.event_name == 'workflow_dispatch') && matrix.no == 1 - run: | - kubectl port-forward svc/arcadia-apiserver -n arcadia 8888:8081 --address 0.0.0.0 >/dev/null 2>&1 & - export GRAPH_API_ENDPOINT="http://0.0.0.0:8888/bff" - make bff-sdk-generator - env: - GRL_SDK_TEST_ONLY: ${{ env.GRL_SDK_TEST_ONLY }} diff --git a/Makefile b/Makefile index 75654a19b..2f59cf812 100644 --- a/Makefile +++ b/Makefile @@ -257,8 +257,8 @@ gql-gen: @go run github.com/99designs/gqlgen@v0.17.44 generate build-apiserver: gql-gen @CGO_ENABLED=0 GOOS=linux go build -o bin/apiserver apiserver/main.go -run-apiserver: - POD_NAMESPACE=arcadia go run apiserver/main.go --enable-playground=true & +run-apiserver: gql-gen + nohup go run apiserver/main.go --enable-playground=true --debug > apiserver.log 2>&1 & # sdk for apiserver api GRL_SDK_GENERATOR_IMAGE ?= yuntijs/gql-sdk-generator:latest diff --git a/apiserver/config/config.go b/apiserver/config/config.go index 3d1f2260a..e46ce62cb 100644 --- a/apiserver/config/config.go +++ b/apiserver/config/config.go @@ -33,20 +33,29 @@ import ( var s = &ServerConfig{} type ServerConfig struct { + Scheme *runtime.Scheme + + // Debug mode which only have graphql server running + Debug bool + // SystemNamespace which hosts system resources SystemNamespace string - Host string - Port int + Host string + Port int + + PlaygroundEndpointPrefix string + + // EnablePlayground is true when graphql playground is going to be utilized EnablePlayground bool - EnableSwagger bool - EnableOIDC bool + // EnableSwagger is true when swagger is going to be utilized + EnableSwagger bool - PlaygroundEndpointPrefix string + // OIDC configurations + EnableOIDC bool IssuerURL, MasterURL, ClientID, ClientSecret string + // DataProcessURL is the URL of the data process service DataProcessURL string - - Scheme *runtime.Scheme } func NewServerFlags() ServerConfig { @@ -62,6 +71,7 @@ func NewServerFlags() ServerConfig { flag.StringVar(&s.ClientID, "client-id", "", "oidc client id(required when enable odic)") flag.StringVar(&s.ClientSecret, "client-secret", "", "oidc client secret(required when enable odic)") flag.StringVar(&s.DataProcessURL, "data-processing-url", "http://127.0.0.1:28888", "url to access data processing server") + flag.BoolVar(&s.Debug, "debug", false, "debug model for apiserver") klog.InitFlags(nil) flag.Parse() diff --git a/apiserver/graph/schema/application.gql b/apiserver/graph/schema/application.gql index 9ecb77d6a..935a34e43 100644 --- a/apiserver/graph/schema/application.gql +++ b/apiserver/graph/schema/application.gql @@ -1,3 +1,4 @@ +# commoe mutation createApplication($input:CreateApplicationMetadataInput!){ Application{ createApplication(input: $input) { diff --git a/apiserver/service/router.go b/apiserver/service/router.go index 97ccd577b..28b03669c 100644 --- a/apiserver/service/router.go +++ b/apiserver/service/router.go @@ -57,24 +57,28 @@ func NewServerAndRun(conf config.ServerConfig) { } bffGroup := r.Group("/bff") - // for file operations - registerMinIOAPI(bffGroup, conf) + // for ops apis with graphql registerGraphQL(r, bffGroup, conf) - ragGroup := r.Group("/rags") - registerRAG(ragGroup, conf) + // when debug model is disabled + if !config.GetConfig().Debug { + // for file operations + registerMinIOAPI(bffGroup, conf) + ragGroup := r.Group("/rags") + registerRAG(ragGroup, conf) - // for admin chat server with Restful apis - chatGroup := r.Group("/chat") - registerChat(chatGroup, conf) + // for admin chat server with Restful apis + chatGroup := r.Group("/chat") + registerChat(chatGroup, conf) - // for gpts chat server with Restful apis - gptsGroup := r.Group("/gpts/chat") - registerGptsChat(gptsGroup, conf) + // for gpts chat server with Restful apis + gptsGroup := r.Group("/gpts/chat") + registerGptsChat(gptsGroup, conf) - fg := r.Group("/forward") - registerForward(fg, conf) + fg := r.Group("/forward") + registerForward(fg, conf) + } // for swagger if conf.EnableSwagger { diff --git a/tests/build-bff-sdk.sh b/tests/build-bff-sdk.sh new file mode 100755 index 000000000..3c37a8877 --- /dev/null +++ b/tests/build-bff-sdk.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright contributors to the KubeAGI project +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./tests/scripts/utils.sh + +if [[ $RUNNER_DEBUG -eq 1 ]] || [[ $GITHUB_RUN_ATTEMPT -gt 1 ]]; then + # use [debug logging](https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging) + # or run the same test multiple times. + set -x +fi +export TERM=xterm-color + +info "Start kubeagi apiserver with debug mode" +make build-apiserver +nohup ./bin/apiserver --enable-playground=true --debug > apiserver.log 2>&1 & + +sleep 5 +cat apiserver.log + +info "build bff sdk" +export GRAPH_API_ENDPOINT="http://0.0.0.0:8081/bff" +make bff-sdk-generator \ No newline at end of file diff --git a/tests/scripts/utils.sh b/tests/scripts/utils.sh new file mode 100755 index 000000000..e6fc081c9 --- /dev/null +++ b/tests/scripts/utils.sh @@ -0,0 +1,335 @@ +#!/bin/bash +# +# Copyright contributors to the KubeAGI project +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#################################################################################### +# Following functions are for debugging purpose # +#################################################################################### + +TimeoutSeconds=${TimeoutSeconds:-"300"} + +function debugInfo { + if [[ $? -eq 0 ]]; then + exit 0 + fi + if [[ $debug -ne 0 ]]; then + exit 1 + fi + if [[ $GITHUB_ACTIONS == "true" ]]; then + warning "debugInfo start 🧐" + mkdir -p $LOG_DIR || true + df -h + + warning "1. Try to get all resources " + kubectl api-resources --verbs=list -o name | xargs -n 1 kubectl get -A --ignore-not-found=true --show-kind=true >$LOG_DIR/get-all-resources-list.log + kubectl api-resources --verbs=list -o name | xargs -n 1 kubectl get -A -oyaml --ignore-not-found=true --show-kind=true >$LOG_DIR/get-all-resources-yaml.log + + warning "2. Try to describe all resources " + kubectl api-resources --verbs=list -o name | xargs -n 1 kubectl describe -A >$LOG_DIR/describe-all-resources.log + + warning "3. Try to export kind logs to $LOG_DIR..." + kind export logs --name=${KindName} $LOG_DIR + sudo chown -R $USER:$USER $LOG_DIR + + warning "debugInfo finished ! " + warning "This means that some tests have failed. Please check the log. 🌚" + debug=1 + fi + exit 1 +} + + +function cecho() { + declare -A colors + colors=( + ['black']='\E[0;47m' + ['red']='\E[0;31m' + ['green']='\E[0;32m' + ['yellow']='\E[0;33m' + ['blue']='\E[0;34m' + ['magenta']='\E[0;35m' + ['cyan']='\E[0;36m' + ['white']='\E[0;37m' + ) + local defaultMSG="No message passed." + local defaultColor="black" + local defaultNewLine=true + while [[ $# -gt 1 ]]; do + key="$1" + case $key in + -c | --color) + color="$2" + shift + ;; + -n | --noline) + newLine=false + ;; + *) + # unknown option + ;; + esac + shift + done + message=${1:-$defaultMSG} # Defaults to default message. + color=${color:-$defaultColor} # Defaults to default color, if not specified. + newLine=${newLine:-$defaultNewLine} + echo -en "${colors[$color]}" + echo -en "$message" + if [ "$newLine" = true ]; then + echo + fi + tput sgr0 # Reset text attributes to normal without clearing screen. + return +} + +function warning() { + cecho -c 'yellow' "$@" +} + +function error() { + cecho -c 'red' "$@" +} + +function info() { + cecho -c 'blue' "$@" +} + + +#################################################################################### +# Following functions are used to check pod status,CRD status,chat status,etc... # +#################################################################################### + +function waitPodReady() { + namespace=$1 + podLabel=$2 + START_TIME=$(date +%s) + while true; do + readStatus=$(kubectl -n${namespace} get po -l ${podLabel} --ignore-not-found=true -o json | jq -r '.items[0].status.conditions[] | select(."type"=="Ready") | .status') + if [[ $readStatus == "True" ]]; then + info "Pod:${podLabel} ready" + break + fi + kubectl -n${namespace} get po -l ${podLabel} + + CURRENT_TIME=$(date +%s) + ELAPSED_TIME=$((CURRENT_TIME - START_TIME)) + if [ $ELAPSED_TIME -gt $TimeoutSeconds ]; then + error "Timeout reached" + kubectl describe po -n${namespace} -l ${podLabel} + kubectl get po -n${namespace} --show-labels + exit 1 + fi + sleep 5 + done +} + +function EnableAPIServerPortForward() { + waitPodReady "arcadia" "app=arcadia-apiserver" + if [ $portal_pid -ne 0 ]; then + kill $portal_pid >/dev/null 2>&1 + fi + echo "re port-forward apiserver..." + kubectl port-forward svc/arcadia-apiserver -n arcadia 8081:8081 >/dev/null 2>&1 & + portal_pid=$! + sleep 3 + info "port-forward apiserver in pid: $portal_pid" +} + +function waitCRDStatusReady() { + source=$1 + namespace=$2 + name=$3 + START_TIME=$(date +%s) + while true; do + readStatus=$(kubectl -n${namespace} get ${source} ${name} --ignore-not-found=true -o json | jq -r '.status.conditions[0].status') + message=$(kubectl -n${namespace} get ${source} ${name} --ignore-not-found=true -o json | jq -r '.status.conditions[0].message') + if [[ $readStatus == "True" ]]; then + info $message + if [[ ${source} == "KnowledgeBase" ]]; then + fileStatus=$(kubectl get knowledgebase -n $namespace $name -o json | jq -r '.status.fileGroupDetail[0].fileDetails[0].phase') + if [[ $fileStatus != "Succeeded" ]]; then + kubectl get knowledgebase -n $namespace $name -o json | jq -r '.status.fileGroupDetail[0].fileDetails' + exit 1 + fi + fi + break + fi + + CURRENT_TIME=$(date +%s) + ELAPSED_TIME=$((CURRENT_TIME - START_TIME)) + if [[ ${source} == "Worker" ]]; then + if [ $ELAPSED_TIME -gt 1800 ]; then + error "Timeout reached" + exit 1 + fi + else + if [ $ELAPSED_TIME -gt $TimeoutSeconds ]; then + error "Timeout reached" + exit 1 + fi + fi + sleep 5 + done +} + +function getRespInAppChat() { + appname=$1 + namespace=$2 + query=$3 + conversationID=$4 + testStream=$5 + attempt=0 + while true; do + info "sleep 3 seconds" + sleep 3 + data=$(jq -n --arg appname "$appname" --arg query "$query" --arg conversationID "$conversationID" '{"query":$query,"response_mode":"blocking","conversation_id":$conversationID,"app_name":$appname}') + resp=$(curl --max-time $TimeoutSeconds -s --show-error -XPOST http://127.0.0.1:8081/chat --data "$data" -H "namespace: ${namespace}") + ai_data=$(echo $resp | jq -r '.message') + references=$(echo $resp | jq -r '.references') + if [ -z "$ai_data" ] || [ "$ai_data" = "null" ]; then + echo $resp + EnableAPIServerPortForward + if [[ $resp == *"googleapi: Error"* ]]; then + echo "google api error, will retry after 60s" + sleep 60 + fi + attempt=$((attempt + 1)) + if [ $attempt -gt $RETRY_COUNT ]; then + echo "❌: Failed. Retry count exceeded." + exit 1 + fi + echo "πŸ”„: Failed. Attempt $attempt/$RETRY_COUNT" + continue + fi + echo "πŸ‘€: ${query}" + echo "πŸ€–: ${ai_data}" + echo "πŸ”—: ${references}" + break + done + resp_conversation_id=$(echo $resp | jq -r '.conversation_id') + + if [ $testStream == "true" ]; then + attempt=0 + while true; do + info "sleep 5 seconds" + sleep 5 + info "just test stream mode" + data=$(jq -n --arg appname "$appname" --arg query "$query" --arg conversationID "$conversationID" '{"query":$query,"response_mode":"streaming","conversation_id":$conversationID,"app_name":$appname}') + curl --max-time $TimeoutSeconds -s --show-error -XPOST http://127.0.0.1:8081/chat --data "$data" -H "namespace: ${namespace}" + if [[ $? -ne 0 ]]; then + attempt=$((attempt + 1)) + if [ $attempt -gt $RETRY_COUNT ]; then + echo "❌: Failed. Retry count exceeded." + exit 1 + fi + echo "πŸ”„: Failed. Attempt $attempt/$RETRY_COUNT" + EnableAPIServerPortForward + echo "and wait 60s for google api error" + sleep 60 + continue + fi + break + done + fi +} + +function fileUploadSummarise() { + appname=$1 + namespace=$2 + filename=$3 + attempt=0 + while true; do + info "sleep 3 seconds" + sleep 3 + resp=$(curl --max-time $TimeoutSeconds -s --show-error -XPOST --form file=@$filename --form app_name=$appname -H "namespace: ${namespace}" -H "Content-Type: multipart/form-data" http://127.0.0.1:8081/chat/conversations/file) + doc_data=$(echo $resp | jq -r '.document') + if [ -z "$doc_data" ]; then + echo $resp + EnableAPIServerPortForward + if [[ $resp == *"googleapi: Error"* ]]; then + echo "google api error, will retry after 60s" + sleep 60 + fi + attempt=$((attempt + 1)) + if [ $attempt -gt $RETRY_COUNT ]; then + echo "❌: Failed. Retry count exceeded." + exit 1 + fi + echo "πŸ”„: Failed. Attempt $attempt/$RETRY_COUNT" + continue + fi + echo "πŸ‘€: ${filename}" + echo "πŸ€–: ${doc_data}" + break + done + file_id=$(echo $resp | jq -r '.document.object') + resp_conversation_id=$(echo $resp | jq -r '.conversation_id') + attempt=0 + while true; do + info "sleep 3 seconds to sumerize doc" + sleep 3 + data=$(jq -n --arg fileid "$file_id" --arg appname "$appname" --arg query "ζ€»η»“δΈ€δΈ‹" --arg conversationID "$resp_conversation_id" '{"query":$query,"response_mode":"blocking","conversation_id":$conversationID,"app_name":$appname, "files": [$fileid]}') + resp=$(curl --max-time $TimeoutSeconds -s --show-error -XPOST http://127.0.0.1:8081/chat --data "$data" -H "namespace: ${namespace}") + ai_data=$(echo $resp | jq -r '.message') + references=$(echo $resp | jq -r '.references') + if [ -z "$ai_data" ] || [ "$ai_data" = "null" ]; then + echo $resp + EnableAPIServerPortForward + if [[ $resp == *"googleapi: Error"* ]]; then + echo "google api error, will retry after 60s" + sleep 60 + fi + attempt=$((attempt + 1)) + if [ $attempt -gt $RETRY_COUNT ]; then + echo "❌: Failed. Retry count exceeded." + exit 1 + fi + echo "πŸ”„: Failed. Attempt $attempt/$RETRY_COUNT" + continue + fi + echo "πŸ‘€: ζ€»η»“δΈ€δΈ‹" + echo "πŸ€–: ${ai_data}" + echo "πŸ”—: ${references}" + break + done + resp_conversation_id=$(echo $resp | jq -r '.conversation_id') + + if [ $testStream == "true" ]; then + attempt=0 + while true; do + info "sleep 5 seconds" + sleep 5 + info "just test stream mode" + data=$(jq -n --arg fileid "$file_id" --arg appname "$appname" --arg query "ζ€»η»“δΈ€δΈ‹" --arg conversationID "$resp_conversation_id" '{"query":$query,"response_mode":"blocking","conversation_id":$conversationID,"app_name":$appname, "files": [$fileid]}') + curl --max-time $TimeoutSeconds -s --show-error -XPOST http://127.0.0.1:8081/chat --data "$data" -H "namespace: ${namespace}" + if [[ $? -ne 0 ]]; then + attempt=$((attempt + 1)) + if [ $attempt -gt $RETRY_COUNT ]; then + echo "❌: Failed. Retry count exceeded." + exit 1 + fi + echo "πŸ”„: Failed. Attempt $attempt/$RETRY_COUNT" + EnableAPIServerPortForward + echo "and wait 60s for google api error" + sleep 60 + continue + fi + break + done + fi +}