diff --git a/.github/workflows/build_stage.yml b/.github/workflows/build_stage.yml index 537cf25d8..e61049b75 100644 --- a/.github/workflows/build_stage.yml +++ b/.github/workflows/build_stage.yml @@ -17,20 +17,23 @@ jobs: uses: actions/upload-artifact@v2 with: name: keploy-binary - path: kTestBuild + path: keployB - test: +# Add Jobs for testing different applications here. + test-go-mongo-1: needs: build - uses: ./.github/workflows/test_stage.yml + uses: ./.github/workflows/test-go-mongo-1.yml + + test-go-mongo-2: + needs: build + uses: ./.github/workflows/test-go-mongo-2.yml - test-ginApp: - needs: build - uses: ./.github/workflows/test-ginApp.yml - check_status: - needs: test-ginApp +# Wait for all the test workflows before getting the coverage + coverage: + needs: [test-go-mongo-1, test-go-mongo-2] runs-on: ubuntu-latest steps: - name: Get Test Status - run: echo "Test script status code ${{ needs.test.outputs.test_status }}" + run: echo "All workflows are completed, Now calculating the coverage" diff --git a/.github/workflows/content.yml b/.github/workflows/content.yml deleted file mode 100644 index 29adf3f4a..000000000 --- a/.github/workflows/content.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Display Workflow Content - -on: [pull_request] - -jobs: - show-workflow-file: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with : - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.head_ref }} - - - name: Display content of the test-stage.yml - run: cat .github/workflows/test_stage.yml \ No newline at end of file diff --git a/.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-1.sh b/.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-1.sh deleted file mode 100644 index 5fa1a84f6..000000000 --- a/.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-1.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -# In this script, latest version of keploy is used for recording, build version of keploy is used for testing - -source ./../../.github/workflows/test_workflow_scripts/test-iid.sh - - -delete_if_exists() { - local path=$1 - if [ -e "$path" ]; then - sudo rm -rf "$path" - fi -} - - -check_test_status() { - local path=$1 - local fixed_index=$2 # Boolean to determine if index should be fixed to 0 - local overallStatus=1 # true - local idx=0 # Initialize index - - for dir in $test_sets; do - if [ "$fixed_index" -eq 1 ]; then - local report_file="$path/keploy/reports/test-run-0/$dir-report.yaml" - else - local report_file="$path/keploy/reports/test-run-$idx/$dir-report.yaml" - idx=$((idx + 1)) - fi - - local test_status=$(grep 'status:' "$report_file" | head -n 1 | awk '{print $2}') - - if [ "$test_status" != "PASSED" ]; then - overallStatus=0 # false - fi - done - echo $overallStatus -} - - -# Checkout a different branch -git fetch origin -git checkout native-linux - -# Check and remove keploy config file if exists -delete_if_exists "./keploy.yml" - -# Generate the keploy-config file -kTestBuild config --generate - -# # Update the global noise to ts. -# config_file="./keploy.yml" -# sed -i 's/global: {}/global: {"body": {"ts":[]}}/' "$config_file" - -# Build the binary -go build -o ginApp - -#### Recording Phase of test-bench #### -pre_rec="." - -# Delete the reports directory if it exists -delete_if_exists "$pre_rec/keploy/reports" - -# Get all directories except the 'reports' directory -test_sets=$(find "$pre_rec/keploy/" -mindepth 1 -maxdepth 1 -type d ! -name "reports" -exec basename {} \;) - -# Loop over each directory stored in 'test_sets' -for dir in $test_sets; do - echo "Recording and replaying for (test-set): $dir" - sudo -E env PATH=$PATH kRecordHosted record -c "sudo -E env PATH=$PATH kTestBuild test -c 'go run main.go handler.go' --proxyPort 56789 --dnsPort 46789 --delay=10 --testsets $dir --path '$pre_rec' --enableTesting --generateGithubActions=false" --path "./test-bench/" --proxyPort=36789 --dnsPort 26789 --enableTesting --generateGithubActions=false - # Wait for 1 second before new test-set - sleep 1 -done - -sleep 2 - - -# Check whether the original tests passed or failed -overallStatus=$(check_test_status "." 0) -echo "Overall TestRun status for pre-recorded testscase ran via test-bench: $overallStatus" -if [ "$overallStatus" -eq 0 ]; then - exit 1 -fi - - -#### Testing Phase of test-bench #### -test_bench_rec="./test-bench" - -## Test assertion -pilot -test-assert -preRecPath . -testBenchPath $test_bench_rec -exit_status=$? -if [ $exit_status -ne 0 ]; then - echo "Test assertion failed with exit status $exit_status." - exit 1 -fi - -echo "Tests are asserted successfully." - -## Mock assertion preparation - -pilot -mock-assert -preRecPath . -testBenchPath $test_bench_rec -exit_status=$? -if [ $exit_status -ne 0 ]; then - echo "Mock assertion preparation failed with exit status $exit_status." - exit 1 -fi - -echo "Mock assertion prepared successfully." - -## Now run the tests both for pre-recorded test cases and test-bench recorded test cases to compare the mocks (mock assertion) -delete_if_exists "$pre_rec/keploy/reports" - -## Run tests for pre-recorded test cases -sudo -E env PATH=$PATH kRecordHosted test -c "./ginApp" --delay=7 --generateGithubActions=false - -sleep 2 - -overallStatus=$(check_test_status "." 1) -echo "Overall TestRun status for pre-recorded testscase (after mock assertion): $overallStatus" -if [ "$overallStatus" -eq 0 ]; then - echo "Newly recorded mocks are not consistent with the pre-recorded mocks." - exit 1 -fi -echo "New mocks are consistent with the pre-recorded mocks." - - -## Run tests for test-bench-recorded test cases -sudo -E env PATH=$PATH kRecordHosted test -c "./ginApp" --path "$test_bench_rec" --delay=7 --generateGithubActions=false - -sleep 2 - -overallStatus=$(check_test_status "$test_bench_rec" 1) -echo "Overall TestRun status for test-bench-recorded testscase (after mock assertion): $overallStatus" -if [ "$overallStatus" -eq 0 ]; then - echo "Old recorded mocks are not consistent with the test-bench-recorded mocks." - exit 1 -fi -echo "Old mocks are consistent with the test-bench-recorded mocks." - -# Delete the tests and mocks generated via test-bench. -delete_if_exists "$test_bench_rec" - -echo "Tests and mocks are consistent for this application." -exit 0 diff --git a/.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-2.sh b/.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-2.sh deleted file mode 100644 index fbf46ae66..000000000 --- a/.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-2.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash - -# In this script, latest version of keploy is used for testing, build version of keploy is used for recording - -source ./../../.github/workflows/test_workflow_scripts/test-iid.sh - -delete_if_exists() { - local path=$1 - if [ -e "$path" ]; then - sudo rm -rf "$path" - fi -} - - -check_test_status() { - local path=$1 - local fixed_index=$2 # Boolean to determine if index should be fixed to 0 - local overallStatus=1 # true - local idx=0 # Initialize index - - for dir in $test_sets; do - if [ "$fixed_index" -eq 1 ]; then - local report_file="$path/keploy/reports/test-run-0/$dir-report.yaml" - else - local report_file="$path/keploy/reports/test-run-$idx/$dir-report.yaml" - idx=$((idx + 1)) - fi - - local test_status=$(grep 'status:' "$report_file" | head -n 1 | awk '{print $2}') - - if [ "$test_status" != "PASSED" ]; then - overallStatus=0 # false - fi - done - echo $overallStatus -} - - -# Checkout a different branch -git fetch origin -git checkout native-linux - -# Get the hosted version of keploy -curl --silent --location "https://github.com/keploy/keploy/releases/latest/download/keploy_linux_amd64.tar.gz" | tar xz -C /tmp -sudo mkdir -p /usr/local/bin && sudo mv /tmp/keploy /usr/local/bin/kTestHosted - -# Check and remove keploy config file if exists -delete_if_exists "./keploy.yml" - -# Generate the keploy-config file -kRecordBuild config --generate - -# # Update the global noise to ts. -# config_file="./keploy.yml" -# sed -i 's/global: {}/global: {"body": {"ts":[]}}/' "$config_file" - -# Build the binary -go build -o ginApp - -#### Recording Phase of test-bench #### -pre_rec="." - -# Delete the reports directory if it exists -delete_if_exists "$pre_rec/keploy/reports" - -# Get all directories except the 'reports' directory -test_sets=$(find "$pre_rec/keploy/" -mindepth 1 -maxdepth 1 -type d ! -name "reports" -exec basename {} \;) - -# Loop over each directory stored in 'test_sets' -for dir in $test_sets; do - echo "Recording and replaying for (test-set): $dir" - sudo -E env PATH=$PATH kRecordBuild record -c "sudo -E env PATH=$PATH kTestHosted test -c ./ginApp --proxyPort 56789 --dnsPort 46789 --delay=10 --testsets $dir --enableTesting --generateGithubActions=false" --path "./test-bench/" --proxyPort=36789 --dnsPort 26789 --enableTesting --generateGithubActions=false - # Wait for 1 second before new test-set - sleep 1 -done - -sleep 2 - - -# Check whether the original tests passed or failed -overallStatus=$(check_test_status "." 0) -echo "Overall TestRun status for pre-recorded testscase ran via test-bench: $overallStatus" -if [ "$overallStatus" -eq 0 ]; then - exit 1 -fi - -#### Testing Phase of test-bench #### - -## Get the pilot for tests and mocks assertion -curl --silent -o pilot --location "https://github.com/keploy/pilot/releases/latest/download/pilot_linux_amd64" && -sudo chmod a+x pilot && sudo mkdir -p /usr/local/bin && sudo mv pilot /usr/local/bin - -## Test assertion -pilot -test-assert -preRecPath . -testBenchPath ./test-bench -exit_status=$? -if [ $exit_status -ne 0 ]; then - echo "Test assertion failed with exit status $exit_status." - exit 1 -fi - -echo "Tests are asserted successfully." - -## Mock assertion preparation -pilot -mock-assert -preRecPath . -testBenchPath ./test-bench -exit_status=$? -if [ $exit_status -ne 0 ]; then - echo "Mock assertion preparation failed with exit status $exit_status." - exit 1 -fi - -echo "Mock assertion prepared successfully." - -## Now run the tests both for pre-recorded test cases and test-bench recorded test cases to compare the mocks (mock assertion) -delete_if_exists "$pre_rec/keploy/reports" - -## Run tests for pre-recorded test cases -sudo -E env PATH=$PATH kTestHosted test -c "./ginApp" --delay=7 --generateGithubActions=false - -sleep 2 - -overallStatus=$(check_test_status "." 1) -echo "Overall TestRun status for pre-recorded testscase (after mock assertion): $overallStatus" -if [ "$overallStatus" -eq 0 ]; then - echo "Newly recorded mocks are not consistent with the pre-recorded mocks." - exit 1 -fi -echo "New mocks are consistent with the pre-recorded mocks." - -test_bench_rec="./test-bench" - -## Run tests for test-bench-recorded test cases -sudo -E env PATH=$PATH kTestHosted test -c "./ginApp" --path "$test_bench_rec" --delay=7 --generateGithubActions=false - -sleep 2 - -overallStatus=$(check_test_status "$test_bench_rec" 1) -echo "Overall TestRun status for test-bench-recorded testscase (after mock assertion): $overallStatus" -if [ "$overallStatus" -eq 0 ]; then - echo "Old recorded mocks are not consistent with the test-bench-recorded mocks." - exit 1 -fi -echo "Old mocks are consistent with the test-bench-recorded mocks." - -# Delete the tests and mocks generated via test-bench. -delete_if_exists "$test_bench_rec" - -echo "Tests and mocks are consistent for this application." -exit 0 diff --git a/.github/workflows/test_stage.yml b/.github/workflows/test-go-mongo-1.yml similarity index 63% rename from .github/workflows/test_stage.yml rename to .github/workflows/test-go-mongo-1.yml index 8fdfde847..8ba6cac6b 100644 --- a/.github/workflows/test_stage.yml +++ b/.github/workflows/test-go-mongo-1.yml @@ -1,16 +1,10 @@ -name: Test Stage +name: Test GinApp 1 on: workflow_call: - outputs: - test_status: - description: "Exit status of the test script" - value: ${{ jobs.test-go-mongo.outputs.test_result }} jobs: test-go-mongo: runs-on: ubuntu-latest - outputs: - test_result: ${{ steps.test_script.outcome }} steps: - name: Checkout repository uses: actions/checkout@v2 @@ -23,12 +17,12 @@ jobs: - name: Giving permission to binary run: | - sudo chmod +x /usr/local/bin/kTestBuild + sudo chmod +x /usr/local/bin/keployB - name: Download the latest released binary of keploy run: | curl --silent --location "https://github.com/keploy/keploy/releases/latest/download/keploy_linux_amd64.tar.gz" | tar xz -C /tmp - sudo mkdir -p /usr/local/bin && sudo mv /tmp/keploy /usr/local/bin/kRecordHosted + sudo mkdir -p /usr/local/bin && sudo mv /tmp/keploy /usr/local/bin/keployH # Get the pilot for tests and mocks assertion - name: Get the pilot for tests and mocks assertion @@ -36,19 +30,34 @@ jobs: curl --silent -o pilot --location "https://github.com/keploy/pilot/releases/latest/download/pilot_linux_amd64" && sudo chmod a+x pilot && sudo mkdir -p /usr/local/bin && sudo mv pilot /usr/local/bin - - name: Checkout the samples-go repository + - name: Checkout to the samples-go repository uses: actions/checkout@v2 with: repository: keploy/samples-go + ref: native-linux path: samples-go + + - name: Build the gin-mongo application + run: | + cd samples-go/gin-mongo + go build -o ginApp . + # add the noisy fields in the config file if any. - name: Add noisy fields in the config file run: | echo "You can add the noisy fields in the config file if any." - - name: Run samples-go application - id: test_script + + - name: Run testing script for gin-mongo application + uses: keploy/testgpt@test + with: + working-directory: samples-go/gin-mongo + command: ./ginApp + delay: 7 + ci-mode: 0 + + - name: Get and Upload the coverage data run: | - cd samples-go/gin-mongo - chmod +x ./../../.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-1.sh - ./../../.github/workflows/test-bench-workflow-scripts/mongo/go-mongo-linux-1.sh - echo "::set-output name=outcome::${?}" + echo "You can get and upload the coverage data here." +# echo "You can use the following command to get the coverage data:" +# echo "go tool cover -html=coverage.out -o coverage.html" + diff --git a/.github/workflows/test-ginApp.yml b/.github/workflows/test-go-mongo-2.yml similarity index 97% rename from .github/workflows/test-ginApp.yml rename to .github/workflows/test-go-mongo-2.yml index c485dace9..f3266717e 100644 --- a/.github/workflows/test-ginApp.yml +++ b/.github/workflows/test-go-mongo-2.yml @@ -1,4 +1,4 @@ -name: Test GinApp +name: Test GinApp 2 on: workflow_call: @@ -53,4 +53,4 @@ jobs: working-directory: samples-go/gin-mongo command: ./ginApp delay: 7 - ci-mode: 0 \ No newline at end of file + ci-mode: 1 \ No newline at end of file