use work directory in klx-training-pre-pr-check.yml

dynamicheart · dynamicheart · commit 4892099cd6c1 · 2023-11-16T08:02:02.000Z
diff --git a/.github/workflows/klx-training-pre-pr-check.yml b/.github/workflows/klx-training-pre-pr-check.yml
@@ -23,78 +23,108 @@ jobs:
         uses: actions/checkout@master
         with:
           fetch-depth: 1
+      
+      # 2. Create tmp directory
+      - name: Retrieve the current timestamp
+        id: work-timestamp
+        run: echo "WORK_TIMESTAMP=$(date +"%Y-%m%d-%H%M%S")"  >> "$GITHUB_OUTPUT"
 
-      # 2. Setup basic information in test_conf.py
+      - name: Set work directory variable
+        id: work-directory
+        env:
+          WORK_TIMESTAMP: ${{ steps.work-timestamp.outputs.WORK_TIMESTAMP }}
+        run: echo "WORK_DIRECTORY=$(realpath ${PWD}/..)/FlagPerf-$WORK_TIMESTAMP"  >> "$GITHUB_OUTPUT"
+
+      - name: Create work directory and copy the whole directory
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
+        run: |
+          mkdir ${WORK_DIRECTORY} &&
+          cp -r * ${WORK_DIRECTORY}
+
+      # 3. Setup basic information in test_conf.py
       - name: Setup test_conf.py
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
-          echo "VENDOR = 'kunlunxin'" >> training/run_benchmarks/config/test_conf.py &&
-          echo "ACCE_CONTAINER_OPT = '--device=/dev/xpu0 --device=/dev/xpu1 --device=/dev/xpu2 --device=/dev/xpu3 --device=/dev/xpu4 --device=/dev/xpu5 --device=/dev/xpu6 --device=/dev/xpu7 --device=/dev/xpuctrl'" >> training/run_benchmarks/config/test_conf.py &&
-          echo "ACCE_VISIBLE_DEVICE_ENV_NAME = 'XPU_VISIBLE_DEVICES'" >> training/run_benchmarks/config/test_conf.py &&
+          export TEST_CONF_FILEPATH=${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py
+          echo "VENDOR = 'kunlunxin'" >> ${TEST_CONF_FILEPATH} &&
+          echo "ACCE_CONTAINER_OPT = '--device=/dev/xpu0 --device=/dev/xpu1 --device=/dev/xpu2 --device=/dev/xpu3 --device=/dev/xpu4 --device=/dev/xpu5 --device=/dev/xpu6 --device=/dev/xpu7 --device=/dev/xpuctrl'" >> ${TEST_CONF_FILEPATH} &&
+          echo "ACCE_VISIBLE_DEVICE_ENV_NAME = 'XPU_VISIBLE_DEVICES'" >> ${TEST_CONF_FILEPATH} &&
           echo "PIP_SOURCE = 'https://pypi.tuna.tsinghua.edu.cn/simple'" &&
-          echo "FLAGPERF_PATH = '${PWD}/training'" >> training/run_benchmarks/config/test_conf.py &&
-          echo "FLAGPERF_LOG_PATH = '${PWD}/training/result/'" >> training/run_benchmarks/config/test_conf.py &&
-          cat training/run_benchmarks/config/test_conf.py
+          echo "FLAGPERF_PATH = '${WORK_DIRECTORY}/training'" >> ${TEST_CONF_FILEPATH} &&
+          echo "FLAGPERF_LOG_PATH = '${WORK_DIRECTORY}/training/result/'" >> ${TEST_CONF_FILEPATH} &&
+          cat ${TEST_CONF_FILEPATH}
 
-      # 3. Setup cases in test_conf.py
+      # 4. Setup cases in test_conf.py
       - name: Setup 1x1 case and 1x8 case in test_conf.py
         if: ${{ inputs.case1x1 != '' && inputs.case1x8 != '' }}
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
-          echo "CASES = { '${{ inputs.case1x1 }}' : '${{ inputs.dataset_path }}', '${{ inputs.case1x8 }}' : '${{ inputs.dataset_path }}' }" >> training/run_benchmarks/config/test_conf.py &&
-          cat training/run_benchmarks/config/test_conf.py
+          echo "CASES = { '${{ inputs.case1x1 }}' : '${{ inputs.dataset_path }}', '${{ inputs.case1x8 }}' : '${{ inputs.dataset_path }}' }" >> ${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py &&
+          cat ${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py
       - name: Setup 1x1 case in test_conf.py
         if: ${{ inputs.case1x1 != '' && inputs.case1x8 == '' }}
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
-          echo "CASES = { '${{ inputs.case1x1 }}' : '${{ inputs.dataset_path }}' }" >> training/run_benchmarks/config/test_conf.py &&
-          cat training/run_benchmarks/config/test_conf.py
+          echo "CASES = { '${{ inputs.case1x1 }}' : '${{ inputs.dataset_path }}' }" >> ${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py &&
+          cat ${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py
       - name: Setup 1x8 case in test_conf.py
         if: ${{ inputs.case1x1 == '' && inputs.case1x8 != '' }}
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
-          echo "CASES = { '${{ inputs.case1x1 }}' : '${{ inputs.dataset_path }}' }" >> training/run_benchmarks/config/test_conf.py &&
-          cat training/run_benchmarks/config/test_conf.py
+          echo "CASES = { '${{ inputs.case1x1 }}' : '${{ inputs.dataset_path }}' }" >> ${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py &&
+          cat ${WORK_DIRECTORY}/training/run_benchmarks/config/test_conf.py
       - name: Setup empty cases in test_conf.py
         if: ${{ inputs.case1x1 == '' && inputs.case1x8 == '' }}
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
           echo "At least one of case1x1 and case1x8 is not empty！" &&
           ls non_existent_file.txt
 
-      # 4. Setup cluster_conf.py
+      # 5. Setup cluster_conf.py
       - name: Setup cluster_conf.py
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
-          echo "HOSTS = ['127.0.0.1']" >> training/run_benchmarks/config/cluster_conf.py &&
-          cat training/run_benchmarks/config/cluster_conf.py
+          echo "HOSTS = ['127.0.0.1']" >> ${WORK_DIRECTORY}/training/run_benchmarks/config/cluster_conf.py &&
+          cat ${WORK_DIRECTORY}/training/run_benchmarks/config/cluster_conf.py
 
-      # 5. Run test
+      # 6. Run test
       - name: Run test
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
         run: |
-          pushd training &&
+          pushd ${WORK_DIRECTORY}/training &&
           python3 run_benchmarks/run.py 2>&1 | tee tmp_run.log &&
           popd
 
-      # 6. Verify test result
+      # 7. Verify test result
       # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
       - name: Get run result name
         id: run-result-name
-        run: echo "RUN_RESULT_NAME=$(cat training/tmp_run.log | grep 'Initialize logger with log path:' | egrep -o 'run[0-9]+')"  >> "$GITHUB_OUTPUT"
+        env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
+        run: echo "RUN_RESULT_NAME=$(cat ${WORK_DIRECTORY}/training/tmp_run.log | grep 'Initialize logger with log path:' | egrep -o 'run[0-9]+')"  >> "$GITHUB_OUTPUT"
 
       - name: Verify 1x1 case result
         if: ${{ inputs.case1x1 != '' }}
         env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
           RUN_RESULT_NAME: ${{ steps.run-result-name.outputs.RUN_RESULT_NAME }}
         run: |
-          cat training/result/${RUN_RESULT_NAME}/${{ inputs.case1x1 != '' }}/round1/127.0.0.1_noderank0/rank0.out.log
-          grep "FINISHED" "training/result/${RUN_RESULT_NAME}/${{ inputs.case1x1 != '' }}/round1/127.0.0.1_noderank0/rank0.out.log"
+          cat ${WORK_DIRECTORY}/training/result/${RUN_RESULT_NAME}/${{ inputs.case1x1 }}/round1/127.0.0.1_noderank0/rank0.out.log
+          grep "FINISHED" "${WORK_DIRECTORY}/training/result/${RUN_RESULT_NAME}/${{ inputs.case1x1 }}/round1/127.0.0.1_noderank0/rank0.out.log"
 
       - name: Verify 1x8 case result
         if: ${{ inputs.case1x8 != '' }}
         env:
+          WORK_DIRECTORY: ${{ steps.work-directory.outputs.WORK_DIRECTORY }}
           RUN_RESULT_NAME: ${{ steps.run-result-name.outputs.RUN_RESULT_NAME }}
         run: |
-          cat training/result/${RUN_RESULT_NAME}/${{ inputs.case1x8 != '' }}/round1/127.0.0.1_noderank0/rank0.out.log
-          grep "FINISHED" "training/result/${RUN_RESULT_NAME}/${{ inputs.case1x8 != '' }}/round1/127.0.0.1_noderank0/rank0.out.log"
-
-      # 7. Clean job
-      - name: Remove tmp_run.log, restore test_conf.py and cluster_conf.py
-        run: |
-          rm -f training/tmp_run.log &&
-          git checkout training/run_benchmarks/config/test_conf.py &&
-          git checkout training/run_benchmarks/config/cluster_conf.py
+          cat ${WORK_DIRECTORY}/training/result/${RUN_RESULT_NAME}/${{ inputs.case1x8 }}/round1/127.0.0.1_noderank0/rank0.out.log
+          grep "FINISHED" "${WORK_DIRECTORY}/training/result/${RUN_RESULT_NAME}/${{ inputs.case1x8 }}/round1/127.0.0.1_noderank0/rank0.out.log"
