Free disk space before CI runs using docker (ucbds-infra#897)

* debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * free disk space in CI runs using docker to prevent image build failures * add comment about source * reorg workflow, add support for docker gha cache in CI * undo gha caching changes since it's already done in conftest.py * debugging failed builds * debugging failed builds * remove alternate_file * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds * debugging failed builds
chrispyles · Jan 1, 2025 · 581bb3e · 581bb3e
1 parent 8cb8449
commit 581bb3e
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 22 deletions.
diff --git a/.github/workflows/run-docker-tests-beta.yml b/.github/workflows/run-docker-tests-beta.yml
@@ -83,12 +83,17 @@ jobs:
         run: |
           micromamba run -n otter-grader Rscript -e 'install.packages("ottr", dependencies=TRUE, repos="https://cran.r-project.org/")'
 
-      - name: Maximize build space
+      # This solution comes from this comment: https://github.com/orgs/community/discussions/25678#discussioncomment-5242487
+      - name: Free disk space
         if: ${{ steps.check-branch.outputs.run == 'true' }}
         run: |
+          echo "Before cleanup:"
+          df -h
+          sudo rm -rf /opt/hostedtoolcache
           sudo swapoff -a
-          sudo rm -f /swapfile
+          sudo rm -rf /swapfile /usr/local/.ghcup /usr/local/lib/android
           sudo apt clean
+          echo "After cleanup:"
           df -h
 
       - name: Run tests

diff --git a/.github/workflows/run-docker-tests.yml b/.github/workflows/run-docker-tests.yml
@@ -65,11 +65,16 @@ jobs:
         run: |
           micromamba run -n otter-grader Rscript -e 'install.packages("ottr", dependencies=TRUE, repos="https://cran.r-project.org/")'
 
-      - name: Maximize build space
+      # This solution comes from this comment: https://github.com/orgs/community/discussions/25678#discussioncomment-5242487
+      - name: Free disk space
         run: |
+          echo "Before cleanup:"
+          df -h
+          sudo rm -rf /opt/hostedtoolcache
           sudo swapoff -a
-          sudo rm -f /swapfile
+          sudo rm -rf /swapfile /usr/local/.ghcup /usr/local/lib/android
           sudo apt clean
+          echo "After cleanup:"
           df -h
 
       - name: Run tests

diff --git a/test/conftest.py b/test/conftest.py
@@ -52,7 +52,7 @@ def pytest_configure(config):
     )
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def cleanup_enabled(pytestconfig):
     """
     A fixture indicating whether test cleanup is enabled.
@@ -93,7 +93,7 @@ def create_fake_pdf(src, dest, **kwargs):
 
 
 @pytest.fixture(autouse=True, scope="session")
-def update_grade_dockerfile():
+def update_grade_dockerfile(cleanup_enabled):
     """
     Update the Dockerfile used in otter grade to install the contents of this repo.
     """
@@ -108,8 +108,9 @@ def update_grade_dockerfile():
 
     yield
 
-    with open("otter/grade/Dockerfile", "w") as f:
-        f.write(contents)
+    if cleanup_enabled:
+        with open("otter/grade/Dockerfile", "w") as f:
+            f.write(contents)
 
 
 def add_repo_dir_to_context_then_build(*args, **kwargs):

diff --git a/test/test_grade/test_integration.py b/test/test_grade/test_integration.py
@@ -1,11 +1,9 @@
 """Tests for ``otter.grade``"""
 
-import logging
 import os
 import pandas as pd
 import pytest
 import re
-import shutil
 import zipfile
 
 from contextlib import ExitStack
@@ -20,7 +18,7 @@
 from otter.run import AutograderConfig
 from otter.test_files import GradingResults
 
-from ..utils import TestFileManager
+from ..utils import delete_paths, TestFileManager
 
 
 ASSIGNMENT_NAME = "otter-grade-test"
@@ -33,18 +31,18 @@
 def cleanup_output(cleanup_enabled):
     yield
     if cleanup_enabled:
-        if os.path.exists("test/final_grades.csv"):
-            os.remove("test/final_grades.csv")
-        if os.path.exists("test/submission_pdfs"):
-            shutil.rmtree("test/submission_pdfs")
-        if os.path.exists(ZIP_SUBM_PATH):
-            os.remove(ZIP_SUBM_PATH)
-        if os.path.exists("test/grading-summaries"):
-            shutil.rmtree("test/grading-summaries")
+        delete_paths(
+            [
+                "test/final_grades.csv",
+                "test/grading-summaries",
+                "test/submission_pdfs",
+                ZIP_SUBM_PATH,
+            ]
+        )
 
 
 @pytest.fixture(autouse=True, scope="module")
-def generate_zip_file():
+def generate_zip_file(cleanup_enabled):
     """
     Generate an autograder zip file for use in these tests.
     """
@@ -59,8 +57,8 @@ def generate_zip_file():
     with logging.level_context(logging.DEBUG):
         yield
 
-    if os.path.isfile(AG_ZIP_PATH):
-        os.remove(AG_ZIP_PATH)
+    if cleanup_enabled:
+        delete_paths([AG_ZIP_PATH])
 
 
 @pytest.fixture