full path for python path

.gitignore

@@ -160,3 +160,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+main.py

Dockerfile

@@ -16,7 +16,7 @@ RUN pip3 install -r requirements.txt
 
 # Run the Python script
 # Note the env variable GITHUB_ACCESS_TOKEN will need to be set in order to authenticate with the GitHub API
-CMD ["python3", "main.py"]
+CMD ["python3", "/home/watcloud-slurm-ci/main.py"]
 # ENTRYPOINT ["python3", "/app/main.py"]
 
 # Custom user for running the CI

allocate-ephemeral-runner-with-stargz.sh

@@ -16,9 +16,9 @@ record_timing() {
 }
 
 # Check if all required arguments are provided
-if [ $# -lt 4 ]; then
+if [ $# -lt 5 ]; then
     log "ERROR: Missing required arguments"
-    log "Usage: $0 <repo-url> <registration-token> <removal-token> <label>"
+    log "Usage: $0 <repo-url> <registration-token> <removal-token> <label> <run_id>"
     exit 1
 fi
 
@@ -71,19 +71,21 @@ record_timing "Start stargz" $duration
 export DOCKER_HOST=unix:///tmp/run/docker.sock
 
 # Define the parent directory for GitHub Actions in the host machine
-PARENT_DIR="/tmp/runner-${SLURMD_NODENAME}-${SLURM_JOB_ID}"
-PROVISIONER_DIR="/mnt/wato-drive/alexboden/provisioner-cache/"
+PARENT_DIR="/dev/shm/docker/runner-${SLURMD_NODENAME}-${SLURM_JOB_ID}"
+PROVISIONER_DIR="/mnt/wato-drive2/alexboden/provisioner-cache/$RUN_ID"
 log "INFO Parent directory for GitHub Actions: $PARENT_DIR"
 
 start_time=$(date +%s)
 GITHUB_ACTIONS_WKDIR="$PARENT_DIR/_work"
 mkdir -p $PARENT_DIR
+ls -l /mnt/wato-drive2/alexboden/provisioner-cache
+mkdir -p $PROVISIONER_DIR
 chown -R $(id -u):$(id -g) $PARENT_DIR
 chmod -R 777 $PARENT_DIR
 end_time=$(date +%s)
 duration=$((end_time - start_time))
-log "INFO Created and set permissions for parent directory (Duration: $duration seconds)"
-record_timing "Create Parent Directory" $duration
+log "INFO Created and set permissions for parent and provisioner directories (Duration: $duration seconds)"
+record_timing "Create Directories" $duration
 
 # Start the actions runner container
 log "INFO Starting actions runner container"
@@ -98,12 +100,13 @@ record_timing "Start Container" $duration
 log "INFO Configuring container"
 start_time=$(date +%s)
 ./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chmod 666 /var/run/docker.sock" # Allows the runner to access the docker socket
-./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "mkdir \"$PARENT_DIR\"" 
+./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "mkdir -p \"$PARENT_DIR\"" 
 ./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chown -R runner:runner \"$PARENT_DIR\""
-docker exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chmod -R 755 \"$PARENT_DIR\"" 
+./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chmod -R 755 \"$PARENT_DIR\"" 
 
-./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chown -R runner:runner \"$PROVISIONER_DIR\""
-docker exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chmod -R 755 \"$PROVISIONER_DIR\"" 
+# ./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "mkdir -p \"$PROVISIONER_DIR\""
+# ./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chown -R root:root \"$PROVISIONER_DIR\""
+./bin/nerdctl exec $DOCKER_CONTAINER_ID /bin/bash -c "sudo chmod -R 777 \"$PROVISIONER_DIR\"" 
 
 end_time=$(date +%s)
 duration=$((end_time - start_time))
@@ -138,8 +141,8 @@ record_timing "Remove Runner" $duration
 # Clean up
 log "INFO Stopping and removing Docker container"
 start_time=$(date +%s)
-docker stop $DOCKER_CONTAINER_ID
-docker rm $DOCKER_CONTAINER_ID
+./bin/nerdctl stop $DOCKER_CONTAINER_ID
+./bin/nerdctl rm $DOCKER_CONTAINER_ID
 end_time=$(date +%s)
 duration=$((end_time - start_time))
 log "INFO Docker container removed (Duration: $duration seconds)"
@@ -155,9 +158,9 @@ kill $SNAPSHOTTER_PID
 
 wait $CONTAINERD_PID
 wait $SNAPSHOTTER_PID
-end_time=$(date +%s)
 
 rootlesskit rm -rf /tmp/{run,config,containerd*,nerdctl}
+end_time=$(date +%s)
 duration=$((end_time - start_time))
 log "INFO Containerd and Stargz cleaned up (Duration: $duration seconds)"
 
@@ -176,4 +179,4 @@ for duration in "${timings[@]}"; do
 done
 log "Total Time: $total_time seconds"
 
-exit 0
+exit 0

apptainer-helper.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Check if the correct number of arguments is provided
+if [ "$#" -ne 7 ]; then
+    echo "Usage: $0 <GITHUB_ACTIONS_WKDIR> <REPO_URL> <REGISTRATION_TOKEN> <LABELS> <SLURMD_NODENAME> <SLURM_JOB_ID> <REMOVAL_TOKEN>"
+    exit 1
+fi
+
+# Assign input arguments to variables
+GITHUB_ACTIONS_WKDIR=$1
+REPO_URL=$2
+REGISTRATION_TOKEN=$3
+LABELS=$4
+SLURMD_NODENAME=$5
+SLURM_JOB_ID=$6
+REMOVAL_TOKEN=$7
+
+# Run the commands inside the Apptainer shell
+export DOCKER_HOST=unix:///tmp/run/docker.sock
+export RUNNER_ALLOW_RUNASROOT=1
+export PYTHONPATH=/home/runner/.local/lib/python3.10/site-packages
+
+/home/runner/config.sh --work "${GITHUB_ACTIONS_WKDIR}" --url "${REPO_URL}" --token "${REGISTRATION_TOKEN}" --labels "${LABELS}" --name "slurm-${SLURMD_NODENAME}-${SLURM_JOB_ID}" --unattended --ephemeral
+/home/runner/run.sh
+/home/runner/config.sh remove --token "${REMOVAL_TOKEN}"

containerd-config.toml

@@ -0,0 +1,4 @@
+[proxy_plugins]
+  [proxy_plugins.stargz]
+    type = "snapshot"
+    address = "/tmp/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"

echo.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ls -l /mnt/wato-drive/alexboden
+ls -l /mnt/wato-drive/alexboden/provisioner-cache
+mkdir /mnt/wato-drive/alexboden/provisioner-cache/111312526769

notes.md

@@ -0,0 +1,119 @@
+# Issues encountered
+1. Docker in docker
+2. Cloudflare 502 errors
+3. Caching docker images 
+
+
+# 3
+Does caching occur in the the machine?
+1. Testing 
+
+# Testing basic
+Average times for each task:
+Register Runner: 4.02 seconds
+Configure Container: 0.37 seconds
+Run Runner: 128.38 seconds
+Start Container: 69.29 seconds
+Start Docker: 4.65 seconds
+Remove Container: 12.67 seconds
+Create Parent Directory: 0.02 seconds
+Remove Runner: 0.44 seconds
+
+Average times for each task:
+Register Runner: 19.64 seconds
+Configure Container: 3.00 seconds
+Run Runner: 258.25 seconds
+Start Container: 33.52 seconds
+Start Docker: 10.39 seconds
+Remove Container: 0.02 seconds
+Create Parent Directory: 0.02 seconds
+Remove Runner: 0.72 seconds
+
+
+Received status code: 401 Unauthorized. Refreshing creds...
+
+Options:
+Apptainer:
+- sif file in apptainer
+- cvmfs with apptainer
+
+stargz-snapshotter:
+- stargz-snapshotter
+
+Docker:
+try docker load from wato-drive
+
+look at the preprovision to see how to provision the image
+debugging
+- ./kuberenetes.sh k9s -A
+
+make new uid for container and user.
+base on slurm-dist, hard code version
+
+clean up root directory 
+
+- try caching on ram for client image
+
+
+
+2024-10-28 01:25:02,144 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:25:02,144 - ERROR - Command stderr: sbatch: error: Invalid generic resource (gres) specification                                           │
+│ 2024-10-28 01:25:02,144 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:25:02,144 - INFO - Allocating runner for job 32130643423                                                                                   │
+│ 2024-10-28 01:25:03,729 - INFO - Job labels: ['alexboden-gh-arc-runners-small']                                                                          │
+│ 2024-10-28 01:25:03,729 - INFO - Using runner size label: gh-arc-runners-small                                                                           │
+│ 2024-10-28 01:25:03,730 - INFO - Running command: /opt/slurm/bin/sbatch --job-name=slurm-gh-arc-runners-small-32130643423 --mem-per-cpu=2G --cpus-per-ta │
+│ sk=1 --gres=tmpdisk:4096 --time=00:15:00 allocate-ephemeral-runner-from-apptainer.sh https://github.com/WATonomous/infra-config AXDJFCYENLFJI55HPB22O2DH │
+│ D3274 AXDJFC2DYCLCLQXM3OOJB3THD3276 alexboden-gh-arc-runners-small 11544717605                                                                           │
+│ 2024-10-28 01:25:03,732 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:25:03,732 - ERROR - Command stderr: sbatch: error: Invalid generic resource (gres) specification                                           │
+│ 2024-10-28 01:25:03,732 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:25:03,732 - INFO - Allocating runner for job 32130643423                                                                                   │
+│ 2024-10-28 01:25:05,279 - INFO - Job labels: ['alexboden-gh-arc-runners-small']                                                                          │
+│ 2024-10-28 01:25:05,280 - INFO - Using runner size label: gh-arc-runners-small                                                                           │
+│ 2024-10-28 01:25:05,280 - INFO - Running command: /opt/slurm/bin/sbatch --job-name=slurm-gh-arc-runners-small-32130643423 --mem-per-cpu=2G --cpus-per-ta │
+│ sk=1 --gres=tmpdisk:4096 --time=00:15:00 allocate-ephemeral-runner-from-apptainer.sh https://github.com/WATonomous/infra-config AXDJFC2DYCLCLQXM3OOJB3TH │
+│ D3276 AXDJFC7P3EWSGPA4Y5PRRD3HD33AC alexboden-gh-arc-runners-small 11544717605                                                                           │
+│ 2024-10-28 01:25:05,282 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:25:05,283 - ERROR - Command stderr: sbatch: error: Invalid generic resource (gres) specification                                           │
+│ 2024-10-28 01:25:05,283 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:25:05,283 - INFO - Allocating runner for job 32130643423                                                                                   │
+│ 2024-10-28 01:25:06,829 - INFO - Job labels: ['alexboden-gh-arc-runners-small']                                                                          │
+│ 2024-10-28 01:25:06,830 - INFO - Using runner size label: gh-arc-runners-small                                                                           │
+│ 2024-10-28 01:25:06,830 - INFO - Running command: /opt/slurm/bin/sbatch --job-name=slurm-gh-arc-runners-small-32130643423 --mem-per-cpu=2G --cpus-per-ta │
+│ sk=1 --gres=tmpdisk:4096 --time=00:15:00 allocate-ephemeral-runner-from-apptainer.sh https://github.com/WATonomous/infra-config AXDJFC7P3EWSGPA4Y5PRRD3H │
+│ D33AC AXDJFC5SRLVJL3S4LVYTS4LHD33AE alexboden-gh-arc-runners-small 11544717605                                                                           │
+│ 2024-10-28 01:25:06,832 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:25:06,832 - ERROR - Command stderr: sbatch: error: Invalid generic resource (gres) specification                                           │
+│ 2024-10-28 01:25:06,832 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:25:06,832 - INFO - Allocating runner for job 32130643423                                                                                   │
+│ 2024-10-28 01:25:08,369 - INFO - Job labels: ['alexboden-gh-arc-runners-small']                                                                          │
+│ 2024-10-28 01:25:08,370 - INFO - Using runner size label: gh-arc-runners-small                                                                           │
+│ 2024-10-28 01:25:08,370 - INFO - Running command: /opt/slurm/bin/sbatch --job-name=slurm-gh-arc-runners-small-32130643423 --mem-per-cpu=2G --cpus-per-ta │
+│ sk=1 --gres=tmpdisk:4096 --time=00:15:00 allocate-ephemeral-runner-from-apptainer.sh https://github.com/WATonomous/infra-config AXDJFC2AIX67XCOEAKT4O73H │
+│ D33AG AXDJFCZRNEIKQBDF6GNAVODHD33AI alexboden-gh-arc-runners-small 11544717605                                                                           │
+│ 2024-10-28 01:25:08,372 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:25:08,372 - ERROR - Command stderr: sbatch: error: Invalid generic resource (gres) specification                                           │
+│ 2024-10-28 01:25:08,372 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:25:08,372 - INFO - Allocating runner for job 32130643423
+
+
+│ 2024-10-28 01:30:31,364 - INFO - Using runner size label: gh-arc-runners-small                                                                           │
+│ 2024-10-28 01:30:31,364 - INFO - Running command: /opt/slurm/bin/sbatch --job-name=slurm-gh-arc-runners-small-32130643423 --mem-per-cpu=2G --cpus-per-ta │
+│ sk=1 --gres tmpdisk:4096 --time=00:15:00 allocate-ephemeral-runner-from-apptainer.sh https://github.com/WATonomous/infra-config AXDJFC2CHXCIDKEJK6PEUP3H │
+│ D33UK AXDJFCZXPFZXWCVBSJCORWDHD33UO alexboden-gh-arc-runners-small 11544717605                                                                           │
+│ 2024-10-28 01:30:31,366 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:30:31,366 - ERROR - Command stderr: /opt/slurm/bin/sbatch: unrecognized option '--gres tmpdisk:4096'                                       │
+│ Try "sbatch --help" for more information                                                                                                                 │
+│ 2024-10-28 01:30:31,367 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:30:31,367 - INFO - Allocating runner for job 32130643423                                                                                   │
+│ 2024-10-28 01:30:32,897 - INFO - Job labels: ['alexboden-gh-arc-runners-small']                                                                          │
+│ 2024-10-28 01:30:32,897 - INFO - Using runner size label: gh-arc-runners-small                                                                           │
+│ 2024-10-28 01:30:32,897 - INFO - Running command: /opt/slurm/bin/sbatch --job-name=slurm-gh-arc-runners-small-32130643423 --mem-per-cpu=2G --cpus-per-ta │
+│ sk=1 --gres tmpdisk:4096 --time=00:15:00 allocate-ephemeral-runner-from-apptainer.sh https://github.com/WATonomous/infra-config AXDJFCZXPFZXWCVBSJCORWDH │
+│ D33UO AXDJFC3LWBBWJ2RFY323UX3HD33UQ alexboden-gh-arc-runners-small 11544717605                                                                           │
+│ 2024-10-28 01:30:32,899 - INFO - Command stdout:                                                                                                         │
+│ 2024-10-28 01:30:32,900 - ERROR - Command stderr: /opt/slurm/bin/sbatch: unrecognized option '--gres tmpdisk:4096'                                       │
+│ Try "sbatch --help" for more information                                                                                                                 │
+│ 2024-10-28 01:30:32,900 - ERROR - Failed to parse SLURM job ID from command output: . Error: list index out of range                                     │
+│ 2024-10-28 01:30:32,900 - INFO - Allocating runner for job 32130643423                                                                                   │

poll_slurm_status.py

@@ -0,0 +1,61 @@
+# Description: This script is used to poll the status of SLURM jobs that have been submitted by the user.
+# It was used mainly for debugging purposes to ensure that the jobs were being submitted and running as expected.
+
+import time
+
+def check_slurm_status():
+    if not allocated_jobs:
+        return
+
+    to_remove = []
+    frozen_jobs = allocated_jobs.copy()
+    for job_id, runningjob in frozen_jobs.items():
+        if not runningjob or not runningjob.slurm_job_id:
+            continue
+
+        # Use 'sacct' to get job status and start time for a single job
+        sacct_cmd = ['sacct', '-n', '-P', '-o', 'JobID,State,Start,End', '--jobs', str(runningjob.slurm_job_id)]
+        try:
+            sacct_result = subprocess.run(sacct_cmd, capture_output=True, text=True)
+            sacct_output = sacct_result.stdout.strip()
+
+            if sacct_result.returncode != 0:
+                logging.error(f"sacct command failed with return code {sacct_result.returncode}")
+                logging.error(f"Error output: {sacct_result.stderr}")
+                continue
+
+            for line in sacct_output.split('\n'):
+                parts = line.split('|')
+                if line == '' or len(parts) < 4:
+                    continue # Sometimes it takes a while for the job to appear in the sacct output
+
+                job_component = parts[0]  # e.g., '3840.batch'
+                status = parts[1]
+                start_time_str = parts[2]
+                end_time_str = parts[3]
+
+                # Focus only on the main job ID and ignore '.batch' or '.extern' components
+                if '.batch' in job_component or '.extern' in job_component:
+                    continue
+
+                # Convert time strings to datetime objects
+                start_time = datetime.strptime(start_time_str, '%Y-%m-%dT%H:%M:%S') if start_time_str != 'Unknown' else None
+                end_time = datetime.strptime(end_time_str, '%Y-%m-%dT%H:%M:%S') if end_time_str != 'Unknown' else None
+
+                if status in ['COMPLETED', 'FAILED', 'CANCELLED', 'TIMEOUT']: # otherwise job is not finished running
+                    duration = "[Unknown Duration]"
+                    if start_time and end_time:
+                        duration = end_time - start_time
+                    logging.info(f"Slurm job {job_component} {status} in {duration}. Running Job Info: {str(runningjob)}")
+                    to_remove.append(job_id)
+
+        except Exception as e:
+            logging.error(f"Error querying SLURM job details for job ID {runningjob.slurm_job_id}: {e}")
+
+    for job_id in to_remove:
+        del allocated_jobs[job_id]
+
+def poll_slurm_statuses(sleep_time=1):
+    while True:
+        check_slurm_status()
+        time.sleep(sleep_time)