Improve reproducibility and consistency of benchmarking workflow

.github/workflows/ucc-benchmarks.yml

@@ -12,6 +12,10 @@ on:
         required: false
         default: 'main'
 
+# Ensure only one instance of the workflow is running at a time
+concurrency:
+  group: ${{ github.workflow }}
+
 jobs:
   run-benchmarks:
     runs-on: ucc-benchmarks-8-core-U22.04

Dockerfile

@@ -23,9 +23,6 @@ POETRY_VIRTUALENVS_PATH=/venv
 # Install (rarely changing) dependencies only using Poetry to leverage docker caching
 RUN --mount=type=cache,target=/tmp/poetry_cache poetry install --no-root
 
-# Update to the latest version of the compiler libraries dependent libraries
-RUN --mount=type=cache,target=/tmp/poetry_cache poetry update
-
 # Install the `ucc` package itself
 RUN poetry install
 

benchmarks/results/expval_2025-02-26_20.csv

@@ -0,0 +1,26 @@
+# Compiler versions: qiskit=1.3.3, cirq=1.4.1, pytket=1.40.0, ucc=0.4.2 # OS: Linux #26~22.04.1-Ubuntu SMP Thu Jul 11 22:33:04 UTC 2024, Architecture: 64bit, CPU Cores: 8
+compiler,circuit_name,observable,expval,absolute_error,relative_error,ideal_expval
+ucc,qaoa_barabasi_albert,ZZZZZZZZZZ,-1.7437351558201308e-06,0.008098073486562062,1.0002153735345967,0.008096329751406242
+pytket,qaoa_barabasi_albert,ZZZZZZZZZZ,4.684878798699645e-05,0.008049480963419245,0.9942135770867213,0.008096329751406242
+qiskit,qaoa_barabasi_albert,ZZZZZZZZZZ,4.684878798699526e-05,0.008049480963419247,0.9942135770867215,0.008096329751406242
+cirq,qaoa_barabasi_albert,ZZZZZZZZZZ,4.684878806294861e-05,0.008049480963343294,0.9942135770773404,0.008096329751406242
+qiskit,qft,ZZZZZZZZZZ,2.2768245622195593e-18,2.0599821909922125e-18,9.499906219123742,2.1684237122734696e-19
+pytket,qft,ZZZZZZZZZZ,-7.589415207398531e-19,9.757838919672e-19,4.499968739707914,2.1684237122734696e-19
+ucc,qft,ZZZZZZZZZZ,6.505213034913027e-19,4.336789322639557e-19,1.999973205463926,2.1684237122734696e-19
+cirq,qft,ZZZZZZZZZZ,1.5178830414797062e-18,1.3010406702523593e-18,5.999937479415828,2.1684237122734696e-19
+ucc,square_heisenberg,ZZZZZZZZZ,0.04475291753159164,0.95524708246841,0.9552470824684085,1.0000000000000016
+qiskit,square_heisenberg,ZZZZZZZZZ,0.07860580325637458,0.921394196743627,0.9213941967436255,1.0000000000000016
+pytket,square_heisenberg,ZZZZZZZZZ,0.00038839342611028866,0.9996116065738913,0.9996116065738897,1.0000000000000016
+cirq,square_heisenberg,ZZZZZZZZZ,0.028044187211763102,0.9719558127882385,0.971955812788237,1.0000000000000016
+qiskit,qv,HOP,-0.5249757623146565,0.0890272567197925,0.14499481917823856,-0.614003019034449
+ucc,qv,HOP,-0.5990277230958924,0.011920036093324371,0.019510728886449055,-0.6109477591892167
+ucc,qcnn,ZZZZZZZZZZ,-0.0001766335755311934,0.0032003725161868306,0.9476952156040286,-0.003377006091718024
+qiskit,qcnn,ZZZZZZZZZZ,-0.0006837729802561487,0.002693233111461875,0.7975209514921885,-0.003377006091718024
+pytket,qv,HOP,-0.4711597689440996,0.1459901701544617,0.23655543151750436,-0.6171499390985613
+pytket,qcnn,ZZZZZZZZZZ,-0.000454789650867551,0.002922216440850473,0.8653275598220255,-0.003377006091718024
+cirq,qcnn,ZZZZZZZZZZ,-0.0004547896509797271,0.0029222164407382966,0.8653275597888078,-0.003377006091718024
+cirq,qv,HOP,-0.49506950264725713,0.11975315085362848,0.19477673792879524,-0.6148226535008856
+qiskit,prep_select,ZZZZZZZZZZ,7.589415207398531e-19,2.893639478316343e-18,1.355526419110587,-2.13469795757649e-18
+ucc,prep_select,ZZZZZZZZZZ,4.445228907190568e-18,6.579926864767058e-18,3.0823690262191517,-2.13469795757649e-18
+pytket,prep_select,ZZZZZZZZZZ,1.0842021724855044e-19,2.2431181748250405e-18,1.0507894884443696,-2.13469795757649e-18
+cirq,prep_select,ZZZZZZZZZZ,1.3519025308939003e-14,1.352116000689658e-14,6333.992103616885,-2.13469795757649e-18

benchmarks/results/gates_2025-02-26_20.csv

@@ -0,0 +1,26 @@
+# Compiler versions: qiskit=1.3.3, cirq=1.4.1, pytket=1.40.0, ucc=0.4.2 # OS: Linux #26~22.04.1-Ubuntu SMP Thu Jul 11 22:33:04 UTC 2024, Architecture: 64bit, CPU Cores: 8
+compiler,circuit_name,raw_multiq_gates,compile_time,compiled_multiq_gates
+qiskit,qaoa_barabasi_albert,1176,0.22294974327087402,1176
+pytket,qaoa_barabasi_albert,1176,0.0227963924407959,1176
+ucc,qaoa_barabasi_albert,1176,0.6848599910736084,1176
+qiskit,qft,10050,0.5481865406036377,3244
+ucc,qft,10050,2.1165647506713867,2740
+pytket,qft,10050,0.3009669780731201,5890
+ucc,square_heisenberg,2160,0.43895673751831055,540
+cirq,qaoa_barabasi_albert,1176,11.067381858825684,1176
+qiskit,square_heisenberg,2160,0.4898693561553955,540
+ucc,qv,15000,10.083852052688599,14856
+qiskit,qv,15000,10.940625429153442,14856
+pytket,square_heisenberg,2160,0.09422016143798828,2160
+qiskit,prep_select,9744,1.6928818225860596,9708
+pytket,prep_select,9744,0.3097376823425293,9712
+ucc,qcnn,388,0.16162467002868652,388
+ucc,prep_select,9744,6.585411787033081,9702
+qiskit,qcnn,388,0.10459160804748535,388
+pytket,qcnn,388,0.02329230308532715,388
+cirq,qcnn,388,5.043301820755005,388
+cirq,square_heisenberg,2160,7.571340560913086,540
+pytket,qv,15000,4.507982969284058,15000
+cirq,qft,10050,39.714821100234985,4648
+cirq,prep_select,9744,92.13228917121887,9712
+cirq,qv,15000,224.17456364631653,14856

benchmarks/scripts/expval_benchmark.py

@@ -83,7 +83,9 @@ def simulate_density_matrix(circuit: qiskit.QuantumCircuit) -> np.ndarray:
         circuit, SINGLE_QUBIT_ERROR_RATE, TWO_QUBIT_ERROR_RATE
     )
     simulator = AerSimulator(
-        method="density_matrix", noise_model=depolarizing_noise
+        method="density_matrix",
+        noise_model=depolarizing_noise,
+        max_parallel_threads=1,
     )
     return simulator.run(circuit).result().data()["density_matrix"]
 
@@ -161,7 +163,7 @@ def fetch_pre_post_compiled_circuits(
 
 def get_heavy_bitstrings(circuit: qiskit.QuantumCircuit) -> Set[str]:
     """ "Determine the heavy bitstrings of the circuit."""
-    simulator = AerSimulator(method="statevector")
+    simulator = AerSimulator(method="statevector", max_parallel_threads=1)
     result = simulator.run(circuit, shots=1024).result()
     counts = list(result.get_counts().items())
     median = np.median([c for (_, c) in counts])
@@ -190,11 +192,10 @@ def estimate_heavy_output_prob(
             noise_model=create_depolarizing_noise_model(
                 circuit, SINGLE_QUBIT_ERROR_RATE, TWO_QUBIT_ERROR_RATE
             ),
+            max_parallel_threads=1,
         )
     else:
-        simulator = AerSimulator(
-            method="statevector",
-        )
+        simulator = AerSimulator(method="statevector", max_parallel_threads=1)
     result = simulator.run(circuit).result()
 
     heavy_counts = sum(

benchmarks/scripts/run_benchmarks.sh

@@ -38,13 +38,17 @@ for qasm_file in "${QASM_FILES[@]}"; do
     for compiler in "${COMPILERS[@]}"; do
         # Combine the common folder path with the QASM file
         full_qasm_file="${QASM_FOLDER}${qasm_file}"
-        
+
         # Build the command, passing the results folder as an argument
         command="python3 $(dirname "$0")/benchmark_script.py \"$full_qasm_file\" \"$compiler\" \"$RESULTS_FOLDER\""
 
         commands+=("$command")
     done
 done
+# Execute the timing benchmarks in parallel up to the specified number of parallel jobs
+# Intentionally run these first to avoid contention with expected value benchmarks below
+# that could impact timing results
+parallel -j "$PARALLELISM" ::: "${commands[@]}"
 
 QASM_EXPVAL_FILES=(
     "benchpress/qaoa_barabasi_albert_N10_3reps_basis_rz_rx_ry_cx.qasm"
@@ -55,17 +59,18 @@ QASM_EXPVAL_FILES=(
     "ucc/qcnn_N10_4layers_basis_rz_rx_ry_h_cx.qasm"
 )
 LOG_EXPVAL=false
+commands=()
 for qasm_file in "${QASM_EXPVAL_FILES[@]}"; do
     for compiler in "${COMPILERS[@]}"; do
         # Combine the common folder path with the QASM file
         full_qasm_file="${QASM_FOLDER}${qasm_file}"
-        
+
         # Build the command, passing the results folder as an argument
         command="poetry run python $(dirname "$0")/expval_benchmark.py \"$full_qasm_file\" \"$compiler\" \"$RESULTS_FOLDER\" $LOG_EXPVAL"
 
         commands+=("$command")
     done
 done
 
-# Execute all the commands in parallel up to the specified number of parallel jobs
+# Execute the expected value benchmarks in parallel up to the specified number of parallel jobs
 parallel -j "$PARALLELISM" ::: "${commands[@]}"