Allow number of nodes in CUDA graph to change

[agray3]

Previously the code would have failed to cope in the case that the number of nodes changes in an existing CUDA graph. This fixes the issue by removing an unnecessary conditional.

[agray3]

See discussion at #7302

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 554 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=8418.9ms p(95)=20007.87ms fails=, finish reason: stop=507 truncated=47
• Prompt processing (pp): avg=99.28tk/s p(95)=415.3tk/s
• Token generation (tg): avg=45.58tk/s p(95)=48.39tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=ag_fix_numnodes_cuda_graph commit=a2fb00de88522f6ba2e4e9f79d6b8ee046ac9ca4

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 554 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1717515149 --> 1717515775
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 392.7, 392.7, 392.7, 392.7, 392.7, 853.39, 853.39, 853.39, 853.39, 853.39, 837.25, 837.25, 837.25, 837.25, 837.25, 843.3, 843.3, 843.3, 843.3, 843.3, 891.38, 891.38, 891.38, 891.38, 891.38, 903.09, 903.09, 903.09, 903.09, 903.09, 897.99, 897.99, 897.99, 897.99, 897.99, 919.69, 919.69, 919.69, 919.69, 919.69, 911.11, 911.11, 911.11, 911.11, 911.11, 922.4, 922.4, 922.4, 922.4, 922.4, 917.46, 917.46, 917.46, 917.46, 917.46, 939.74, 939.74, 939.74, 939.74, 939.74, 859.3, 859.3, 859.3, 859.3, 859.3, 862.99, 862.99, 862.99, 862.99, 862.99, 820.39, 820.39, 820.39, 820.39, 820.39, 778.75, 778.75, 778.75, 778.75, 778.75, 785.87, 785.87, 785.87, 785.87, 785.87, 784.9, 784.9, 784.9, 784.9, 784.9, 802.01, 802.01, 802.01, 802.01, 802.01, 813.09, 813.09, 813.09, 813.09, 813.09, 814.57, 814.57, 814.57, 814.57, 814.57, 821.16, 821.16, 821.16, 821.16, 821.16, 825.26, 825.26, 825.26, 825.26, 825.26, 800.08, 800.08, 800.08, 800.08, 800.08, 801.7, 801.7, 801.7, 801.7, 801.7, 806.89, 806.89, 806.89, 806.89, 806.89, 812.14, 812.14, 812.14, 812.14, 812.14, 812.13, 812.13, 812.13, 812.13, 812.13, 813.12, 813.12, 813.12, 813.12, 813.12, 818.78, 818.78, 818.78, 818.78, 818.78, 819.54, 819.54, 819.54, 819.54, 819.54, 816.88, 816.88, 816.88, 816.88, 816.88, 822.44, 822.44, 822.44, 822.44, 822.44, 832.12, 832.12, 832.12, 832.12, 832.12, 836.44, 836.44, 836.44, 836.44, 836.44, 837.94, 837.94, 837.94, 837.94, 837.94, 820.05, 820.05, 820.05, 820.05, 820.05, 817.37, 817.37, 817.37, 817.37, 817.37, 817.61, 817.61, 817.61, 817.61, 817.61, 820.96, 820.96, 820.96, 820.96, 820.96, 822.34, 822.34, 822.34, 822.34, 822.34, 817.44, 817.44, 817.44, 817.44, 817.44, 825.06, 825.06, 825.06, 825.06, 825.06, 825.29, 825.29, 825.29, 825.29, 825.29, 824.25, 824.25, 824.25, 824.25, 824.25, 822.61, 822.61, 822.61, 822.61, 822.61, 821.89, 821.89, 821.89, 821.89, 821.89, 823.69, 823.69, 823.69, 823.69, 823.69, 823.08, 823.08, 823.08, 823.08, 823.08, 825.58, 825.58, 825.58, 825.58, 825.58, 825.47, 825.47, 825.47, 825.47, 825.47, 829.28, 829.28, 829.28, 829.28, 829.28, 824.88, 824.88, 824.88, 824.88, 824.88, 832.31, 832.31, 832.31, 832.31, 832.31, 833.62, 833.62, 833.62, 833.62, 833.62, 833.67, 833.67, 833.67, 833.67, 833.67, 835.42, 835.42, 835.42, 835.42, 835.42, 835.02, 835.02, 835.02, 835.02, 835.02, 833.26, 833.26, 833.26, 833.26, 833.26, 836.44, 836.44, 836.44, 836.44, 836.44, 837.22, 837.22, 837.22, 837.22]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 554 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1717515149 --> 1717515775
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 45.3, 45.3, 45.3, 45.3, 45.3, 41.67, 41.67, 41.67, 41.67, 41.67, 29.35, 29.35, 29.35, 29.35, 29.35, 27.5, 27.5, 27.5, 27.5, 27.5, 28.76, 28.76, 28.76, 28.76, 28.76, 30.76, 30.76, 30.76, 30.76, 30.76, 32.48, 32.48, 32.48, 32.48, 32.48, 32.98, 32.98, 32.98, 32.98, 32.98, 33.38, 33.38, 33.38, 33.38, 33.38, 33.18, 33.18, 33.18, 33.18, 33.18, 32.95, 32.95, 32.95, 32.95, 32.95, 32.98, 32.98, 32.98, 32.98, 32.98, 32.96, 32.96, 32.96, 32.96, 32.96, 32.44, 32.44, 32.44, 32.44, 32.44, 31.45, 31.45, 31.45, 31.45, 31.45, 30.08, 30.08, 30.08, 30.08, 30.08, 29.76, 29.76, 29.76, 29.76, 29.76, 29.82, 29.82, 29.82, 29.82, 29.82, 29.84, 29.84, 29.84, 29.84, 29.84, 29.84, 29.84, 29.84, 29.84, 29.84, 30.17, 30.17, 30.17, 30.17, 30.17, 30.26, 30.26, 30.26, 30.26, 30.26, 30.37, 30.37, 30.37, 30.37, 30.37, 30.55, 30.55, 30.55, 30.55, 30.55, 30.42, 30.42, 30.42, 30.42, 30.42, 30.8, 30.8, 30.8, 30.8, 30.8, 30.61, 30.61, 30.61, 30.61, 30.61, 30.58, 30.58, 30.58, 30.58, 30.58, 30.72, 30.72, 30.72, 30.72, 30.72, 31.01, 31.01, 31.01, 31.01, 31.01, 31.06, 31.06, 31.06, 31.06, 31.06, 31.28, 31.28, 31.28, 31.28, 31.28, 31.41, 31.41, 31.41, 31.41, 31.41, 31.33, 31.33, 31.33, 31.33, 31.33, 31.05, 31.05, 31.05, 31.05, 31.05, 30.74, 30.74, 30.74, 30.74, 30.74, 30.49, 30.49, 30.49, 30.49, 30.49, 30.19, 30.19, 30.19, 30.19, 30.19, 30.25, 30.25, 30.25, 30.25, 30.25, 30.54, 30.54, 30.54, 30.54, 30.54, 30.64, 30.64, 30.64, 30.64, 30.64, 30.66, 30.66, 30.66, 30.66, 30.66, 30.48, 30.48, 30.48, 30.48, 30.48, 30.15, 30.15, 30.15, 30.15, 30.15, 29.78, 29.78, 29.78, 29.78, 29.78, 28.69, 28.69, 28.69, 28.69, 28.69, 28.63, 28.63, 28.63, 28.63, 28.63, 28.63, 28.63, 28.63, 28.63, 28.63, 28.6, 28.6, 28.6, 28.6, 28.6, 28.65, 28.65, 28.65, 28.65, 28.65, 28.72, 28.72, 28.72, 28.72, 28.72, 28.8, 28.8, 28.8, 28.8, 28.8, 28.79, 28.79, 28.79, 28.79, 28.79, 28.83, 28.83, 28.83, 28.83, 28.83, 28.82, 28.82, 28.82, 28.82, 28.82, 28.88, 28.88, 28.88, 28.88, 28.88, 29.11, 29.11, 29.11, 29.11, 29.11, 29.17, 29.17, 29.17, 29.17, 29.17, 29.26, 29.26, 29.26, 29.26, 29.26, 29.34, 29.34, 29.34, 29.34, 29.34, 29.41, 29.41, 29.41, 29.41]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 554 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1717515149 --> 1717515775
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11, 0.11, 0.11, 0.11, 0.11, 0.37, 0.37, 0.37, 0.37, 0.37, 0.26, 0.26, 0.26, 0.26, 0.26, 0.14, 0.14, 0.14, 0.14, 0.14, 0.21, 0.21, 0.21, 0.21, 0.21, 0.08, 0.08, 0.08, 0.08, 0.08, 0.12, 0.12, 0.12, 0.12, 0.12, 0.2, 0.2, 0.2, 0.2, 0.2, 0.22, 0.22, 0.22, 0.22, 0.22, 0.22, 0.22, 0.22, 0.22, 0.22, 0.2, 0.2, 0.2, 0.2, 0.2, 0.15, 0.15, 0.15, 0.15, 0.15, 0.34, 0.34, 0.34, 0.34, 0.34, 0.36, 0.36, 0.36, 0.36, 0.36, 0.4, 0.4, 0.4, 0.4, 0.4, 0.2, 0.2, 0.2, 0.2, 0.2, 0.24, 0.24, 0.24, 0.24, 0.24, 0.14, 0.14, 0.14, 0.14, 0.14, 0.29, 0.29, 0.29, 0.29, 0.29, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.23, 0.23, 0.23, 0.23, 0.23, 0.12, 0.12, 0.12, 0.12, 0.12, 0.27, 0.27, 0.27, 0.27, 0.27, 0.14, 0.14, 0.14, 0.14, 0.14, 0.19, 0.19, 0.19, 0.19, 0.19, 0.23, 0.23, 0.23, 0.23, 0.23, 0.16, 0.16, 0.16, 0.16, 0.16, 0.13, 0.13, 0.13, 0.13, 0.13, 0.16, 0.16, 0.16, 0.16, 0.16, 0.17, 0.17, 0.17, 0.17, 0.17, 0.12, 0.12, 0.12, 0.12, 0.12, 0.16, 0.16, 0.16, 0.16, 0.16, 0.3, 0.3, 0.3, 0.3, 0.3, 0.38, 0.38, 0.38, 0.38, 0.38, 0.5, 0.5, 0.5, 0.5, 0.5, 0.34, 0.34, 0.34, 0.34, 0.34, 0.12, 0.12, 0.12, 0.12, 0.12, 0.14, 0.14, 0.14, 0.14, 0.14, 0.11, 0.11, 0.11, 0.11, 0.11, 0.13, 0.13, 0.13, 0.13, 0.13, 0.32, 0.32, 0.32, 0.32, 0.32, 0.6, 0.6, 0.6, 0.6, 0.6, 0.57, 0.57, 0.57, 0.57, 0.57, 0.57, 0.57, 0.57, 0.57, 0.57, 0.43, 0.43, 0.43, 0.43, 0.43, 0.23, 0.23, 0.23, 0.23, 0.23, 0.27, 0.27, 0.27, 0.27, 0.27, 0.19, 0.19, 0.19, 0.19, 0.19, 0.17, 0.17, 0.17, 0.17, 0.17, 0.13, 0.13, 0.13, 0.13, 0.13, 0.19, 0.19, 0.19, 0.19, 0.19, 0.14, 0.14, 0.14, 0.14, 0.14, 0.26, 0.26, 0.26, 0.26, 0.26, 0.11, 0.11, 0.11, 0.11, 0.11, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.14, 0.14, 0.14, 0.14, 0.14, 0.13, 0.13, 0.13, 0.13, 0.13, 0.19, 0.19, 0.19, 0.19, 0.19, 0.14, 0.14, 0.14, 0.14]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 554 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1717515149 --> 1717515775
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 3.0, 3.0, 3.0, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 4.0, 4.0, 4.0, 4.0, 4.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 6.0, 6.0, 6.0, 6.0, 6.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 2.0, 2.0, 2.0, 2.0, 8.0, 8.0, 8.0, 8.0, 8.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 3.0, 3.0, 3.0, 3.0]
                    

Loading