-
Notifications
You must be signed in to change notification settings - Fork 70
321 lines (284 loc) · 13.6 KB
/
e2e-preset-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
name: E2E Preset Test
on:
workflow_run:
workflows: ["Build and Push Preset Models"]
types:
- completed
workflow_dispatch:
inputs:
image_tag:
description: 'Image Tag'
required: true
env:
GO_VERSION: "1.20"
VERSION: 0.0.1
permissions:
id-token: write
contents: read
jobs:
setup:
if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
runs-on: self-hosted
outputs:
image_tag: ${{ steps.set_tag.outputs.image_tag }}
steps:
- name: Set Image Tag
id: set_tag
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.image_tag }}" ]]; then
echo "Using workflow dispatch to set image tag"
echo "image_tag=${{ github.event.inputs.image_tag }}" >> $GITHUB_OUTPUT
else
echo "Setting image tag based on version set"
echo "image_tag=${{ env.VERSION }}" >> $GITHUB_OUTPUT
fi
e2e-preset-tests:
if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
needs: setup
runs-on: [self-hosted, 'username:runner-3']
strategy:
fail-fast: false
matrix:
image:
- name: falcon-7b
node-count: 1
node-vm-size: Standard_NC12s_v3
node-osdisk-size: 100
- name: falcon-7b-instruct
node-count: 1
node-vm-size: Standard_NC12s_v3
node-osdisk-size: 100
- name: falcon-40b
node-count: 1
node-vm-size: Standard_NC96ads_A100_v4
node-osdisk-size: 400
- name: falcon-40b-instruct
node-count: 1
node-vm-size: Standard_NC96ads_A100_v4
node-osdisk-size: 400
- name: llama-2-7b
node-count: 1
node-vm-size: Standard_NC12s_v3
node-osdisk-size: 100
- name: llama-2-13b
node-count: 2
node-vm-size: Standard_NC12s_v3
node-osdisk-size: 150
# Uncomment once service/deployment made
# - name: llama-2-70b
# node-count: 2
# node-vm-size: Standard_NC96ads_A100_v4
# node-osdisk-size: 400
- name: llama-2-7b-chat
node-count: 1
node-vm-size: Standard_NC12s_v3
node-osdisk-size: 100
- name: llama-2-13b-chat
node-count: 2
node-vm-size: Standard_NC12s_v3
node-osdisk-size: 150
# Uncomment once service/deployment made
# - name: llama-2-70b-chat
# node-count: 2
# node-vm-size: Standard_NC96ads_A100_v4
# node-osdisk-size: 400
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- name: Get ACR Name
id: get_acr_name
run: |
# Set the ACR based on the tag value
if [[ "${{ needs.setup.outputs.image_tag }}" == "latest" ]]; then
echo "ACR_NAME=aimodelsregistry" >> $GITHUB_OUTPUT
else
echo "ACR_NAME=aimodelsregistrytest" >> $GITHUB_OUTPUT
fi
- name: Install Azure CLI latest
run: |
if ! which az > /dev/null; then
echo "Azure CLI not found. Installing..."
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
else
echo "Azure CLI already installed."
fi
- name: 'Az CLI login'
uses: azure/login@v1.6.0
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
allow-no-subscriptions: true
- name: 'Set subscription'
run: az account set --subscription ${{secrets.AZURE_SUBSCRIPTION_ID}}
- name: 'Check if Image exists in ACR'
id: check_image
run: |
ACR_NAME=${{ steps.get_acr_name.outputs.ACR_NAME }}
IMAGE_NAME=${{ matrix.image.name }}
TAG=${{ needs.setup.outputs.image_tag }}
TAGS=$(az acr repository show-tags -n $ACR_NAME --repository $IMAGE_NAME --output tsv)
if echo "$TAGS" | grep -q "^$TAG$"; then
echo "IMAGE_EXISTS=true" >> $GITHUB_OUTPUT
else
echo "IMAGE_EXISTS=false" >> $GITHUB_OUTPUT
echo "Image $IMAGE_NAME:$TAG not found in $ACR_NAME."
fi
- name: Set up kubectl context
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
az aks get-credentials --resource-group llm-test --name GitRunner
- name: Get Nodepool Name
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
id: get_nodepool_name
run: |
NAME_SUFFIX=${{ matrix.image.name }}
NAME_SUFFIX_WITHOUT_DASHES=${NAME_SUFFIX//-/} # Removing all '-' symbols
if [ ${#NAME_SUFFIX_WITHOUT_DASHES} -gt 12 ]; then
TRUNCATED_NAME_SUFFIX=${NAME_SUFFIX_WITHOUT_DASHES: -12}
else
TRUNCATED_NAME_SUFFIX=$NAME_SUFFIX_WITHOUT_DASHES
fi
echo "Nodepool Name: $TRUNCATED_NAME_SUFFIX"
echo "NODEPOOL_NAME=$TRUNCATED_NAME_SUFFIX" >> $GITHUB_OUTPUT
- name: Create Nodepool
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
NODEPOOL_EXIST=$(az aks nodepool show \
--name ${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }} \
--cluster-name GitRunner \
--resource-group llm-test \
--query 'name' -o tsv || echo "")
echo "NODEPOOL_EXIST: $NODEPOOL_EXIST"
if [ -z "$NODEPOOL_EXIST" ]; then
az aks nodepool add \
--name ${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }} \
--cluster-name GitRunner \
--resource-group llm-test \
--node-count ${{ matrix.image.node-count }} \
--node-vm-size ${{ matrix.image.node-vm-size }} \
--node-osdisk-size ${{ matrix.image.node-osdisk-size }} \
--labels pool=${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }} \
--node-taints sku=gpu:NoSchedule \
--aks-custom-headers UseGPUDedicatedVHD=true
else
NODEPOOL_STATE=$(az aks nodepool show \
--name ${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }} \
--cluster-name GitRunner \
--resource-group llm-test \
--query 'provisioningState' -o tsv)
echo "NODEPOOL_STATE: $NODEPOOL_STATE"
if [ "$NODEPOOL_STATE" != "Succeeded" ]; then
echo "Nodepool exists but is not in a Succeeded state. Please check manually."
exit 1
else
echo "Nodepool already exists and is in a running state."
fi
fi
- name: Create Service
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: kubectl apply -f presets/test/manifests/${{ matrix.image.name }}/${{ matrix.image.name }}-service.yaml
- name: Retrieve External Service IP
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
id: get_ip
run: |
while [[ -z $SERVICE_IP ]]; do
SERVICE_IP=$(kubectl get svc ${{ matrix.image.name }} -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
sleep 5
done
echo "Service IP is $SERVICE_IP"
echo "SERVICE_IP=$SERVICE_IP" >> $GITHUB_OUTPUT
- name: Replace IP and Deploy Statefulset to K8s
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
sed -i "s/MASTER_ADDR_HERE/${{ steps.get_ip.outputs.SERVICE_IP }}/g" presets/test/manifests/${{ matrix.image.name }}/${{ matrix.image.name }}-statefulset.yaml
sed -i "s/TAG_HERE/${{ needs.setup.outputs.image_tag }}/g" presets/test/manifests/${{ matrix.image.name }}/${{ matrix.image.name }}-statefulset.yaml
sed -i "s/REPO_HERE/${{ steps.get_acr_name.outputs.ACR_NAME }}/g" presets/test/manifests/${{ matrix.image.name }}/${{ matrix.image.name }}-statefulset.yaml
kubectl apply -f presets/test/manifests/${{ matrix.image.name }}/${{ matrix.image.name }}-statefulset.yaml
- name: Wait for Statefulset to be ready
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
kubectl rollout status statefulset/${{ matrix.image.name }}
- name: Test home endpoint
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
curl http://${{ steps.get_ip.outputs.SERVICE_IP }}:80/
- name: Test healthz endpoint
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
curl http://${{ steps.get_ip.outputs.SERVICE_IP }}:80/healthz
- name: Test inference endpoint
if: steps.check_image.outputs.IMAGE_EXISTS == 'true'
run: |
if [[ "${{ matrix.image.name }}" == *"llama"* && "${{ matrix.image.name }}" == *"-chat"* ]]; then
echo "Testing inference for ${{ matrix.image.name }}"
curl -X POST \
-H "Content-Type: application/json" \
-d '{
"input_data": {
"input_string": [
[
{
"role": "system",
"content": "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe."
},
{
"role": "user",
"content": "Write a brief birthday message to John"
}
]
]
}
}' \
http://${{ steps.get_ip.outputs.SERVICE_IP }}:80/chat
elif [[ "${{ matrix.image.name }}" == *"llama"* ]]; then
echo "Testing inference for ${{ matrix.image.name }}"
curl -X POST \
-H "Content-Type: application/json" \
-d '{
"prompts": [
"I believe the meaning of life is",
"Simply put, the theory of relativity states that ",
"A brief message congratulating the team on the launch: Hi everyone, I just ",
"Translate English to French: sea otter => loutre de mer, peppermint => menthe poivrée, plush girafe => girafe peluche, cheese =>"
],
"parameters": {
"max_gen_len": 128
}
}' \
http://${{ steps.get_ip.outputs.SERVICE_IP }}:80/generate
elif [[ "${{ matrix.image.name }}" == *"falcon"* ]]; then
echo "Testing inference for ${{ matrix.image.name }}"
curl -X POST \
-H "accept: application/json" \
-H "Content-Type: application/json" \
-d '{"prompt":"Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:","max_length":200,"min_length":0,"do_sample":true,"early_stopping":false,"num_beams":1,"num_beam_groups":1,"diversity_penalty":0.0,"temperature":1.0,"top_k":10,"top_p":1,"typical_p":1,"repetition_penalty":1,"length_penalty":1,"no_repeat_ngram_size":0,"encoder_no_repeat_ngram_size":0,"bad_words_ids":null,"num_return_sequences":1,"output_scores":false,"return_dict_in_generate":false,"forced_bos_token_id":null,"forced_eos_token_id":null,"remove_invalid_values":null}' \
http://${{ steps.get_ip.outputs.SERVICE_IP }}:80/chat
fi
- name: Cleanup
if: always()
run: |
# Check and Delete K8s Service if it exists
if kubectl get svc ${{ matrix.image.name }} > /dev/null 2>&1; then
kubectl delete svc ${{ matrix.image.name }}
fi
# Check and Delete K8s StatefulSet if it exists
if kubectl get statefulset ${{ matrix.image.name }} > /dev/null 2>&1; then
kubectl delete statefulset ${{ matrix.image.name }}
fi
# Check and Delete AKS Nodepool if it exists
if [ -n "${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }}" ]; then
NODEPOOL_EXIST=$(az aks nodepool show \
--name ${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }} \
--cluster-name GitRunner \
--resource-group llm-test \
--query 'name' -o tsv || echo "")
if [ -n "$NODEPOOL_EXIST" ]; then
az aks nodepool delete \
--name ${{ steps.get_nodepool_name.outputs.NODEPOOL_NAME }} \
--cluster-name GitRunner \
--resource-group llm-test
fi
fi