Skip to content

Commit

Permalink
chore: dont install and run metrics stack on kind network smoke (#11366)
Browse files Browse the repository at this point in the history
  • Loading branch information
Maddiaa0 authored Jan 20, 2025
1 parent b32d9a1 commit f66db63
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 36 deletions.
9 changes: 8 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,14 @@ jobs:
docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
cd yarn-project/end-to-end
NAMESPACE=smoke FRESH_INSTALL=true VALUES_FILE=ci-smoke.yaml ./scripts/network_test.sh ./src/spartan/smoke.test.ts
export INSTALL_CHAOS_MESH=false
export INSTALL_METRICS=false
export NAMESPACE=smoke
export FRESH_INSTALL=true
export VALUES_FILE=ci-smoke.yaml
./scripts/network_test.sh ./src/spartan/smoke.test.ts
- name: Copy Network Logs
if: always()
run: scripts/copy_from_tester yarn-project/end-to-end/scripts/network-test.log network-test.log || true
Expand Down
16 changes: 14 additions & 2 deletions spartan/scripts/setup_local_k8s.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ set -e

SCRIPT_DIR="$(dirname $(realpath -s "${BASH_SOURCE[0]}"))"

# Selectively install metrics and chaos mesh
INSTALL_METRICS=${INSTALL_METRICS:-true}
INSTALL_CHAOS_MESH=${INSTALL_CHAOS_MESH:-true}

# exit if we are not on linux amd64
if [ "$(uname)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then
echo "This script is only supported on Linux amd64"
Expand Down Expand Up @@ -61,5 +65,13 @@ fi

kubectl config use-context kind-kind || true

"$SCRIPT_DIR"/../chaos-mesh/install.sh
"$SCRIPT_DIR"/../metrics/install-kind.sh
if [ "$INSTALL_CHAOS_MESH" = "true" ]; then
echo "Installing chaos mesh"
"$SCRIPT_DIR"/../chaos-mesh/install.sh
fi

if [ "$INSTALL_METRICS" = "true" ]; then
echo "Installing metrics"
"$SCRIPT_DIR"/../metrics/install-kind.sh
fi

26 changes: 18 additions & 8 deletions yarn-project/end-to-end/scripts/network_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ FRESH_INSTALL="${FRESH_INSTALL:-false}"
AZTEC_DOCKER_TAG=${AZTEC_DOCKER_TAG:-$(git rev-parse HEAD)}
INSTALL_TIMEOUT=${INSTALL_TIMEOUT:-30m}
CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-false}
export INSTALL_CHAOS_MESH=${INSTALL_CHAOS_MESH:-true}
export INSTALL_METRICS=${INSTALL_METRICS:-true}

# Check required environment variable
if [ -z "${NAMESPACE:-}" ]; then
Expand Down Expand Up @@ -125,7 +127,7 @@ function cleanup() {
trap cleanup SIGINT SIGTERM EXIT

# if we don't have a chaos values, remove any existing chaos experiments
if [ -z "${CHAOS_VALUES:-}" ]; then
if [ -z "${CHAOS_VALUES:-}" ] && [ "$INSTALL_CHAOS_MESH" = "true" ]; then
echo "Deleting existing network chaos experiments..."
kubectl delete networkchaos --all --all-namespaces
fi
Expand All @@ -152,20 +154,28 @@ PXE_PORT=$(echo $FREE_PORTS | awk '{print $1}')
ANVIL_PORT=$(echo $FREE_PORTS | awk '{print $2}')
METRICS_PORT=$(echo $FREE_PORTS | awk '{print $3}')

GRAFANA_PASSWORD=$(kubectl get secrets -n metrics metrics-grafana -o jsonpath='{.data.admin-password}' | base64 --decode)
if [ "$INSTALL_METRICS" = "true" ]; then
GRAFANA_PASSWORD=$(kubectl get secrets -n metrics metrics-grafana -o jsonpath='{.data.admin-password}' | base64 --decode)
else
GRAFANA_PASSWORD=""
fi

# Namespace variable (assuming it's set)
NAMESPACE=${NAMESPACE:-default}

# If we are unable to apply network shaping, as we cannot change existing chaos configurations, then delete existing configurations and try again
if ! handle_network_shaping; then
echo "Deleting existing network chaos experiments..."
kubectl delete networkchaos --all --all-namespaces

if [ "$INSTALL_CHAOS_MESH" = "true" ]; then
if ! handle_network_shaping; then
echo "Error: failed to apply network shaping configuration!"
exit 1
echo "Deleting existing network chaos experiments..."
kubectl delete networkchaos --all --all-namespaces

if ! handle_network_shaping; then
echo "Error: failed to apply network shaping configuration!"
exit 1
fi
fi
else
echo "Skipping network chaos configuration (INSTALL_CHAOS_MESH=false)"
fi

# Get the values from the values file
Expand Down
25 changes: 1 addition & 24 deletions yarn-project/end-to-end/src/spartan/smoke.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,12 @@ import { RollupAbi } from '@aztec/l1-artifacts';
import { createPublicClient, getAddress, getContract, http } from 'viem';
import { foundry } from 'viem/chains';

import { type AlertConfig } from '../quality_of_service/alert_checker.js';
import { isK8sConfig, runAlertCheck, setupEnvironment, startPortForward } from './utils.js';
import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js';

const config = setupEnvironment(process.env);

const debugLogger = createLogger('e2e:spartan-test:smoke');

// QoS alerts for when we are running in k8s
const qosAlerts: AlertConfig[] = [
{
alert: 'SequencerTimeToCollectAttestations',
expr: 'avg_over_time(aztec_sequencer_time_to_collect_attestations[2m]) > 2500',
labels: { severity: 'error' },
for: '10m',
annotations: {},
},
];

describe('smoke test', () => {
let pxe: PXE;
beforeAll(async () => {
Expand All @@ -35,23 +23,12 @@ describe('smoke test', () => {
hostPort: config.HOST_PXE_PORT,
});
PXE_URL = `http://127.0.0.1:${config.HOST_PXE_PORT}`;

await startPortForward({
resource: `svc/metrics-grafana`,
namespace: 'metrics',
containerPort: config.CONTAINER_METRICS_PORT,
hostPort: config.HOST_METRICS_PORT,
});
} else {
PXE_URL = config.PXE_URL;
}
pxe = await createCompatibleClient(PXE_URL, debugLogger);
});

afterAll(async () => {
await runAlertCheck(config, qosAlerts, debugLogger);
});

it('should be able to get node enr', async () => {
const info = await pxe.getNodeInfo();
expect(info).toBeDefined();
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/end-to-end/src/spartan/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const k8sLocalConfigSchema = z.object({
CONTAINER_ETHEREUM_PORT: z.coerce.number().default(8545),
HOST_METRICS_PORT: z.coerce.number().min(1, 'HOST_METRICS_PORT env variable must be set'),
CONTAINER_METRICS_PORT: z.coerce.number().default(80),
GRAFANA_PASSWORD: z.string().min(1, 'GRAFANA_PASSWORD env variable must be set'),
GRAFANA_PASSWORD: z.string().optional(),
METRICS_API_PATH: z.string().default('/api/datasources/proxy/uid/spartan-metrics-prometheus/api/v1'),
SPARTAN_DIR: z.string().min(1, 'SPARTAN_DIR env variable must be set'),
K8S: z.literal('local'),
Expand Down

0 comments on commit f66db63

Please sign in to comment.