From 28f53a77063bd865f883fb559014f2c87ff5e05e Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 14:54:59 -0500 Subject: [PATCH 01/25] chore(ci): test docker images --- .circleci/config.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f3ecbd75..9b999d4ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker, ubuntu_musl, macos, windows] + platform: [ubuntu_gnu_docker, ubuntu_gnu_docker_dind, ubuntu_musl, macos, windows] rust_channel: [stable] command: [unit-test] @@ -37,6 +37,16 @@ workflows: requires: - Cargo tests (<< matrix.rust_channel >> rust on ubuntu_gnu_docker) + - xtask: + name: Supergraph demo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) + matrix: + parameters: + platform: [ubuntu_gnu_docker_dind] + rust_channel: [stable] + command: [integration-test] + requires: + - Cargo tests (<< matrix.rust_channel >> rust on ubuntu_gnu_docker_dind) + jobs: xtask: parameters: @@ -51,10 +61,12 @@ jobs: enum: ["lint", "unit-test", "integration-test"] executor: << parameters.platform >> steps: - - checkout - install_system_deps: platform: << parameters.platform >> rust_channel: << parameters.rust_channel >> + - run: + command: ldd --version + - checkout - exec_xtask: command: << parameters.command >> platform: << parameters.platform >> @@ -67,6 +79,12 @@ executors: resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" + ubuntu_gnu_docker_dind: &ubuntu_gnu_docker_dind_executor + docker: + - image: circleci/buildpack-deps:buster-curl-dind + resource_class: xlarge + environment: + XTASK_TARGET: "x86_64-unknown-linux-gnu" # This is only used to run supergraph-demo since you can't run Docker from Docker ubuntu_gnu_vm: &ubuntu_gnu_vm_executor From 3529b65d155ac67eac98416eb8c88a8917c32dbe Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 14:57:27 -0500 Subject: [PATCH 02/25] try without dind --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b999d4ac..e77ed1c34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker, ubuntu_gnu_docker_dind, ubuntu_musl, macos, windows] + platform: [ubuntu_gnu_docker, buster_docker, ubuntu_musl, macos, windows] rust_channel: [stable] command: [unit-test] @@ -41,11 +41,11 @@ workflows: name: Supergraph demo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker_dind] + platform: [buster_docker] rust_channel: [stable] command: [integration-test] requires: - - Cargo tests (<< matrix.rust_channel >> rust on ubuntu_gnu_docker_dind) + - Cargo tests (<< matrix.rust_channel >> rust on buster_docker) jobs: xtask: @@ -79,9 +79,9 @@ executors: resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" - ubuntu_gnu_docker_dind: &ubuntu_gnu_docker_dind_executor + buster_docker: &ubuntu_gnu_docker_executor docker: - - image: circleci/buildpack-deps:buster-curl-dind + - image: circleci/buildpack-deps:buster-curl resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" From 47540fb3aefcef50e9d65a5ca2141a68993f483a Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 14:58:29 -0500 Subject: [PATCH 03/25] put checkout back where it was --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e77ed1c34..385609c77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,12 +61,12 @@ jobs: enum: ["lint", "unit-test", "integration-test"] executor: << parameters.platform >> steps: + - checkout - install_system_deps: platform: << parameters.platform >> rust_channel: << parameters.rust_channel >> - run: command: ldd --version - - checkout - exec_xtask: command: << parameters.command >> platform: << parameters.platform >> From 7b71de5af79e9e9cf156c5282fb6071ca97f952c Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 15:02:33 -0500 Subject: [PATCH 04/25] dont duplicate executor names --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 385609c77..d9f7e720c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ executors: resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" - buster_docker: &ubuntu_gnu_docker_executor + buster_docker: &buster_docker_executor docker: - image: circleci/buildpack-deps:buster-curl resource_class: xlarge @@ -130,6 +130,7 @@ commands: condition: or: - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] + - equal: [ *buster_docker_executor, << parameters.platform >> ] - equal: [ *ubuntu_gnu_vm_executor, << parameters.platform >> ] - equal: [ *ubuntu_musl_executor, << parameters.platform >> ] steps: @@ -148,7 +149,9 @@ commands: command: sudo apt-get install -y musl-tools - when: condition: - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] + or: + - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] + - equal: [ *buster_docker_executor, << parameters.platform >> ] steps: - node/install: node-version: "14.17.1" From 8c17980a831498e702c4ebd0dc8d16ea66909ac7 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 15:06:48 -0500 Subject: [PATCH 05/25] install build-essential --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9f7e720c..5dc9a6f2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,6 +140,9 @@ commands: - run: name: Install OpenSSL command: sudo apt-get install -y libssl-dev + - run: + name: Install build-essential + command: sudo apt-get install -y build-essential - when: condition: equal: [ *ubuntu_musl_executor, << parameters.platform >> ] From e0f024bf9c8130643549208f5ff7045e7b7f3bbc Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 15:10:05 -0500 Subject: [PATCH 06/25] install pkg-config --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5dc9a6f2f..a0363786e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,9 +140,14 @@ commands: - run: name: Install OpenSSL command: sudo apt-get install -y libssl-dev + - when: + condition: + equal: [ *buster_docker_executor, << parameters.platform >> ] + steps: - run: - name: Install build-essential - command: sudo apt-get install -y build-essential + name: Install build-essential and pkg-config + command: sudo apt-get install -y build-essential pkg-config + - when: condition: equal: [ *ubuntu_musl_executor, << parameters.platform >> ] From 3c7d74fb20a1331eb44b803404f0028751f60269 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 15:26:25 -0500 Subject: [PATCH 07/25] chore(ci): test rover's glibc version --- .circleci/config.yml | 15 ++++++++-- check_glibc.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100755 check_glibc.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a0363786e..eca8cb798 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,8 +65,6 @@ jobs: - install_system_deps: platform: << parameters.platform >> rust_channel: << parameters.rust_channel >> - - run: - command: ldd --version - exec_xtask: command: << parameters.command >> platform: << parameters.platform >> @@ -137,6 +135,9 @@ commands: - run: name: Update apt repositories command: sudo apt-get update + - run: + name: Check glibc version + command: ldd --version - run: name: Install OpenSSL command: sudo apt-get install -y libssl-dev @@ -177,6 +178,16 @@ commands: - install_rust_toolchain: rust_channel: << parameters.rust_channel >> platform: << parameters.platform >> + - when: + condition: + or: + - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] + - equal: [ *buster_docker_executor, << parameters.platform >> ] + - equal: [ *ubuntu_gnu_vm_executor, << parameters.platform >> ] + steps: + - run: + name: Check Rover's glibc version + command: ./check_glibc.sh ./target/debug/$XTASK_TARGET/rover install_rust_toolchain: parameters: diff --git a/check_glibc.sh b/check_glibc.sh new file mode 100755 index 000000000..4756fb9ad --- /dev/null +++ b/check_glibc.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# This scripts checks if Rover's compiled executable only asks for +# supported versions of glibc + +# source: https://gist.github.com/fasterthanlime/17e002a8f5e0f189861c + +# usage: ./check_glibc.sh ./target/debug/rover + +MAX_VER=2.18 + +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +BINARY=$1 + +# Version comparison function in bash +vercomp () { + if [[ $1 == $2 ]] + then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +IFS=" +" +VERS=$(objdump -T $BINARY | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -u) + +for VER in $VERS; do + vercomp $VER $MAX_VER + COMP=$? + if [[ $COMP -eq 1 ]]; then + echo "Error! ${BINARY} requests GLIBC ${VER}, which is higher than target ${MAX_VER}" + echo "Affected symbols:" + objdump -T $BINARY | grep GLIBC_${VER} + echo "Looking for symbols in libraries..." + for LIBRARY in $(ldd $BINARY | cut -d ' ' -f 3); do + echo $LIBRARY + objdump -T $LIBRARY | fgrep GLIBC_${VER} + done + exit 27 + else + echo "Found version ${VER}" + fi +done \ No newline at end of file From 2683f0507618020ba9c130199c9027c08bbbf340 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 15:48:29 -0500 Subject: [PATCH 08/25] wip: add glibc checker to tests --- .circleci/config.yml | 20 -------------------- xtask/src/commands/unit_test.rs | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eca8cb798..aa857e99f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,16 +37,6 @@ workflows: requires: - Cargo tests (<< matrix.rust_channel >> rust on ubuntu_gnu_docker) - - xtask: - name: Supergraph demo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) - matrix: - parameters: - platform: [buster_docker] - rust_channel: [stable] - command: [integration-test] - requires: - - Cargo tests (<< matrix.rust_channel >> rust on buster_docker) - jobs: xtask: parameters: @@ -178,16 +168,6 @@ commands: - install_rust_toolchain: rust_channel: << parameters.rust_channel >> platform: << parameters.platform >> - - when: - condition: - or: - - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] - - equal: [ *buster_docker_executor, << parameters.platform >> ] - - equal: [ *ubuntu_gnu_vm_executor, << parameters.platform >> ] - steps: - - run: - name: Check Rover's glibc version - command: ./check_glibc.sh ./target/debug/$XTASK_TARGET/rover install_rust_toolchain: parameters: diff --git a/xtask/src/commands/unit_test.rs b/xtask/src/commands/unit_test.rs index de75979c8..9d6cb5a9c 100644 --- a/xtask/src/commands/unit_test.rs +++ b/xtask/src/commands/unit_test.rs @@ -1,8 +1,12 @@ use anyhow::Result; +use camino::Utf8PathBuf; use structopt::StructOpt; use crate::target::{Target, POSSIBLE_TARGETS}; -use crate::tools::CargoRunner; +use crate::tools::{CargoRunner, Runner}; +use crate::utils::PKG_PROJECT_ROOT; + +use std::str::FromStr; #[derive(Debug, StructOpt)] pub struct UnitTest { @@ -15,6 +19,15 @@ impl UnitTest { pub fn run(&self, verbose: bool) -> Result<()> { let mut cargo_runner = CargoRunner::new(verbose)?; cargo_runner.test(&self.target)?; + + let check_glibc_script = "./check_glibc.sh".to_string(); + let runner = Runner { + verbose, + tool_name: check_glibc_script.clone(), + tool_exe: Utf8PathBuf::from_str(&check_glibc_script)?, + }; + let bin_path = format!("./target/{}/debug/rover", &self.target); + runner.exec(&[&bin_path], &PKG_PROJECT_ROOT, None)?; Ok(()) } } From b056b2e7af2e3349fe36b05b15399eae9f0e5038 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 15:53:34 -0500 Subject: [PATCH 09/25] wip: only check glibc in CI --- xtask/src/commands/unit_test.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/xtask/src/commands/unit_test.rs b/xtask/src/commands/unit_test.rs index 9d6cb5a9c..bb9dd31ea 100644 --- a/xtask/src/commands/unit_test.rs +++ b/xtask/src/commands/unit_test.rs @@ -6,7 +6,7 @@ use crate::target::{Target, POSSIBLE_TARGETS}; use crate::tools::{CargoRunner, Runner}; use crate::utils::PKG_PROJECT_ROOT; -use std::str::FromStr; +use std::{env, str::FromStr}; #[derive(Debug, StructOpt)] pub struct UnitTest { @@ -20,14 +20,19 @@ impl UnitTest { let mut cargo_runner = CargoRunner::new(verbose)?; cargo_runner.test(&self.target)?; - let check_glibc_script = "./check_glibc.sh".to_string(); - let runner = Runner { - verbose, - tool_name: check_glibc_script.clone(), - tool_exe: Utf8PathBuf::from_str(&check_glibc_script)?, - }; - let bin_path = format!("./target/{}/debug/rover", &self.target); - runner.exec(&[&bin_path], &PKG_PROJECT_ROOT, None)?; + if let Target::GnuLinux = self.target { + if env::var_os("CI").is_some() { + let check_glibc_script = "./check_glibc.sh".to_string(); + let runner = Runner { + verbose, + tool_name: check_glibc_script.clone(), + tool_exe: Utf8PathBuf::from_str(&check_glibc_script)?, + }; + let bin_path = format!("./target/{}/debug/rover", &self.target); + runner.exec(&[&bin_path], &PKG_PROJECT_ROOT, None)?; + } + } + Ok(()) } } From f6af50228684d984a770cd2594ca2ce6b8377f0e Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Mon, 13 Sep 2021 16:00:31 -0500 Subject: [PATCH 10/25] wip: add xenial image to matrix --- .circleci/config.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aa857e99f..0d15f5cf3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker, buster_docker, ubuntu_musl, macos, windows] + platform: [ubuntu_gnu_docker, buster_docker, xenial_docker, ubuntu_musl, macos, windows] rust_channel: [stable] command: [unit-test] @@ -73,6 +73,12 @@ executors: resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" + xenial_docker: &xenial_docker_executor + docker: + - image: circleci/buildpack-deps:xenial-curl + resource_class: xlarge + environment: + XTASK_TARGET: "x86_64-unknown-linux-gnu" # This is only used to run supergraph-demo since you can't run Docker from Docker ubuntu_gnu_vm: &ubuntu_gnu_vm_executor @@ -119,6 +125,7 @@ commands: or: - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] - equal: [ *buster_docker_executor, << parameters.platform >> ] + - equal: [ *xenial_docker_executor, << parameters.platform >> ] - equal: [ *ubuntu_gnu_vm_executor, << parameters.platform >> ] - equal: [ *ubuntu_musl_executor, << parameters.platform >> ] steps: @@ -133,7 +140,9 @@ commands: command: sudo apt-get install -y libssl-dev - when: condition: - equal: [ *buster_docker_executor, << parameters.platform >> ] + or: + - equal: [ *buster_docker_executor, << parameters.platform >> ] + - equal: [ *xenial_docker_executor, << parameters.platform >> ] steps: - run: name: Install build-essential and pkg-config @@ -150,7 +159,6 @@ commands: condition: or: - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] - - equal: [ *buster_docker_executor, << parameters.platform >> ] steps: - node/install: node-version: "14.17.1" From 2508d9d89b97cf323216dba8e372162fd8082d77 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 10:17:58 -0500 Subject: [PATCH 11/25] try centos image --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d15f5cf3..0b3db5fd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker, buster_docker, xenial_docker, ubuntu_musl, macos, windows] + platform: [ubuntu_gnu_docker, buster_docker, xenial_docker, centos_docker, ubuntu_musl, macos, windows] rust_channel: [stable] command: [unit-test] @@ -79,6 +79,12 @@ executors: resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" + centos_docker: ¢os_docker_executor + docker: + - image: centos/7 + resource_class: xlarge + environment: + XTASK_TARGET: "x86_64-unknown-linux-gnu" # This is only used to run supergraph-demo since you can't run Docker from Docker ubuntu_gnu_vm: &ubuntu_gnu_vm_executor @@ -128,6 +134,7 @@ commands: - equal: [ *xenial_docker_executor, << parameters.platform >> ] - equal: [ *ubuntu_gnu_vm_executor, << parameters.platform >> ] - equal: [ *ubuntu_musl_executor, << parameters.platform >> ] + - equal: [ *centos_docker_executor, << parameters.platform >> ] steps: - run: name: Update apt repositories @@ -143,6 +150,7 @@ commands: or: - equal: [ *buster_docker_executor, << parameters.platform >> ] - equal: [ *xenial_docker_executor, << parameters.platform >> ] + - equal: [ *centos_docker_executor, << parameters.platform >> ] steps: - run: name: Install build-essential and pkg-config From 66ea7a64d2367b6aeb30b62da82790522166bd2b Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 10:59:13 -0500 Subject: [PATCH 12/25] try centos:7 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b3db5fd3..4aa241000 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,7 @@ executors: XTASK_TARGET: "x86_64-unknown-linux-gnu" centos_docker: ¢os_docker_executor docker: - - image: centos/7 + - image: centos:7 resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" From 43ee88fc66d96d34b9217c431e5f4f27fd561239 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:14:10 -0500 Subject: [PATCH 13/25] chore(ci): use yum for centos packages --- .circleci/config.yml | 75 ++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4aa241000..bf4cf7d87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ workflows: name: Lint (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker] + platform: [centos] rust_channel: [stable] command: [lint] test: @@ -23,7 +23,7 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_docker, buster_docker, xenial_docker, centos_docker, ubuntu_musl, macos, windows] + platform: [centos, musl, macos, windows] rust_channel: [stable] command: [unit-test] @@ -31,11 +31,11 @@ workflows: name: Supergraph demo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [ubuntu_gnu_vm] + platform: [ubuntu] rust_channel: [stable] command: [integration-test] requires: - - Cargo tests (<< matrix.rust_channel >> rust on ubuntu_gnu_docker) + - Cargo tests (<< matrix.rust_channel >> rust on centos) jobs: xtask: @@ -56,45 +56,19 @@ jobs: platform: << parameters.platform >> rust_channel: << parameters.rust_channel >> - exec_xtask: - command: << parameters.command >> platform: << parameters.platform >> + command: << parameters.command >> # The machines we use to run our workflows on executors: - ubuntu_gnu_docker: &ubuntu_gnu_docker_executor - docker: - - image: cimg/base:stable - resource_class: xlarge - environment: - XTASK_TARGET: "x86_64-unknown-linux-gnu" - buster_docker: &buster_docker_executor - docker: - - image: circleci/buildpack-deps:buster-curl - resource_class: xlarge - environment: - XTASK_TARGET: "x86_64-unknown-linux-gnu" - xenial_docker: &xenial_docker_executor - docker: - - image: circleci/buildpack-deps:xenial-curl - resource_class: xlarge - environment: - XTASK_TARGET: "x86_64-unknown-linux-gnu" - centos_docker: ¢os_docker_executor + centos: ¢os_executor docker: - image: centos:7 resource_class: xlarge environment: - XTASK_TARGET: "x86_64-unknown-linux-gnu" + XTASK_TARGET: "x86_64-unknown-linux-ubuntu" - # This is only used to run supergraph-demo since you can't run Docker from Docker - ubuntu_gnu_vm: &ubuntu_gnu_vm_executor - machine: - image: ubuntu-1604:202104-01 - resource_class: xlarge - environment: - XTASK_TARGET: "x86_64-unknown-linux-gnu" - - ubuntu_musl: &ubuntu_musl_executor + musl: &musl_executor docker: - image: cimg/base:stable resource_class: xlarge @@ -116,6 +90,14 @@ executors: environment: XTASK_TARGET: "x86_64-pc-windows-msvc" + # This is only used to run supergraph-demo since you can't run Docker from Docker + ubuntu: &ubuntu_executor + machine: + image: ubuntu-2004:202010-01 + resource_class: xlarge + environment: + XTASK_TARGET: "x86_64-unknown-linux-ubuntu" + # reusable command snippets can be referred to in any `steps` object commands: install_system_deps: @@ -129,12 +111,8 @@ commands: - when: condition: or: - - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] - - equal: [ *buster_docker_executor, << parameters.platform >> ] - - equal: [ *xenial_docker_executor, << parameters.platform >> ] - - equal: [ *ubuntu_gnu_vm_executor, << parameters.platform >> ] - - equal: [ *ubuntu_musl_executor, << parameters.platform >> ] - - equal: [ *centos_docker_executor, << parameters.platform >> ] + - equal: [ *ubuntu_executor, << parameters.platform >> ] + - equal: [ *musl_executor, << parameters.platform >> ] steps: - run: name: Update apt repositories @@ -145,20 +123,21 @@ commands: - run: name: Install OpenSSL command: sudo apt-get install -y libssl-dev + - when: condition: - or: - - equal: [ *buster_docker_executor, << parameters.platform >> ] - - equal: [ *xenial_docker_executor, << parameters.platform >> ] - - equal: [ *centos_docker_executor, << parameters.platform >> ] + equal: [ *centos_executor, << parameters.platform >> ] steps: - run: - name: Install build-essential and pkg-config - command: sudo apt-get install -y build-essential pkg-config + name: Upgrade yum packages + command: yum update + - run: + name: Check glibc version + command: ldd --version - when: condition: - equal: [ *ubuntu_musl_executor, << parameters.platform >> ] + equal: [ *musl_executor, << parameters.platform >> ] steps: - run: name: Install musl-tools @@ -166,7 +145,7 @@ commands: - when: condition: or: - - equal: [ *ubuntu_gnu_docker_executor, << parameters.platform >> ] + - equal: [ *ubuntu_executor, << parameters.platform >> ] steps: - node/install: node-version: "14.17.1" From 69fe1e4ecde33bbc567f6343409d6a2aa8052366 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:16:24 -0500 Subject: [PATCH 14/25] chore(ci): add -y to yum commands --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf4cf7d87..727382cda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,8 +129,8 @@ commands: equal: [ *centos_executor, << parameters.platform >> ] steps: - run: - name: Upgrade yum packages - command: yum update + name: Update and upgrade yum packages + command: yum -y update && yum -y upgrade - run: name: Check glibc version command: ldd --version From 815ce437bdeb83d54825aba784e1de2c13b35750 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:18:11 -0500 Subject: [PATCH 15/25] chore(ci): fix up bad find and replace --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 727382cda..cf5459e44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,7 +66,7 @@ executors: - image: centos:7 resource_class: xlarge environment: - XTASK_TARGET: "x86_64-unknown-linux-ubuntu" + XTASK_TARGET: "x86_64-unknown-linux-gnu" musl: &musl_executor docker: @@ -96,7 +96,7 @@ executors: image: ubuntu-2004:202010-01 resource_class: xlarge environment: - XTASK_TARGET: "x86_64-unknown-linux-ubuntu" + XTASK_TARGET: "x86_64-unknown-linux-gnu" # reusable command snippets can be referred to in any `steps` object commands: From 19ae9b92c2427dec7fc1127803ce7c4147a32c3e Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:20:29 -0500 Subject: [PATCH 16/25] chore(ci): install cmake for centos --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf5459e44..0cbc3fd8c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,6 +131,9 @@ commands: - run: name: Update and upgrade yum packages command: yum -y update && yum -y upgrade + - run: + name: Install cmake + command: yum -y install cmake - run: name: Check glibc version command: ldd --version From afa74a5d18a4da9f4e1e01e19cdeaf6b3b5176f1 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:28:15 -0500 Subject: [PATCH 17/25] chore(ci): install gcc instead of cmake --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0cbc3fd8c..366ca22be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,8 +132,8 @@ commands: name: Update and upgrade yum packages command: yum -y update && yum -y upgrade - run: - name: Install cmake - command: yum -y install cmake + name: Install gcc + command: yum -y install gcc - run: name: Check glibc version command: ldd --version From 950dbb9b7d1ba3d98a9542d1353efc7c1d07a2dc Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:30:52 -0500 Subject: [PATCH 18/25] chore(ci): install openssl on centos --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 366ca22be..317437f11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,8 +132,8 @@ commands: name: Update and upgrade yum packages command: yum -y update && yum -y upgrade - run: - name: Install gcc - command: yum -y install gcc + name: Install gcc and OpenSSL + command: yum -y install gcc openssl-devel - run: name: Check glibc version command: ldd --version From 85fd2422627f74df80fd25cb190964d4961b2c13 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:36:21 -0500 Subject: [PATCH 19/25] chore(ci): groupinstall dev tools centos --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 317437f11..429cb8dc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,6 +131,9 @@ commands: - run: name: Update and upgrade yum packages command: yum -y update && yum -y upgrade + - run: + name: Install development tools + command: yum groupinstall -y "Development Tools" - run: name: Install gcc and OpenSSL command: yum -y install gcc openssl-devel From 60c4981cafdb901e3c7245b4f3e36b8e5f1849a3 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 11:40:53 -0500 Subject: [PATCH 20/25] chore(ci): use ubuntu for lint --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 429cb8dc5..3680ee339 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ workflows: name: Lint (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [centos] + platform: [ubuntu] rust_channel: [stable] command: [lint] test: From 5d2e4453ae09a89f55aff026f3495f55466e2393 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 12:02:01 -0500 Subject: [PATCH 21/25] chore(ci): persist target directory for integration tests --- .circleci/config.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3680ee339..1e5cac51a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -223,6 +223,16 @@ commands: - restore_cache: keys: - rust-target-v1-<< parameters.platform >>-{{ checksum "Cargo.lock" }} + - when: + condition: + and: + - equal: [ *ubuntu_executor, << parameters.platform >> ] + - equal: [ "integration-test", << parameters.command >> ] + steps: + - run: + command: ls -la + - attach_workspace: + at: target/ - when: condition: # cargo xtask lint is the only xtask command that doesn't take a target @@ -243,4 +253,16 @@ commands: key: rust-target-v1-<< parameters.platform >>-{{ checksum "Cargo.lock" }} paths: - target/ + - when: + condition: + and: + - equal: [ *centos_executor, << parameters.platform >> ] + - equal: [ "unit-test", << parameters.command >> ] + steps: + - run: + command: ls -la + - persist_to_workspace: + root: workspace + paths: + - target/ \ No newline at end of file From 391d042164790fc284be9fa68ffec7ffe6982cba Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 12:06:36 -0500 Subject: [PATCH 22/25] chore(ci): persist correct directory --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e5cac51a..ec0326ba4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -262,7 +262,7 @@ commands: - run: command: ls -la - persist_to_workspace: - root: workspace + root: ./ paths: - target/ \ No newline at end of file From 4e2ebed5320f642f9c35ddb87d4f473a7c2b0618 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 12:15:44 -0500 Subject: [PATCH 23/25] chore(ci): dont persist directory, it's slow --- .circleci/config.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec0326ba4..3680ee339 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -223,16 +223,6 @@ commands: - restore_cache: keys: - rust-target-v1-<< parameters.platform >>-{{ checksum "Cargo.lock" }} - - when: - condition: - and: - - equal: [ *ubuntu_executor, << parameters.platform >> ] - - equal: [ "integration-test", << parameters.command >> ] - steps: - - run: - command: ls -la - - attach_workspace: - at: target/ - when: condition: # cargo xtask lint is the only xtask command that doesn't take a target @@ -253,16 +243,4 @@ commands: key: rust-target-v1-<< parameters.platform >>-{{ checksum "Cargo.lock" }} paths: - target/ - - when: - condition: - and: - - equal: [ *centos_executor, << parameters.platform >> ] - - equal: [ "unit-test", << parameters.command >> ] - steps: - - run: - command: ls -la - - persist_to_workspace: - root: ./ - paths: - - target/ \ No newline at end of file From eedc6784d155f9f630adee0904678deb54bdaff1 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 12:17:10 -0500 Subject: [PATCH 24/25] chore(ci): run unit tests and integration tests on ubuntu --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3680ee339..4f7b2ca32 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [centos, musl, macos, windows] + platform: [centos, ubuntu, musl, macos, windows] rust_channel: [stable] command: [unit-test] @@ -35,7 +35,7 @@ workflows: rust_channel: [stable] command: [integration-test] requires: - - Cargo tests (<< matrix.rust_channel >> rust on centos) + - Cargo tests (<< matrix.rust_channel >> rust on ubuntu) jobs: xtask: From 6f09ffe1fbd7450aca952cd2822cf6935a6ba0f7 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Tue, 14 Sep 2021 12:31:58 -0500 Subject: [PATCH 25/25] chore(ci): run e2e tests in the same job --- .circleci/config.yml | 13 ++++++------- xtask/src/commands/unit_test.rs | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f7b2ca32..a95c34c5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,19 +23,17 @@ workflows: name: Cargo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: - platform: [centos, ubuntu, musl, macos, windows] + platform: [centos, musl, macos, windows] rust_channel: [stable] command: [unit-test] - xtask: - name: Supergraph demo tests (<< matrix.rust_channel >> rust on << matrix.platform >>) + name: Cargo tests and supergraph demo (<< matrix.rust_channel >> rust on << matrix.platform >>) matrix: parameters: platform: [ubuntu] rust_channel: [stable] - command: [integration-test] - requires: - - Cargo tests (<< matrix.rust_channel >> rust on ubuntu) + command: [test] jobs: xtask: @@ -48,7 +46,7 @@ jobs: type: executor command: type: enum - enum: ["lint", "unit-test", "integration-test"] + enum: ["lint", "unit-test", "integration-test", "test", "dist"] executor: << parameters.platform >> steps: - checkout @@ -67,6 +65,7 @@ executors: resource_class: xlarge environment: XTASK_TARGET: "x86_64-unknown-linux-gnu" + CHECK_GLIBC: "true" musl: &musl_executor docker: @@ -216,7 +215,7 @@ commands: parameters: command: type: enum - enum: [lint, integration-test, unit-test] + enum: [lint, integration-test, unit-test, "test", "dist"] platform: type: executor steps: diff --git a/xtask/src/commands/unit_test.rs b/xtask/src/commands/unit_test.rs index bb9dd31ea..87ad2aa73 100644 --- a/xtask/src/commands/unit_test.rs +++ b/xtask/src/commands/unit_test.rs @@ -21,7 +21,7 @@ impl UnitTest { cargo_runner.test(&self.target)?; if let Target::GnuLinux = self.target { - if env::var_os("CI").is_some() { + if env::var_os("CHECK_GLIBC").is_some() { let check_glibc_script = "./check_glibc.sh".to_string(); let runner = Runner { verbose,