civi-test-run

#!/usr/bin/env bash
set -e
TIMER=vtime

function getPhpunitVer() {
  if [ -n "$PHPUNIT" ]; then
    case "$PHPUNIT" in
      phpunit5)
        echo "$PHPUNIT --stderr --tap"
        return
        ;;
      phpunit6|phpunit7|phpunit8)
        echo "$PHPUNIT --stderr --printer \Civi\Test\TAP"
        return
        ;;
      phpunit9)
        echo "$PHPUNIT --stderr --printer \Civi\Test\TAP"
        return
        ;;
      phpunit10)
         ## FIXME Don't have TAP support yet, and --debug is super-noisy
        echo "$PHPUNIT --stderr"
        return
        ;;
    esac
  fi

  CIVIVER=$(getCiviVer)
  if [ $(php -r "echo version_compare('$CIVIVER', '5.65', '>=');") ]; then
    echo "phpunit9 --stderr --printer \Civi\Test\TAP"
  elif [ $(php -r "echo version_compare('$CIVIVER', '5.39', '>=');") ]; then
    echo "phpunit8 --stderr --printer \Civi\Test\TAP"
  elif [ $(php -r "echo version_compare('$CIVIVER', '5.28', '>=');") ]; then
    echo "phpunit7 --stderr --printer \Civi\Test\TAP"
  elif [ $(php -r "echo version_compare('$CIVIVER', '5.22', '>=');") ]; then
    echo "phpunit6 --stderr --printer \Civi\Test\TAP"
  else
    echo 'phpunit5 --tap'
  fi
}

#################################################
## Helpers
function fatal() {
  echo "Fatal: $@"
  exit 1
}

function getCiviVer() {
  pushd "$CIVI_CORE" >> /dev/null
    if [ -f xml/version.xml ]; then
      ## Works in any git-based build, even if gencode hasn't run yet.
      php -r 'echo simplexml_load_file("xml/version.xml")->version_no;'
    else
      ## works in any tar-based build.
      php -r 'require "civicrm-version.php"; $a = civicrmVersion(); echo $a["version"];'
    fi
  popd >> /dev/null
}

function show_help() {
  PROG=$(basename "$0")
  echo "about: Execute the CiviCRM test suites"
  echo
  echo "usage: $PROG [options] <suites>"
  echo
  echo "options:"
  echo "  -h                  Display help"
  echo "  -b <build-name>     The name of a local site produced by civibuild (required)"
  echo "  -j <junit-dir>      The path to a folder for storing results in JUnit XML (required)"
  echo "  --exclude-group <g> Exclude tests with a particular @group (if supported by suite)"
  echo "  --group <g>         Focus on tests with a particular @group (if supported by suite)"
  echo "  --filter <regex>    Filter which test functions to run (if supported by suite)"
  echo "  <suites>            A list of one more of the following:"
  echo "                        - all"
  echo "                        - karma"
  echo "                        - mixin"
  echo "                        - phpunit-api3"
  echo "                        - phpunit-api3-{1,2}"
  echo "                        - phpunit-api4"
  echo "                        - phpunit-civi"
  echo "                        - phpunit-core-exts"
  echo "                        - phpunit-crm"
  echo "                        - phpunit-crm-{1,2,3,4}"
  echo "                        - phpunit-e2e"
  echo "                        - phpunit-drupal"
  echo "                        - phpunit-wordpress"
  echo "                        - phpunit-backdrop"
  echo "                        - phpunit-uf (alias for phpunit-{drupal,wordpress,backdrop})"
  echo "                        - phpunit-creditnotes"
  echo "                        - phpunit-flexmailer"
  echo "                        - upgrade"
  echo "                        - upgrade@4.7.30:10"
  echo
  echo "example: Run the KarmaJS and DB-upgrade tests on D7/Civi-Master build"
  echo "  $PROG -b dmaster -j /tmp/junit-output karma upgrade"
  echo
}

#################################################
## Test tasks
##
## Each task is responsible for depositing a JUnit XML file under
## "$JUNITDIR".

#################################################
## Run a PHPUnit test suite (based on a flexible expression).
##
## In the future, it will get progressively harder to run tests based on class-name.
## File-names, paths, and groups will work better.
##
## usage: task_phpunit_expr <logical-suite-name> <pwd> [...phpunit file/filter options...]
## example: task_phpunit_expr phpunit-foo "$CIVI_CORE" tests/phpunit/Foo/
function task_phpunit_expr() {
  local PHPUNIT_BASE=$(getPhpunitVer)
  local SUITE_NAME="$1"
  local WORK_DIR="$2"
  shift 2

  [ ! -d "$JUNITDIR" ] && mkdir -p "$JUNITDIR"

  pushd "$WORK_DIR"
    if ! $GUARD $TIMER $PHPUNIT_BASE ${PHPUNIT_ARGS[@]} --log-junit "$JUNITDIR/$SUITE_NAME.xml" "$@" ; then
      EXITCODES="$EXITCODES $SUITE_NAME"
    fi
    echo "Found EXITCODES=\"$EXITCODES\""
  popd

  $GUARD phpunit-xml-cleanup "$JUNITDIR/$SUITE_NAME.xml"
}

## Run PHPUnit. Find all subsdirs matching a pattern and put them in a suite together.
##
## usage: task_phpunit_find <logical-suite-name> <work-dir> <find-expr...>
## example: task_phpunit_find CRM_1 "$CIVI_CORE" ./tests/phpunit/CRM -name '[ABC]*' ;;
function task_phpunit_find() {
  local SUITE_NAME="$1"
  local WORK_DIR="$2"
  shift 2

  pushd "$WORK_DIR"
    local DIRS=$( find "$@" -type d )
    if [ ! -f phpunit.xml.dist ]; then
      fatal "($WORK_DIR) Cannot generate local phpunit.xml for $SUITE_NAME. The template phpunit.xml.dist is missing."
    fi
    php "$PRJDIR/src/phpunit-add-suite.php" "$SUITE_NAME" $( find "$@" | sort ) < phpunit.xml.dist > phpunit.xml.tmp
  popd

  task_phpunit_expr "$SUITE_NAME" "$WORK_DIR" -c phpunit.xml.tmp --testsuite "$SUITE_NAME"
  rm -f "$WORK_DIR/phpunit.xml.tmp"
}

## usage: task_phpunit_core_extension <extension-dirname> <junit-name>
function task_phpunit_core_extension() {
  EXTENSION="$1"
  JUNIT_NAME=${2:-$EXTENSION}

  pushd "$CIVI_CORE/ext/$EXTENSION"
    for GROUP in headless e2e ; do
      if ! $GUARD $TIMER $(getPhpunitVer) --log-junit="$JUNITDIR/junit-${JUNIT_NAME}-${GROUP}.xml" --group "$GROUP" ; then
       EXITCODES="$EXITCODES $EXTENSION-$GROUP"
     fi
     echo "Found EXITCODES=\"$EXITCODES\""
   done
  popd

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
}

function task_mixin() {
  pushd "$CIVI_CORE"
    if [ -f tools/mixin/bin/test-all ]; then
      if ! $GUARD $TIMER ./tools/mixin/bin/test-all "$JUNITDIR" ; then
        EXITCODES="$EXITCODES mixin"
      fi
      $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
    else
      junit_placeholder
    fi
  popd
}

function test_core_extenions() {
  for DIR in $CIVI_CORE/ext/*/
  do
    if [ "$(basename $DIR)" = "afform" ]; then
      test_afform_extensions $DIR
    elif [ -f "$DIR/phpunit.xml.dist" ]; then
      task_phpunit_core_extension "$(basename $DIR)"
      $GUARD civibuild restore "$BLDNAME"
      $GUARD cv flush --cwd="$CIVI_CORE" ## This is smelly. It begs the question of "what is leaking between tests".
    fi
  done
}

function test_afform_extensions() {
  DIR="$1"
  pushd "$DIR"
    task_phpunit_core_extension "$(basename $DIR)/core" "afform-core"
    $GUARD civibuild restore "$BLDNAME"
    $GUARD cv flush --cwd="$CIVI_CORE" ## This is smelly. It begs the question of "what is leaking between tests".
    cv en afform
    cv en --ignore-missing authx
    task_phpunit_core_extension "$(basename $DIR)/mock" "afform-mock"
    $GUARD civibuild restore "$BLDNAME"
    $GUARD cv flush --cwd="$CIVI_CORE" ## This is smelly. It begs the question of "what is leaking between tests".
  popd
}

function task_karma() {
  $GUARD pushd "$CIVI_CORE"
    ## Work-around: ensure pradmin user has sa contact
    ADMIN_USER=$(cv ev 'echo $GLOBALS["_CV"]["ADMIN_USER"];')
    cv api -U "$ADMIN_USER" contact.get id=user_contact_id
    if [ -f karma.conf.js ]; then
      if ! $GUARD $TIMER env OPENSSL_CONF=/dev/null karma start --single-run --reporters dots,junit ; then
        EXITCODES="$EXITCODES karma"
      fi
      echo "Found EXITCODES=\"$EXITCODES\""
      $GUARD cp tests/output/karma.xml "$JUNITDIR/"
    fi
  $GUARD popd
}

## Execute the upgrade test suite (via 'phpunit' or 'civicrm-upgrade-test' or whatever is available).
##
## Usage: task_upgrade [<version-filter>]
## Ex: task_upgrade @4.3..4.6.30
function task_upgrade() {
  if [ -e "$CIVI_CORE/tests/phpunit/Upgrade/" -a "$(ls $CIVI_CORE/tests/phpunit/Upgrade/*.php)"  ]; then
    export UPGRADE_TEST_FILTER="$@"
    task_phpunit_expr UpgradeTest "$CIVI_CORE" tests/phpunit/Upgrade/
  else
    if [ -z "$1" ]; then
      CIVIVER=$(getCiviVer)
      task_upgrade_legacy 5.13.3-multilingual_af_bg_en* "@4.7.30..$CIVIVER:10"
    else
      task_upgrade_legacy "$@"
    fi
  fi

  $GUARD civibuild restore "$BLDNAME"
  $GUARD cv flush --cwd="$CIVI_CORE" ## This is smelly. It begs the question of "what is leaking between tests".
}

## Execute the upgrade test suite (via 'civicrm-upgrade-test' bash script).
##
## Usage: task_upgrade <version-filter>
## Ex: task_upgrade @4.3..4.6.30
function task_upgrade_legacy() {
  ## Run the tests -- DB upgrade tests
  if  ! $GUARD $TIMER civibuild upgrade-test $BLDNAME $@ ; then
    EXITCODES="$EXITCODES upgrade-test"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""
  if [ -d "$PRJDIR/build/$BLDNAME/.civibuild" ]; then
    $GUARD cp "$PRJDIR/build/$BLDNAME/.civibuild/debug/civicrm-upgrade-test.xml" "$JUNITDIR/"
  else
    $GUARD cp "$PRJDIR/app/debug/$BLDNAME/civicrm-upgrade-test.xml" "$JUNITDIR/"
  fi
}

#################################################
## Cleanup left-overs from previous test-runs
function junit_cleanup() {
  [ -d "$JUNITDIR" ] && $GUARD rm -rf "$JUNITDIR"
  [ ! -d "$JUNITDIR" ] && $GUARD mkdir "$JUNITDIR"
}

#################################################
## Create a stub JUnit XML file. This is useful if we've decided to explicitly *not* run tests.
function junit_placeholder() {
  echo '<?xml version="1.0" encoding="UTF-8"?>' > "$JUNITDIR/none.xml"
  echo '<testsuites>' >> "$JUNITDIR/none.xml"
  echo '</testsuites>' >> "$JUNITDIR/none.xml"
}

function task_phpunit_uf() {
  $GUARD pushd "$CIVI_CORE"
    $GUARD civibuild restore "$BLDNAME"
    $GUARD cv flush --cwd="$CIVI_CORE"
    local uf=$(cv ev 'echo CIVICRM_UF;')
    case "$uf" in
      Backdrop) task_phpunit_backdrop ;;
      Drupal) task_phpunit_drupal ;;
      WordPress) task_phpunit_wordpress ;;
      *) echo "Skipped phpunit-uf. No suites known for CIVICRM_UF=[$uf]" ;;
    esac
  $GUARD popd
}

function task_phpunit_drupal() {
  $GUARD pushd "$CIVI_CORE/drupal"
  if ! $GUARD $(getPhpunitVer) --log-junit "$JUNITDIR/phpunit_drupal.xml" ${PHPUNIT_ARGS[@]} ; then
    EXITCODES="$EXITCODES phpunit-drupal"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
  $GUARD popd
}

function task_phpunit_wordpress() {
  $GUARD pushd "$CIVI_CORE/.."
  if ! $GUARD $(getPhpunitVer) --log-junit "$JUNITDIR/phpunit_wordpress.xml" ${PHPUNIT_ARGS[@]} ; then
    EXITCODES="$EXITCODES phpunit-wordpress"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
  $GUARD popd
}

function task_phpunit_backdrop() {
  $GUARD pushd "$CIVI_CORE/backdrop"
  if ! $GUARD $(getPhpunitVer) --log-junit "$JUNITDIR/phpunit_backdrop.xml" ${PHPUNIT_ARGS[@]} ; then
    EXITCODES="$EXITCODES phpunit-backdrop"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
  $GUARD popd
}

## Determine the absolute path of the directory with the file
## absdirname <file-path>
function absdirname() {
  pushd $(dirname $0) >> /dev/null
    pwd
  popd >> /dev/null
}

BINDIR=$(absdirname "$0")
PRJDIR=$(dirname "$BINDIR")
## Note: Don't override TMPDIR. It's not actually needed by `civi-test-run`, and it can sometimes interfere with tests.
[ -z "$CIVIBUILD_HOME" ] && BLDDIR="$PRJDIR/build" || BLDDIR="$CIVIBUILD_HOME"
EXITCODES=
SUITES=
PHPUNIT_ARGS=()
GUARD=


[ -f "$PRJDIR/app/civibuild.conf" ] && source "$PRJDIR/app/civibuild.conf"

#################################################
## Parse inputs
while [ -n "$1" ] ; do
  OPTION="$1"
  shift

  case "$OPTION" in
    -b)
      BLDNAME="$1"
      [ -d "$BLDDIR/$BLDNAME/web" ] && CMS_ROOT="$BLDDIR/$BLDNAME/web" || CMS_ROOT="$BLDDIR/$BLDNAME"
      pushd "$CMS_ROOT" >> /dev/null
        CIVI_CORE=$(cv ev 'echo $GLOBALS["civicrm_root"];')
        ## or: cv path -d '[civicrm.root]'
      popd >> /dev/null
      shift
      ;;

    -j)
      JUNITDIR="$1"
      junit_cleanup
      shift
      ;;

    --exclude-group|--group|--filter)
      PHPUNIT_ARGS+=("$OPTION" "$1")
      shift
      ;;

    ## TODO: Not currently supported by phpunit-each
    #--stop-on-defect|--stop-on-error|--stop-on-failure|--stop-on-warning|--stop-on-risky|--stop-on-skipped|--stop-on-incomplete|--fail-on-warning|--fail-on-risky)
    #  PHPUNIT_ARGS+=("$OPTION")
    #  ;;

    -h|--help|-?)
      show_help
      exit 2
      ;;

    *)
      SUITES=$(echo "$SUITES" "$OPTION" | xargs)
      # note: xargs strips leading/trailing spaces
      ;;
  esac
done

## Validate
if [ -z "$SUITES" -o -z "$BLDNAME" -o -z "$JUNITDIR" ]; then
  echo "Error: Missing required argument"
  echo
  show_help
  exit 2
fi

if [ "$SUITES" = "all" ]; then
  pushd "$CIVI_CORE" >> /dev/null
    SUITES="upgrade"
    if [ -f "karma.conf.js" ]; then
      SUITES="karma $SUITES"
    else
      echo "Skip unavailable suite: karma"
    fi

    if [ -f "tests/phpunit/E2E/AllTests.php" ]; then
      SUITES="$SUITES phpunit-e2e"
    else
      echo "Skip unavailable suite: phpunit-e2e"
    fi

    SUITES="$SUITES phpunit-crm phpunit-api3 phpunit-civi"

    if [ -f "tests/phpunit/api/v4/AllTests.php" ]; then
      SUITES="$SUITES phpunit-api4"
    fi

    if [ -d "ext" ]; then
      SUITES="$SUITES phpunit-core-exts"
    fi

    if [ -f "tools/mixin/bin/test-all" ]; then
      SUITES="$SUITES mixin"
    else
      echo "Skip unavailable suite: mixin"
    fi

  popd >> /dev/null
elif [[ " $SUITES " =~ \ all\  ]]; then
  echo "The \"all\" target should not be mixed with other targets."
  exit 3
fi

## dev/core#1781: Some tests exhibit flakiness due to the timing/pacing/happenstance of execution. TIME_FUNC helps to deterministically simulate some of this variation.
if [ -n "$TIME_FUNC" ]; then
  echo "(Clock simulation) Tests will use inherited option TIME_FUNC=$TIME_FUNC"
else
  TIME_FUNC=$( php -r '$a=$argv; array_shift($a); shuffle($a); echo $a[0];' natural natural frozen linear:500 linear:1250 prng:500 prng:666 prng:1000 prng:1500 )
  echo "(Clock simulation) No TIME_FUNC was specified. Using TIME_FUNC=$TIME_FUNC"
fi
export TIME_FUNC

## Main
for TESTTYPE in $SUITES ; do
  case "$TESTTYPE" in
    karma)             task_karma ;;
    mixin)             task_mixin ;;
    upgrade)           task_upgrade ;;
    upgrade@*)         task_upgrade $(echo $TESTTYPE | sed s'/^upgrade//' ) ;;
    phpunit-e2e)       PHP_APC_CLI=on task_phpunit_expr "$TESTTYPE" "$CIVI_CORE" tests/phpunit/E2E/ ; civibuild restore "$BLDNAME" ;  cv flush --cwd="$CIVI_CORE" ; ;;
    phpunit-crm)       CIVICRM_UF=UnitTests task_phpunit_expr "$TESTTYPE" "$CIVI_CORE" tests/phpunit/CRM/ ;;
    ## Alternatively, we may run the CRM suite as 3 roughly-equal sub-suites.
    phpunit-crm-1)     CIVICRM_UF=UnitTests task_phpunit_find "$TESTTYPE" "$CIVI_CORE" ./tests/phpunit/CRM -mindepth 1 -maxdepth 1 -name 'Core' ;;
    phpunit-crm-2)     CIVICRM_UF=UnitTests task_phpunit_find "$TESTTYPE" "$CIVI_CORE" ./tests/phpunit/CRM -mindepth 1 -maxdepth 1 -name 'C*' ! -name Core ;;
    phpunit-crm-3)     CIVICRM_UF=UnitTests task_phpunit_find "$TESTTYPE" "$CIVI_CORE" ./tests/phpunit/CRM -mindepth 1 -maxdepth 1 -name '[ABEFGHIJKLM]*' ;;
    phpunit-crm-4)     CIVICRM_UF=UnitTests task_phpunit_find "$TESTTYPE" "$CIVI_CORE" ./tests/phpunit/CRM -mindepth 1 -maxdepth 1 -name '[DNOPQRSTUVWXYZ]*' ;;
    # Handle legacy call for phpunit-api being equivilant to phpunit-api3
    phpunit-api)       CIVICRM_UF=UnitTests task_phpunit_expr "$TESTTYPE" "$CIVI_CORE" tests/phpunit/api/v3 ;;
    phpunit-api3)      CIVICRM_UF=UnitTests task_phpunit_expr "$TESTTYPE" "$CIVI_CORE" tests/phpunit/api/v3 ;;
    phpunit-api3-1)    CIVICRM_UF=UnitTests task_phpunit_find "$TESTTYPE" "$CIVI_CORE" ./tests/phpunit/api/v3 -mindepth 1 -maxdepth 1 -name '[ABCDEFGHIKLMNO]*' ;;
    phpunit-api3-2)    CIVICRM_UF=UnitTests task_phpunit_find "$TESTTYPE" "$CIVI_CORE" ./tests/phpunit/api/v3 -mindepth 1 -maxdepth 1 -name '[JPQRSTUVWXYZ]*' ;;
    phpunit-api4)      CIVICRM_UF=UnitTests task_phpunit_expr "$TESTTYPE" "$CIVI_CORE" tests/phpunit/api/v4 ;;
    phpunit-civi)      CIVICRM_UF=UnitTests task_phpunit_expr "$TESTTYPE" "$CIVI_CORE" tests/phpunit/Civi ;;
    phpunit-uf)        task_phpunit_uf ;;
    phpunit-drupal)    task_phpunit_drupal ;;
    phpunit-wordpress) task_phpunit_wordpress ;;
    phpunit-backdrop)  task_phpunit_backdrop ;;
    phpunit-creditnotes) task_phpunit_core_extension sequentialcreditnotes ;;
    phpunit-flexmailer) task_phpunit_core_extension flexmailer ;;
    phpunit-financialacls) task_phpunit_core_extension financialacls ;;
    phpunit-eway) task_phpunit_core_extension ewaysingle ;;
    phpunit-afform) test_afform_extensions "$CIVI_CORE/ext/afform" ;;
    phpunit-authx) task_phpunit_core_extension authx ;;
    phpunit-searchkit) task_phpunit_core_extension search_kit ;;
    phpunit-core-exts) test_core_extenions ;;
    *) echo "unrecognized suite: $TESTTYPE"; exit 2 ;;
  esac
done

## Check test results and set exit code
echo "Check test results and set exit code"
cat "$JUNITDIR"/*.xml | grep '<failure' -q && fatal "Found <failure> in XML"
cat "$JUNITDIR"/*.xml | grep '<error' -q && fatal "Found <error> in XML"
for TESTTYPE in $SUITES ; do
  if  [ $TESTTYPE = "upgrade" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/civicrm-upgrade-test.xml" ] && fatal "Missing XML: civicrm-upgrade-test.xml"
  fi
  if  [ $TESTTYPE = "phpunit-crm" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/phpunit-crm.xml" ] && fatal "Missing XML: phpunit-crm.xml"
  fi
  if [[ $TESTTYPE =~ ^(phpunit-crm-[1-9]|phpunit-api3-[1-9])$ ]]; then
    ## NOTE: These are NOT expected to run when TESTTYPE="all". Instead, "all" has the larger phpunit-crm and phpunit-api3 suites.
    [ ! -f "$JUNITDIR/$TESTTYPE.xml" ] && fatal "Missing XML: $TESTTYPE.xml"
  fi
  if  [ $TESTTYPE = "phpunit-api" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/phpunit-api.xml" ] && fatal "Missing XML: phpunit-api.xml"
  fi
  if  [ $TESTTYPE = "phpunit-api3" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/phpunit-api3.xml" ] && fatal "Missing XML: phpunit-api3.xml"
  fi
  if  [ $TESTTYPE = "phpunit-api4" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/phpunit-api4.xml" ] && fatal "Missing XML: phpunit-api4.xml"
  fi
  if  [ $TESTTYPE = "phpunit-drupal" ] ; then
    [ ! -f "$JUNITDIR/phpunit_drupal.xml" ] && fatal "Missing XML: phpunit_drupal.xml"
  fi
  if  [ $TESTTYPE = "phpunit-wordpress" ] ; then
    [ ! -f "$JUNITDIR/phpunit_wordpress.xml" ] && fatal "Missing XML: phpunit_wordpress.xml"
  fi
  if  [ $TESTTYPE = "phpunit-backdrop" ] ; then
    [ ! -f "$JUNITDIR/phpunit_backdrop.xml" ] && fatal "Missing XML: phpunit_backdrop.xml"
  fi
  if  [ $TESTTYPE = "phpunit-civi" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/phpunit-civi.xml" ] && fatal "Missing XML: phpunit-civi.xml"
  fi
  if  [ $TESTTYPE = "phpunit-e2e" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/phpunit-e2e.xml" ] && fatal "Missing XML: phpunit-e2e.xml"
  fi
  if  [ $TESTTYPE = "karma" ] || [ $TESTTYPE = "all" ] ; then
    [ -d "$CIVI_CORE/tests/karma" -a ! -f "$JUNITDIR/karma.xml" ] && fatal "Missing XML: karma.xml"
  fi
done
[ -n "$EXITCODES" ] && fatal "At least one command failed abnormally [$EXITCODES]"
echo "Exit normally"