ui-test.sh - mozsearch

Enable keyboard shortcuts

#!/usr/bin/env bash

# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# This script does the following:

# 1. Retrieves glcoud service account token

# 2. Activates gcloud service account

# 3. Connects to google Firebase (using TestArmada's Flank tool)

# 4. Executes UI tests

# 5. Puts test artifacts into the test_artifacts folder

# NOTE:

# Flank supports sharding across multiple devices at a time, but gcloud API

# only supports 1 defined APK per test run.

# If a command fails then do not proceed and fail this script too.

set -e

#########################

# The command line help #

#########################

display_help() {

    echo "Usage: $0 Component_Name Build_Variant [Number_Shards...]"

    echo

    echo "Examples:"

    echo "To run component/browser tests on ARM device shard (1 test / shard)"

    echo "$ execute-firebase-test.sh component arm"

    echo

    echo "To run component/feature tests on X86 device (on 3 shards)"

    echo "$ execute-firebase-test.sh feature x86 3"

    echo

    echo "To run UI samples/sampleName tests"

    echo "$ execute-firebase-test.sh sample-sampleName arm 1"

    echo

# Basic parameter check

if [[ $# -lt 2 ]]; then

    echo "Your command line contains $# arguments"

    display_help

    exit 1

fi

component="$1" # browser, concept, feature

device_type="$2"  # arm | x86

if [[ ! -z "$3" ]]; then

    num_shards=$3

fi

JAVA_BIN="/usr/bin/java"

PATH_TEST="./automation/taskcluster/androidTest"

FLANK_BIN="/builds/worker/test-tools/flank.jar"

FLANK_CONF_ARM="${PATH_TEST}/flank-arm.yml"

FLANK_CONF_X86="${PATH_TEST}/flank-x86.yml"

ARTIFACT_DIR="/builds/worker/artifacts"

RESULTS_DIR="${ARTIFACT_DIR}/results"

echo

echo "ACTIVATE SERVICE ACCT"

echo

# this is where the Google Testcloud project ID is set

gcloud config set project "$GOOGLE_PROJECT"

echo

gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"

echo

echo

# From now on disable exiting on error. If the tests fail we want to continue

# and try to download the artifacts. We will exit with the actual error code later.

set +e

if [[ "${device_type,,}" == "x86" ]]

then

    flank_template="$FLANK_CONF_X86"

else

    flank_template="$FLANK_CONF_ARM"

fi

# Remove samples- from the component for each APK path

samples=${component//samples-}

# If tests are for components, the path is different than for samples

if [[ "${component}" != samples-* ]]

then

    # Case 1: tests for any component (but NOT samples, NOT real UI tests)

    APK_APP="./samples/browser/build/outputs/apk/gecko/debug/samples-browser-gecko-debug.apk"

    if [[ "${component}" == *"-"* ]]

    then

      regex='([a-z]*)-(.*)'

      [[ "$component" =~ $regex ]]

      APK_TEST="./components/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/build/outputs/apk/androidTest/debug/${component}-debug-androidTest.apk"

      else

        APK_TEST="./components/${component}/engine-gecko/build/outputs/apk/androidTest/debug/browser-engine-gecko-debug-androidTest.apk"

fi

elif [[ "${component}" == "samples-browser" ]]

then

    # Case 2: tests for browser sample (gecko sample only)

    APK_APP="./samples/${samples}/build/outputs/apk/gecko/debug/samples-${samples}-gecko-debug.apk"

    APK_TEST="./samples/${samples}/build/outputs/apk/androidTest/gecko/debug/samples-{$samples}-gecko-debug-androidTest.apk"

else

    # Case 3: tests for non-browser samples (i.e.  samples-glean)

    APK_APP="./samples/${samples}/build/outputs/apk/debug/samples-${samples}-debug.apk"

    APK_TEST="./samples/${samples}/build/outputs/apk/androidTest/debug/samples-${samples}-debug-androidTest.apk"

fi

# function to exit script with exit code from test run.

# (Only 0 if all test executions passed)

function failure_check() {

    echo

    echo

    if [[ $exitcode -ne 0 ]]; then

        echo "ERROR: UI test run failed, please check above URL"

    else

    echo "All UI test(s) have passed!"

fi

    echo

    echo "RESULTS"

    echo

    mkdir -p ${ARTIFACT_DIR}/github

    chmod +x ${PATH_TEST}/parse-ui-test.py

    chmod +x ${PATH_TEST}/parse-ui-test-fromfile.py

    ${PATH_TEST}/parse-ui-test-fromfile.py \

        --results "${RESULTS_DIR}"

    if [[ $? -ne 0 ]]; then

        ${PATH_TEST}/parse-ui-test.py \

            --exit-code "${exitcode}" \

            --log flank.log \

            --results "${RESULTS_DIR}" \

            --output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \

            --device-type "${device_type}"

    else

        ${PATH_TEST}/parse-ui-test.py \

            --exit-code "${exitcode}" \

            --log flank.log \

            --results "${RESULTS_DIR}" \

            --output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \

            --device-type "${device_type}" \

            --report-treeherder-failures

fi

    echo

    echo

echo

echo "EXECUTE TEST(S)"

echo

set -o pipefail && $JAVA_BIN -jar $FLANK_BIN android run \

	--config=$flank_template \

	--max-test-shards=$num_shards \

	--app=$APK_APP --test=$APK_TEST \

	--local-result-dir="${RESULTS_DIR}" \

	--project=$GOOGLE_PROJECT \

	--client-details=commit=${MOBILE_HEAD_REV:-None},pullRequest=${PULL_REQUEST_NUMBER:-None} \

	| tee flank.log

exitcode=$?

failure_check

exit $exitcode