diff --git a/bamboo-specs/test.yaml b/bamboo-specs/test.yaml index 807892ce4..41a494da2 100644 --- a/bamboo-specs/test.yaml +++ b/bamboo-specs/test.yaml @@ -5,9 +5,12 @@ 'key': 'AHBRTSPECS' 'name': 'AdGuard Home - Build and run tests' 'variables': + # This variable is used to override Docker caching, for example to rerun a + # flaky test suite. + 'cacheBuster': '0' + 'channel': 'development' 'dockerFrontend': 'adguard/home-js-builder:4.0' 'dockerGo': 'adguard/go-builder:1.25.7--1' - 'channel': 'development' 'stages': - 'Tests': @@ -36,10 +39,8 @@ - 'Test e2e' 'Test frontend': - 'docker': - 'image': '${bamboo.dockerFrontend}' - 'volumes': - '${system.NPM_DIR}': '${bamboo.cacheNpm}' + 'final-tasks': + - 'clean' 'key': 'JSTEST' 'other': 'clean-working-dir': true @@ -54,19 +55,18 @@ set -e -f -u -x - make VERBOSE=1 js-deps js-typecheck js-lint js-test - 'final-tasks': - - 'clean' - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'tester' \ + -f ./docker/frontend.Dockerfile \ + . -# TODO(e.burkov): Add the linting stage for markdown docs and shell scripts. 'Test backend': - 'docker': - 'image': '${bamboo.dockerGo}' - 'volumes': - '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' - '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' 'final-tasks': - 'test-parser': # The default pattern, '**/test-reports/*.xml', works, so don't set @@ -88,35 +88,21 @@ set -e -f -u -x - # Do not sort the commands below, as they are arranged - # intentionally to keep faster commands first. - make \ - GOMAXPROCS=1 \ - VERBOSE=1 \ - go-deps \ - md-lint \ - sh-lint \ - txt-lint \ - go-lint \ - ; + docker info - make \ - TEST_REPORTS_DIR="./test-reports/" \ - VERBOSE=1 \ - go-test \ - ; + docker build \ + --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'tester-exporter' \ + -f ./docker/ci.Dockerfile \ + . exit_code="$(cat ./test-reports/test-exit-code.txt)" readonly exit_code - make VERBOSE=1 \ - go-fuzz \ - go-bench \ - ; - exit "$exit_code" - 'requirements': - - 'golang': true 'Build frontend': 'artifacts': @@ -124,10 +110,6 @@ 'pattern': 'build/**' 'shared': true 'required': true - 'docker': - 'image': '${bamboo.dockerFrontend}' - 'volumes': - '${system.NPM_DIR}': '${bamboo.cacheNpm}' 'key': 'BF' 'other': 'clean-working-dir': true @@ -142,11 +124,16 @@ set -e -f -u -x - make\ - VERBOSE=1\ - js-deps js-build - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/frontend.Dockerfile \ + . 'Artifact': 'artifact-subscriptions': @@ -164,11 +151,6 @@ 'pattern': 'dist/AdGuardHome_linux_amd64.tar.gz' 'shared': true 'required': true - 'docker': - 'image': '${bamboo.dockerGo}' - 'volumes': - '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' - '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' 'key': 'ART' 'other': 'clean-working-dir': true @@ -183,26 +165,28 @@ set -e -f -u -x - make\ - ARCH="amd64"\ - CHANNEL=${bamboo.channel}\ - FRONTEND_PREBUILT=1\ - OS="windows darwin linux"\ - PARALLELISM=1\ - SIGN=0\ - VERBOSE=2\ - build-release - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "ARCH=amd64" \ + --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ + --build-arg "BRANCH=${bamboo_planRepository_branchName}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --build-arg "CHANNEL=${bamboo_channel}" \ + --build-arg "OS=windows darwin linux" \ + --build-arg "REVISION=${bamboo_repository_revision_number}" \ + --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ + --build-arg "VERSION=${bamboo_buildNumber}" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/ci.Dockerfile \ + . 'Test e2e': 'artifact-subscriptions': - 'artifact': 'AdGuardHome_linux_amd64' - 'artifact': 'AdGuardHome frontend' - 'docker': - 'image': '${bamboo.dockerFrontend}' - 'volumes': - '${system.NPM_DIR}': '${bamboo.cacheNpm}' 'key': 'E2ETEST' 'other': 'clean-working-dir': true @@ -217,15 +201,20 @@ set -e -f -u -x - export CI=true - tar -xzf dist/AdGuardHome_linux_amd64.tar.gz -C /tmp mv /tmp/AdGuardHome/AdGuardHome ./AdGuardHome - make VERBOSE=1 js-deps js-test-e2e - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'e2etester' \ + -f ./docker/frontend.Dockerfile \ + . 'branches': 'create': 'for-pull-request' diff --git a/docker/Dockerfile b/docker/build.Dockerfile similarity index 100% rename from docker/Dockerfile rename to docker/build.Dockerfile diff --git a/.dockerignore b/docker/build.Dockerfile.dockerignore similarity index 100% rename from .dockerignore rename to docker/build.Dockerfile.dockerignore diff --git a/docker/ci.Dockerfile b/docker/ci.Dockerfile new file mode 100644 index 000000000..f7a0ff9f9 --- /dev/null +++ b/docker/ci.Dockerfile @@ -0,0 +1,191 @@ +# syntax=docker/dockerfile:1 + +# This comment is used to simplify checking local copies of the Dockerfile. +# Bump this number every time a significant change is made to this Dockerfile. +# +# AdGuard-Project-Version: 10 + +# Dockerfile guidelines: +# +# 1. Make sure that Docker correctly caches layers, on a second build attempt it +# must not run lint / test second time when it's not required. +# +# 2. Use BuildKit to improve the build performance (--mount=type=cache, etc). +# +# 3. Prefer using ARG instead of ENV when appropriate, as ARG does not create a +# layer in the final image. However, be careful with what you use ARG for. +# Also, prefer to give ARGs sensible default values. +# +# 4. Use --output and the export stage if you need to get any output on the host +# machine. +# +# NOTE: Only use --output with FROM scratch. +# +# 5. Use .dockerignore to prevent unnecessary files from being sent to the +# Docker daemon, which can invalidate the cache. +# +# 6. Add a CACHE_BUSTER argument to stages to be able to rerun the stages if +# needed. Keep it in sync with bamboo-specs/bamboo.yaml. + +# NOTE: Keep in sync with bamboo-specs/bamboo.yaml. +ARG BASE_IMAGE=adguard/go-builder:1.25.7--1 + +# The dependencies stage is needed to install packages and tool dependencies. +# This is also where binaries like osslsigncode, which may be required for tests +# in some projects, must be installed. +# +# Use fake BRANCH and REVISION values to both prevent git calls and also not +# ruin the caching with ARGs. +# +# NOTE: Only ADD the files required to install the dependencies. +FROM "$BASE_IMAGE" AS dependencies +ADD Makefile go.mod go.sum /app/ +ADD scripts /app/scripts +WORKDIR /app +RUN \ + --mount=type=cache,id=gocache,target=/root/.cache/go-build \ + --mount=type=cache,id=gopath,target=/go \ +<<-'EOF' +set -e -f -u -x +make \ + BRANCH='master' \ + REVISION='0000000000000000000000000000000000000000' \ + VERBOSE=1 \ + go-env \ + go-deps \ + ; +EOF + +# The linter stage is separated from the tester stage to make catching test +# failures easier. +# +# Use fake BRANCH and REVISION values to both prevent git calls and also not +# ruin the caching with ARGs. IGNORE_NON_REPRODUCIBLE is set to 1 to make this +# stage reproducible even when linters that query external sources fail. +FROM dependencies AS linter +ADD . /app +WORKDIR /app +RUN \ + --mount=type=cache,id=gocache,target=/root/.cache/go-build \ + --mount=type=cache,id=gopath,target=/go \ +<<-'EOF' +set -e -f -u -x +export GOMAXPROCS=2 +make \ + BRANCH='master' \ + IGNORE_NON_REPRODUCIBLE='1' \ + REVISION='0000000000000000000000000000000000000000' \ + VERBOSE=1 \ + go-lint \ + md-lint \ + sh-lint \ + txt-lint \ + ; +EOF + +# The test stage. TEST_REPORTS_DIR is set to create JUnit reports for the +# tester-exporter stage; run with --build-arg TEST_REPORTS_DIR='' if you don't +# need them on your machine. +# +# Use fake BRANCH and REVISION values to both prevent git calls and also not +# ruin the caching with ARGs. +# +# To run the tests: +# +# docker build --target tester -t 'app' . +# +# Projects that have go-bench and/or go-fuzz targets should add them here as +# well. +FROM linter AS tester +ARG CACHE_BUSTER=0 +ARG TEST_REPORTS_DIR=/test-reports +RUN \ + --mount=type=cache,id=gocache,target=/root/.cache/go-build \ + --mount=type=cache,id=gopath,target=/go \ +<<-'EOF' +set -e -f -u -x +export GOMAXPROCS=2 + +make \ + BRANCH='master' \ + REVISION='0000000000000000000000000000000000000000' \ + TEST_REPORTS_DIR="$TEST_REPORTS_DIR" \ + VERBOSE=1 \ + go-test \ + ; + +exit_code="$(cat "${TEST_REPORTS_DIR}/test-exit-code.txt")" +readonly exit_code + +make \ + BRANCH='master' \ + REVISION='0000000000000000000000000000000000000000' \ + VERBOSE=1 \ + go-fuzz \ + go-bench \ + ; + +exit "$exit_code" +EOF + +# tester-exporter exports the test result to the host machine so that it could +# parse and analyze it. This stage should only used in a CI. +# +# It the file test-report.xml, which contains test results in the JUnit format. +# +# Run the following command to export the test result: +# +# docker build \ +# --output . \ +# --progress plain \ +# --target tester-exporter \ +# . +FROM scratch AS tester-exporter +ARG CACHE_BUSTER=0 +ARG TEST_REPORTS_DIR=/test-reports +COPY --from=tester "$TEST_REPORTS_DIR" "$TEST_REPORTS_DIR" + +# The builder stage is used to build release artifacts. Real BRANCH and +# REVISION must be used here. +FROM dependencies AS builder +ARG ARCH="" +ARG BRANCH=master +ARG CACHE_BUSTER=0 +ARG CHANNEL=development +ARG DEPLOY_SCRIPT_PATH=not/a/real/path +ARG GPG_KEY_PASSPHRASE +ARG GPG_SECRET_KEY +ARG OS="" +ARG REVISION=0000000000000000000000000000000000000000 +ARG SIGNER_API_KEY +ARG SOURCE_DATE_EPOCH=0 +ARG VERSION="" +ADD . /app +WORKDIR /app +RUN \ + --mount=type=cache,id=gocache,target=/root/.cache/go-build \ + --mount=type=cache,id=gopath,target=/go \ +<<-'EOF' +set -e -f -u -x + +make \ + ARCH="${ARCH}" \ + BRANCH="${BRANCH}" \ + CHANNEL="${CHANNEL}" \ + FRONTEND_PREBUILT=1 \ + OS="${OS}" \ + PARALLELISM=1 \ + REVISION="${REVISION}" \ + SOURCE_DATE_EPOCH="$SOURCE_DATE_EPOCH" \ + SIGN=0 \ + VERBOSE=2 \ + VERSION="${VERSION}" \ + build-release \ + ; +EOF + +# builder-exporter exports the build artifacts to the host machine so that they +# could be published. This stage should only be used in a CI. +FROM scratch AS builder-exporter +ARG CACHE_BUSTER=0 +COPY --from=builder /app/dist /dist diff --git a/docker/ci.Dockerfile.dockerignore b/docker/ci.Dockerfile.dockerignore new file mode 100644 index 000000000..e93ebfc68 --- /dev/null +++ b/docker/ci.Dockerfile.dockerignore @@ -0,0 +1,8 @@ +# This comment is used to simplify checking local copies of the file. Bump this +# number every time a significant change is made to this file. +# +# AdGuard-Project-Version: 2 +.git +/bin/ +/tmp/ +/client/ diff --git a/docker/frontend.Dockerfile b/docker/frontend.Dockerfile new file mode 100644 index 000000000..0f4d3a9f3 --- /dev/null +++ b/docker/frontend.Dockerfile @@ -0,0 +1,118 @@ +# syntax=docker/dockerfile:1 + +# This comment is used to simplify checking local copies of the Dockerfile. +# Bump this number every time a significant change is made to this Dockerfile. +# +# AdGuard-Project-Version: 11 + +# Dockerfile guidelines: +# +# 1. Make sure that Docker correctly caches layers, on a second build attempt it +# must not run lint / test second time when it's not required. +# +# 2. Use BuildKit to improve the build performance (--mount=type=cache, etc). +# +# 3. Prefer using ARG instead of ENV when appropriate, as ARG does not create a +# layer in the final image. However, be careful with what you use ARG for. +# Also, prefer to give ARGs sensible default values. +# +# 4. Use --output and the export stage if you need to get any output on the host +# machine. +# +# NOTE: Only use --output with FROM scratch. +# +# 5. Use .dockerignore to prevent unnecessary files from being sent to the +# Docker daemon, which can invalidate the cache. +# +# 6. Add a CACHE_BUSTER argument to stages to be able to rerun the stages if +# needed. Keep it in sync with bamboo-specs/bamboo.yaml. + +# NOTE: Keep in sync with bamboo-specs/bamboo.yaml. +ARG BASE_IMAGE=adguard/home-js-builder:4.0 + +# The dependencies stage is needed to install packages and tool dependencies. +# This is also where binaries like osslsigncode, which may be required for tests +# in some projects, must be installed. +# +# NOTE: Only ADD the files required to install the dependencies. +FROM "$BASE_IMAGE" AS dependencies +ADD Makefile /app/ +ADD scripts /app/scripts +ADD client /app/client +WORKDIR /app +RUN \ + --mount=type=cache,id=npm-root-cache,target=/root/.npm \ +<<-'EOF' +set -e -f -u -x +make \ + VERBOSE=1 \ + js-deps \ + ; +EOF + +# The linter stage is separated from the tester stage to make catching test +# failures easier. +FROM dependencies AS linter +ARG CACHE_BUSTER=0 +ADD . /app +WORKDIR /app +RUN \ + --mount=type=cache,id=npm-root-cache,target=/root/.npm \ +<<-'EOF' +set -e -f -u -x +make \ + VERBOSE=1 \ + js-typecheck \ + js-lint \ + ; +EOF + +# The test stage. +FROM linter AS tester +ARG CACHE_BUSTER=0 +RUN \ + --mount=type=cache,id=npm-root-cache,target=/root/.npm \ +<<-'EOF' +set -e -f -u -x +make \ + VERBOSE=1 \ + js-test \ + ; +EOF + +# The e2e test stage. +FROM dependencies AS e2etester +ARG CACHE_BUSTER=0 +ADD . /app +WORKDIR /app +RUN \ + --mount=type=cache,id=npm-root-cache,target=/root/.npm \ +<<-'EOF' +set -e -f -u -x +make \ + CI='true' \ + VERBOSE=1 \ + js-test-e2e \ + ; +EOF + +# The builder stage. +FROM dependencies AS builder +ARG CACHE_BUSTER=0 +ADD . /app +WORKDIR /app +RUN \ + --mount=type=cache,id=npm-root-cache,target=/root/.npm \ +<<-'EOF' +set -e -f -u -x +make \ + VERBOSE=1 \ + js-build \ + ; +EOF + +# builder-exporter exports the build artifacts to the host machine so that they +# could be published. This stage should only be used in a CI. +FROM scratch AS builder-exporter +ARG CACHE_BUSTER=0 +COPY --from=builder /app/build /build diff --git a/docker/frontend.Dockerfile.dockerignore b/docker/frontend.Dockerfile.dockerignore new file mode 100644 index 000000000..1d09ccb2e --- /dev/null +++ b/docker/frontend.Dockerfile.dockerignore @@ -0,0 +1,8 @@ +# This comment is used to simplify checking local copies of the file. Bump this +# number every time a significant change is made to this file. +# +# AdGuard-Project-Version: 2 +.git +/bin/ +/tmp/ +/client/node_modules diff --git a/scripts/make/build-docker.sh b/scripts/make/build-docker.sh index 63386fa6c..eb4303130 100644 --- a/scripts/make/build-docker.sh +++ b/scripts/make/build-docker.sh @@ -137,7 +137,7 @@ docker_build_opt_tag() { set -- \ "$@" \ -f \ - ./docker/Dockerfile \ + ./docker/build.Dockerfile \ . \ ;