diff --git a/bamboo-specs/release.yaml b/bamboo-specs/release.yaml index d8ac23e98..78100ff8d 100644 --- a/bamboo-specs/release.yaml +++ b/bamboo-specs/release.yaml @@ -6,9 +6,12 @@ 'name': 'AdGuard Home - Build and publish release' # Make sure to sync any changes with the branch overrides below. 'variables': + # This variable is used to override Docker caching, for example to rerun a + # flaky test suite. + 'cacheBuster': '0' 'channel': 'edge' 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.25.7--1' + 'dockerGo': 'adguard/go-builder:1.25.7--2' 'stages': - 'Build frontend': @@ -47,10 +50,6 @@ 'pattern': 'build/**' 'shared': true 'required': true - 'docker': - 'image': '${bamboo.dockerFrontend}' - 'volumes': - '${system.NPM_DIR}': '${bamboo.cacheNpm}' 'key': 'BF' 'other': 'clean-working-dir': true @@ -60,16 +59,21 @@ - 'script': 'interpreter': 'SHELL' 'scripts': - - | - #!/bin/sh + - |- + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - make\ - VERBOSE=1\ - js-deps js-build - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/frontend.Dockerfile \ + . 'Make release': 'artifact-subscriptions': @@ -80,11 +84,6 @@ 'pattern': 'dist/**' 'shared': true 'required': true - 'docker': - 'image': '${bamboo.dockerGo}' - 'volumes': - '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' - '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' 'key': 'MR' 'other': 'clean-working-dir': true @@ -107,32 +106,46 @@ # Explicitly checkout the revision that we need. git checkout "${bamboo.repository.revision.number}" - # Run the build with the specified channel. - echo "${bamboo.gpgSecretKeyPart1}${bamboo.gpgSecretKeyPart2}"\ - | awk '{ gsub(/\\n/, "\n"); print; }'\ - | gpg --import --batch --yes + # Copy the deploy script to the working repository. + cp ../bamboo-deploy-publisher/deploy.sh ./deploy.sh - make\ - CHANNEL=${bamboo.channel}\ - GPG_KEY_PASSPHRASE=${bamboo.gpgPassword}\ - DEPLOY_SCRIPT_PATH="./bamboo-deploy-publisher/deploy.sh"\ - SIGNER_API_KEY="${bamboo.adguardHomeWinSignerSecretApiKey}"\ - FRONTEND_PREBUILT=1\ - PARALLELISM=1\ - VERBOSE=2\ - build-release - 'requirements': - - 'golang': true + version="$(env CHANNEL=${bamboo_channel} sh ./scripts/make/version.sh)" + readonly version + + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ + --build-arg "BRANCH=${bamboo_planRepository_branchName}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --build-arg "CHANNEL=${bamboo_channel}" \ + --build-arg "DEPLOY_SCRIPT_PATH=./deploy.sh" \ + --build-arg "GPG_SECRET_KEY=${bamboo_gpgSecretKeyPart1}${bamboo_gpgSecretKeyPart2}" \ + --build-arg "GPG_KEY_PASSPHRASE=${bamboo_gpgPassword}" \ + --build-arg "REVISION=${bamboo_repository_revision_number}" \ + --build-arg "SIGN=1" \ + --build-arg "SIGNER_API_KEY=${bamboo_adguardHomeWinSignerSecretApiKey}" \ + --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ + --build-arg "VERSION=$version" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/ci.Dockerfile \ + . 'Make and publish docker': 'key': 'MPD' 'other': 'clean-working-dir': true + 'final-tasks': + - 'clean' 'tasks': - 'checkout': 'force-clean-build': true - 'script': 'interpreter': 'SHELL' + 'environment': + DOCKER_CLI_EXPERIMENTAL=enabled 'scripts': - | #!/bin/sh @@ -149,7 +162,8 @@ docker buildx inspect --bootstrap # Login to DockerHub. - docker login -u="${bamboo.dockerHubUsername}" \ + docker login \ + -u="${bamboo.dockerHubUsername}" \ -p="${bamboo.dockerHubPassword}" # Boot the builder. @@ -168,17 +182,13 @@ DOCKER_PUSH='1' \ VERBOSE='1' \ sh ./scripts/make/build-docker.sh - 'environment': - DOCKER_CLI_EXPERIMENTAL=enabled - 'final-tasks': - - 'clean' - 'requirements': - - 'golang': true 'Publish to static storage': 'key': 'PUB' 'other': 'clean-working-dir': true + 'final-tasks': + - 'clean' 'tasks': - 'clean' - 'checkout': @@ -199,15 +209,13 @@ export CHANNEL ../bamboo-deploy-publisher/deploy.sh adguard-home-"$CHANNEL" - 'final-tasks': - - 'clean' - 'requirements': - - 'golang': true 'Publish to GitHub Releases': 'key': 'PTGR' 'other': 'clean-working-dir': true + 'final-tasks': + - 'clean' 'tasks': - 'clean' - 'checkout': @@ -237,10 +245,6 @@ env\ GITHUB_TOKEN="${bamboo.githubPublicRepoPassword}"\ ../bamboo-deploy-publisher/deploy.sh adguard-home-github - 'final-tasks': - - 'clean' - 'requirements': - - 'golang': true 'triggers': # Don't use minute values that end with a zero or a five as these are often @@ -279,7 +283,7 @@ 'variables': 'channel': 'beta' 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.25.7--1' + 'dockerGo': 'adguard/go-builder:1.25.7--2' # release-vX.Y.Z branches are the branches from which the actual final # release is built. - '^release-v[0-9]+\.[0-9]+\.[0-9]+': @@ -295,4 +299,4 @@ 'variables': 'channel': 'release' 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.25.7--1' + 'dockerGo': 'adguard/go-builder:1.25.7--2' diff --git a/bamboo-specs/snapcraft.yaml b/bamboo-specs/snapcraft.yaml index 02f70fd8c..2aabcd7c2 100644 --- a/bamboo-specs/snapcraft.yaml +++ b/bamboo-specs/snapcraft.yaml @@ -9,17 +9,14 @@ 'name': 'AdGuard Home - Build and publish Snapcraft release' # Make sure to sync any changes with the branch overrides below. 'variables': + # This variable is used to override Docker caching, for example to rerun a + # flaky test suite. + 'cacheBuster': '0' 'channel': 'edge' 'dockerSnap': 'adguard/snap-builder:2.1' 'snapcraftChannel': 'edge' 'stages': - - 'Download release': - 'manual': false - 'final': false - 'jobs': - - 'Download release' - - 'Build packages': 'manual': false 'final': false @@ -32,55 +29,7 @@ 'jobs': - 'Publish to Snapstore' -# TODO(a.garipov): Consider using the Artifact Downloader Task if it ever learns -# about plan branches. -'Download release': - 'artifacts': - - 'name': 'i386_binary' - 'pattern': 'AdGuardHome_i386' - 'shared': true - 'required': true - - 'name': 'amd64_binary' - 'pattern': 'AdGuardHome_amd64' - 'shared': true - 'required': true - - 'name': 'armhf_binary' - 'pattern': 'AdGuardHome_armhf' - 'shared': true - 'required': true - - 'name': 'arm64_binary' - 'pattern': 'AdGuardHome_arm64' - 'shared': true - 'required': true - 'docker': - 'image': '${bamboo.dockerSnap}' - 'key': 'DR' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh - - set -e -f -u -x - - env \ - CHANNEL="${bamboo.channel}" \ - VERBOSE='1' \ - sh ./scripts/snap/download.sh - 'requirements': - - 'golang': true - 'Build packages': - 'artifact-subscriptions': - - 'artifact': 'i386_binary' - - 'artifact': 'amd64_binary' - - 'artifact': 'armhf_binary' - - 'artifact': 'arm64_binary' 'artifacts': - 'name': 'i386_snap' 'pattern': 'AdGuardHome_i386.snap' @@ -98,8 +47,6 @@ 'pattern': 'AdGuardHome_arm64.snap' 'shared': true 'required': true - 'docker': - 'image': '${bamboo.dockerSnap}' 'key': 'BP' 'other': 'clean-working-dir': true @@ -114,11 +61,18 @@ set -e -f -u -x - env \ - VERBOSE='1' \ - sh ./scripts/snap/build.sh - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo_dockerSnap}" + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --build-arg "CHANNEL=${bamboo_channel}" \ + --build-arg "VERSION=${bamboo_buildNumber}" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/snapcraft.Dockerfile \ + . 'Publish to Snapstore': 'artifact-subscriptions': @@ -126,11 +80,11 @@ - 'artifact': 'amd64_snap' - 'artifact': 'armhf_snap' - 'artifact': 'arm64_snap' - 'docker': - 'image': '${bamboo.dockerSnap}' 'key': 'PTS' 'other': 'clean-working-dir': true + 'final-tasks': + - 'clean' 'tasks': - 'checkout': 'force-clean-build': true @@ -142,15 +96,19 @@ set -e -f -u -x - env \ - SNAPCRAFT_CHANNEL="${bamboo.snapcraftChannel}" \ - SNAPCRAFT_STORE_CREDENTIALS="${bamboo.snapcraftMacaroonPassword}" \ - VERBOSE='1' \ - sh ./scripts/snap/upload.sh - 'final-tasks': - - 'clean' - 'requirements': - - 'golang': true + docker info + + docker build \ + --build-arg "BASE_IMAGE=${bamboo_dockerSnap}" + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --build-arg "SNAPCRAFT_CHANNEL=${bamboo_snapcraftChannel}" \ + --build-arg "SNAPCRAFT_STORE_CREDENTIALS=${bamboo_snapcraftMacaroonPassword}" \ + --build-arg "VERSION=${bamboo_buildNumber}" \ + --output '.' \ + --progress 'plain' \ + --target 'publisher' \ + -f ./docker/snapcraft.Dockerfile \ + . 'triggers': # Don't use minute values that end with a zero or a five as these are often diff --git a/bamboo-specs/test.yaml b/bamboo-specs/test.yaml index 41a494da2..c81e2a10d 100644 --- a/bamboo-specs/test.yaml +++ b/bamboo-specs/test.yaml @@ -10,7 +10,7 @@ 'cacheBuster': '0' 'channel': 'development' 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.25.7--1' + 'dockerGo': 'adguard/go-builder:1.25.7--2' 'stages': - 'Tests': @@ -165,6 +165,9 @@ set -e -f -u -x + version="$(env CHANNEL=${bamboo_channel} sh ./scripts/make/version.sh)" + readonly version + docker info docker build \ @@ -175,8 +178,9 @@ --build-arg "CHANNEL=${bamboo_channel}" \ --build-arg "OS=windows darwin linux" \ --build-arg "REVISION=${bamboo_repository_revision_number}" \ + --build-arg "SIGN=0" \ --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ - --build-arg "VERSION=${bamboo_buildNumber}" \ + --build-arg "VERSION=$version" \ --output '.' \ --progress 'plain' \ --target 'builder-exporter' \ @@ -247,5 +251,5 @@ # may need to build a few of these. 'variables': 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.25.7--1' + 'dockerGo': 'adguard/go-builder:1.25.7--2' 'channel': 'candidate' diff --git a/docker/ci.Dockerfile b/docker/ci.Dockerfile index f7a0ff9f9..a8016fbdd 100644 --- a/docker/ci.Dockerfile +++ b/docker/ci.Dockerfile @@ -3,7 +3,7 @@ # This comment is used to simplify checking local copies of the Dockerfile. # Bump this number every time a significant change is made to this Dockerfile. # -# AdGuard-Project-Version: 10 +# AdGuard-Project-Version: 11 # Dockerfile guidelines: # @@ -28,7 +28,7 @@ # needed. Keep it in sync with bamboo-specs/bamboo.yaml. # NOTE: Keep in sync with bamboo-specs/bamboo.yaml. -ARG BASE_IMAGE=adguard/go-builder:1.25.7--1 +ARG BASE_IMAGE=adguard/go-builder:1.25.7--2 # The dependencies stage is needed to install packages and tool dependencies. # This is also where binaries like osslsigncode, which may be required for tests @@ -153,11 +153,12 @@ ARG BRANCH=master ARG CACHE_BUSTER=0 ARG CHANNEL=development ARG DEPLOY_SCRIPT_PATH=not/a/real/path -ARG GPG_KEY_PASSPHRASE -ARG GPG_SECRET_KEY +ARG GPG_KEY_PASSPHRASE=not-a-real-passphrase +ARG GPG_SECRET_KEY="" ARG OS="" ARG REVISION=0000000000000000000000000000000000000000 -ARG SIGNER_API_KEY +ARG SIGN=0 +ARG SIGNER_API_KEY=not-a-real-key ARG SOURCE_DATE_EPOCH=0 ARG VERSION="" ADD . /app @@ -168,16 +169,24 @@ RUN \ <<-'EOF' set -e -f -u -x +# Import GPG key if provided. +if [ "${GPG_SECRET_KEY:-}" != '' ]; then + echo "$GPG_SECRET_KEY" | awk '{ gsub(/\\n/, "\n"); print; }' | gpg --import --batch --yes +fi + make \ ARCH="${ARCH}" \ BRANCH="${BRANCH}" \ CHANNEL="${CHANNEL}" \ + DEPLOY_SCRIPT_PATH="${DEPLOY_SCRIPT_PATH}" \ FRONTEND_PREBUILT=1 \ + GPG_KEY_PASSPHRASE="${GPG_KEY_PASSPHRASE}" \ OS="${OS}" \ PARALLELISM=1 \ REVISION="${REVISION}" \ SOURCE_DATE_EPOCH="$SOURCE_DATE_EPOCH" \ - SIGN=0 \ + SIGN="${SIGN}" \ + SIGNER_API_KEY="${SIGNER_API_KEY}" \ VERBOSE=2 \ VERSION="${VERSION}" \ build-release \ diff --git a/docker/ci.Dockerfile.dockerignore b/docker/ci.Dockerfile.dockerignore index e93ebfc68..4fa8a01ad 100644 --- a/docker/ci.Dockerfile.dockerignore +++ b/docker/ci.Dockerfile.dockerignore @@ -1,8 +1,9 @@ # This comment is used to simplify checking local copies of the file. Bump this # number every time a significant change is made to this file. # -# AdGuard-Project-Version: 2 +# AdGuard-Project-Version: 3 .git /bin/ +/test-reports/ /tmp/ /client/ diff --git a/docker/frontend.Dockerfile.dockerignore b/docker/frontend.Dockerfile.dockerignore index 1d09ccb2e..ff2803fe8 100644 --- a/docker/frontend.Dockerfile.dockerignore +++ b/docker/frontend.Dockerfile.dockerignore @@ -4,5 +4,6 @@ # AdGuard-Project-Version: 2 .git /bin/ +/test-reports/ /tmp/ /client/node_modules diff --git a/docker/snapcraft.Dockerfile b/docker/snapcraft.Dockerfile new file mode 100644 index 000000000..6340aa394 --- /dev/null +++ b/docker/snapcraft.Dockerfile @@ -0,0 +1,87 @@ +# syntax=docker/dockerfile:1 + +# This comment is used to simplify checking local copies of the Dockerfile. +# Bump this number every time a significant change is made to this Dockerfile. +# +# AdGuard-Project-Version: 11 + +# Dockerfile guidelines: +# +# 1. Make sure that Docker correctly caches layers, on a second build attempt it +# must not run lint / test second time when it's not required. +# +# 2. Use BuildKit to improve the build performance (--mount=type=cache, etc). +# +# 3. Prefer using ARG instead of ENV when appropriate, as ARG does not create a +# layer in the final image. However, be careful with what you use ARG for. +# Also, prefer to give ARGs sensible default values. +# +# 4. Use --output and the export stage if you need to get any output on the host +# machine. +# +# NOTE: Only use --output with FROM scratch. +# +# 5. Use .dockerignore to prevent unnecessary files from being sent to the +# Docker daemon, which can invalidate the cache. +# +# 6. Add a CACHE_BUSTER argument to stages to be able to rerun the stages if +# needed. Keep it in sync with bamboo-specs/snapcraft.yaml. + +# NOTE: Keep in sync with bamboo-specs/snapcraft.yaml. +ARG BASE_IMAGE=adguard/snap-builder:2.1 + +# builder downloads the release artifacts and builds snap artifacts. +FROM "$BASE_IMAGE" AS builder +ARG CACHE_BUSTER=0 +ARG CHANNEL=development +ARG VERSION="" +ADD snap /app/snap +ADD scripts /app/scripts +WORKDIR /app +RUN \ +<<-'EOF' +set -e -f -u -x + +export VERBOSE='1' + +env \ + CHANNEL="${CHANNEL}" \ + sh ./scripts/snap/download.sh \ + ; + +sh ./scripts/snap/build.sh +EOF + +# builder-exporter exports the build artifacts to the host machine so that they +# could be published. This stage should only be used in a CI. +FROM scratch AS builder-exporter +ARG CACHE_BUSTER=0 +ARG VERSION="" +COPY --from=builder /app/AdGuardHome_amd64.snap /AdGuardHome_amd64.snap +COPY --from=builder /app/AdGuardHome_arm64.snap /AdGuardHome_arm64.snap +COPY --from=builder /app/AdGuardHome_armhf.snap /AdGuardHome_armhf.snap +COPY --from=builder /app/AdGuardHome_i386.snap /AdGuardHome_i386.snap + +# publisher uploads the release artifacts to the Snap Store. +FROM "$BASE_IMAGE" AS publisher +ARG CACHE_BUSTER=0 +ARG SNAPCRAFT_CHANNEL=0 +ARG SNAPCRAFT_STORE_CREDENTIALS=0 +ARG VERSION="" +ADD snap /app/snap +ADD scripts /app/scripts +ADD AdGuardHome_amd64.snap /app/AdGuardHome_amd64.snap +ADD AdGuardHome_arm64.snap /app/AdGuardHome_arm64.snap +ADD AdGuardHome_armhf.snap /app/AdGuardHome_armhf.snap +ADD AdGuardHome_i386.snap /app/AdGuardHome_i386.snap +WORKDIR /app +RUN \ +<<-'EOF' +set -e -f -u -x + +env \ + SNAPCRAFT_CHANNEL="${SNAPCRAFT_CHANNEL}" \ + SNAPCRAFT_STORE_CREDENTIALS="${SNAPCRAFT_STORE_CREDENTIALS}" \ + VERBOSE='1' \ + sh ./scripts/snap/upload.sh +EOF diff --git a/docker/snapcraft.Dockerfile.dockerignore b/docker/snapcraft.Dockerfile.dockerignore new file mode 100644 index 000000000..e93ebfc68 --- /dev/null +++ b/docker/snapcraft.Dockerfile.dockerignore @@ -0,0 +1,8 @@ +# This comment is used to simplify checking local copies of the file. Bump this +# number every time a significant change is made to this file. +# +# AdGuard-Project-Version: 2 +.git +/bin/ +/tmp/ +/client/