From 48e52787a864b3d2879d3a31dd28597ba2ae0d28 Mon Sep 17 00:00:00 2001 From: Jonathan Cohen Date: Tue, 9 Feb 2021 13:34:32 -0500 Subject: [PATCH] CI Updates (#804) * Fix broken Travis config * Doesn't also build the Linux or Mac manager on server deploy * Correctly builds a Windows binary on manager release (although we still have to manually upload build artifacts) * Renames the Shadowbox deployment step * Run Linux unit tests on Github Actions * Run manager tests on mac, linux, and windows * Fix syntax, use correct node version * Don't test Shadowbox on Windows or Mac * Don't cancel other jobs in the matrix when the build on one platform fails * Use bash on all platforms. This uses Git for Windows bash on Windows * Remove Windows tests for now. This is not a downgrade as we were only testing on linux before anyways * Reduce code duplication * Remove test config from Travis * Respond to review comments: * Remove redundant build step in the shadowbox unit testing workflow * Add a testing step to the shadowbox release Travis workflow * Pin to Ubuntu 16.04 to match our release infrastructure on Travis * Run daily releases of the manager on all platforms * Verbose mode on docker-compose * try turning off Docker Content Trust * Pin to a version of distroless Python and stop removing temp files at the end * Remove verbose from compose * Use Ubuntu 16.04 * Comment the pinned python image * Add temp state cleanup back --- .github/workflows/test.yml | 66 +++++++++++++++++ .travis.yml | 71 ++++++------------- .../integration_test/target/Dockerfile | 5 +- 3 files changed, 90 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..d6e213ce --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,66 @@ +name: Build and Test + +on: push + +jobs: + manager: + name: Build and Test + + runs-on: ${{ matrix.os }} + strategy: + matrix: + # This matches the Ubuntu version used in our Travis releases + os: [ubuntu-16.04] + # Don't cancel jobs on other platforms if one fails + fail-fast: false + + steps: + - name: Checkout repo + uses: actions/checkout@v1 + + - name: Install Node.js, NPM and Yarn + uses: actions/setup-node@v1 + with: + node-version: 12 + + - name: Show Environment Info + run: yarn -v + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: Check yarn cache + uses: actions/cache@v2 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: ${{ runner.os }}-yarn- + + - name: Install dependencies + run: yarn --prefer-offline + + - name: Lint + run: yarn lint + + - name: Manager + run: | + yarn do server_manager/electron_app/build + yarn do server_manager/test + + - name: Shadowbox + if: runner.os == 'Linux' + run: | + yarn do shadowbox/server/build + yarn do shadowbox/test + yarn do shadowbox/integration_test/run + + - name: Metrics Server + if: runner.os == 'Linux' + run: | + yarn do metrics_server/build + yarn do metrics_server/test + + - name: Sentry Webhook + if: runner.os == 'Linux' + run: yarn do sentry_webhook/build diff --git a/.travis.yml b/.travis.yml index 1c3420bf..e5344924 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,7 @@ before_install: stages: - name: "Server Daily Release" if: type = cron - - name: Test - if: type != cron - - name: "deploy" + - name: "Deploy Server" if: tag =~ ^server- - name: "Manager Release" if: tag =~ ^v[0-9] @@ -37,41 +35,27 @@ jobs: include: - stage: "Server Daily Release" script: - - RELEASE_NAME=server-$(date -I) - - curl --data '{"tag_name":"'$RELEASE_NAME'","name":"'$RELEASE_NAME'","prerelease":true}' https://api.github.com/repos/Jigsaw-Code/outline-server/releases?access_token=$CI_USER_TOKEN - - # Ideally, we would split this stage in some way, e.g. by component or by - # build/test commands, to make it clearer in the Travis UI exactly which - # command failed. However, since each stage incurs a significantly start-up - # cost, we combine test and build commands for all components into one fast - # stage. - - stage: Test - name: Unit Tests - script: - - yarn lint - - yarn do metrics_server/build - - yarn do metrics_server/test - - yarn do sentry_webhook/build - - yarn do shadowbox/server/build + - CREATE_RELEASE_URL=https://api.github.com/repos/Jigsaw-Code/outline-server/releases?access_token=$CI_USER_TOKEN + - SERVER_RELEASE_NAME=server-$(date -I) + - curl --data '{"tag_name":"'$SERVER_RELEASE_NAME'","name":"'$SERVER_RELEASE_NAME'","prerelease":true}' $CREATE_RELEASE_URL + - MANAGER_RELEASE_NAME=v$(date -I) + - curl --data '{"tag_name":"'$MANAGER_RELEASE_NAME'","name":"'$MANAGER_RELEASE_NAME'","prerelease":true}' $CREATE_RELEASE_URL + - stage: "Deploy Server" + name: Server Testing + sudo: required + services: docker + script: + # https://docs.travis-ci.com/user/docker/ + - | + sudo rm -f /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.25.5/docker-compose-$(uname -s)-$(uname -m) > docker-compose + chmod +x docker-compose + sudo mv docker-compose /usr/local/bin - yarn do shadowbox/test - - yarn do server_manager/electron_app/build - - yarn do server_manager/web_app/build - - yarn do server_manager/test - - - stage: Test - name: Server Integration Test - sudo: required - services: docker - script: - # https://docs.travis-ci.com/user/docker/ - - | - sudo rm -f /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/1.25.5/docker-compose-$(uname -s)-$(uname -m) > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - yarn do shadowbox/docker/build && cd src/shadowbox/integration_test && ./test.sh - - - stage: deploy + + + - stage: "Deploy Server" name: Server Docker Image sudo: required services: docker @@ -83,16 +67,8 @@ jobs: - docker tag outline/shadowbox quay.io/outline/shadowbox:daily - docker push quay.io/outline/shadowbox:daily - - stage: deploy - name: Manager Linux - addons: - apt: - packages: - - rpm - script: yarn do server_manager/electron_app/package_linux - # https://www.electron.build/multi-platform-build - - stage: deploy + - stage: "Manager Release" name: Manager Windows sudo: required services: docker @@ -106,11 +82,6 @@ jobs: electronuserland/builder:wine /bin/bash -c "yarn do server_manager/electron_app/package_only_windows" || travis_terminate $? - - stage: deploy - name: Manager macOS - os: osx - script: yarn do server_manager/electron_app/package_macos - # Note that because we cannot currently sign Windows binaries on Travis, # these must be manually built and uploaded to the releases page. - stage: "Manager Release" diff --git a/src/shadowbox/integration_test/target/Dockerfile b/src/shadowbox/integration_test/target/Dockerfile index b72df952..44f7ced2 100644 --- a/src/shadowbox/integration_test/target/Dockerfile +++ b/src/shadowbox/integration_test/target/Dockerfile @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/distroless/python3 +# Pin to a known good signed image to avoid failures from the Docker notary service +FROM gcr.io/distroless/python3@sha256:58087520b3c929fe77e1ef3fc95062dbe80bbda265e0e7966c4997c71a9636ea COPY index.html . -ENTRYPOINT ["python", "-m", "http.server", "80"] \ No newline at end of file +ENTRYPOINT ["python", "-m", "http.server", "80"]