chore(server): remove docker-compose and cleanup (#1484)

package-lock.json

@@ -26,6 +26,7 @@
         "generate-license-file": "^1.2.0",
         "husky": "^1.3.1",
         "jasmine": "^3.5.0",
+        "minimist": "^1.2.8",
         "prettier": "^2.4.1",
         "pretty-quick": "^3.1.1",
         "typescript": "^4.9.5"
@@ -2999,6 +3000,12 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/args/node_modules/minimist": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+      "integrity": "sha512-7Wl+Jz+IGWuSdgsQEJ4JunV0si/iMhg42MnQQG6h1R6TNeVenp4U9x5CC5v/gYqz/fENLQITAWXidNtVL0NNbw==",
+      "dev": true
+    },
     "node_modules/args/node_modules/strip-ansi": {
       "version": "3.0.1",
       "dev": true,
@@ -12450,9 +12457,13 @@
       }
     },
     "node_modules/minimist": {
-      "version": "1.2.0",
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
       "dev": true,
-      "license": "MIT"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
     },
     "node_modules/minipass": {
       "version": "3.3.6",
@@ -12541,11 +12552,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/mkdirp/node_modules/minimist": {
-      "version": "1.2.6",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/mri": {
       "version": "1.2.0",
       "dev": true,
@@ -18166,12 +18172,6 @@
         "strip-bom": "^3.0.0"
       }
     },
-    "node_modules/tsconfig-paths/node_modules/minimist": {
-      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
-      "dev": true
-    },
     "node_modules/tsconfig-paths/node_modules/strip-bom": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -22055,6 +22055,12 @@
             "supports-color": "^2.0.0"
           }
         },
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha512-7Wl+Jz+IGWuSdgsQEJ4JunV0si/iMhg42MnQQG6h1R6TNeVenp4U9x5CC5v/gYqz/fENLQITAWXidNtVL0NNbw==",
+          "dev": true
+        },
         "strip-ansi": {
           "version": "3.0.1",
           "dev": true,
@@ -28625,7 +28631,9 @@
       }
     },
     "minimist": {
-      "version": "1.2.0",
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
       "dev": true
     },
     "minipass": {
@@ -28692,12 +28700,6 @@
       "dev": true,
       "requires": {
         "minimist": "^1.2.6"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "1.2.6",
-          "dev": true
-        }
       }
     },
     "mkdirp-classic": {
@@ -32820,12 +32822,6 @@
         "strip-bom": "^3.0.0"
       },
       "dependencies": {
-        "minimist": {
-          "version": "1.2.6",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-          "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
-          "dev": true
-        },
         "strip-bom": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",

package.json

@@ -19,6 +19,7 @@
     "generate-license-file": "^1.2.0",
     "husky": "^1.3.1",
     "jasmine": "^3.5.0",
+    "minimist": "^1.2.8",
     "prettier": "^2.4.1",
     "pretty-quick": "^3.1.1",
     "typescript": "^4.9.5"

src/shadowbox/README.md

@@ -31,8 +31,7 @@ Shadowbox supports running on linux and macOS hosts.
 
 Besides [Node](https://nodejs.org/en/download/) you will also need:
 
-1. [Docker 1.13+](https://docs.docker.com/engine/installation/)
-2. [docker-compose 1.11+](https://docs.docker.com/compose/install/)
+1. [Docker Engine](https://docs.docker.com/engine/installation/)
 
 ### Running Shadowbox as a Node.js app
 
@@ -69,7 +68,7 @@ npm run action shadowbox/docker/build
 Debug image:
 
 ```
-docker run --rm -it --entrypoint=sh outline/shadowbox
+docker run --rm -it --entrypoint=sh localhost/outline/shadowbox
 ```
 
 Or a running container:

src/shadowbox/docker/Dockerfile

@@ -25,7 +25,7 @@ WORKDIR /
 # Don't copy node_modules and other things not needed for install.
 COPY package.json package-lock.json ./
 COPY src/shadowbox/package.json src/shadowbox/
-RUN npm ci
+RUN --mount=type=cache,target=/root/.npm npm ci
 
 # We copy the source code only after npm ci, so that source code changes don't trigger re-installs.
 COPY scripts scripts/
@@ -46,8 +46,11 @@ LABEL shadowbox.node_version=16.18.0
 ARG GITHUB_RELEASE
 LABEL shadowbox.github.release="${GITHUB_RELEASE}"
 
+# The user management service doesn't quit with SIGTERM.
+STOPSIGNAL SIGKILL
+
 # We use curl to detect the server's public IP. We need to use the --date option in `date` to
-# safely grab the ip-to-country database
+# safely grab the ip-to-country database.
 RUN apk add --no-cache --upgrade coreutils curl
 
 COPY src/shadowbox/scripts scripts/

src/shadowbox/docker/build.action.sh

@@ -35,14 +35,10 @@ readonly NODE_IMAGE=$(
     fi
 )
 
-# Doing an explicit `docker pull` of the container base image to work around an issue where
-# Travis fails to pull the base image when using BuildKit. Seems to be related to:
-# https://github.com/moby/buildkit/issues/606 and https://github.com/moby/buildkit/issues/1397
-docker pull "${NODE_IMAGE}"
 docker build --force-rm \
     --build-arg ARCH="${ARCH}" \
     --build-arg NODE_IMAGE="${NODE_IMAGE}" \
     --build-arg GITHUB_RELEASE="${TRAVIS_TAG:-none}" \
     -f src/shadowbox/docker/Dockerfile \
-    -t "${SB_IMAGE:-outline/shadowbox}" \
+    -t "${SB_IMAGE:-localhost/outline/shadowbox}" \
     "${ROOT_DIR}"

src/shadowbox/docker/start.action.sh

@@ -47,7 +47,7 @@ declare -ar docker_bindings=(
   -e "SB_METRICS_URL=${SB_METRICS_URL:-https://dev.metrics.getoutline.org}"
 )
 
-readonly IMAGE="${SB_IMAGE:-outline/shadowbox}"
+readonly IMAGE="${SB_IMAGE:-localhost/outline/shadowbox}"
 echo "Running image ${IMAGE}"
 
 declare -a NET_BINDINGS=("--network=host")

src/shadowbox/integration_test/README.md

@@ -0,0 +1,15 @@
+# Outline Server Image Integration Test
+
+This folder contains the integration test for the Outline Server image.
+
+To build and test the image:
+
+```sh
+npm run action shadowbox/integration_test/run
+```
+
+For development of the test, or to test a specific image, you may prefer calling the test directly, without the build step:
+
+```sh
+./src/shadowbox/integration_test/test.sh localhost/outline/shadowbox:latest
+```

src/shadowbox/integration_test/client/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1-alpine
+FROM docker.io/golang:1-alpine
 
 # curl for fetching pages using the local proxy
 RUN apk add --no-cache curl git

src/shadowbox/integration_test/docker-compose.yml

@@ -1,60 +0,0 @@
-version: "2.1"
-
-networks:
-  open:
-  censored:
-
-services:
-  target:
-    build:
-      context: ./target
-    ports:
-      - "10080:80"
-    networks:
-      - open
-    # The python SimpleHTTPServer doesn't quit with SIGTERM.
-    stop_signal: SIGKILL
-
-  shadowbox:
-    image: ${SB_IMAGE:-outline/shadowbox}
-    environment:
-      - SB_API_PORT=443
-      - SB_API_PREFIX=${SB_API_PREFIX}
-      - LOG_LEVEL=debug
-      - SB_CERTIFICATE_FILE=/root/shadowbox/test.crt
-      - SB_PRIVATE_KEY_FILE=/root/shadowbox/test.key
-    ports:
-      - "20443:443"
-    links:
-      - target
-    networks:
-      - open
-      - censored
-    volumes:
-      - ${SB_CERTIFICATE_FILE}:/root/shadowbox/test.crt
-      - ${SB_PRIVATE_KEY_FILE}:/root/shadowbox/test.key
-      - ${TMP_STATE_DIR}:/root/shadowbox/persisted-state
-    # The user management service doesn't quit with SIGTERM
-    stop_signal: SIGKILL
-
-  client:
-    build:
-      context: ./client
-    ports:
-      - "30555:555"
-    # Keep the container running
-    stdin_open: true
-    tty: true
-    links:
-      - shadowbox
-    networks:
-      - censored
-
-  util:
-    build:
-      context: ./util
-    networks:
-      - open
-    # Keep the container running
-    stdin_open: true
-    tty: true

src/shadowbox/integration_test/run.action.sh

@@ -24,7 +24,7 @@ cd src/shadowbox/integration_test
 
 declare -i result=0
 
-if ./test.sh > "${LOGFILE}" 2>&1 ; then
+if ./test.sh "${SB_IMAGE:-localhost/outline/shadowbox:latest}" > "${LOGFILE}" 2>&1 ; then
   echo "Test Passed!"
   # Removing the log file sometimes fails on Travis.  There's no point in us cleaning it up
   # on a CI build anyways.

src/shadowbox/integration_test/target/Dockerfile

@@ -14,5 +14,7 @@
 
 # Pin to a known good signed image to avoid failures from the Docker notary service
 FROM gcr.io/distroless/python3@sha256:58087520b3c929fe77e1ef3fc95062dbe80bbda265e0e7966c4997c71a9636ea
+# The python SimpleHTTPServer doesn't quit with SIGTERM, so we use SIGKILL.
+STOPSIGNAL SIGKILL
 COPY index.html .
 ENTRYPOINT ["python", "-m", "http.server", "80"]

src/shadowbox/integration_test/test.sh

@@ -32,16 +32,23 @@
 
 set -x
 
-export DOCKER_CONTENT_TRUST="${DOCKER_CONTENT_TRUST:-1}"
-
 OUTPUT_DIR="$(mktemp -d)"
 readonly OUTPUT_DIR
 # TODO(fortuna): Make it possible to run multiple tests in parallel by adding a
 # run id to the container names.
-readonly TARGET_CONTAINER='integrationtest_target_1'
-readonly SHADOWBOX_CONTAINER='integrationtest_shadowbox_1'
-readonly CLIENT_CONTAINER='integrationtest_client_1'
-readonly UTIL_CONTAINER='integrationtest_util_1'
+readonly NAMESPACE='integrationtest'
+readonly TARGET_CONTAINER="${NAMESPACE}_target"
+readonly TARGET_IMAGE="${TARGET_CONTAINER}"
+readonly SHADOWBOX_IMAGE="${1?Must pass image name in the command line}"
+readonly SHADOWBOX_CONTAINER="${NAMESPACE}_shadowbox"
+readonly CLIENT_CONTAINER="${NAMESPACE}_client"
+readonly CLIENT_IMAGE="${CLIENT_CONTAINER}"
+readonly UTIL_IMAGE="${NAMESPACE}_util"
+
+readonly NET_OPEN="${NAMESPACE}_open"
+readonly NET_BLOCKED="${NAMESPACE}_blocked"
+
+
 readonly INTERNET_TARGET_URL="http://www.gstatic.com/generate_204"
 echo "Test output at ${OUTPUT_DIR}"
 # Set DEBUG=1 to not kill the stack when the test is finished so you can query
@@ -55,7 +62,7 @@ function wait_for_resource() {
 }
 
 function util_jq() {
-  docker exec -i "${UTIL_CONTAINER}" jq "$@"
+  docker run --rm -i --entrypoint jq "${UTIL_IMAGE}" "$@"
 }
 
 # Takes the JSON from a /access-keys POST request and returns the appropriate
@@ -79,10 +86,61 @@ function fail() {
   exit 1
 }
 
+function setup() {
+  remove_containers
+
+  docker network create -d bridge "${NET_OPEN}"
+  docker network create -d bridge --internal "${NET_BLOCKED}"
+
+  # Target service.
+  docker build --force-rm -t "${TARGET_IMAGE}" "$(dirname "$0")/target"
+  docker run -d --rm -p "10080:80" --network="${NET_OPEN}" --network-alias="target" --name="${TARGET_CONTAINER}" "${TARGET_IMAGE}"
+  
+  # Shadowsocks service.
+  declare -ar shadowbox_flags=(
+    -d
+    --rm
+    --network="${NET_BLOCKED}"
+    --network-alias="shadowbox"
+    -p "20443:443"
+    -e "SB_API_PORT=443"
+    -e "SB_API_PREFIX=${SB_API_PREFIX}"
+    -e "LOG_LEVEL=debug"
+    -e "SB_CERTIFICATE_FILE=/root/shadowbox/test.crt"
+    -e "SB_PRIVATE_KEY_FILE=/root/shadowbox/test.key"
+    -v "${SB_CERTIFICATE_FILE}:/root/shadowbox/test.crt"
+    -v "${SB_PRIVATE_KEY_FILE}:/root/shadowbox/test.key"
+    -v "${TMP_STATE_DIR}:/root/shadowbox/persisted-state"
+    --name "${SHADOWBOX_CONTAINER}"
+    "${SHADOWBOX_IMAGE}"
+  )
+  docker run "${shadowbox_flags[@]}"
+  # docker network connect --alias shadowbox "${NET_BLOCKED}" "${SHADOWBOX_CONTAINER}"
+  docker network connect "${NET_OPEN}" "${SHADOWBOX_CONTAINER}"
+
+  # Client service.
+  docker build --force-rm -t "${CLIENT_IMAGE}" "$(dirname "$0")/client"
+  # Use -i to keep the container running.
+  docker run -d --rm -it -p "30555:555" --network "${NET_BLOCKED}" --name "${CLIENT_CONTAINER}" "${CLIENT_IMAGE}"
+
+  # Utilities
+  docker build --force-rm -t "${UTIL_IMAGE}" "$(dirname "$0")/util"
+}
+
+function remove_containers() {
+  # Force remove (-f) running containers and `|| true` to not trigger a shell error
+  # in case the container or network doesn't exist.
+  docker rm -f -v "${TARGET_CONTAINER}" || true
+  docker rm -f -v "${SHADOWBOX_CONTAINER}" || true
+  docker rm -f -v "${CLIENT_CONTAINER}" || true
+  docker network rm "${NET_OPEN}" || true
+  docker network rm "${NET_BLOCKED}" || true
+}
+
 function cleanup() {
   local -i status=$?
   if ((DEBUG != 1)); then
-    docker-compose --project-name=integrationtest down
+    remove_containers
     rm -rf "${TMP_STATE_DIR}" || echo "Failed to cleanup files at ${TMP_STATE_DIR}"
   fi
   return "${status}"
@@ -96,16 +154,16 @@ function cleanup() {
   # Ensure proper shut down on exit if not in debug mode
   trap "cleanup" EXIT
 
-  # Make the certificate
-  source ../scripts/make_test_certificate.sh /tmp
-
   # Sets everything up
   export SB_API_PREFIX='TestApiPrefix'
   readonly SB_API_URL="https://shadowbox/${SB_API_PREFIX}"
   TMP_STATE_DIR="$(mktemp -d)"
   export TMP_STATE_DIR
   echo '{"hostname": "shadowbox"}' > "${TMP_STATE_DIR}/shadowbox_server_config.json"
-  docker-compose --project-name=integrationtest up --build -d
+  # Make the certificates. This exports SB_CERTIFICATE_FILE and SB_PRIVATE_KEY_FILE.
+  # shellcheck source=../scripts/make_test_certificate.sh
+  source "$(dirname "$0")/../scripts/make_test_certificate.sh" "${TMP_STATE_DIR}"
+  setup
 
   # Wait for target to come up.
   wait_for_resource localhost:10080
@@ -113,13 +171,13 @@ function cleanup() {
   readonly TARGET_IP
 
   # Verify that the client cannot access or even resolve the target
-  # Exit code 28 for "Connection timed out".
-  (docker exec "${CLIENT_CONTAINER}" curl --silent --connect-timeout 5 "${TARGET_IP}" > /dev/null && \
-    fail "Client should not have access to target IP") || (($? == 28))
+  # Exit code 7 is "Failed to connect to host" and 28 is "Connection timed out".
+  (docker exec "${CLIENT_CONTAINER}" curl --silent --connect-timeout 5 "http://${TARGET_IP}" > /dev/null && \
+    fail "Client should not have access to target IP") || (($? == 7 || $? == 28))
 
   # Exit code 6 for "Could not resolve host".  In some environments, curl reports a timeout
   # error (28) instead, which is surprising.  TODO: Investigate and fix.
-  (docker exec "${CLIENT_CONTAINER}" curl --connect-timeout 5 http://target > /dev/null && \
+  (docker exec "${CLIENT_CONTAINER}" curl --silent --connect-timeout 5 http://target > /dev/null && \
     fail "Client should not have access to target host") || (($? == 6 || $? == 28))
 
   # Wait for shadowbox to come up.