diff --git a/src/build/download_file.mjs b/src/build/download_file.mjs index 30f7d5f7..591292c3 100644 --- a/src/build/download_file.mjs +++ b/src/build/download_file.mjs @@ -13,21 +13,46 @@ // limitations under the License. import {createWriteStream} from 'node:fs'; +import {mkdir} from 'node:fs/promises'; import {pipeline} from 'node:stream/promises'; +import * as path from 'path' +import url from 'url'; + import fetch from 'node-fetch'; +import minimist from 'minimist'; + +import {getFileChecksum} from './get_file_checksum.mjs' /** * Download a remote file from `fileUrl` and save it to `filepath`, using HTTPS protocol. * This function will also follow HTTP redirects. * @param {string} fileUrl The full URL of the remote resource. * @param {string} filepath The full path of the target file. + * @param {string} sha256Checksum The SHA256 checksum of the file to use for verification. * @returns {Promise} A task that will be completed once the download is completed. */ -export async function downloadHttpsFile(fileUrl, filepath) { +export async function downloadFile(fileUrl, filepath, sha256Checksum) { + await mkdir(path.dirname(filepath), { recursive: true }); + const response = await fetch(fileUrl); if (!response.ok) { throw new Error(`failed to download "${fileUrl}": ${response.status} ${response.statusText}`); } const target = createWriteStream(filepath); await pipeline(response.body, target); + + const actualChecksum = await getFileChecksum(filepath, 'sha256'); + if (actualChecksum !== sha256Checksum) { + throw new Error(`failed to verify "${filepath}". ` + + `Expected checksum ${sha256Checksum}, but found ${actualChecksum}`); + } +} + +async function main(...args) { + const {url, sha256, out} = minimist(args); + await downloadFile(url, out, sha256) +} + +if (import.meta.url === url.pathToFileURL(process.argv[1]).href) { + await main(...process.argv.slice(2)); } diff --git a/third_party/jsign/index.mjs b/third_party/jsign/index.mjs index d5cf3e10..d6777371 100644 --- a/third_party/jsign/index.mjs +++ b/third_party/jsign/index.mjs @@ -15,7 +15,7 @@ import {spawn} from 'node:child_process'; import {resolve} from 'node:path'; -import {downloadHttpsFile} from '../../src/build/download_file.mjs'; +import {downloadFile} from '../../src/build/download_file.mjs'; import {getFileChecksum} from '../../src/build/get_file_checksum.mjs'; import {getRootDir} from '../../src/build/get_root_dir.mjs'; @@ -58,13 +58,7 @@ async function ensureJsignJar() { } console.debug(`downloading jsign from "${JSIGN_DOWNLOAD_URL}" to "${jsignPath}"`); - await downloadHttpsFile(JSIGN_DOWNLOAD_URL, jsignPath); - - const actualChecksum = await getFileChecksum(jsignPath, 'sha256'); - if (actualChecksum !== JSIGN_SHA256_CHECKSUM) { - throw new Error(`failed to verify "${jsignPath}". ` + - `Expected checksum ${JSIGN_SHA256_CHECKSUM}, but found ${actualChecksum}`); - } + await downloadFile(JSIGN_DOWNLOAD_URL, jsignPath, JSIGN_SHA256_CHECKSUM); console.debug(`successfully downloaded "${jsignPath}"`); return jsignPath; diff --git a/third_party/outline-ss-server/Makefile b/third_party/outline-ss-server/Makefile index 49bf51d0..b678c2dc 100644 --- a/third_party/outline-ss-server/Makefile +++ b/third_party/outline-ss-server/Makefile @@ -3,13 +3,19 @@ VERSION=1.4.0 .PHONY: all all: bin/linux/outline-ss-server bin/macos/outline-ss-server -bin/linux/outline-ss-server: BASENAME=outline-ss-server_$(VERSION)_linux_x86_64 -bin/macos/outline-ss-server: BASENAME=outline-ss-server_$(VERSION)_macos_x86_64 -bin/linux/outline-ss-server bin/macos/outline-ss-server: checksums.txt - VERSION=$(VERSION) BASENAME="$(BASENAME)" OUTPUT="$@" ./download.sh +bin/linux/outline-ss-server: OS=linux +bin/linux/outline-ss-server: SHA256=f51bcb6391cca0ae828620c429e698a3b7c409de2374c52f113ca9a525e021a8 -checksums.txt: - wget --quiet https://github.com/Jigsaw-Code/outline-ss-server/releases/download/v$(VERSION)/checksums.txt -O $@ +bin/macos/outline-ss-server: OS=macos +bin/macos/outline-ss-server: SHA256=c85b2e8ae2d48482cbc101e54dcb7eed074a22c14a3a7301993e5f786b34081d + +TEMPFILE := $(shell mktemp) +bin/%/outline-ss-server: + node ../../src/build/download_file.mjs --url="https://github.com/Jigsaw-Code/outline-ss-server/releases/download/v$(VERSION)/outline-ss-server_$(VERSION)_$(OS)_x86_64.tar.gz" --out="$(TEMPFILE)" --sha256=$(SHA256) + mkdir -p "$(dir $@)" + tar -zx -f "$(TEMPFILE)" -C "$(dir $@)" "$(notdir $@)" + chmod +x "$@" + rm -f $(TEMPFILE) .PHONY: clean clean: diff --git a/third_party/outline-ss-server/checksums.txt b/third_party/outline-ss-server/checksums.txt deleted file mode 100644 index 59732ba1..00000000 --- a/third_party/outline-ss-server/checksums.txt +++ /dev/null @@ -1,12 +0,0 @@ -0e6cfa1a62ccd5060cf2ea0d1c77946412c50136ecf628b02df85cc382a46620 outline-ss-server_1.4.0_windows_i386.tar.gz -14ae581414c9aab04253a385ef1854c003d09f545f6f8a3a55aa987f0c6d3859 outline-ss-server_1.4.0_linux_arm64.tar.gz -1b331c90f1707304dc9599975a66c92fa0182fb7d9e35677da4802d2d88d282f outline-ss-server_1.4.0_linux_i386.tar.gz -2c3d1e954e31ee31a54f85d205ed45e5a16e4f2c41c057ef4f37ab78f3e58d8c outline-ss-server_1.4.0_windows_armv7.tar.gz -2c854a3d31463911c5a9ae24b44142a324c95e5be2ee750d0c59d0c205cea11d outline-ss-server_1.4.0_windows_x86_64.tar.gz -3c49a6b497628fea4ac4d887988e6856b8fed1594cf8061fe16bc8beb0921be9 outline-ss-server_1.4.0_linux_armv7.tar.gz -50d67c9b97cb2bd3229895c134d3049e782ebbc355c97e8886fefa4775231f5e outline-ss-server_1.4.0_windows_armv6.tar.gz -5b15e4bb2a29061674b6e675e7f69d20071aefae6bedf8177c9ca44d7a42ef15 outline-ss-server_1.4.0_linux_armv6.tar.gz -9647712a7c732184f98b1e2e7f74281855afed2245ec922c4a24b54f0eb0ce72 outline-ss-server_1.4.0_macos_arm64.tar.gz -c85b2e8ae2d48482cbc101e54dcb7eed074a22c14a3a7301993e5f786b34081d outline-ss-server_1.4.0_macos_x86_64.tar.gz -f3ad870aeba4220f69b4d28f8eaac0197627b70893d083a725e4807b129cf92d outline-ss-server_1.4.0_windows_arm64.tar.gz -f51bcb6391cca0ae828620c429e698a3b7c409de2374c52f113ca9a525e021a8 outline-ss-server_1.4.0_linux_x86_64.tar.gz diff --git a/third_party/outline-ss-server/download.sh b/third_party/outline-ss-server/download.sh deleted file mode 100755 index a7450431..00000000 --- a/third_party/outline-ss-server/download.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -eu -# -# Copyright 2022 The Outline Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Downloads and verifies Prometheus binaries. -# Requires VERSION, OUTPUT and BASENAME to be defined. - -declare -r ARCHIVE="${BASENAME}.tar.gz" -# We use wget instead of curl because it's already available on Alpine distros. -wget --quiet "https://github.com/Jigsaw-Code/outline-ss-server/releases/download/v${VERSION}/${ARCHIVE}" -O "${ARCHIVE}" -shasum -a 256 --check --ignore-missing checksums.txt -mkdir -p $(dirname "${OUTPUT}") -tar -zx -f "${BASENAME}.tar.gz" -C $(dirname "${OUTPUT}") "outline-ss-server" -chmod +x "${OUTPUT}" -rm "${ARCHIVE}" - diff --git a/third_party/prometheus/Makefile b/third_party/prometheus/Makefile index 6863bf33..85221e2b 100644 --- a/third_party/prometheus/Makefile +++ b/third_party/prometheus/Makefile @@ -3,13 +3,19 @@ VERSION=2.37.1 .PHONY: all all: bin/linux/prometheus bin/macos/prometheus -bin/linux/prometheus: BASENAME=prometheus-$(VERSION).linux-amd64 -bin/macos/prometheus: BASENAME=prometheus-$(VERSION).darwin-amd64 -bin/linux/prometheus bin/macos/prometheus: sha256sums.txt - VERSION=$(VERSION) BASENAME="$(BASENAME)" OUTPUT="$@" ./download.sh +bin/linux/prometheus: OS=linux +bin/linux/prometheus: SHA256=753f66437597cf52ada98c2f459aa8c03745475c249c9f2b40ac7b3919131ba6 -sha256sums.txt: - wget --quiet https://github.com/prometheus/prometheus/releases/download/v$(VERSION)/sha256sums.txt -O $@ +bin/macos/prometheus: OS=darwin +bin/macos/prometheus: SHA256=e03a43d98955ac3500f57353ea74b5df829074205f195ea6b3b88f55c4575c79 + +bin/%/prometheus: TEMPFILE := $(shell mktemp) +bin/%/prometheus: + node ../../src/build/download_file.mjs --url="https://github.com/prometheus/prometheus/releases/download/v$(VERSION)/prometheus-$(VERSION).$(OS)-amd64.tar.gz" --out="$(TEMPFILE)" --sha256=$(SHA256) + mkdir -p "$(dir $@)" + tar -zx -f "$(TEMPFILE)" --strip-components=1 -C "$(dir $@)" prometheus-$(VERSION).$(OS)-amd64/prometheus + chmod +x "$@" + rm -f $(TEMPFILE) .PHONY: clean clean: diff --git a/third_party/prometheus/download.sh b/third_party/prometheus/download.sh deleted file mode 100755 index 8d8274df..00000000 --- a/third_party/prometheus/download.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -eu -# -# Copyright 2022 The Outline Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Downloads and verifies Prometheus binaries. -# Requires VERSION, OUTPUT and BASENAME to be defined. - -declare -r ARCHIVE="${BASENAME}.tar.gz" -# We use wget instead of curl because it's already available on Alpine distros. -wget --quiet "https://github.com/prometheus/prometheus/releases/download/v${VERSION}/${ARCHIVE}" -O "${ARCHIVE}" -shasum -a 256 --check --ignore-missing sha256sums.txt -mkdir -p $(dirname "${OUTPUT}") -tar -zx -f "${BASENAME}.tar.gz" --strip-components=1 -C $(dirname "${OUTPUT}") "${BASENAME}/prometheus" -chmod +x "${OUTPUT}" -rm "${ARCHIVE}" - diff --git a/third_party/prometheus/sha256sums.txt b/third_party/prometheus/sha256sums.txt deleted file mode 100644 index ec1be74d..00000000 --- a/third_party/prometheus/sha256sums.txt +++ /dev/null @@ -1,34 +0,0 @@ -e03a43d98955ac3500f57353ea74b5df829074205f195ea6b3b88f55c4575c79 prometheus-2.37.1.darwin-amd64.tar.gz -eb8a174c82a0fb6c84e81d9a73214318fb4a605115ad61505d7883d02e5a6f52 prometheus-2.37.1.darwin-arm64.tar.gz -70e10e65cd03719ef8163d35dc5acb10fbeb1bb659af743492e9ae59d335c16a prometheus-2.37.1.dragonfly-amd64.tar.gz -5ba6dcc8f645a502193db6d2a07733d9a70b8777a92e3ff076ab2ec174bac6f5 prometheus-2.37.1.freebsd-386.tar.gz -72b2e0426bc7a58a7d8f30c7bd2c55ab928612457f1e8f0ae218988f68e91ac3 prometheus-2.37.1.freebsd-amd64.tar.gz -5e88d74079252463ea6eed4f91ab5fa3a8dc9179cad4538ef642b2c87281870b prometheus-2.37.1.freebsd-arm64.tar.gz -f9232f7bcf54cbb6af26d423984cf02aaea8a27b28e447f0b06bfecba2213cd0 prometheus-2.37.1.freebsd-armv6.tar.gz -200f744e7ab825739560bc236fed7be75f4112ce177fed700a29fce24065851c prometheus-2.37.1.freebsd-armv7.tar.gz -5332b59b2d070f1334628068c6f7f110daff4c9039367a36eaeb8850b14e7df1 prometheus-2.37.1.illumos-amd64.tar.gz -9208fe856313ab49873565a9a09b624c376517f58d229f365d26a05881325225 prometheus-2.37.1.linux-386.tar.gz -753f66437597cf52ada98c2f459aa8c03745475c249c9f2b40ac7b3919131ba6 prometheus-2.37.1.linux-amd64.tar.gz -b59a66fb5c7ec5acf6bf426793528a5789a1478a0dad8c64edc2843caf31b1b8 prometheus-2.37.1.linux-arm64.tar.gz -5ca315caed408c95fc04381d73374e1b1c65e4315db23c9d57765213bddf8cd3 prometheus-2.37.1.linux-armv5.tar.gz -e38349c8becebc847008cc262101fc1bb869374f60885a86272068ea339676bb prometheus-2.37.1.linux-armv6.tar.gz -3eaa01248879fb14524ea09ace1b4fabbbdc5ffc6b288bb101395681aa3c77f0 prometheus-2.37.1.linux-armv7.tar.gz -0cc32f08088e8a475fb787de9d475c1f46d4a0cdb53647bca0f229a4e829d04f prometheus-2.37.1.linux-mips.tar.gz -75ca2852240d0b05c880d74489d4508bd0e59dbf48fb14853fe87a4997804047 prometheus-2.37.1.linux-mips64.tar.gz -7d6a00f27148be97525386445ec3eac8724cf97bdaff348434b532b7ec3dd7b7 prometheus-2.37.1.linux-mips64le.tar.gz -9ea7c96896ba7e18e1c92608a122e83ee595eff976390796d3f1149a42241959 prometheus-2.37.1.linux-mipsle.tar.gz -137c4ff9183c8136d59750623ff811e5f50d81a9fc1057e61b5bd72927f64f1c prometheus-2.37.1.linux-ppc64.tar.gz -74a21c476ac783e31fa63087a76bce81a1c783f775bf8ffba2137737e1b1a945 prometheus-2.37.1.linux-ppc64le.tar.gz -f4653be8f571aeef033871b5f404107a725d505f1042eb5056595c077464bbf0 prometheus-2.37.1.linux-s390x.tar.gz -11d976643b2b4eed78737008bd8361b86ba72b562d42700768bc7d0d8b4db65f prometheus-2.37.1.netbsd-386.tar.gz -66d155e295f6e6b4ec502058fb0b01f86b04c5b8a5a6fdeb11a08fd9cabb760a prometheus-2.37.1.netbsd-amd64.tar.gz -15a861c711ec8d3b7c6c407748516a984fc49aba55732f67b21b94335e962f82 prometheus-2.37.1.netbsd-arm64.tar.gz -66eaafcfda1483d1cbaaea03ffbae446ecb11e66531fa91d36cd23216fffa15f prometheus-2.37.1.netbsd-armv6.tar.gz -687a939803517401091347b3b5996e6f4a94b187b018b2c473f79e778f1dcc0d prometheus-2.37.1.netbsd-armv7.tar.gz -0afbce7a0458059626226013e38681e9b3c7410e76c8ef1c07406475c4ea45e0 prometheus-2.37.1.windows-386.tar.gz -b19e093bf4f078770122c16afaf5586af74ee73d3d573a89619f148a06057c7c prometheus-2.37.1.windows-386.zip -099ea240418febf4bcf714c2a754f5b8cc414bda9c98c8b717fb911184af1a56 prometheus-2.37.1.windows-amd64.tar.gz -c93c5f38a00ae1550f9df76d448ae1ba9fdb1f4d2bd23ebd4c162ec2a128c175 prometheus-2.37.1.windows-amd64.zip -9e2abbafdd1c26fcf57e72b6a08e824f3693bb459b6f9a9208ae98d6a7c75a69 prometheus-2.37.1.windows-arm64.tar.gz -56c50f20b14b7b900ac864c718f85cb22c267c48abac55c50260381541ffc9dd prometheus-2.37.1.windows-arm64.zip -52ef6e3b4964efd93b9eee18c7fc28fadbf8155bb99e5f93512175c3cbd4a1a3 prometheus-web-ui-2.37.1.tar.gz diff --git a/third_party/shellcheck/run.sh b/third_party/shellcheck/run.sh index 9fffc4a1..1fe7dca0 100755 --- a/third_party/shellcheck/run.sh +++ b/third_party/shellcheck/run.sh @@ -20,34 +20,22 @@ readonly VERSION='v0.7.1' DOWNLOAD_DIR="$(dirname "$0")/download" readonly DOWNLOAD_DIR -# `sha256sum` is part of GNU coreutils but is not available in macOS. -# macOS does have `shasum` (a Perl script designed to match the behavior -# of `sha256sum`) in the default install. -function sha256wrapper() { - if command -v sha256sum &> /dev/null; then - sha256sum "$@" - else - shasum -a 256 "$@" - fi -} - declare file="shellcheck-${VERSION}" # Name of the file to download declare cmd="${DOWNLOAD_DIR}/shellcheck-${VERSION}" # Path to the executable +declare sha256='' # SHA256 checksum case "$(uname -s)" in - Linux) file+='.linux.x86_64.tar.xz'; cmd+='/shellcheck';; - Darwin) file+='.darwin.x86_64.tar.xz'; cmd+='/shellcheck';; - *) file+='.zip'; cmd+='.exe';; # Presume Windows/Cygwin + Linux) file+='.linux.x86_64.tar.xz'; cmd+='/shellcheck'; sha256='64f17152d96d7ec261ad3086ed42d18232fcb65148b44571b564d688269d36c8';; + Darwin) file+='.darwin.x86_64.tar.xz'; cmd+='/shellcheck'; sha256='b080c3b659f7286e27004aa33759664d91e15ef2498ac709a452445d47e3ac23' ;; + *) file+='.zip'; cmd+='.exe'; sha256='1763f8f4a639d39e341798c7787d360ed79c3d68a1cdbad0549c9c0767a75e98';; # Presume Windows/Cygwin esac readonly file cmd if [[ ! -s "${cmd}" ]]; then mkdir -p "${DOWNLOAD_DIR}" - readonly url="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/${file}" - curl --location --fail --output "${DOWNLOAD_DIR}/${file}" "${url}" + node "$(dirname "$0")/../../src/build/download_file.mjs" --url="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/${file}" --out="${DOWNLOAD_DIR}/${file}" --sha256="${sha256}" pushd "${DOWNLOAD_DIR}" - sha256wrapper --check --ignore-missing ../hashes.sha256 if [[ "${file}" == *'.tar.xz' ]]; then tar xf "${file}" else