diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5cfeb9e5..e6fa9b9e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,135 +1,135 @@ -'name': 'build' +"name": "build" # Permissions are disabled for all the jobs by default, and then overridden for # specific jobs if needed. This is a recommended practice for security reasons, # and also allows to avoid mistakes with permissions when creating new jobs. # # See https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#permissions. -'permissions': {} -'env': - 'GO_VERSION': '1.26.2' - 'NODE_VERSION': '20' -'on': - 'push': - 'branches': - - '*' - 'tags': - - 'v*' - 'pull_request': -'jobs': - 'test': - 'runs-on': '${{ matrix.os }}' - 'permissions': - 'contents': 'read' - 'env': - 'GO111MODULE': 'on' - 'GOPROXY': 'https://goproxy.cn' - 'strategy': - 'fail-fast': false - 'matrix': - 'os': - - 'ubuntu-latest' - - 'macOS-latest' - - 'windows-latest' - 'steps': - - 'name': 'Checkout' - 'uses': 'actions/checkout@v2' - 'with': - 'fetch-depth': 0 +"permissions": {} +"env": + "GO_VERSION": "1.26.4" + "NODE_VERSION": "20" +"on": + "push": + "branches": + - "*" + "tags": + - "v*" + "pull_request": +"jobs": + "test": + "runs-on": "${{ matrix.os }}" + "permissions": + "contents": "read" + "env": + "GO111MODULE": "on" + "GOPROXY": "https://goproxy.cn" + "strategy": + "fail-fast": false + "matrix": + "os": + - "ubuntu-latest" + - "macOS-latest" + - "windows-latest" + "steps": + - "name": "Checkout" + "uses": "actions/checkout@v2" + "with": + "fetch-depth": 0 # Set to false to avoid GITHUB_TOKEN leaking. # # See https://github.com/actions/checkout/issues/485#issuecomment-934422611. - 'persist-credentials': false - - 'name': 'Set up Go' - 'uses': 'actions/setup-go@v3' - 'with': - 'go-version': '${{ env.GO_VERSION }}' - - 'name': 'Set up Node' - 'uses': 'actions/setup-node@v1' - 'with': - 'node-version': '${{ env.NODE_VERSION }}' - - 'name': 'Set up Go modules cache' - 'uses': 'actions/cache@v4' - 'with': - 'path': '~/go/pkg/mod' - 'key': "${{ runner.os }}-go-${{ hashFiles('go.sum') }}" - 'restore-keys': '${{ runner.os }}-go-' - - 'name': 'Get npm cache directory' - 'id': 'npm-cache' - 'run': 'echo "::set-output name=dir::$( npm config get cache )"' - - 'name': 'Set up npm cache' - 'uses': 'actions/cache@v4' - 'with': - 'path': '${{ steps.npm-cache.outputs.dir }}' - 'key': "${{ runner.os }}-node-${{ hashFiles('client/package-lock.json') }}" - 'restore-keys': '${{ runner.os }}-node-' - - 'name': 'Run tests' - 'shell': 'bash' - 'run': 'make VERBOSE=1 deps test go-bench go-fuzz' - - 'name': 'Upload coverage' - 'uses': 'codecov/codecov-action@v1' - 'if': "success() && matrix.os == 'ubuntu-latest'" - 'with': - 'token': '${{ secrets.CODECOV_TOKEN }}' - 'file': './coverage.txt' - 'build-release': - 'runs-on': 'ubuntu-latest' - 'needs': 'test' - 'permissions': - 'contents': 'read' - 'steps': - - 'name': 'Checkout' - 'uses': 'actions/checkout@v2' - 'with': - 'fetch-depth': 0 + "persist-credentials": false + - "name": "Set up Go" + "uses": "actions/setup-go@v3" + "with": + "go-version": "${{ env.GO_VERSION }}" + - "name": "Set up Node" + "uses": "actions/setup-node@v1" + "with": + "node-version": "${{ env.NODE_VERSION }}" + - "name": "Set up Go modules cache" + "uses": "actions/cache@v4" + "with": + "path": "~/go/pkg/mod" + "key": "${{ runner.os }}-go-${{ hashFiles('go.sum') }}" + "restore-keys": "${{ runner.os }}-go-" + - "name": "Get npm cache directory" + "id": "npm-cache" + "run": 'echo "::set-output name=dir::$( npm config get cache )"' + - "name": "Set up npm cache" + "uses": "actions/cache@v4" + "with": + "path": "${{ steps.npm-cache.outputs.dir }}" + "key": "${{ runner.os }}-node-${{ hashFiles('client/package-lock.json') }}" + "restore-keys": "${{ runner.os }}-node-" + - "name": "Run tests" + "shell": "bash" + "run": "make VERBOSE=1 deps test go-bench go-fuzz" + - "name": "Upload coverage" + "uses": "codecov/codecov-action@v1" + "if": "success() && matrix.os == 'ubuntu-latest'" + "with": + "token": "${{ secrets.CODECOV_TOKEN }}" + "file": "./coverage.txt" + "build-release": + "runs-on": "ubuntu-latest" + "needs": "test" + "permissions": + "contents": "read" + "steps": + - "name": "Checkout" + "uses": "actions/checkout@v2" + "with": + "fetch-depth": 0 # Set to false to avoid GITHUB_TOKEN leaking. # # See https://github.com/actions/checkout/issues/485#issuecomment-934422611. - 'persist-credentials': false - - 'name': 'Set up Go' - 'uses': 'actions/setup-go@v3' - 'with': - 'go-version': '${{ env.GO_VERSION }}' - - 'name': 'Set up Node' - 'uses': 'actions/setup-node@v1' - 'with': - 'node-version': '${{ env.NODE_VERSION }}' - - 'name': 'Set up Go modules cache' - 'uses': 'actions/cache@v4' - 'with': - 'path': '~/go/pkg/mod' - 'key': "${{ runner.os }}-go-${{ hashFiles('go.sum') }}" - 'restore-keys': '${{ runner.os }}-go-' - - 'name': 'Get npm cache directory' - 'id': 'npm-cache' - 'run': 'echo "::set-output name=dir::$(npm config get cache)"' - - 'name': 'Set up npm cache' - 'uses': 'actions/cache@v4' - 'with': - 'path': '${{ steps.npm-cache.outputs.dir }}' - 'key': "${{ runner.os }}-node-${{ hashFiles('client/package-lock.json') }}" - 'restore-keys': '${{ runner.os }}-node-' - - 'name': 'Set up Snapcraft' - 'run': 'sudo snap install snapcraft --classic' - - 'name': 'Set up QEMU' - 'uses': 'docker/setup-qemu-action@v3' - - 'name': 'Set up Docker Buildx' - 'uses': 'docker/setup-buildx-action@v3' - 'with': - 'install': true - - 'name': 'Run snapshot build' + "persist-credentials": false + - "name": "Set up Go" + "uses": "actions/setup-go@v3" + "with": + "go-version": "${{ env.GO_VERSION }}" + - "name": "Set up Node" + "uses": "actions/setup-node@v1" + "with": + "node-version": "${{ env.NODE_VERSION }}" + - "name": "Set up Go modules cache" + "uses": "actions/cache@v4" + "with": + "path": "~/go/pkg/mod" + "key": "${{ runner.os }}-go-${{ hashFiles('go.sum') }}" + "restore-keys": "${{ runner.os }}-go-" + - "name": "Get npm cache directory" + "id": "npm-cache" + "run": 'echo "::set-output name=dir::$(npm config get cache)"' + - "name": "Set up npm cache" + "uses": "actions/cache@v4" + "with": + "path": "${{ steps.npm-cache.outputs.dir }}" + "key": "${{ runner.os }}-node-${{ hashFiles('client/package-lock.json') }}" + "restore-keys": "${{ runner.os }}-node-" + - "name": "Set up Snapcraft" + "run": "sudo snap install snapcraft --classic" + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "Set up Docker Buildx" + "uses": "docker/setup-buildx-action@v3" + "with": + "install": true + - "name": "Run snapshot build" # Set a custom version string, since the checkout@v2 action does not seem # to know about the master branch, while the version script uses it to # count the number of commits within the branch. - 'run': 'make SIGN=0 VERBOSE=1 VERSION="v0.0.0-github" build-release build-docker' - 'notify': - 'needs': - - 'build-release' + "run": 'make SIGN=0 VERBOSE=1 VERSION="v0.0.0-github" build-release build-docker' + "notify": + "needs": + - "build-release" # Secrets are not passed to workflows that are triggered by a pull request # from a fork. # # Use always() to signal to the runner that this job must run even if the # previous ones failed. - 'if': | + "if": | ${{ always() && github.repository_owner == 'AdguardTeam' && @@ -138,19 +138,19 @@ github.event.pull_request.head.repo.full_name == github.repository ) }} - 'runs-on': 'ubuntu-latest' - 'permissions': + "runs-on": "ubuntu-latest" + "permissions": # Required by workflow-conclusion-action to check workflow status. - 'actions': 'read' - 'contents': 'read' - 'steps': - - 'name': 'Conclusion' - 'uses': 'technote-space/workflow-conclusion-action@v1' - - 'name': 'Send Slack notif' - 'uses': '8398a7/action-slack@v3' - 'with': - 'status': '${{ env.WORKFLOW_CONCLUSION }}' - 'fields': 'repo, message, commit, author, workflow' - 'env': - 'GITHUB_TOKEN': '${{ secrets.GITHUB_TOKEN }}' - 'SLACK_WEBHOOK_URL': '${{ secrets.SLACK_WEBHOOK_URL }}' + "actions": "read" + "contents": "read" + "steps": + - "name": "Conclusion" + "uses": "technote-space/workflow-conclusion-action@v1" + - "name": "Send Slack notif" + "uses": "8398a7/action-slack@v3" + "with": + "status": "${{ env.WORKFLOW_CONCLUSION }}" + "fields": "repo, message, commit, author, workflow" + "env": + "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}" + "SLACK_WEBHOOK_URL": "${{ secrets.SLACK_WEBHOOK_URL }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7353f08e1..a6c93fb04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,10 +18,35 @@ See also the [v0.107.78 GitHub milestone][ms-v0.107.78]. NOTE: Add new changes BELOW THIS COMMENT. --> +### Added + +- Improved updater logging to give users more insight into the problem with version updating ([#8410]). + +[#8410]: https://github.com/AdguardTeam/AdGuardHome/issues/8410 + +### Security + +- Go version has been updated to prevent the possibility of exploiting the Go vulnerabilities fixed in [1.26.4][go-1.26.4]. + +- The H2C connection establishment via HTTP/1.1 request upgrade is no longer supported. See [RFC 9113][rfc9113]. + +- The size of rulelists is limited. This is necessary to prevent a user's machine from becoming overloaded if the filter source misbehaves. + +### Changed + +- The interval of filter updates can now be set to any number of ours between 0 and 365 days in the configuration file. + +#### Configuration changes + +- The `filtering` object of the YAML configuration now includes a new property, `max_http_size`, which defines the maximum size of the HTTP request for rulelists. To disable the limitation, set a large size, such as `1 TB`. + ### Fixed - Blocked services check on the Custom filtering rules page does not work properly without specifying of a client. +[rfc9113]: https://datatracker.ietf.org/doc/html/rfc9113 +[go-1.26.4]: https://groups.google.com/g/golang-announce/c/tKs3rmcBcKw + diff --git a/Makefile b/Makefile index 4abd0ff0b..da06918a7 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ DIST_DIR = dist GOAMD64 = v1 GOPROXY = https://proxy.golang.org|direct GOTELEMETRY = off -GOTOOLCHAIN = go1.26.3 +GOTOOLCHAIN = go1.26.4 GPG_KEY = devteam@adguard.com GPG_KEY_PASSPHRASE = not-a-real-password NPM = npm diff --git a/bamboo-specs/release.yaml b/bamboo-specs/release.yaml index 948900d53..3aab6b4ba 100644 --- a/bamboo-specs/release.yaml +++ b/bamboo-specs/release.yaml @@ -1,299 +1,298 @@ --- -'version': 2 -'plan': - 'project-key': 'AGH' - 'key': 'AGHBSNAPSPECS' - 'name': 'AdGuard Home - Build and publish release' +"version": 2 +"plan": + "project-key": "AGH" + "key": "AGHBSNAPSPECS" + "name": "AdGuard Home - Build and publish release" # Make sure to sync any changes with the branch overrides below. -'variables': - # This variable is used to override Docker caching, for example to rerun a - # flaky test suite. - 'cacheBuster': '0' - 'channel': 'edge' - 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.26.2--1' +"variables": + # This variable is used to override Docker caching, for example to rerun a + # flaky test suite. + "cacheBuster": "0" + "channel": "edge" + "dockerFrontend": "adguard/home-js-builder:4.0" + "dockerGo": "adguard/go-builder:1.26.4--1" -'stages': - - 'Build frontend': - 'manual': false - 'final': false - 'jobs': - - 'Build frontend' +"stages": + - "Build frontend": + "manual": false + "final": false + "jobs": + - "Build frontend" - - 'Make release': - 'manual': false - 'final': false - 'jobs': - - 'Make release' + - "Make release": + "manual": false + "final": false + "jobs": + - "Make release" - - 'Make and publish docker': - 'manual': false - 'final': false - 'jobs': - - 'Make and publish docker' + - "Make and publish docker": + "manual": false + "final": false + "jobs": + - "Make and publish docker" - - 'Publish to static storage': - 'manual': false - 'final': false - 'jobs': - - 'Publish to static storage' + - "Publish to static storage": + "manual": false + "final": false + "jobs": + - "Publish to static storage" - - 'Publish to GitHub Releases': - 'manual': false - 'final': false - 'jobs': - - 'Publish to GitHub Releases' + - "Publish to GitHub Releases": + "manual": false + "final": false + "jobs": + - "Publish to GitHub Releases" -'Build frontend': - 'artifacts': - - 'name': 'AdGuardHome frontend' - 'pattern': 'build/**' - 'shared': true - 'required': true - 'key': 'BF' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - |- - #!/bin/sh +"Build frontend": + "artifacts": + - "name": "AdGuardHome frontend" + "pattern": "build/**" + "shared": true + "required": true + "key": "BF" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - |- + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - docker info + docker info - docker build \ - --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --output '.' \ - --progress 'plain' \ - --target 'builder-exporter' \ - -f ./docker/frontend.Dockerfile \ - . + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/frontend.Dockerfile \ + . -'Make release': - 'artifact-subscriptions': - - 'artifact': 'AdGuardHome frontend' - # TODO(a.garipov): Use more fine-grained artifact rules. - 'artifacts': - - 'name': 'AdGuardHome dists' - 'pattern': 'dist/**' - 'shared': true - 'required': true - 'key': 'MR' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'checkout': - 'repository': 'bamboo-deploy-publisher' - # The paths are always relative to the working directory. - 'path': 'bamboo-deploy-publisher' - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh +"Make release": + "artifact-subscriptions": + - "artifact": "AdGuardHome frontend" + # TODO(a.garipov): Use more fine-grained artifact rules. + "artifacts": + - "name": "AdGuardHome dists" + "pattern": "dist/**" + "shared": true + "required": true + "key": "MR" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "checkout": + "repository": "bamboo-deploy-publisher" + # The paths are always relative to the working directory. + "path": "bamboo-deploy-publisher" + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - # Explicitly checkout the revision that we need. - git checkout "${bamboo.repository.revision.number}" + # Explicitly checkout the revision that we need. + git checkout "${bamboo.repository.revision.number}" - version="$(env CHANNEL=${bamboo_channel} sh ./scripts/make/version.sh)" - readonly version + version="$(env CHANNEL=${bamboo_channel} sh ./scripts/make/version.sh)" + readonly version - docker info + docker info - docker build \ - --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ - --build-arg "BRANCH=${bamboo_planRepository_branchName}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --build-arg "CHANNEL=${bamboo_channel}" \ - --build-arg "DEPLOY_SCRIPT_PATH=./bamboo-deploy-publisher/deploy.sh" \ - --build-arg "GPG_SECRET_KEY=${bamboo_gpgSecretKeyPart1}${bamboo_gpgSecretKeyPart2}" \ - --build-arg "GPG_KEY_PASSPHRASE=${bamboo_gpgPassword}" \ - --build-arg "REVISION=${bamboo_repository_revision_number}" \ - --build-arg "SIGN=1" \ - --build-arg "SIGNER_API_KEY=${bamboo_adguardHomeWinSignerSecretApiKey}" \ - --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ - --build-arg "VERSION=$version" \ - --output '.' \ - --progress 'plain' \ - --target 'builder-exporter' \ - -f ./docker/ci.Dockerfile \ - . + docker build \ + --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ + --build-arg "BRANCH=${bamboo_planRepository_branchName}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --build-arg "CHANNEL=${bamboo_channel}" \ + --build-arg "DEPLOY_SCRIPT_PATH=./bamboo-deploy-publisher/deploy.sh" \ + --build-arg "GPG_SECRET_KEY=${bamboo_gpgSecretKeyPart1}${bamboo_gpgSecretKeyPart2}" \ + --build-arg "GPG_KEY_PASSPHRASE=${bamboo_gpgPassword}" \ + --build-arg "REVISION=${bamboo_repository_revision_number}" \ + --build-arg "SIGN=1" \ + --build-arg "SIGNER_API_KEY=${bamboo_adguardHomeWinSignerSecretApiKey}" \ + --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ + --build-arg "VERSION=$version" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/ci.Dockerfile \ + . -'Make and publish docker': - 'key': 'MPD' - 'other': - 'clean-working-dir': true - 'final-tasks': - - 'clean' - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'environment': - DOCKER_CLI_EXPERIMENTAL=enabled - 'scripts': - - | - #!/bin/sh +"Make and publish docker": + "key": "MPD" + "other": + "clean-working-dir": true + "final-tasks": + - "clean" + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "environment": DOCKER_CLI_EXPERIMENTAL=enabled + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - # Install Qemu, create builder. - docker version -f '{{ .Server.Experimental }}' - docker buildx rm buildx-builder || : - docker buildx create \ - --name buildx-builder \ - --driver docker-container \ - --use - docker buildx inspect --bootstrap + # Install Qemu, create builder. + docker version -f '{{ .Server.Experimental }}' + docker buildx rm buildx-builder || : + docker buildx create \ + --name buildx-builder \ + --driver docker-container \ + --use + docker buildx inspect --bootstrap - # Login to DockerHub. - docker login \ - -u="${bamboo.dockerHubUsername}" \ - -p="${bamboo.dockerHubPassword}" + # Login to DockerHub. + docker login \ + -u="${bamboo.dockerHubUsername}" \ + -p="${bamboo.dockerHubPassword}" - # Boot the builder. - docker buildx inspect --bootstrap + # Boot the builder. + docker buildx inspect --bootstrap - # Print Docker info. - docker info - docker buildx version + # Print Docker info. + docker info + docker buildx version - # Prepare and push the build. - env \ - CHANNEL="${bamboo.channel}" \ - REVISION="${bamboo.repository.revision.number}" \ - DIST_DIR='dist' \ - DOCKER_IMAGE_NAME='adguard/adguardhome' \ - DOCKER_PUSH='1' \ - VERBOSE='1' \ - sh ./scripts/make/build-docker.sh + # Prepare and push the build. + env \ + CHANNEL="${bamboo.channel}" \ + REVISION="${bamboo.repository.revision.number}" \ + DIST_DIR='dist' \ + DOCKER_IMAGE_NAME='adguard/adguardhome' \ + DOCKER_PUSH='1' \ + VERBOSE='1' \ + sh ./scripts/make/build-docker.sh -'Publish to static storage': - 'key': 'PUB' - 'other': - 'clean-working-dir': true - 'final-tasks': - - 'clean' - 'tasks': - - 'clean' - - 'checkout': - 'repository': 'bamboo-deploy-publisher' - 'path': 'bamboo-deploy-publisher' - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh +"Publish to static storage": + "key": "PUB" + "other": + "clean-working-dir": true + "final-tasks": + - "clean" + "tasks": + - "clean" + - "checkout": + "repository": "bamboo-deploy-publisher" + "path": "bamboo-deploy-publisher" + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - cd ./dist/ + cd ./dist/ - CHANNEL="${bamboo.channel}" - export CHANNEL + CHANNEL="${bamboo.channel}" + export CHANNEL - ../bamboo-deploy-publisher/deploy.sh adguard-home-"$CHANNEL" + ../bamboo-deploy-publisher/deploy.sh adguard-home-"$CHANNEL" -'Publish to GitHub Releases': - 'key': 'PTGR' - 'other': - 'clean-working-dir': true - 'final-tasks': - - 'clean' - 'tasks': - - 'clean' - - 'checkout': - 'repository': 'bamboo-deploy-publisher' - 'path': 'bamboo-deploy-publisher' - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh +"Publish to GitHub Releases": + "key": "PTGR" + "other": + "clean-working-dir": true + "final-tasks": + - "clean" + "tasks": + - "clean" + - "checkout": + "repository": "bamboo-deploy-publisher" + "path": "bamboo-deploy-publisher" + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - channel="${bamboo.channel}" - readonly channel + channel="${bamboo.channel}" + readonly channel - if [ "$channel" != 'release' ] && [ "${channel}" != 'beta' ] - then - echo "don't publish to GitHub Releases for this channel" + if [ "$channel" != 'release' ] && [ "${channel}" != 'beta' ] + then + echo "don't publish to GitHub Releases for this channel" - exit 0 - fi + exit 0 + fi - cd ./dist/ + cd ./dist/ - env\ - GITHUB_TOKEN="${bamboo.githubPublicRepoPassword}"\ - ../bamboo-deploy-publisher/deploy.sh adguard-home-github + env\ + GITHUB_TOKEN="${bamboo.githubPublicRepoPassword}"\ + ../bamboo-deploy-publisher/deploy.sh adguard-home-github -'triggers': - # Don't use minute values that end with a zero or a five as these are often - # used in CI and so resources during these minutes can be quite busy. - - 'cron': '0 42 13 ? * MON-FRI *' -'branches': - 'create': 'manually' - 'delete': - 'after-deleted-days': 1 - 'after-inactive-days': 30 - 'integration': - 'push-on-success': false - 'merge-from': 'AdGuard Home - Build and publish release' - 'link-to-jira': true +"triggers": + # Don't use minute values that end with a zero or a five as these are often + # used in CI and so resources during these minutes can be quite busy. + - "cron": "0 42 13 ? * MON-FRI *" +"branches": + "create": "manually" + "delete": + "after-deleted-days": 1 + "after-inactive-days": 30 + "integration": + "push-on-success": false + "merge-from": "AdGuard Home - Build and publish release" + "link-to-jira": true -'notifications': - - 'events': - - 'plan-completed' - 'recipients': - - 'webhook': - 'name': 'Build webhook' - 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa-dns-builds' +"notifications": + - "events": + - "plan-completed" + "recipients": + - "webhook": + "name": "Build webhook" + "url": "http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa-dns-builds" -'labels': [] -'other': - 'concurrent-build-plugin': 'system-default' +"labels": [] +"other": + "concurrent-build-plugin": "system-default" -'branch-overrides': - # beta-vX.Y branches are the branches into which the commits that are needed - # to release a new patch version are initially cherry-picked. +"branch-overrides": + # beta-vX.Y branches are the branches into which the commits that are needed + # to release a new patch version are initially cherry-picked. - '^beta-v[0-9]+\.[0-9]+': - # Build betas on release branches manually. - 'triggers': [] - # Set the default release channel on the release branch to beta, as we may - # need to build a few of these. - 'variables': - 'channel': 'beta' - 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.26.2--1' + # Build betas on release branches manually. + "triggers": [] + # Set the default release channel on the release branch to beta, as we may + # need to build a few of these. + "variables": + "channel": "beta" + "dockerFrontend": "adguard/home-js-builder:4.0" + "dockerGo": "adguard/go-builder:1.26.4--1" # release-vX.Y.Z branches are the branches from which the actual final # release is built. - '^release-v[0-9]+\.[0-9]+\.[0-9]+': - # Disable integration branches for release branches. - 'branch-config': - 'integration': - 'push-on-success': false - 'merge-from': 'beta-v0.107' - # Build final releases on release branches manually. - 'triggers': [] - # Set the default release channel on the final branch to release, as these - # are the ones that actually get released. - 'variables': - 'channel': 'release' - 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.26.2--1' + # Disable integration branches for release branches. + "branch-config": + "integration": + "push-on-success": false + "merge-from": "beta-v0.107" + # Build final releases on release branches manually. + "triggers": [] + # Set the default release channel on the final branch to release, as these + # are the ones that actually get released. + "variables": + "channel": "release" + "dockerFrontend": "adguard/home-js-builder:4.0" + "dockerGo": "adguard/go-builder:1.26.4--1" diff --git a/bamboo-specs/test.yaml b/bamboo-specs/test.yaml index c53dfd0e8..e011c51d4 100644 --- a/bamboo-specs/test.yaml +++ b/bamboo-specs/test.yaml @@ -1,259 +1,259 @@ --- -'version': 2 -'plan': - 'project-key': 'AGH' - 'key': 'AHBRTSPECS' - 'name': 'AdGuard Home - Build and run tests' -'variables': - # This variable is used to override Docker caching, for example to rerun a - # flaky test suite. - 'cacheBuster': '0' - 'channel': 'development' - 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.26.2--1' +"version": 2 +"plan": + "project-key": "AGH" + "key": "AHBRTSPECS" + "name": "AdGuard Home - Build and run tests" +"variables": + # This variable is used to override Docker caching, for example to rerun a + # flaky test suite. + "cacheBuster": "0" + "channel": "development" + "dockerFrontend": "adguard/home-js-builder:4.0" + "dockerGo": "adguard/go-builder:1.26.4--1" -'stages': - - 'Tests': - 'manual': false - 'final': false - 'jobs': - - 'Test frontend' - - 'Test backend' +"stages": + - "Tests": + "manual": false + "final": false + "jobs": + - "Test frontend" + - "Test backend" - - 'Frontend': - manual: false - final: false - jobs: - - 'Build frontend' + - "Frontend": + manual: false + final: false + jobs: + - "Build frontend" - - 'Artifact': - manual: false - final: false - jobs: - - 'Artifact' + - "Artifact": + manual: false + final: false + jobs: + - "Artifact" - - 'E2E': + - "E2E": manual: true final: false jobs: - - 'Test e2e' + - "Test e2e" -'Test frontend': - 'final-tasks': - - 'clean' - 'key': 'JSTEST' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh +"Test frontend": + "final-tasks": + - "clean" + "key": "JSTEST" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - docker info + docker info - docker build \ - --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --output '.' \ - --progress 'plain' \ - --target 'tester' \ - -f ./docker/frontend.Dockerfile \ - . + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'tester' \ + -f ./docker/frontend.Dockerfile \ + . -'Test backend': - 'final-tasks': - - 'test-parser': - # The default pattern, '**/test-reports/*.xml', works, so don't set - # the test-results property. - 'type': 'junit' - 'ignore-time': true - - 'clean' - 'key': 'GOTEST' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh +"Test backend": + "final-tasks": + - "test-parser": + # The default pattern, '**/test-reports/*.xml', works, so don't set + # the test-results property. + "type": "junit" + "ignore-time": true + - "clean" + "key": "GOTEST" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - docker info + docker info - docker build \ - --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --output '.' \ - --progress 'plain' \ - --target 'tester-exporter' \ - -f ./docker/ci.Dockerfile \ - . + docker build \ + --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'tester-exporter' \ + -f ./docker/ci.Dockerfile \ + . - exit_code="$(cat ./test-reports/test-exit-code.txt)" - readonly exit_code + exit_code="$(cat ./test-reports/test-exit-code.txt)" + readonly exit_code - exit "$exit_code" + exit "$exit_code" -'Build frontend': - 'artifacts': - - 'name': 'AdGuardHome frontend' - 'pattern': 'build/**' - 'shared': true - 'required': true - 'key': 'BF' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - |- - #!/bin/sh +"Build frontend": + "artifacts": + - "name": "AdGuardHome frontend" + "pattern": "build/**" + "shared": true + "required": true + "key": "BF" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - |- + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - docker info + docker info - docker build \ - --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --output '.' \ - --progress 'plain' \ - --target 'builder-exporter' \ - -f ./docker/frontend.Dockerfile \ - . + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/frontend.Dockerfile \ + . -'Artifact': - 'artifact-subscriptions': - - 'artifact': 'AdGuardHome frontend' - 'artifacts': - - 'name': 'AdGuardHome_windows_amd64' - 'pattern': 'dist/AdGuardHome_windows_amd64.zip' - 'shared': true - 'required': true - - 'name': 'AdGuardHome_darwin_amd64' - 'pattern': 'dist/AdGuardHome_darwin_amd64.zip' - 'shared': true - 'required': true - - 'name': 'AdGuardHome_darwin_arm64' - 'pattern': 'dist/AdGuardHome_darwin_arm64.zip' - 'shared': true - 'required': true - - 'name': 'AdGuardHome_linux_amd64' - 'pattern': 'dist/AdGuardHome_linux_amd64.tar.gz' - 'shared': true - 'required': true - 'key': 'ART' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - |- - #!/bin/sh +"Artifact": + "artifact-subscriptions": + - "artifact": "AdGuardHome frontend" + "artifacts": + - "name": "AdGuardHome_windows_amd64" + "pattern": "dist/AdGuardHome_windows_amd64.zip" + "shared": true + "required": true + - "name": "AdGuardHome_darwin_amd64" + "pattern": "dist/AdGuardHome_darwin_amd64.zip" + "shared": true + "required": true + - "name": "AdGuardHome_darwin_arm64" + "pattern": "dist/AdGuardHome_darwin_arm64.zip" + "shared": true + "required": true + - "name": "AdGuardHome_linux_amd64" + "pattern": "dist/AdGuardHome_linux_amd64.tar.gz" + "shared": true + "required": true + "key": "ART" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - |- + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - version="$(env CHANNEL=${bamboo_channel} sh ./scripts/make/version.sh)" - readonly version + version="$(env CHANNEL=${bamboo_channel} sh ./scripts/make/version.sh)" + readonly version - docker info + docker info - docker build \ - --build-arg "ARCH=amd64 arm64" \ - --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ - --build-arg "BRANCH=${bamboo_planRepository_branchName}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --build-arg "CHANNEL=${bamboo_channel}" \ - --build-arg "OS=windows darwin linux" \ - --build-arg "REVISION=${bamboo_repository_revision_number}" \ - --build-arg "SIGN=0" \ - --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ - --build-arg "VERSION=$version" \ - --output '.' \ - --progress 'plain' \ - --target 'builder-exporter' \ - -f ./docker/ci.Dockerfile \ - . + docker build \ + --build-arg "ARCH=amd64 arm64" \ + --build-arg "BASE_IMAGE=${bamboo_dockerGo}" \ + --build-arg "BRANCH=${bamboo_planRepository_branchName}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --build-arg "CHANNEL=${bamboo_channel}" \ + --build-arg "OS=windows darwin linux" \ + --build-arg "REVISION=${bamboo_repository_revision_number}" \ + --build-arg "SIGN=0" \ + --build-arg "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" \ + --build-arg "VERSION=$version" \ + --output '.' \ + --progress 'plain' \ + --target 'builder-exporter' \ + -f ./docker/ci.Dockerfile \ + . -'Test e2e': - 'artifact-subscriptions': - - 'artifact': 'AdGuardHome_linux_amd64' - - 'artifact': 'AdGuardHome frontend' - 'key': 'E2ETEST' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh +"Test e2e": + "artifact-subscriptions": + - "artifact": "AdGuardHome_linux_amd64" + - "artifact": "AdGuardHome frontend" + "key": "E2ETEST" + "other": + "clean-working-dir": true + "tasks": + - "checkout": + "force-clean-build": true + - "script": + "interpreter": "SHELL" + "scripts": + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - tar -xzf dist/AdGuardHome_linux_amd64.tar.gz -C /tmp + tar -xzf dist/AdGuardHome_linux_amd64.tar.gz -C /tmp - mv /tmp/AdGuardHome/AdGuardHome ./AdGuardHome + mv /tmp/AdGuardHome/AdGuardHome ./AdGuardHome - docker info + docker info - docker build \ - --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ - --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ - --output '.' \ - --progress 'plain' \ - --target 'e2etester' \ - -f ./docker/frontend.Dockerfile \ - . + docker build \ + --build-arg "BASE_IMAGE=${bamboo.dockerFrontend}" \ + --build-arg "CACHE_BUSTER=${bamboo_cacheBuster}" \ + --output '.' \ + --progress 'plain' \ + --target 'e2etester' \ + -f ./docker/frontend.Dockerfile \ + . -'branches': - 'create': 'for-pull-request' - 'delete': - 'after-deleted-days': 1 - 'after-inactive-days': 5 - 'integration': - 'push-on-success': false - 'merge-from': 'AdGuard Home - Build and run tests' - 'link-to-jira': true +"branches": + "create": "for-pull-request" + "delete": + "after-deleted-days": 1 + "after-inactive-days": 5 + "integration": + "push-on-success": false + "merge-from": "AdGuard Home - Build and run tests" + "link-to-jira": true -'notifications': - - 'events': - - 'plan-status-changed' - 'recipients': - - 'webhook': - 'name': 'Build webhook' - 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo' +"notifications": + - "events": + - "plan-status-changed" + "recipients": + - "webhook": + "name": "Build webhook" + "url": "http://prod.jirahub.service.eu.consul/v1/webhook/bamboo" -'labels': [] -'other': - 'concurrent-build-plugin': 'system-default' +"labels": [] +"other": + "concurrent-build-plugin": "system-default" -'branch-overrides': - # rc-vX.Y.Z branches are the release candidate branches. They are created - # from the release branch and are used to build the release candidate - # images. +"branch-overrides": + # rc-vX.Y.Z branches are the release candidate branches. They are created + # from the release branch and are used to build the release candidate + # images. - '^rc-v[0-9]+\.[0-9]+\.[0-9]+': - # Set the default release channel on the release branch to beta, as we - # may need to build a few of these. - 'variables': - 'dockerFrontend': 'adguard/home-js-builder:4.0' - 'dockerGo': 'adguard/go-builder:1.26.2--1' - 'channel': 'candidate' + # Set the default release channel on the release branch to beta, as we + # may need to build a few of these. + "variables": + "dockerFrontend": "adguard/home-js-builder:4.0" + "dockerGo": "adguard/go-builder:1.26.4--1" + "channel": "candidate" diff --git a/client/src/components/Settings/FiltersConfig/index.tsx b/client/src/components/Settings/FiltersConfig/index.tsx index 944a6f9d4..a5bfc18ef 100644 --- a/client/src/components/Settings/FiltersConfig/index.tsx +++ b/client/src/components/Settings/FiltersConfig/index.tsx @@ -4,20 +4,17 @@ import { Trans, useTranslation } from 'react-i18next'; import i18next from 'i18next'; import { toNumber } from '../../../helpers/form'; -import { DAY, FILTERS_INTERVALS_HOURS, FILTERS_RELATIVE_LINK } from '../../../helpers/constants'; +import { DAY_HOURS, FILTERS_INTERVALS_HOURS, FILTERS_RELATIVE_LINK } from '../../../helpers/constants'; import { Checkbox } from '../../ui/Controls/Checkbox'; import { Select } from '../../ui/Controls/Select'; -const THREE_DAYS_INTERVAL = DAY * 3; -const SEVEN_DAYS_INTERVAL = DAY * 7; - const getTitleForInterval = (interval: number) => { if (interval === 0) { return i18next.t('disabled'); } - if (interval === THREE_DAYS_INTERVAL || interval === SEVEN_DAYS_INTERVAL) { - return i18next.t('interval_days', { count: interval / DAY }); + if (interval % DAY_HOURS === 0) { + return i18next.t('interval_days', { count: interval / DAY_HOURS }); } return i18next.t('interval_hours', { count: interval }); @@ -58,6 +55,10 @@ export const FiltersConfig = ({ initialValues, setFiltersConfig, processing }: P a: , }; + const options = FILTERS_INTERVALS_HOURS.includes(initialValues.interval) + ? FILTERS_INTERVALS_HOURS + : [...FILTERS_INTERVALS_HOURS, initialValues.interval]; + return ( <>
@@ -99,7 +100,7 @@ export const FiltersConfig = ({ initialValues, setFiltersConfig, processing }: P const { value } = e.target; field.onChange(toNumber(value)); }}> - {FILTERS_INTERVALS_HOURS.map((interval) => ( + {options.map((interval) => ( diff --git a/client/src/helpers/constants.ts b/client/src/helpers/constants.ts index d4e7c9409..e4d7cbe96 100644 --- a/client/src/helpers/constants.ts +++ b/client/src/helpers/constants.ts @@ -169,7 +169,9 @@ export const DISABLED_STATS_INTERVAL = 0; export const HOUR = 60 * 60 * 1000; -export const DAY = HOUR * 24; +export const DAY_HOURS = 24; + +export const DAY = HOUR * DAY_HOURS; export const STATS_INTERVALS_DAYS = [DAY, DAY * 7, DAY * 30, DAY * 90]; diff --git a/docker/ci.Dockerfile b/docker/ci.Dockerfile index 17ac91e73..1467c97f6 100644 --- a/docker/ci.Dockerfile +++ b/docker/ci.Dockerfile @@ -28,7 +28,7 @@ # needed. Keep it in sync with bamboo-specs/bamboo.yaml. # NOTE: Keep in sync with bamboo-specs/bamboo.yaml. -ARG BASE_IMAGE=adguard/go-builder:1.26.2--1 +ARG BASE_IMAGE=adguard/go-builder:1.26.4--1 # The dependencies stage is needed to install packages and tool dependencies. # This is also where binaries like osslsigncode, which may be required for tests diff --git a/go.mod b/go.mod index c8a40f9fa..6b313f00e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/AdguardTeam/AdGuardHome -go 1.26.3 +go 1.26.4 require ( github.com/AdguardTeam/dnsproxy v0.81.4 diff --git a/internal/filtering/filter.go b/internal/filtering/filter.go index 167b8d1f1..6ff3b9a37 100644 --- a/internal/filtering/filter.go +++ b/internal/filtering/filter.go @@ -19,6 +19,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/golibs/container" "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/ioutil" "github.com/AdguardTeam/golibs/logutil/slogutil" ) @@ -30,7 +31,8 @@ const filterDir = "filters" // // TODO(e.burkov): Investigate if the field ordering is important. type FilterYAML struct { - Enabled bool + Enabled bool + // TODO(m.kazantsev): Refactor. URL string // URL or a file path Name string `yaml:"name"` RulesCount int `yaml:"-"` @@ -495,7 +497,7 @@ func (d *DNSFilter) update(filter *FilterYAML) (b bool, err error) { } // updateIntl updates the flt rewriting it's actual file. It returns true if -// the actual update has been performed. +// the actual update has been performed. flt must not be nil. func (d *DNSFilter) updateIntl(ctx context.Context, flt *FilterYAML) (ok bool, err error) { d.logger.DebugContext(ctx, "downloading update for filter", "id", flt.ID, "url", flt.URL) @@ -503,24 +505,78 @@ func (d *DNSFilter) updateIntl(ctx context.Context, flt *FilterYAML) (ok bool, e tmpFile, err := aghrenameio.NewPendingFile(flt.Path(d.conf.DataDir), aghos.DefaultPermFile) if err != nil { + // Don't wrap the error because it's informative enough as is. return false, err } defer func() { err = d.finalizeUpdate(ctx, tmpFile, flt, res, err, ok) }() - r, err := d.reader(flt.URL) + if filepath.IsAbs(flt.URL) { + // Initialise this variable to avoid any confusion. + path := flt.URL + + res, err = d.readFromFile(tmpFile, path) + } else { + res, err = d.readFromHTTP(tmpFile, flt.URL) + } + if err != nil { - // Don't wrap the error since it's informative enough as is. + // Don't wrap the error because it's informative enough as is. return false, err } - defer func() { err = errors.WithDeferred(err, r.Close()) }() + + return res.Checksum != flt.checksum, nil +} + +// readFromHTTP reads filter data from urlStr via HTTP and parses it into the +// tmpFile file. tmpFile must not be nil. urlStr must be a valid URL. +func (d *DNSFilter) readFromHTTP( + tmpFile aghrenameio.PendingFile, + urlStr string, +) (res *rulelist.ParseResult, err error) { + resp, err := d.conf.HTTPClient.Get(urlStr) + if err != nil { + // Don't wrap the error because it's informative enough as is. + return nil, err + } + defer func() { err = errors.WithDeferred(err, resp.Body.Close()) }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK) + } bufPtr := d.bufPool.Get() defer d.bufPool.Put(bufPtr) p := rulelist.NewParser() - res, err = p.Parse(tmpFile, r, *bufPtr) + httpBody := ioutil.LimitReader(resp.Body, d.conf.MaxHTTPSize.Bytes()) - return res.Checksum != flt.checksum && err == nil, err + return p.Parse(tmpFile, httpBody, *bufPtr) +} + +// readFromFile reads filter data from a file located at path and parses it into +// the tmpFile file. tmpFile must not be nil. path must be a valid filepath. +func (d *DNSFilter) readFromFile( + tmpFile aghrenameio.PendingFile, + path string, +) (res *rulelist.ParseResult, err error) { + path = filepath.Clean(path) + + if !pathMatchesAny(d.safeFSPatterns, path) { + return nil, fmt.Errorf("path %q does not match safe patterns", path) + } + + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("opening file: %w", err) + } + defer func() { err = errors.WithDeferred(err, file.Close()) }() + + bufPtr := d.bufPool.Get() + defer d.bufPool.Put(bufPtr) + + p := rulelist.NewParser() + + return p.Parse(tmpFile, file, *bufPtr) } // finalizeUpdate closes and gets rid of temporary file f with filter's content @@ -566,47 +622,6 @@ func (d *DNSFilter) finalizeUpdate( return nil } -// reader returns an io.ReadCloser reading filtering-rule list data form either -// a file on the filesystem or the filter's HTTP URL. -func (d *DNSFilter) reader(fltURL string) (r io.ReadCloser, err error) { - if !filepath.IsAbs(fltURL) { - r, err = d.readerFromURL(fltURL) - if err != nil { - return nil, fmt.Errorf("reading from url: %w", err) - } - - return r, nil - } - - fltURL = filepath.Clean(fltURL) - if !pathMatchesAny(d.safeFSPatterns, fltURL) { - return nil, fmt.Errorf("path %q does not match safe patterns", fltURL) - } - - r, err = os.Open(fltURL) - if err != nil { - return nil, fmt.Errorf("opening file: %w", err) - } - - return r, nil -} - -// readerFromURL returns an io.ReadCloser reading filtering-rule list data form -// the filter's URL. -func (d *DNSFilter) readerFromURL(fltURL string) (r io.ReadCloser, err error) { - resp, err := d.conf.HTTPClient.Get(fltURL) - if err != nil { - // Don't wrap the error since it's informative enough as is. - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK) - } - - return resp.Body, nil -} - // loads filter contents from the file in dataDir func (d *DNSFilter) load(ctx context.Context, flt *FilterYAML) (err error) { fileName := flt.Path(d.conf.DataDir) diff --git a/internal/filtering/filter_internal_test.go b/internal/filtering/filter_internal_test.go index 9e7165cae..77cf95b67 100644 --- a/internal/filtering/filter_internal_test.go +++ b/internal/filtering/filter_internal_test.go @@ -11,6 +11,7 @@ import ( "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/AdguardTeam/golibs/testutil" + "github.com/c2h5oh/datasize" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -76,6 +77,9 @@ func updateAndAssert( require.NoError(tb, err) } +// testFilterSize is a test size of filters. +const testFilterSize = 10 * datasize.MB + // newDNSFilter returns a new properly initialized DNS filter instance. func newDNSFilter(tb testing.TB) (d *DNSFilter) { tb.Helper() @@ -86,6 +90,7 @@ func newDNSFilter(tb testing.TB) (d *DNSFilter) { HTTPClient: &http.Client{ Timeout: testTimeout, }, + MaxHTTPSize: testFilterSize, }, nil) require.NoError(tb, err) diff --git a/internal/filtering/filtering.go b/internal/filtering/filtering.go index 645c4fe53..159d216ca 100644 --- a/internal/filtering/filtering.go +++ b/internal/filtering/filtering.go @@ -32,6 +32,7 @@ import ( "github.com/AdguardTeam/urlfilter" "github.com/AdguardTeam/urlfilter/filterlist" "github.com/AdguardTeam/urlfilter/rules" + "github.com/c2h5oh/datasize" "github.com/miekg/dns" ) @@ -154,6 +155,10 @@ type Config struct { // files can be added. SafeFSPatterns []string `yaml:"safe_fs_patterns"` + // MaxHTTPSize defines the maximum size of the HTTP body. The value must + // not be equal to zero. + MaxHTTPSize datasize.ByteSize `yaml:"max_http_size"` + SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes) SafeSearchCacheSize uint `yaml:"safesearch_cache_size"` // (in bytes) ParentalCacheSize uint `yaml:"parental_cache_size"` // (in bytes) diff --git a/internal/filtering/http.go b/internal/filtering/http.go index 82c053283..a7ffd462e 100644 --- a/internal/filtering/http.go +++ b/internal/filtering/http.go @@ -760,7 +760,10 @@ func (d *DNSFilter) RegisterFilteringHandlers() { registerHTTP(http.MethodGet, "/control/filtering/check_host", d.handleCheckHost) } +// maxUpdateIvlHours is the maximum allowed filter update interval in hours. +const maxUpdateIvlHours = 365 * 24 + // ValidateUpdateIvl returns false if i is not a valid filters update interval. -func ValidateUpdateIvl(i uint32) bool { - return i == 0 || i == 1 || i == 12 || i == 1*24 || i == 3*24 || i == 7*24 +func ValidateUpdateIvl(i uint32) (ok bool) { + return i <= maxUpdateIvlHours } diff --git a/internal/filtering/http_internal_test.go b/internal/filtering/http_internal_test.go index 4ed66c740..f6e82ae71 100644 --- a/internal/filtering/http_internal_test.go +++ b/internal/filtering/http_internal_test.go @@ -119,6 +119,7 @@ func TestDNSFilter_handleFilteringSetURL(t *testing.T) { ConfModifier: confModifier, HTTPReg: aghhttp.EmptyRegistrar{}, DataDir: filtersDir, + MaxHTTPSize: testFilterSize, }, nil) require.NoError(t, err) t.Cleanup(d.Close) diff --git a/internal/filtering/rulelist/rulelist.go b/internal/filtering/rulelist/rulelist.go index 17e8c3cd6..f185519cd 100644 --- a/internal/filtering/rulelist/rulelist.go +++ b/internal/filtering/rulelist/rulelist.go @@ -21,7 +21,7 @@ import ( const DefaultRuleBufSize = 1024 // DefaultMaxRuleListSize is the default maximum filtering-rule list size. -const DefaultMaxRuleListSize = 64 * datasize.MB +const DefaultMaxRuleListSize = 256 * datasize.MB // APIID is the type for the rule-list IDs used in the HTTP API. type APIID int64 diff --git a/internal/home/config.go b/internal/home/config.go index ed4e6008f..31bfaee8a 100644 --- a/internal/home/config.go +++ b/internal/home/config.go @@ -20,6 +20,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" "github.com/AdguardTeam/AdGuardHome/internal/filtering" + "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/AdGuardHome/internal/querylog" "github.com/AdguardTeam/AdGuardHome/internal/schedule" "github.com/AdguardTeam/AdGuardHome/internal/stats" @@ -564,6 +565,7 @@ var config = &configuration{ ParentalEnabled: false, SafeBrowsingEnabled: false, + MaxHTTPSize: rulelist.DefaultMaxRuleListSize, SafeBrowsingCacheSize: 1 * 1024 * 1024, SafeSearchCacheSize: 1 * 1024 * 1024, ParentalCacheSize: 1 * 1024 * 1024, diff --git a/internal/home/controlupdate.go b/internal/home/controlupdate.go index cb14ab62c..d0c080f2e 100644 --- a/internal/home/controlupdate.go +++ b/internal/home/controlupdate.go @@ -77,7 +77,7 @@ func (web *webAPI) handleVersionJSON(w http.ResponseWriter, r *http.Request) { } // requestVersionInfo sets the VersionInfo field of resp if it can reach the -// update server. +// update server. resp must not be nil. func (web *webAPI) requestVersionInfo( ctx context.Context, resp *versionResponse, @@ -109,6 +109,8 @@ func (web *webAPI) requestVersionInfo( } if err != nil { + web.logger.WarnContext(ctx, "getting version info", slogutil.KeyError, err) + return fmt.Errorf("getting version info: %w", err) } @@ -128,7 +130,7 @@ func (web *webAPI) handleUpdate(w http.ResponseWriter, r *http.Request) { r, w, http.StatusBadRequest, - "/update request isn't allowed now", + "update request isn't allowed now", ) return diff --git a/internal/home/web.go b/internal/home/web.go index 1862eff04..29ab155e1 100644 --- a/internal/home/web.go +++ b/internal/home/web.go @@ -24,10 +24,6 @@ import ( "github.com/AdguardTeam/golibs/osutil/executil" "github.com/NYTimes/gziphandler" "github.com/quic-go/quic-go/http3" - "golang.org/x/net/http2" - - //lint:ignore SA1019 See AGDNS-4038. - "golang.org/x/net/http2/h2c" ) // TODO(a.garipov): Make configurable. @@ -277,18 +273,16 @@ func (web *webAPI) start(ctx context.Context) { hdlr = web.auth.middleware().Wrap(hdlr) - // Use an h2c handler to support unencrypted HTTP/2, e.g. for proxies. - // - // NOTE: The auth middleware must be inside the h2c handler to ensure - // it applies to upgraded HTTP/2 connections as well. See AG-51779. - // - //lint:ignore SA1019 See AGDNS-4038. - hdlr = h2c.NewHandler(hdlr, &http2.Server{}) + // Enable unencrypted HTTP/2, e.g. for proxies. + protocols := &http.Protocols{} + protocols.SetUnencryptedHTTP2(true) + protocols.SetHTTP1(true) // Create a new instance, because the Web is not usable after Shutdown. web.httpServer = &http.Server{ Addr: web.conf.BindAddr.String(), Handler: hdlr, + Protocols: protocols, ReadTimeout: web.conf.ReadTimeout, ReadHeaderTimeout: web.conf.ReadHeaderTimeout, WriteTimeout: web.conf.WriteTimeout, diff --git a/internal/home/web_internal_test.go b/internal/home/web_internal_test.go deleted file mode 100644 index cf153da90..000000000 --- a/internal/home/web_internal_test.go +++ /dev/null @@ -1,349 +0,0 @@ -package home - -import ( - "bufio" - "bytes" - "fmt" - "net" - "net/http" - "net/url" - "path" - "strconv" - "testing" - "testing/fstest" - - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" - "github.com/AdguardTeam/AdGuardHome/internal/aghos" - "github.com/AdguardTeam/AdGuardHome/internal/aghuser" - "github.com/AdguardTeam/golibs/netutil" - "github.com/AdguardTeam/golibs/netutil/urlutil" - "github.com/AdguardTeam/golibs/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/crypto/bcrypt" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" -) - -const ( - // clientPreface is the message sent to the server as a final confirmation - // of HTTP2 usage. - clientPreface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" - - // testSettings is a common value of the HTTP2-Settings header for tests. - testSettings = "AAEAABAAAAIAAAABAAQAAP__AAUAAEAAAAgAAAAAAAMAAABkAAYAAQAA" - - // testHpackMaxDynamicTableSize is the common HPACK max dynamic table size - // value for tests. - testHPACKMaxDynamicTableSize = 4096 - - // testTargetStreamID is a common HTTP2 stream ID for sending requests after - // an upgrade. - // - // NOTE: The upgrade request implicitly uses Stream ID 1, so the first - // client-side valid ID is 3. - testTargetStreamID = 3 -) - -// h2c upgrade headers. -// -// TODO(a.garipov): Add to httphdr. -const ( - headerConnection = "Connection" - headerUpgrade = "Upgrade" - headerHTTP2Settings = "HTTP2-Settings" -) - -// h2c upgrade header values for tests. -const ( - testHeaderValueConnection = "Upgrade, HTTP2-Settings" - testHeaderValueUpgrade = "h2c" -) - -// testDecoder implements HTTP2 HPACK-encoded headers decoding for tests. -type testDecoder struct { - decoder *hpack.Decoder - status int -} - -// newTestDecoder returns a properly initialized *testDecoder. -func newTestDecoder(tb testing.TB) (d *testDecoder) { - tb.Helper() - - d = &testDecoder{} - d.decoder = hpack.NewDecoder(testHPACKMaxDynamicTableSize, func(f hpack.HeaderField) { - if f.Name != ":status" { - return - } - - status64, err := strconv.ParseInt(f.Value, 10, 64) - require.NoError(tb, err) - - d.status = int(status64) - }) - - return d -} - -// decodeStatus decodes an HPACK-encoded header block and returns the HTTP -// status code. -func (d *testDecoder) decodeStatus(tb testing.TB, b []byte) (status int) { - tb.Helper() - - d.status = 0 - - _, err := d.decoder.Write(b) - require.NoError(tb, err) - - return d.status -} - -func TestWebAPI_h2cVulnerability(t *testing.T) { - storeGlobals(t) - - stop := make(chan struct{}) - t.Cleanup(func() { - testutil.RequireReceive(t, stop, testTimeout) - }) - - password := "password" - passwordHash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost) - require.NoError(t, err) - - fs := fstest.MapFS{ - "build/static/login.html": &fstest.MapFile{ - Data: []byte("foo"), - Mode: aghos.DefaultPermFile, - }, - } - - user := webUser{ - Name: "foo", - PasswordHash: string(passwordHash), - UserID: aghuser.MustNewUserID(), - } - - mux := http.NewServeMux() - auth, err := newAuth(testutil.ContextWithTimeout(t, testTimeout), &authConfig{ - baseLogger: testLogger, - rateLimiter: emptyRateLimiter{}, - trustedProxies: testTrustedProxies, - dbFilename: path.Join(t.TempDir(), "sessions.db"), - users: []webUser{user}, - sessionTTL: testTimeout, - isGLiNet: false, - mux: mux, - }) - require.NoError(t, err) - - t.Cleanup(func() { - ctx := testutil.ContextWithTimeout(t, testTimeout) - auth.close(ctx) - }) - - mw := &webMw{} - registrar := aghhttp.NewDefaultRegistrar(mux, mw.wrap) - web := newTestWeb(t, &webConfig{ - baseLogger: testLogger, - auth: auth, - mux: mux, - httpReg: registrar, - clientBuildFS: fs, - }) - - mw.set(web) - globalContext.web = web - - port := config.HTTPConfig.Address.Port() - host := fmt.Sprintf("%s:%d", netutil.IPv4Localhost(), port) - - go func() { - ctx := testutil.ContextWithTimeout(t, testTimeout) - web.start(ctx) - close(stop) - }() - - t.Cleanup(func() { - ctx := testutil.ContextWithTimeout(t, testTimeout) - web.close(ctx) - }) - - waitForWebAPIReady(t, host) - performH2CUpgradeAttack(t, host) -} - -// waitForWebAPIReady waits until the [webAPI] server has started and is ready -// to accept connections. -func waitForWebAPIReady(tb testing.TB, host string) { - tb.Helper() - - u := (&url.URL{ - Scheme: urlutil.SchemeHTTP, - Host: host, - Path: "/login.html", - }).String() - - require.EventuallyWithT(tb, func(c *assert.CollectT) { - ctx := testutil.ContextWithTimeout(tb, testTimeout) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - require.NoError(c, err) - - resp, err := http.DefaultClient.Do(req) - require.NoError(c, err) - assert.Equal(c, http.StatusOK, resp.StatusCode) - }, testTimeout, testTimeout/10) -} - -// performH2CUpgradeAttack establishes a TCP connection to the specified host, -// performs an HTTP2 protocol upgrade, and attempts to access a protected -// endpoint without proper authentication, verifying that the server responds -// with [http.StatusUnauthorized]. -func performH2CUpgradeAttack(tb testing.TB, host string) { - tb.Helper() - - dialer := &net.Dialer{} - ctx := testutil.ContextWithTimeout(tb, testTimeout) - - conn, err := dialer.DialContext(ctx, "tcp", host) - require.NoError(tb, err) - testutil.CleanupAndRequireSuccess(tb, conn.Close) - - writer := bufio.NewWriter(conn) - reader := bufio.NewReader(conn) - - u := &url.URL{ - Scheme: urlutil.SchemeHTTP, - Host: host, - Path: "/control/login", - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - require.NoError(tb, err) - - req.Header.Set(headerConnection, testHeaderValueConnection) - req.Header.Set(headerUpgrade, testHeaderValueUpgrade) - req.Header.Set(headerHTTP2Settings, testSettings) - - err = req.Write(writer) - require.NoError(tb, err) - require.NoError(tb, writer.Flush()) - - resp, err := http.ReadResponse(reader, req) - require.NoError(tb, err) - require.Equal(tb, http.StatusSwitchingProtocols, resp.StatusCode) - testutil.CleanupAndRequireSuccess(tb, resp.Body.Close) - - _, err = writer.Write([]byte(clientPreface)) - require.NoError(tb, err) - - framer := http2.NewFramer(writer, reader) - decoder := newTestDecoder(tb) - performH2CSettingsExchange(tb, framer, writer, decoder) - sendH2CRequest(tb, framer, host) - require.NoError(tb, writer.Flush()) - - readH2CResponse(tb, framer, decoder) -} - -// performH2CSettingsExchange performs the HTTP2 settings exchange handshake. It -// sends empty client settings, waits for acknowledgement, and then receives the -// server settings and responds with acknowledgement. framer, writer and -// decoder must not be nil. -func performH2CSettingsExchange( - tb testing.TB, - framer *http2.Framer, - writer *bufio.Writer, - decoder *testDecoder, -) { - tb.Helper() - - err := framer.WriteSettings() - require.NoError(tb, err) - require.NoError(tb, writer.Flush()) - - var ( - gotServerSettings bool - gotSettingsAck bool - ) - for !gotServerSettings || !gotSettingsAck { - var frame http2.Frame - frame, err = framer.ReadFrame() - require.NoError(tb, err) - - switch f := frame.(type) { - case *http2.HeadersFrame: - // NOTE: The decoder must process all headers frames because the - // client and server share the same HPACK dynamic table. Skipping - // frames causes index desynchronization. - decoder.decodeStatus(tb, f.HeaderBlockFragment()) - case *http2.SettingsFrame: - if f.IsAck() { - gotSettingsAck = true - - continue - } - - err = framer.WriteSettingsAck() - require.NoError(tb, err) - require.NoError(tb, writer.Flush()) - - gotServerSettings = true - } - } -} - -// sendH2CRequest writes a request to a protected endpoint into the framer. -// framer must not be nil. -func sendH2CRequest(tb testing.TB, framer *http2.Framer, host string) { - tb.Helper() - - var headerBlockFragment bytes.Buffer - enc := hpack.NewEncoder(&headerBlockFragment) - headers := []hpack.HeaderField{ - {Name: ":method", Value: http.MethodGet}, - {Name: ":path", Value: "/control/status"}, - {Name: ":scheme", Value: urlutil.SchemeHTTP}, - {Name: ":authority", Value: host}, - } - - for _, h := range headers { - require.NoError(tb, enc.WriteField(h)) - } - - err := framer.WriteHeaders(http2.HeadersFrameParam{ - StreamID: testTargetStreamID, - BlockFragment: headerBlockFragment.Bytes(), - EndHeaders: true, - EndStream: true, - }) - require.NoError(tb, err) -} - -// readH2CResponse reads the response from an h2c connection and asserts that -// the server responds with [http.StatusUnauthorized]. framer and decoder must -// not be nil. -func readH2CResponse(tb testing.TB, framer *http2.Framer, decoder *testDecoder) { - tb.Helper() - - for { - frame, err := framer.ReadFrame() - require.NoError(tb, err) - - if frame.Header().StreamID != testTargetStreamID { - headerFrame, ok := frame.(*http2.HeadersFrame) - if ok { - decoder.decodeStatus(tb, headerFrame.HeaderBlockFragment()) - } - - continue - } - - headerFrame := testutil.RequireTypeAssert[*http2.HeadersFrame](tb, frame) - require.True(tb, headerFrame.StreamEnded()) - - status := decoder.decodeStatus(tb, headerFrame.HeaderBlockFragment()) - assert.Equal(tb, http.StatusUnauthorized, status) - - break - } -} diff --git a/internal/updater/check.go b/internal/updater/check.go index bd6325ab5..bd5a41d45 100644 --- a/internal/updater/check.go +++ b/internal/updater/check.go @@ -13,6 +13,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/ioutil" + "github.com/AdguardTeam/golibs/validate" "github.com/c2h5oh/datasize" ) @@ -42,6 +43,8 @@ func (u *Updater) VersionInfo(ctx context.Context, forceRecheck bool) (vi Versio now := time.Now() recheckTime := u.prevCheckTime.Add(versionCheckPeriod) if !forceRecheck && now.Before(recheckTime) { + u.logger.DebugContext(ctx, "version info recheck is not required yet") + return u.prevCheckResult, u.prevCheckError } @@ -55,10 +58,18 @@ func (u *Updater) VersionInfo(ctx context.Context, forceRecheck bool) (vi Versio resp, err := u.client.Do(req) if err != nil { - return VersionInfo{}, fmt.Errorf("requesting %s: %w", vcu, err) + return VersionInfo{}, fmt.Errorf("sending http request to %s: %w", vcu, err) } defer func() { err = errors.WithDeferred(err, resp.Body.Close()) }() + if resp.StatusCode != http.StatusOK { + return VersionInfo{}, fmt.Errorf( + "got status code %d, want %d", + resp.StatusCode, + http.StatusOK, + ) + } + r := ioutil.LimitReader(resp.Body, maxVersionRespSize.Bytes()) // This use of ReadAll is safe, because we just limited the appropriate @@ -74,7 +85,12 @@ func (u *Updater) VersionInfo(ctx context.Context, forceRecheck bool) (vi Versio return u.prevCheckResult, u.prevCheckError } -func (u *Updater) parseVersionResponse(ctx context.Context, data []byte) (VersionInfo, error) { +// parseVersionResponse parses version-related data and unmarshals it into the +// [VersionInfo] structure. +func (u *Updater) parseVersionResponse( + ctx context.Context, + data []byte, +) (vi VersionInfo, err error) { info := VersionInfo{ CanAutoUpdate: aghalg.NBFalse, } @@ -83,14 +99,15 @@ func (u *Updater) parseVersionResponse(ctx context.Context, data []byte) (Versio "announcement": "", "announcement_url": "", } - err := json.Unmarshal(data, &versionJSON) + err = json.Unmarshal(data, &versionJSON) if err != nil { return info, fmt.Errorf("version.json: %w", err) } for k, v := range versionJSON { - if v == "" { - return info, fmt.Errorf("version.json: bad data: value for key %q is empty", k) + err = validate.NotEmpty("version_json_value", v) + if err != nil { + return info, fmt.Errorf("bad value for %q key: %w", k, err) } } @@ -100,10 +117,26 @@ func (u *Updater) parseVersionResponse(ctx context.Context, data []byte) (Versio packageURL, key, found := u.downloadURL(ctx, versionJSON) if !found { - return info, fmt.Errorf("version.json: no package URL: key %q not found in object", key) + return info, fmt.Errorf("version.json: bad key %q: %w", key, errors.ErrNoValue) } - info.CanAutoUpdate = aghalg.BoolToNullBool(info.NewVersion != u.version) + isNewVersion := info.NewVersion != u.version + if isNewVersion { + u.logger.InfoContext( + ctx, + "a new version is available", + "current_version", u.version, + "new_version", info.NewVersion, + ) + } else { + u.logger.DebugContext( + ctx, + "the current version is up-to-date", + "current_version", u.version, + ) + } + + info.CanAutoUpdate = aghalg.BoolToNullBool(isNewVersion) u.newVersion = info.NewVersion u.packageURL = packageURL diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index bbc5b0188..0b1a520f1 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -2,6 +2,12 @@ +## v0.107.78: API changes + +### New `interval` values in `GET /control/filtering/status` and `POST /control/filtering/config` APIs + +The property `interval` of the objects returned from and accepted by the aforementioned APIs can now be any integer between 0 and 8760 (365 days). + ## v0.107.77: API changes ### New `reason` query parameter in 'GET /control/querylog'