From 07917e1893e6bd12e7dcff2bdb341f4e837d5055 Mon Sep 17 00:00:00 2001 From: Tom Winnerl <29701706+Pixelplanet@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:39:45 +0100 Subject: [PATCH] fix(review): quarantine a location behind an error without a line Problem: a custom location with `proxy_set_header X-Test $missing_variable;` still blocked the whole host save. nginx reports `unknown "missing_variable" variable` without file or line (variables are checked after parsing), so the error never reached the per-location quarantine, which needs a line inside a location's markers. The check without this host passed, so the save was rejected as a "conflict with another host"; in bulk mode (startup, repair of another host) the whole host went offline instead of the one location. Fix: lib/nginx-locations.js gets isolateFaultyLocations(). When an error has no line and only this host's config causes it (no file named and the test without the host passes, or the host's file named without a line), configureHost() stubs the rendered locations: if the error stays with all of them stubbed it comes from the host itself and the result is unchanged. Otherwise each location is rendered alone with all the others still stubbed, and every one that brings the same error back is quarantined. Two broken locations with the same message are thus both found, and a working location is never quarantined for them (stubbing one at a time would leave the other's identical error and blame every candidate). Only when none fails alone (they fail only together) is each one stubbed alone and the first that makes the error go away (or change) quarantined, all of them when none does. The loop then tests the full config again, bounded by maxQuarantines as before. In a nested configure it is skipped when the host being saved is the cause (causedByProtected() handles that as before). After the search the config that failed is written back to the host's file, so a host rejected at host level in bulk mode keeps its real config as .err for inspection, not the last stubbed variant. When no location is to blame (the error stays with all of them stubbed, or the host has none), the host level check from the feat(nginx) commit applies as before: the host is only rejected as a conflict with another host when the error goes away with every other host's config moved aside (failsAlone()), otherwise it fails with a host level error such as `Advanced configuration: unknown variable "$x"`. Tests: nginx-location-isolate.test.js with unit tests for the search and configureHost() against a fake nginx -t that reports unknown variables without a line: the bad location is quarantined (503 stub) and the host stays online; two bad locations with different variables are both quarantined; two bad locations with the same variable around a clean one quarantine only the two, the clean one stays ok; the same variable in the host's advanced config still rejects the host at host level with an "Advanced configuration" message (not a conflict with another host), also for a host without locations, and in bulk mode its .err file holds the real config (proxy_pass, the variable), not 503 stubs, and its nginx_err names the advanced configuration. Cypress (ProxyHostLocations): a location with `$missing_variable` is quarantined with an unknown-variable message, the host stays online, its other location serves and the bad one answers 503. Co-Authored-By: Claude Opus 5.5 --- backend/internal/nginx.js | 60 +++- backend/lib/nginx-locations.js | 45 +++ backend/test/nginx-location-isolate.test.js | 285 ++++++++++++++++++ docs/src/advanced-config/locations.md | 5 + test/cypress/e2e/api/ProxyHostLocations.cy.js | 22 ++ 5 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 backend/test/nginx-location-isolate.test.js diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 23c771d22..d97e8125d 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -28,6 +28,7 @@ import { CONFIG_VERSION_MARKER, chooseLocationMode, describeConfigLine, + isolateFaultyLocations, isVariableHost, MESSAGES as LOCATION_MESSAGES, parseLocationPath, @@ -552,6 +553,47 @@ const internalNginx = { let configText = ""; let failure = null; + // An error without a line (eg. `unknown "x" variable`) can still come from a custom location. Before the + // whole host is rejected, the locations are stubbed one by one to find it, see isolateFaultyLocations(). + const isolateLocations = async (parsed) => { + const candidates = (states || []) + .filter((s) => s.status === "ok" || s.status === "unresolved") + .map((s) => s.index); + if (!candidates.length) { + return null; + } + // In a nested configure the host being saved may cause it, causedByProtected() decides that below + if (opts.protect?.length && (await internalNginx.testAside(opts.protect)).ok) { + return null; + } + try { + return await isolateFaultyLocations({ + candidates, + parsed, + testWithStubs: async (indexes) => { + const stubbed = states.map((s) => + indexes.includes(s.index) + ? Object.assign({}, s, { status: "quarantined", mode: null, stub: true }) + : s, + ); + // writes the host's file, see below + const text = await internalNginx.generateConfig(hostType, host, { locationStates: stubbed }); + return internalNginx.runTest({ file: filename, text }); + }, + }); + } finally { + // Put the config that failed back: when no location is to blame, the host is rejected or (bulk + // mode) its file is kept as .err for inspection, which must not show the last stubbed variant + fs.writeFileSync(filename, configText, { encoding: "utf8" }); + } + }; + const quarantineFaulty = (indexes, parsed) => { + for (const index of indexes) { + internalNginx.quarantineLocation(states[index], host, parsed, modeResolver); + } + quarantines += indexes.length; + }; + for (;;) { if (quarantines > maxQuarantines) { failure = { scope: "host", parsed: { file: filename, line: null, message: "", output: "" } }; @@ -607,6 +649,12 @@ const internalNginx = { quarantines++; continue; } + // the file but no line, eg. a log path of a location that can't be opened + const faulty = where.location_index === null && !parsed.line ? await isolateLocations(parsed) : null; + if (faulty) { + quarantineFaulty(faulty, parsed); + continue; + } failure = { scope: "host", parsed }; break; } @@ -638,9 +686,15 @@ const internalNginx = { } if (blame) { if (blame.action === "conflict") { - // When the error stays with every other host's config moved aside, this host's own config - // causes it (eg. its advanced config uses an unknown variable): a host level error, not a - // conflict with another host + // fails only with this host's config: maybe one of its locations + const faulty = !parsed.file ? await isolateLocations(parsed) : null; + if (faulty) { + quarantineFaulty(faulty, parsed); + continue; + } + // No location is to blame. When the error stays with every other host's config moved aside, this + // host's own config causes it (eg. its advanced config uses an unknown variable): a host level + // error, not a conflict with another host if (!parsed.file && (await internalNginx.failsAlone(filename, parsed))) { failure = { scope: "host", parsed }; break; diff --git a/backend/lib/nginx-locations.js b/backend/lib/nginx-locations.js index e7dbbe606..d47b3a9a2 100644 --- a/backend/lib/nginx-locations.js +++ b/backend/lib/nginx-locations.js @@ -383,6 +383,50 @@ const renderLocation = async (engine, template, loc, state) => { return ` ${begin}\n${rendered.replace(/\s+$/, "")}\n ${end}\n`; }; +/** + * Finds the custom locations behind an `nginx -t` error that names no line, eg. `unknown "x" variable` + * (nginx checks variables after parsing the files and doesn't say where they are used). The candidates are + * replaced by `return 503;` stubs, which contain nothing of the user's input: + * + * - still the same error with all of them stubbed: not caused by a location, null + * - otherwise each candidate is rendered alone, with all the others still stubbed: every one that brings the + * same error back fails on its own and is returned (two broken locations with the same message are both + * found, a working one is never blamed for them) + * - none fails alone (eg. two locations only fail together): each candidate is stubbed alone, the first one + * that makes the error go away (or change) is returned + * - none of that: all candidates + * + * @param {Object} opts + * @param {Number[]} opts.candidates indexes of the locations that are rendered (ok / unresolved) + * @param {Object} opts.parsed the parsed error + * @param {Function} opts.testWithStubs async (indexes) => {ok, parsed}: nginx -t with those locations stubbed + * @returns {Promise} the indexes to quarantine + */ +const isolateFaultyLocations = async ({ candidates, parsed, testWithStubs }) => { + const sameError = (result) => !result.ok && result.parsed?.message === parsed.message; + if (!candidates.length || sameError(await testWithStubs(candidates))) { + return null; + } + if (candidates.length === 1) { + return candidates; + } + const failAlone = []; + for (const index of candidates) { + if (sameError(await testWithStubs(candidates.filter((i) => i !== index)))) { + failAlone.push(index); + } + } + if (failAlone.length) { + return failAlone; + } + for (const index of candidates) { + if (!sameError(await testWithStubs([index]))) { + return [index]; + } + } + return candidates; +}; + /** * @returns {String} */ @@ -479,6 +523,7 @@ export { findDuplicatePath, hasDirective, hasVariablePath, + isolateFaultyLocations, isVariableHost, matchSavedLocations, normaliseLocationPath, diff --git a/backend/test/nginx-location-isolate.test.js b/backend/test/nginx-location-isolate.test.js new file mode 100644 index 000000000..211f13260 --- /dev/null +++ b/backend/test/nginx-location-isolate.test.js @@ -0,0 +1,285 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, afterEach, before, beforeEach, describe, it } from "node:test"; +import { isolateFaultyLocations } from "../lib/nginx-locations.js"; +import { db, load, ts } from "./support/backend.js"; + +const internalNginx = await load("internal/nginx.js"); +const proxyHostModel = await load("models/proxy_host.js"); + +describe("isolateFaultyLocations", () => { + const parsed = { file: null, line: null, message: 'unknown "missing" variable' }; + const fail = (message = parsed.message) => ({ ok: false, parsed: { message } }); + // nginx -t fails while any of `bad` is rendered (not stubbed); with `together`, only while all of them are + const tester = (bad, { together = false, other = null } = {}) => { + const calls = []; + const testWithStubs = async (stubbed) => { + calls.push([...stubbed]); + if (other) { + return fail(other); + } + const rendered = bad.filter((i) => !stubbed.includes(i)); + const fails = together ? rendered.length === bad.length : rendered.length > 0; + return fails ? fail() : { ok: true }; + }; + return { calls, testWithStubs }; + }; + + it("finds the one location that causes the error", async () => { + const t = tester([2]); + assert.deepEqual(await isolateFaultyLocations({ candidates: [0, 1, 2, 3], parsed, ...t }), [2]); + // all stubbed, then each one rendered alone + assert.deepEqual(t.calls, [ + [0, 1, 2, 3], + [1, 2, 3], + [0, 2, 3], + [0, 1, 3], + [0, 1, 2], + ]); + }); + + it("returns null when the error stays with every location stubbed, eg. the host's advanced config", async () => { + const t = tester([]); + t.testWithStubs = async (stubbed) => { + t.calls.push(stubbed); + return fail(); + }; + assert.equal(await isolateFaultyLocations({ candidates: [0, 1], parsed, ...t }), null); + assert.deepEqual(t.calls, [[0, 1]]); + }); + + it("finds every location that fails on its own with the same error, never a working one", async () => { + // two broken locations report the same message: stubbing one alone leaves the other's error + const t = tester([0, 2]); + assert.deepEqual(await isolateFaultyLocations({ candidates: [0, 1, 2], parsed, ...t }), [0, 2]); + assert.deepEqual(t.calls, [ + [0, 1, 2], + [1, 2], + [0, 2], + [0, 1], + ]); + }); + + it("stubs one at a time when the locations only fail together", async () => { + // eg. both define the same thing: none fails alone, stubbing the first one is enough + assert.deepEqual( + await isolateFaultyLocations({ candidates: [0, 1, 2], parsed, ...tester([0, 2], { together: true }) }), + [0], + ); + }); + + it("returns all candidates when no single one is the cause", async () => { + // fails while any two of them are rendered: none alone, and stubbing one still leaves two + const testWithStubs = async (stubbed) => + [0, 1, 2].filter((i) => !stubbed.includes(i)).length >= 2 ? fail() : { ok: true }; + assert.deepEqual(await isolateFaultyLocations({ candidates: [0, 1, 2], parsed, testWithStubs }), [0, 1, 2]); + }); + + it("counts a different error as a change", async () => { + assert.deepEqual( + await isolateFaultyLocations({ candidates: [0, 1], parsed, ...tester([0], { other: "something else" }) }), + [0], + ); + }); + + it("does nothing without candidates", async () => { + const t = tester([0]); + assert.equal(await isolateFaultyLocations({ candidates: [], parsed, ...t }), null); + assert.deepEqual(t.calls, []); + }); + + it("needs no second test for a single candidate", async () => { + const t = tester([0]); + assert.deepEqual(await isolateFaultyLocations({ candidates: [0], parsed, ...t }), [0]); + assert.deepEqual(t.calls, [[0]]); + }); +}); + +describe("configure with an error that names no line", () => { + const dir = fs.mkdtempSync(join(tmpdir(), "npm-isolate-")); + const file = (id) => join(dir, `${id}.conf`); + const original = {}; + let tests = 0; + + // nginx checks variables after parsing: it names neither file nor line for an unknown one + const fakeNginxTest = async () => { + tests++; + for (const name of fs.readdirSync(dir).filter((f) => f.endsWith(".conf"))) { + const text = fs + .readFileSync(join(dir, name), "utf8") + .split("\n") + .filter((line) => !line.trim().startsWith("#")) + .join("\n"); + const m = text.match(/\$(missing_[a-z]+)/); + if (m) { + throw new Error( + `nginx: [emerg] unknown "${m[1]}" variable\nnginx: configuration file /etc/nginx/nginx.conf test failed`, + ); + } + } + return ""; + }; + const modeResolver = Object.assign(async () => ({ mode: "static", status: "ok" }), { + hasResolver: false, + dynamicEnabled: false, + }); + + before(() => { + for (const key of ["getConfigName", "test", "listManagedFiles"]) { + original[key] = internalNginx[key]; + } + internalNginx.getConfigName = (_hostType, id) => file(id); + internalNginx.test = fakeNginxTest; + internalNginx.listManagedFiles = () => []; + }); + after(async () => { + Object.assign(internalNginx, original); + fs.rmSync(dir, { recursive: true, force: true }); + await db.destroy(); + }); + beforeEach(() => { + tests = 0; + }); + afterEach(() => { + for (const name of fs.readdirSync(dir)) { + fs.rmSync(join(dir, name)); + } + }); + + let nextId = 1; + const location = (path, advanced_config = "") => ({ + path, + forward_scheme: "http", + forward_host: "app", + forward_port: 80, + forward_path: "", + advanced_config, + }); + const insertHost = async (locations, advanced_config = "") => { + const id = nextId++; + const row = { + id, + owner_user_id: 1, + certificate_id: 0, + enabled: 1, + meta: {}, + domain_names: [`h${id}.example.com`], + forward_scheme: "http", + forward_host: "app", + forward_port: 80, + access_list_id: 0, + advanced_config, + locations, + }; + await db("proxy_host").insert({ + ...row, + meta: "{}", + domain_names: JSON.stringify(row.domain_names), + locations: JSON.stringify(locations), + ...ts, + }); + return row; + }; + const configure = (host) => + internalNginx.configureHost(proxyHostModel, "proxy_host", host, { reload: false, modeResolver }); + + it("quarantines the location that uses an unknown variable, the host stays online", async () => { + const host = await insertHost([ + location("/ok"), + location("/bad", "proxy_set_header X-Test $missing_variable;"), + location("/also-ok"), + ]); + const meta = await configure(host); + + assert.equal(meta.nginx_online, true); + assert.deepEqual( + meta.nginx_locations.map((l) => l.status), + ["ok", "quarantined", "ok"], + ); + assert.match(meta.nginx_locations[1].message, /unknown variable "\$missing_variable"/); + const text = fs.readFileSync(file(host.id), "utf8"); + assert.doesNotMatch(text, /\$missing_variable/); + assert.match(text, /location \/bad \{\n\s+# npm: quarantined[^\n]*\n\s+return 503;/); + assert.match(text, /location \/ok \{/); + const stored = JSON.parse((await db("proxy_host").where("id", host.id).first()).meta); + assert.equal(stored.nginx_locations[1].status, "quarantined"); + }); + + it("quarantines every location when only all of them together are the cause", async () => { + const host = await insertHost([ + location("/a", "add_header X-A $missing_a;"), + location("/b", "add_header X-B $missing_b;"), + ]); + const meta = await configure(host); + assert.equal(meta.nginx_online, true); + assert.deepEqual( + meta.nginx_locations.map((l) => l.status), + ["quarantined", "quarantined"], + ); + }); + + it("quarantines only the broken locations when two of them fail with the same error", async () => { + const host = await insertHost([ + location("/a", "add_header X-A $missing_same;"), + location("/clean"), + location("/b", "add_header X-B $missing_same;"), + ]); + const meta = await configure(host); + assert.equal(meta.nginx_online, true); + assert.deepEqual( + meta.nginx_locations.map((l) => l.status), + ["quarantined", "ok", "quarantined"], + ); + const text = fs.readFileSync(file(host.id), "utf8"); + assert.doesNotMatch(text, /\$missing_same/); + assert.match(text, /location \/clean \{\n(?!\s+# npm: quarantined)/); + }); + + it("keeps the config that failed, not a stubbed variant, as .err in bulk mode", async () => { + const host = await insertHost([location("/ok")], "add_header X-Test $missing_variable;"); + const meta = await internalNginx.configureHost(proxyHostModel, "proxy_host", host, { + reload: false, + modeResolver, + offlineOnError: true, + }); + assert.equal(meta.nginx_online, false); + assert.match(meta.nginx_err, /^Advanced configuration: unknown variable "\$missing_variable"/); + assert.doesNotMatch(meta.nginx_err, /another host/); + assert.equal(fs.existsSync(file(host.id)), false); + const err = fs.readFileSync(`${file(host.id)}.err`, "utf8"); + assert.match(err, /\$missing_variable/); + assert.match(err, /location \/ok \{/); + assert.match(err, /proxy_pass/); + assert.doesNotMatch(err, /return 503;/); + }); + + it("still rejects the host when its own advanced config uses the unknown variable", async () => { + const host = await insertHost([location("/ok")], "add_header X-Test $missing_variable;"); + await assert.rejects(configure(host), (err) => { + assert.equal(err.details.scope, "host"); + assert.equal(err.details.location_index, null); + // the host itself is at fault (the error stays with every other host moved aside), not a conflict + assert.match(err.message, /^Advanced configuration: unknown variable "\$missing_variable"/); + assert.doesNotMatch(err.message, /another host/); + return true; + }); + // the stubbing never touched the location and nothing was left behind + assert.equal(fs.existsSync(file(host.id)), false); + const stored = JSON.parse((await db("proxy_host").where("id", host.id).first()).meta); + assert.equal(stored.nginx_locations, undefined); + // error, without this host, all stubbed, alone: no test per location + assert.equal(tests, 4); + }); + + it("rejects a host without locations whose advanced config uses an unknown variable as its own error", async () => { + const host = await insertHost([], "add_header X-Test $missing_variable;"); + await assert.rejects(configure(host), (err) => { + assert.equal(err.details.scope, "host"); + assert.match(err.message, /^Advanced configuration: unknown variable "\$missing_variable"/); + assert.doesNotMatch(err.message, /another host/); + return true; + }); + }); +}); diff --git a/docs/src/advanced-config/locations.md b/docs/src/advanced-config/locations.md index 4b9fb8344..2ecfcfb7e 100644 --- a/docs/src/advanced-config/locations.md +++ b/docs/src/advanced-config/locations.md @@ -89,6 +89,11 @@ it in `meta.nginx_locations` of the proxy host: | `quarantined` | nginx rejected the location, or NPM knows it would. The message says why, for example an unknown directive in its advanced configuration, a forward path without an nginx variable on a regular-expression or named location (nginx rejects that URI; a capture such as `/v1/$1` or `$request_uri` works), or a forward path with spaces, quotes, `;`, `{`, `}`, `#` or `\` (nginx variables such as `$1` are fine). | Answers `503` for this path. If nginx rejects even that placeholder, for example because the path itself is invalid, the location is left out. | | `disabled` | You switched the location off. | The location isn't in the config. Requests fall through to the host's other locations. | +Some nginx errors name no line, for example `unknown "x" variable` for a misspelled variable (nginx checks variables +after reading all files). Then NPM replaces the host's custom locations with placeholders to find the one that causes +it and quarantines only that one. If the error stays with every location replaced, it comes from the host itself and the +save fails as described below. + Fix the cause and save the host again to bring a quarantined location back. An unresolved location comes back by itself once the upstream name resolves. diff --git a/test/cypress/e2e/api/ProxyHostLocations.cy.js b/test/cypress/e2e/api/ProxyHostLocations.cy.js index 6e0bdbb5b..5892ae29d 100644 --- a/test/cypress/e2e/api/ProxyHostLocations.cy.js +++ b/test/cypress/e2e/api/ProxyHostLocations.cy.js @@ -269,6 +269,28 @@ describe('Proxy Host custom location resilience', () => { expectPage('/cap/dashboard', 'this is the dashboard page'); }); + it('Should quarantine a location that uses an unknown variable, not the host', () => { + cy.task('backendApiPut', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}`, + data: { + locations: [ + location('/dash', 'examplesite', { forward_path: '/dashboard' }), + location('/typo', 'examplesite', { advanced_config: 'proxy_set_header X-Test $missing_variable;' }), + ], + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/nginx/proxy-hosts/{hostID}', data); + expect(data.meta).to.have.property('nginx_online', true); + expect(data.meta.nginx_locations[0]).to.include({ index: 0, status: 'ok' }); + expect(data.meta.nginx_locations[1]).to.include({ index: 1, status: 'quarantined', mode: null }); + expect(data.meta.nginx_locations[1].message).to.contain('unknown variable "$missing_variable"'); + }); + + expectPage('/dash', 'this is the dashboard page'); + expectStatus('/typo', '503'); + }); + it('Should delete the host', () => { cy.task('backendApiDelete', { token: token,