diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 23c771d22..d97e8125d 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -28,6 +28,7 @@ import { CONFIG_VERSION_MARKER, chooseLocationMode, describeConfigLine, + isolateFaultyLocations, isVariableHost, MESSAGES as LOCATION_MESSAGES, parseLocationPath, @@ -552,6 +553,47 @@ const internalNginx = { let configText = ""; let failure = null; + // An error without a line (eg. `unknown "x" variable`) can still come from a custom location. Before the + // whole host is rejected, the locations are stubbed one by one to find it, see isolateFaultyLocations(). + const isolateLocations = async (parsed) => { + const candidates = (states || []) + .filter((s) => s.status === "ok" || s.status === "unresolved") + .map((s) => s.index); + if (!candidates.length) { + return null; + } + // In a nested configure the host being saved may cause it, causedByProtected() decides that below + if (opts.protect?.length && (await internalNginx.testAside(opts.protect)).ok) { + return null; + } + try { + return await isolateFaultyLocations({ + candidates, + parsed, + testWithStubs: async (indexes) => { + const stubbed = states.map((s) => + indexes.includes(s.index) + ? Object.assign({}, s, { status: "quarantined", mode: null, stub: true }) + : s, + ); + // writes the host's file, see below + const text = await internalNginx.generateConfig(hostType, host, { locationStates: stubbed }); + return internalNginx.runTest({ file: filename, text }); + }, + }); + } finally { + // Put the config that failed back: when no location is to blame, the host is rejected or (bulk + // mode) its file is kept as .err for inspection, which must not show the last stubbed variant + fs.writeFileSync(filename, configText, { encoding: "utf8" }); + } + }; + const quarantineFaulty = (indexes, parsed) => { + for (const index of indexes) { + internalNginx.quarantineLocation(states[index], host, parsed, modeResolver); + } + quarantines += indexes.length; + }; + for (;;) { if (quarantines > maxQuarantines) { failure = { scope: "host", parsed: { file: filename, line: null, message: "", output: "" } }; @@ -607,6 +649,12 @@ const internalNginx = { quarantines++; continue; } + // the file but no line, eg. a log path of a location that can't be opened + const faulty = where.location_index === null && !parsed.line ? await isolateLocations(parsed) : null; + if (faulty) { + quarantineFaulty(faulty, parsed); + continue; + } failure = { scope: "host", parsed }; break; } @@ -638,9 +686,15 @@ const internalNginx = { } if (blame) { if (blame.action === "conflict") { - // When the error stays with every other host's config moved aside, this host's own config - // causes it (eg. its advanced config uses an unknown variable): a host level error, not a - // conflict with another host + // fails only with this host's config: maybe one of its locations + const faulty = !parsed.file ? await isolateLocations(parsed) : null; + if (faulty) { + quarantineFaulty(faulty, parsed); + continue; + } + // No location is to blame. When the error stays with every other host's config moved aside, this + // host's own config causes it (eg. its advanced config uses an unknown variable): a host level + // error, not a conflict with another host if (!parsed.file && (await internalNginx.failsAlone(filename, parsed))) { failure = { scope: "host", parsed }; break; diff --git a/backend/lib/nginx-locations.js b/backend/lib/nginx-locations.js index e7dbbe606..d47b3a9a2 100644 --- a/backend/lib/nginx-locations.js +++ b/backend/lib/nginx-locations.js @@ -383,6 +383,50 @@ const renderLocation = async (engine, template, loc, state) => { return ` ${begin}\n${rendered.replace(/\s+$/, "")}\n ${end}\n`; }; +/** + * Finds the custom locations behind an `nginx -t` error that names no line, eg. `unknown "x" variable` + * (nginx checks variables after parsing the files and doesn't say where they are used). The candidates are + * replaced by `return 503;` stubs, which contain nothing of the user's input: + * + * - still the same error with all of them stubbed: not caused by a location, null + * - otherwise each candidate is rendered alone, with all the others still stubbed: every one that brings the + * same error back fails on its own and is returned (two broken locations with the same message are both + * found, a working one is never blamed for them) + * - none fails alone (eg. two locations only fail together): each candidate is stubbed alone, the first one + * that makes the error go away (or change) is returned + * - none of that: all candidates + * + * @param {Object} opts + * @param {Number[]} opts.candidates indexes of the locations that are rendered (ok / unresolved) + * @param {Object} opts.parsed the parsed error + * @param {Function} opts.testWithStubs async (indexes) => {ok, parsed}: nginx -t with those locations stubbed + * @returns {Promise} the indexes to quarantine + */ +const isolateFaultyLocations = async ({ candidates, parsed, testWithStubs }) => { + const sameError = (result) => !result.ok && result.parsed?.message === parsed.message; + if (!candidates.length || sameError(await testWithStubs(candidates))) { + return null; + } + if (candidates.length === 1) { + return candidates; + } + const failAlone = []; + for (const index of candidates) { + if (sameError(await testWithStubs(candidates.filter((i) => i !== index)))) { + failAlone.push(index); + } + } + if (failAlone.length) { + return failAlone; + } + for (const index of candidates) { + if (!sameError(await testWithStubs([index]))) { + return [index]; + } + } + return candidates; +}; + /** * @returns {String} */ @@ -479,6 +523,7 @@ export { findDuplicatePath, hasDirective, hasVariablePath, + isolateFaultyLocations, isVariableHost, matchSavedLocations, normaliseLocationPath, diff --git a/backend/test/nginx-location-isolate.test.js b/backend/test/nginx-location-isolate.test.js new file mode 100644 index 000000000..211f13260 --- /dev/null +++ b/backend/test/nginx-location-isolate.test.js @@ -0,0 +1,285 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, afterEach, before, beforeEach, describe, it } from "node:test"; +import { isolateFaultyLocations } from "../lib/nginx-locations.js"; +import { db, load, ts } from "./support/backend.js"; + +const internalNginx = await load("internal/nginx.js"); +const proxyHostModel = await load("models/proxy_host.js"); + +describe("isolateFaultyLocations", () => { + const parsed = { file: null, line: null, message: 'unknown "missing" variable' }; + const fail = (message = parsed.message) => ({ ok: false, parsed: { message } }); + // nginx -t fails while any of `bad` is rendered (not stubbed); with `together`, only while all of them are + const tester = (bad, { together = false, other = null } = {}) => { + const calls = []; + const testWithStubs = async (stubbed) => { + calls.push([...stubbed]); + if (other) { + return fail(other); + } + const rendered = bad.filter((i) => !stubbed.includes(i)); + const fails = together ? rendered.length === bad.length : rendered.length > 0; + return fails ? fail() : { ok: true }; + }; + return { calls, testWithStubs }; + }; + + it("finds the one location that causes the error", async () => { + const t = tester([2]); + assert.deepEqual(await isolateFaultyLocations({ candidates: [0, 1, 2, 3], parsed, ...t }), [2]); + // all stubbed, then each one rendered alone + assert.deepEqual(t.calls, [ + [0, 1, 2, 3], + [1, 2, 3], + [0, 2, 3], + [0, 1, 3], + [0, 1, 2], + ]); + }); + + it("returns null when the error stays with every location stubbed, eg. the host's advanced config", async () => { + const t = tester([]); + t.testWithStubs = async (stubbed) => { + t.calls.push(stubbed); + return fail(); + }; + assert.equal(await isolateFaultyLocations({ candidates: [0, 1], parsed, ...t }), null); + assert.deepEqual(t.calls, [[0, 1]]); + }); + + it("finds every location that fails on its own with the same error, never a working one", async () => { + // two broken locations report the same message: stubbing one alone leaves the other's error + const t = tester([0, 2]); + assert.deepEqual(await isolateFaultyLocations({ candidates: [0, 1, 2], parsed, ...t }), [0, 2]); + assert.deepEqual(t.calls, [ + [0, 1, 2], + [1, 2], + [0, 2], + [0, 1], + ]); + }); + + it("stubs one at a time when the locations only fail together", async () => { + // eg. both define the same thing: none fails alone, stubbing the first one is enough + assert.deepEqual( + await isolateFaultyLocations({ candidates: [0, 1, 2], parsed, ...tester([0, 2], { together: true }) }), + [0], + ); + }); + + it("returns all candidates when no single one is the cause", async () => { + // fails while any two of them are rendered: none alone, and stubbing one still leaves two + const testWithStubs = async (stubbed) => + [0, 1, 2].filter((i) => !stubbed.includes(i)).length >= 2 ? fail() : { ok: true }; + assert.deepEqual(await isolateFaultyLocations({ candidates: [0, 1, 2], parsed, testWithStubs }), [0, 1, 2]); + }); + + it("counts a different error as a change", async () => { + assert.deepEqual( + await isolateFaultyLocations({ candidates: [0, 1], parsed, ...tester([0], { other: "something else" }) }), + [0], + ); + }); + + it("does nothing without candidates", async () => { + const t = tester([0]); + assert.equal(await isolateFaultyLocations({ candidates: [], parsed, ...t }), null); + assert.deepEqual(t.calls, []); + }); + + it("needs no second test for a single candidate", async () => { + const t = tester([0]); + assert.deepEqual(await isolateFaultyLocations({ candidates: [0], parsed, ...t }), [0]); + assert.deepEqual(t.calls, [[0]]); + }); +}); + +describe("configure with an error that names no line", () => { + const dir = fs.mkdtempSync(join(tmpdir(), "npm-isolate-")); + const file = (id) => join(dir, `${id}.conf`); + const original = {}; + let tests = 0; + + // nginx checks variables after parsing: it names neither file nor line for an unknown one + const fakeNginxTest = async () => { + tests++; + for (const name of fs.readdirSync(dir).filter((f) => f.endsWith(".conf"))) { + const text = fs + .readFileSync(join(dir, name), "utf8") + .split("\n") + .filter((line) => !line.trim().startsWith("#")) + .join("\n"); + const m = text.match(/\$(missing_[a-z]+)/); + if (m) { + throw new Error( + `nginx: [emerg] unknown "${m[1]}" variable\nnginx: configuration file /etc/nginx/nginx.conf test failed`, + ); + } + } + return ""; + }; + const modeResolver = Object.assign(async () => ({ mode: "static", status: "ok" }), { + hasResolver: false, + dynamicEnabled: false, + }); + + before(() => { + for (const key of ["getConfigName", "test", "listManagedFiles"]) { + original[key] = internalNginx[key]; + } + internalNginx.getConfigName = (_hostType, id) => file(id); + internalNginx.test = fakeNginxTest; + internalNginx.listManagedFiles = () => []; + }); + after(async () => { + Object.assign(internalNginx, original); + fs.rmSync(dir, { recursive: true, force: true }); + await db.destroy(); + }); + beforeEach(() => { + tests = 0; + }); + afterEach(() => { + for (const name of fs.readdirSync(dir)) { + fs.rmSync(join(dir, name)); + } + }); + + let nextId = 1; + const location = (path, advanced_config = "") => ({ + path, + forward_scheme: "http", + forward_host: "app", + forward_port: 80, + forward_path: "", + advanced_config, + }); + const insertHost = async (locations, advanced_config = "") => { + const id = nextId++; + const row = { + id, + owner_user_id: 1, + certificate_id: 0, + enabled: 1, + meta: {}, + domain_names: [`h${id}.example.com`], + forward_scheme: "http", + forward_host: "app", + forward_port: 80, + access_list_id: 0, + advanced_config, + locations, + }; + await db("proxy_host").insert({ + ...row, + meta: "{}", + domain_names: JSON.stringify(row.domain_names), + locations: JSON.stringify(locations), + ...ts, + }); + return row; + }; + const configure = (host) => + internalNginx.configureHost(proxyHostModel, "proxy_host", host, { reload: false, modeResolver }); + + it("quarantines the location that uses an unknown variable, the host stays online", async () => { + const host = await insertHost([ + location("/ok"), + location("/bad", "proxy_set_header X-Test $missing_variable;"), + location("/also-ok"), + ]); + const meta = await configure(host); + + assert.equal(meta.nginx_online, true); + assert.deepEqual( + meta.nginx_locations.map((l) => l.status), + ["ok", "quarantined", "ok"], + ); + assert.match(meta.nginx_locations[1].message, /unknown variable "\$missing_variable"/); + const text = fs.readFileSync(file(host.id), "utf8"); + assert.doesNotMatch(text, /\$missing_variable/); + assert.match(text, /location \/bad \{\n\s+# npm: quarantined[^\n]*\n\s+return 503;/); + assert.match(text, /location \/ok \{/); + const stored = JSON.parse((await db("proxy_host").where("id", host.id).first()).meta); + assert.equal(stored.nginx_locations[1].status, "quarantined"); + }); + + it("quarantines every location when only all of them together are the cause", async () => { + const host = await insertHost([ + location("/a", "add_header X-A $missing_a;"), + location("/b", "add_header X-B $missing_b;"), + ]); + const meta = await configure(host); + assert.equal(meta.nginx_online, true); + assert.deepEqual( + meta.nginx_locations.map((l) => l.status), + ["quarantined", "quarantined"], + ); + }); + + it("quarantines only the broken locations when two of them fail with the same error", async () => { + const host = await insertHost([ + location("/a", "add_header X-A $missing_same;"), + location("/clean"), + location("/b", "add_header X-B $missing_same;"), + ]); + const meta = await configure(host); + assert.equal(meta.nginx_online, true); + assert.deepEqual( + meta.nginx_locations.map((l) => l.status), + ["quarantined", "ok", "quarantined"], + ); + const text = fs.readFileSync(file(host.id), "utf8"); + assert.doesNotMatch(text, /\$missing_same/); + assert.match(text, /location \/clean \{\n(?!\s+# npm: quarantined)/); + }); + + it("keeps the config that failed, not a stubbed variant, as .err in bulk mode", async () => { + const host = await insertHost([location("/ok")], "add_header X-Test $missing_variable;"); + const meta = await internalNginx.configureHost(proxyHostModel, "proxy_host", host, { + reload: false, + modeResolver, + offlineOnError: true, + }); + assert.equal(meta.nginx_online, false); + assert.match(meta.nginx_err, /^Advanced configuration: unknown variable "\$missing_variable"/); + assert.doesNotMatch(meta.nginx_err, /another host/); + assert.equal(fs.existsSync(file(host.id)), false); + const err = fs.readFileSync(`${file(host.id)}.err`, "utf8"); + assert.match(err, /\$missing_variable/); + assert.match(err, /location \/ok \{/); + assert.match(err, /proxy_pass/); + assert.doesNotMatch(err, /return 503;/); + }); + + it("still rejects the host when its own advanced config uses the unknown variable", async () => { + const host = await insertHost([location("/ok")], "add_header X-Test $missing_variable;"); + await assert.rejects(configure(host), (err) => { + assert.equal(err.details.scope, "host"); + assert.equal(err.details.location_index, null); + // the host itself is at fault (the error stays with every other host moved aside), not a conflict + assert.match(err.message, /^Advanced configuration: unknown variable "\$missing_variable"/); + assert.doesNotMatch(err.message, /another host/); + return true; + }); + // the stubbing never touched the location and nothing was left behind + assert.equal(fs.existsSync(file(host.id)), false); + const stored = JSON.parse((await db("proxy_host").where("id", host.id).first()).meta); + assert.equal(stored.nginx_locations, undefined); + // error, without this host, all stubbed, alone: no test per location + assert.equal(tests, 4); + }); + + it("rejects a host without locations whose advanced config uses an unknown variable as its own error", async () => { + const host = await insertHost([], "add_header X-Test $missing_variable;"); + await assert.rejects(configure(host), (err) => { + assert.equal(err.details.scope, "host"); + assert.match(err.message, /^Advanced configuration: unknown variable "\$missing_variable"/); + assert.doesNotMatch(err.message, /another host/); + return true; + }); + }); +}); diff --git a/docs/src/advanced-config/locations.md b/docs/src/advanced-config/locations.md index 4b9fb8344..2ecfcfb7e 100644 --- a/docs/src/advanced-config/locations.md +++ b/docs/src/advanced-config/locations.md @@ -89,6 +89,11 @@ it in `meta.nginx_locations` of the proxy host: | `quarantined` | nginx rejected the location, or NPM knows it would. The message says why, for example an unknown directive in its advanced configuration, a forward path without an nginx variable on a regular-expression or named location (nginx rejects that URI; a capture such as `/v1/$1` or `$request_uri` works), or a forward path with spaces, quotes, `;`, `{`, `}`, `#` or `\` (nginx variables such as `$1` are fine). | Answers `503` for this path. If nginx rejects even that placeholder, for example because the path itself is invalid, the location is left out. | | `disabled` | You switched the location off. | The location isn't in the config. Requests fall through to the host's other locations. | +Some nginx errors name no line, for example `unknown "x" variable` for a misspelled variable (nginx checks variables +after reading all files). Then NPM replaces the host's custom locations with placeholders to find the one that causes +it and quarantines only that one. If the error stays with every location replaced, it comes from the host itself and the +save fails as described below. + Fix the cause and save the host again to bring a quarantined location back. An unresolved location comes back by itself once the upstream name resolves. diff --git a/test/cypress/e2e/api/ProxyHostLocations.cy.js b/test/cypress/e2e/api/ProxyHostLocations.cy.js index 6e0bdbb5b..5892ae29d 100644 --- a/test/cypress/e2e/api/ProxyHostLocations.cy.js +++ b/test/cypress/e2e/api/ProxyHostLocations.cy.js @@ -269,6 +269,28 @@ describe('Proxy Host custom location resilience', () => { expectPage('/cap/dashboard', 'this is the dashboard page'); }); + it('Should quarantine a location that uses an unknown variable, not the host', () => { + cy.task('backendApiPut', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}`, + data: { + locations: [ + location('/dash', 'examplesite', { forward_path: '/dashboard' }), + location('/typo', 'examplesite', { advanced_config: 'proxy_set_header X-Test $missing_variable;' }), + ], + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/nginx/proxy-hosts/{hostID}', data); + expect(data.meta).to.have.property('nginx_online', true); + expect(data.meta.nginx_locations[0]).to.include({ index: 0, status: 'ok' }); + expect(data.meta.nginx_locations[1]).to.include({ index: 1, status: 'quarantined', mode: null }); + expect(data.meta.nginx_locations[1].message).to.contain('unknown variable "$missing_variable"'); + }); + + expectPage('/dash', 'this is the dashboard page'); + expectStatus('/typo', '503'); + }); + it('Should delete the host', () => { cy.task('backendApiDelete', { token: token,