Dev-server: fall back between direct and HTTPS_PROXY transports
The local WAF is unpredictable: some windows the gost VPN tunnel at 127.0.0.1:8888 is 503-ing (direct must work), other windows the direct IP is throttled to 403 by Ngenix (VPN must be used). The previous hardcoded `--noproxy '*'` survived one of those states only, which is why the dictionary load surfaced as console 403s as soon as the state flipped. Try direct first (faster when it works, simpler cookie jar), fall back through the system HTTPS_PROXY on 4xx/5xx or curl failure, keep a separate cookie jar per transport so the Ngenix cookies don't cross- contaminate edge nodes.
This commit is contained in:
+94
-52
@@ -21,7 +21,11 @@ const API_TARGET = "https://flights.test.aeroflot.ru";
|
||||
|
||||
// Shared cookie jar so the Ngenix WAF cookie challenge (`ngenix_valid` +
|
||||
// 307-to-self) only runs once per dev-server lifetime, not per request.
|
||||
const COOKIE_JAR = resolve(tmpdir(), `aeroflot-dev-cookies-${process.pid}.txt`);
|
||||
// We keep one jar per transport (direct vs. HTTPS_PROXY) because the WAF
|
||||
// anchors the cookie to the upstream edge node that minted it; mixing
|
||||
// jars across transports re-triggers the challenge on every request.
|
||||
const COOKIE_JAR_DIRECT = resolve(tmpdir(), `aeroflot-dev-cookies-direct-${process.pid}.txt`);
|
||||
const COOKIE_JAR_PROXY = resolve(tmpdir(), `aeroflot-dev-cookies-proxy-${process.pid}.txt`);
|
||||
|
||||
// --- Start Modern.js on internal port ---
|
||||
console.log(`Starting Modern.js on :${MODERNJS_PORT}...`);
|
||||
@@ -80,89 +84,127 @@ app.use("/map", (req, res) => {
|
||||
res.setHeader("Content-Type", contentType);
|
||||
res.setHeader("Cache-Control", "public, max-age=86400");
|
||||
|
||||
const args = [
|
||||
"-sS",
|
||||
"-f", // fail on HTTP 4xx/5xx
|
||||
"-L", // follow Ngenix 307-to-self cookie challenge
|
||||
"--noproxy", "*", // bypass any inherited HTTPS_PROXY — upstream is publicly reachable
|
||||
"-c", COOKIE_JAR, "-b", COOKIE_JAR, // persist `ngenix_valid` cookie across requests
|
||||
const commonHeaders = [
|
||||
"-H", `Accept: image/webp,image/jpeg,image/*,*/*`,
|
||||
"-H", `User-Agent: ${req.headers["user-agent"] || "Mozilla/5.0"}`,
|
||||
"-H", `Referer: ${API_TARGET}/`,
|
||||
targetUrl,
|
||||
];
|
||||
const child = spawn("/usr/bin/curl", args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
|
||||
child.stdout.pipe(res);
|
||||
// Tile transport is binary — fall back to the proxy path only if the
|
||||
// direct attempt emits no data (indicating WAF deny / throttle).
|
||||
const tryTile = (noproxy) => {
|
||||
const args = [
|
||||
"-sS",
|
||||
"-f", // fail on HTTP 4xx/5xx
|
||||
"-L", // follow Ngenix 307-to-self cookie challenge
|
||||
...(noproxy ? ["--noproxy", "*"] : []),
|
||||
"-c", noproxy ? COOKIE_JAR_DIRECT : COOKIE_JAR_PROXY,
|
||||
"-b", noproxy ? COOKIE_JAR_DIRECT : COOKIE_JAR_PROXY,
|
||||
...commonHeaders,
|
||||
targetUrl,
|
||||
];
|
||||
return spawn("/usr/bin/curl", args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
};
|
||||
const child = tryTile(true);
|
||||
let gotData = false;
|
||||
child.stdout.on("data", (chunk) => { gotData = true; res.write(chunk); });
|
||||
child.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
// -f means upstream failed; close cleanly with empty body.
|
||||
res.end();
|
||||
}
|
||||
if (code === 0) { res.end(); return; }
|
||||
if (gotData) { res.end(); return; }
|
||||
// Direct failed with no data — retry through the system HTTPS_PROXY.
|
||||
const retry = tryTile(false);
|
||||
retry.stdout.pipe(res);
|
||||
retry.on("exit", () => res.end());
|
||||
});
|
||||
});
|
||||
|
||||
// --- API proxy via curl (bypasses WAF TLS fingerprinting) ---
|
||||
// Two transports: (1) direct with `--noproxy '*'` — fast path when the
|
||||
// upstream is publicly reachable; (2) through the system HTTPS_PROXY
|
||||
// (e.g. a local gost VPN tunnel) — required when the direct IP hits a
|
||||
// WAF throttle (403 deny page). Try direct first; on 403/4xx/5xx or
|
||||
// network error, retry through the proxy. Each transport keeps its own
|
||||
// cookie jar because the WAF cookie is bound to the edge node that
|
||||
// minted it.
|
||||
app.use(["/api", "/flights"], (req, res) => {
|
||||
const targetUrl = `${API_TARGET}${req.originalUrl}`;
|
||||
|
||||
// Use curl to make the request — it passes through the system proxy
|
||||
// with a proper TLS fingerprint that the WAF accepts.
|
||||
const args = [
|
||||
"-s", // silent
|
||||
"-L", // follow Ngenix 307-to-self cookie challenge
|
||||
"--noproxy", "*", // bypass any inherited HTTPS_PROXY — upstream is publicly reachable
|
||||
"-c", COOKIE_JAR, "-b", COOKIE_JAR, // persist `ngenix_valid` cookie across requests
|
||||
const buildArgs = (noproxy) => [
|
||||
"-s",
|
||||
"-L",
|
||||
...(noproxy ? ["--noproxy", "*"] : []),
|
||||
"-c", noproxy ? COOKIE_JAR_DIRECT : COOKIE_JAR_PROXY,
|
||||
"-b", noproxy ? COOKIE_JAR_DIRECT : COOKIE_JAR_PROXY,
|
||||
"-H", `Accept: ${req.headers.accept || "application/json"}`,
|
||||
"-H", `User-Agent: ${req.headers["user-agent"] || "Mozilla/5.0"}`,
|
||||
"-H", `Accept-Language: ${req.headers["accept-language"] || "ru"}`,
|
||||
"-w", "\n%{http_code}", // append status code at end
|
||||
"-w", "\n%{http_code}",
|
||||
targetUrl,
|
||||
];
|
||||
|
||||
if (req.method === "POST") {
|
||||
args.unshift("-X", "POST");
|
||||
// Read body and pass to curl
|
||||
let body = "";
|
||||
req.on("data", (chunk) => { body += chunk; });
|
||||
req.on("end", () => {
|
||||
if (body) {
|
||||
args.push("-d", body);
|
||||
args.push("-H", "Content-Type: application/json");
|
||||
}
|
||||
execCurl(args, res);
|
||||
const bodyArgs = body ? ["-X", "POST", "-d", body, "-H", "Content-Type: application/json"] : ["-X", "POST"];
|
||||
execCurlWithFallback(buildArgs, bodyArgs, res);
|
||||
});
|
||||
} else {
|
||||
execCurl(args, res);
|
||||
execCurlWithFallback(buildArgs, [], res);
|
||||
}
|
||||
});
|
||||
|
||||
function execCurl(args, res) {
|
||||
execFile("/usr/bin/curl", args, { maxBuffer: 10 * 1024 * 1024, timeout: 30000 }, (err, stdout) => {
|
||||
if (err) {
|
||||
res.status(502).json({ error: err.message });
|
||||
function execCurlWithFallback(buildArgs, extraArgs, res) {
|
||||
runCurl([...extraArgs, ...buildArgs(true)], (direct) => {
|
||||
if (isSuccessfulUpstream(direct)) {
|
||||
respondWithCurlResult(direct, res);
|
||||
return;
|
||||
}
|
||||
|
||||
// stdout = body + "\n" + statusCode (from -w "\n%{http_code}"). When
|
||||
// upstream returns an empty body the stdout starts with "\n", so a
|
||||
// `> 0` check mis-treats the status as body and falls back to 200 —
|
||||
// silently hiding real 4xx/5xx responses. Use `>= 0` and split
|
||||
// unconditionally when the newline is present.
|
||||
const lastNewline = stdout.lastIndexOf("\n");
|
||||
const body = lastNewline >= 0 ? stdout.substring(0, lastNewline) : stdout;
|
||||
const statusStr = lastNewline >= 0 ? stdout.substring(lastNewline + 1).trim() : "200";
|
||||
const status = parseInt(statusStr) || 200;
|
||||
|
||||
const isJson = body.trimStart().startsWith("{") || body.trimStart().startsWith("[");
|
||||
|
||||
res.status(status);
|
||||
res.set("Content-Type", isJson ? "application/json" : "text/html");
|
||||
res.set("Access-Control-Allow-Origin", "*");
|
||||
res.send(body);
|
||||
// Direct hit a WAF deny / throttle / network failure — retry through
|
||||
// the system HTTPS_PROXY (gost VPN tunnel on this host).
|
||||
runCurl([...extraArgs, ...buildArgs(false)], (proxy) => {
|
||||
respondWithCurlResult(isSuccessfulUpstream(proxy) ? proxy : direct, res);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function runCurl(args, cb) {
|
||||
execFile("/usr/bin/curl", args, { maxBuffer: 10 * 1024 * 1024, timeout: 30000 }, (err, stdout) => {
|
||||
cb({ err, stdout: stdout ?? "" });
|
||||
});
|
||||
}
|
||||
|
||||
function isSuccessfulUpstream({ err, stdout }) {
|
||||
if (err) return false;
|
||||
const lastNewline = stdout.lastIndexOf("\n");
|
||||
const statusStr = lastNewline >= 0 ? stdout.substring(lastNewline + 1).trim() : "";
|
||||
const status = parseInt(statusStr) || 0;
|
||||
return status >= 200 && status < 400;
|
||||
}
|
||||
|
||||
function respondWithCurlResult({ err, stdout }, res) {
|
||||
if (err) {
|
||||
res.status(502).json({ error: err.message });
|
||||
return;
|
||||
}
|
||||
|
||||
// stdout = body + "\n" + statusCode (from -w "\n%{http_code}"). When
|
||||
// upstream returns an empty body the stdout starts with "\n", so a
|
||||
// `> 0` check mis-treats the status as body and falls back to 200 —
|
||||
// silently hiding real 4xx/5xx responses. Use `>= 0` and split
|
||||
// unconditionally when the newline is present.
|
||||
const lastNewline = stdout.lastIndexOf("\n");
|
||||
const body = lastNewline >= 0 ? stdout.substring(0, lastNewline) : stdout;
|
||||
const statusStr = lastNewline >= 0 ? stdout.substring(lastNewline + 1).trim() : "200";
|
||||
const status = parseInt(statusStr) || 200;
|
||||
|
||||
const isJson = body.trimStart().startsWith("{") || body.trimStart().startsWith("[");
|
||||
|
||||
res.status(status);
|
||||
res.set("Content-Type", isJson ? "application/json" : "text/html");
|
||||
res.set("Access-Control-Allow-Origin", "*");
|
||||
res.send(body);
|
||||
}
|
||||
|
||||
// --- Everything else → Modern.js ---
|
||||
const modernProxy = createProxyMiddleware({
|
||||
target: `http://localhost:${MODERNJS_PORT}`,
|
||||
|
||||
Reference in New Issue
Block a user