WAF rate-limit mitigation: nginx /api cache + Playwright throttle

(A) Add proxy_cache zone for ui-dashboard.gnerim.ru. /api/ caches 200 for
1m, /map/api/ for 24h. proxy_cache_use_stale serves cached content during
upstream errors (incl. 403 from WAF rate limit). proxy_cache_lock collapses
concurrent fetches for the same URI. Cache zone declared in conf.d/ (must
be in http{} context).

(B) Playwright workers=2, retries=2 in CI. Cuts the parallel burst that
trips the WAF before nginx cache warms up; retries handle the residual
flake.

setup-pve201.sh now installs the conf.d cache file and pre-creates the
cache dir with nginx-user ownership.
This commit is contained in:
2026-04-27 16:40:44 +03:00
parent f17961d523
commit b0e9aafed2
4 changed files with 66 additions and 0 deletions
@@ -0,0 +1,15 @@
# Cache zone for ui-dashboard.gnerim.ru /api/* and /map/api/* upstreams.
# Lives in /etc/nginx/conf.d/ because proxy_cache_path is only valid in the
# http {} context, not inside server {}.
#
# Why we need it: flights.test.aeroflot.ru's WAF has a per-source-IP rate
# limit (~25-30 fresh TCP connections per window) that the parallel e2e
# burst trips. Caching read-only GETs by the customer-facing nginx layer
# absorbs the burst — only one request per (URI, window) reaches the WAF.
proxy_cache_path /var/cache/nginx/flights-api
levels=1:2
keys_zone=flights_api:10m
max_size=200m
inactive=30m
use_temp_path=off;
@@ -1,6 +1,9 @@
# Production vhost for ui-dashboard.gnerim.ru.
# Symlink into /etc/nginx/sites-enabled/ and reload nginx.
# TLS certs assumed to exist via certbot (separate process).
#
# Cache zone `flights_api` is declared in /etc/nginx/conf.d/flights-api-cache.conf
# (proxy_cache_path lives at http context, can't be in server {}).
server {
listen 80;
@@ -37,6 +40,9 @@ server {
# ssh -L tunnel to webzavod which exits via ppp0 with a corp-VPN source IP
# the upstream WAF whitelists. SNI/Host are set explicitly because the
# TCP target is loopback rather than the real hostname.
#
# Cached to absorb e2e bursts that would otherwise trip the upstream
# WAF rate limit. Only GET/HEAD are cached (default proxy_cache_methods).
location /api/ {
auth_basic off;
proxy_pass https://127.0.0.1:8443;
@@ -44,8 +50,19 @@ server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_ssl_server_name on;
proxy_ssl_name flights.test.aeroflot.ru;
proxy_cache flights_api;
proxy_cache_key "$scheme$host$request_uri";
proxy_cache_valid 200 1m;
proxy_cache_valid 404 30s;
proxy_cache_lock on;
proxy_cache_use_stale error timeout updating http_403 http_500 http_502 http_503 http_504;
proxy_cache_bypass $http_cache_control;
add_header X-Cache-Status $upstream_cache_status always;
}
# Map tiles — heavily cacheable (tile data rarely changes for an area).
# Longer TTL than /api/ since these are essentially static.
location /map/api/ {
auth_basic off;
proxy_pass https://127.0.0.1:8443;
@@ -53,5 +70,13 @@ server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_ssl_server_name on;
proxy_ssl_name flights.test.aeroflot.ru;
proxy_cache flights_api;
proxy_cache_key "$scheme$host$request_uri";
proxy_cache_valid 200 24h;
proxy_cache_valid 404 5m;
proxy_cache_lock on;
proxy_cache_use_stale error timeout updating http_403 http_500 http_502 http_503 http_504;
add_header X-Cache-Status $upstream_cache_status always;
}
}
+19
View File
@@ -100,6 +100,25 @@ fi
mkdir -p /etc/nginx/htpasswd
ok "/etc/nginx/htpasswd ensured"
# Install proxy_cache zone declaration (must live in http {} context)
CACHE_CONF_SRC="$REPO_ROOT/deployment/nginx/conf.d/flights-api-cache.conf"
CACHE_CONF_DST="/etc/nginx/conf.d/flights-api-cache.conf"
if [ -f "$CACHE_CONF_DST" ] && cmp -s "$CACHE_CONF_SRC" "$CACHE_CONF_DST"; then
ok "$CACHE_CONF_DST already up-to-date"
else
cp "$CACHE_CONF_SRC" "$CACHE_CONF_DST"
ok "installed $CACHE_CONF_DST"
fi
# Cache directory — nginx auto-creates with proper perms on first start, but
# we pre-create with the right ownership so reload picks it up cleanly.
CACHE_DIR="/var/cache/nginx/flights-api"
NGINX_USER="$(awk '/^user / {gsub(";",""); print $2}' /etc/nginx/nginx.conf 2>/dev/null | head -1)"
NGINX_USER="${NGINX_USER:-www-data}"
mkdir -p "$CACHE_DIR"
chown -R "$NGINX_USER":"$NGINX_USER" "$CACHE_DIR"
ok "$CACHE_DIR ensured (owner: $NGINX_USER)"
# ---------- 4. htpasswd ----------
step "4. htpasswd"