feat: add property-based fuzz tests for advisory parsing, semver matc… (#69)

* feat: add property-based fuzz tests for advisory parsing, semver matching, and suppression config

* fix(ci): install deps before fuzz test jobs

.github/workflows/ci.yml

@@ -101,6 +101,8 @@ jobs:
       - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
         with:
           node-version: '20'
+          cache: 'npm'
+      - run: npm ci
       - name: Feed Verification Tests
         run: node skills/clawsec-suite/test/feed_verification.test.mjs
       - name: Guarded Install Tests
@@ -109,6 +111,10 @@ jobs:
         run: node skills/clawsec-suite/test/advisory_suppression.test.mjs
       - name: Path Resolution Tests
         run: node skills/clawsec-suite/test/path_resolution.test.mjs
+      - name: Fuzz Property Tests
+        run: node skills/clawsec-suite/test/fuzz_properties.test.mjs
+      - name: Semver/Scope/Suppression Fuzz Tests
+        run: node skills/clawsec-suite/test/fuzz_semver_scope_suppression.test.mjs
       - name: Advisory Application Scope Tests
         run: node skills/clawsec-suite/test/advisory_application_scope.test.mjs
 
@@ -120,7 +126,11 @@ jobs:
       - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
         with:
           node-version: '20'
+          cache: 'npm'
+      - run: npm ci
       - name: Suppression Config Tests
         run: node skills/openclaw-audit-watchdog/test/suppression_config.test.mjs
+      - name: Suppression Config Fuzz Tests
+        run: node skills/openclaw-audit-watchdog/test/suppression_config_fuzz.test.mjs
       - name: Render Report Suppression Tests
         run: node skills/openclaw-audit-watchdog/test/render_report_suppression.test.mjs

.github/workflows/codeql.yml

@@ -5,6 +5,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  workflow_dispatch:
   schedule:
     - cron: "17 3 * * 1"
 

.github/workflows/poll-nvd-cves.yml

@@ -30,6 +30,7 @@ jobs:
   poll-and-update:
     runs-on: ubuntu-latest
     permissions:
+      actions: write
       contents: write
       pull-requests: write
     steps:
@@ -654,6 +655,44 @@ jobs:
             ${{ env.SKILL_FEED_PATH }}
             ${{ env.SKILL_FEED_SIG_PATH }}
 
+      - name: Run CodeQL on generated PR branch
+        if: steps.create-pr.outputs.pull-request-number != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+
+          BRANCH="${{ steps.create-pr.outputs.pull-request-branch }}"
+          if [ -z "$BRANCH" ]; then
+            echo "::error::Missing pull-request-branch output from create-pull-request"
+            exit 1
+          fi
+
+          echo "Dispatching CodeQL for branch: $BRANCH"
+          gh workflow run codeql.yml --ref "$BRANCH"
+
+          RUN_ID=""
+          for _ in $(seq 1 30); do
+            RUN_ID=$(gh run list \
+              --workflow "CodeQL" \
+              --branch "$BRANCH" \
+              --event workflow_dispatch \
+              --json databaseId,createdAt \
+              --jq 'sort_by(.createdAt) | last | .databaseId // empty')
+            if [ -n "$RUN_ID" ]; then
+              break
+            fi
+            sleep 5
+          done
+
+          if [ -z "$RUN_ID" ]; then
+            echo "::error::Unable to locate dispatched CodeQL run for branch $BRANCH"
+            exit 1
+          fi
+
+          echo "Waiting for CodeQL run id: $RUN_ID"
+          gh run watch "$RUN_ID" --exit-status
+
       - name: Summary
         run: |
           echo "## NVD CVE Poll Summary" >> $GITHUB_STEP_SUMMARY

metadata.json

@@ -1,4 +1,4 @@
 {
   "name": "ClawSec",
-  "description": "A security-first skill distribution platform for OpenClaw agents (and some clones), featuring verified audit skills, hardening feeds, and guardian mode protocols."
+  "description": "A security-first skill distribution platform for OpenClaw and NanoClaw agents, featuring verified audit skills, hardening feeds, and guardian mode protocols."
 }

package-lock.json

@@ -25,6 +25,7 @@
         "eslint": "^9.39.2",
         "eslint-plugin-react": "^7.37.5",
         "eslint-plugin-react-hooks": "^7.0.1",
+        "fast-check": "^4.5.3",
         "typescript": "~5.8.2",
         "vite": "^7.3.1"
       }
@@ -2690,6 +2691,28 @@
       "version": "3.0.2",
       "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
     },
+    "node_modules/fast-check": {
+      "version": "4.5.3",
+      "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.5.3.tgz",
+      "integrity": "sha512-IE9csY7lnhxBnA8g/WI5eg/hygA6MGWJMSNfFRrBlXUciADEhS1EDB0SIsMSvzubzIlOBbVITSsypCsW717poA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/dubzzz"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fast-check"
+        }
+      ],
+      "dependencies": {
+        "pure-rand": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=12.17.0"
+      }
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -4793,6 +4816,22 @@
         "node": ">=6"
       }
     },
+    "node_modules/pure-rand": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz",
+      "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/dubzzz"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fast-check"
+        }
+      ]
+    },
     "node_modules/react": {
       "version": "19.2.4",
       "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==",

package.json

@@ -26,6 +26,7 @@
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-react-hooks": "^7.0.1",
+    "fast-check": "^4.5.3",
     "typescript": "~5.8.2",
     "vite": "^7.3.1"
   },

skills/clawsec-suite/test/fuzz_properties.js

@@ -0,0 +1,62 @@
+import assert from "node:assert/strict";
+import path from "node:path";
+import fc from "fast-check";
+import { parseAffectedSpecifier } from "../hooks/clawsec-advisory-guardian/lib/feed.mjs";
+import { normalizeSkillName, resolveConfiguredPath, uniqueStrings } from "../hooks/clawsec-advisory-guardian/lib/utils.mjs";
+
+const SAFE_SEGMENT = fc
+  .array(fc.constantFrom(...("abcdefghijklmnopqrstuvwxyz0123456789-_")), { minLength: 1, maxLength: 24 })
+  .map((chars) => chars.join(""));
+
+/**
+ * Runs property-based fuzz checks for advisory parsing and utility behavior.
+ */
+export function runFuzzProperties() {
+  fc.assert(
+    fc.property(fc.string(), (raw) => {
+      const expected = String(raw ?? "")
+        .trim()
+        .toLowerCase();
+      assert.equal(normalizeSkillName(raw), expected);
+    }),
+    { numRuns: 300 },
+  );
+
+  fc.assert(
+    fc.property(fc.array(fc.string(), { maxLength: 40 }), (values) => {
+      const deduped = uniqueStrings(values);
+      assert.deepEqual(deduped, Array.from(new Set(values)));
+    }),
+    { numRuns: 200 },
+  );
+
+  fc.assert(
+    fc.property(fc.string(), fc.string(), (left, right) => {
+      const rawSpecifier = `${left}@${right}`;
+      const specifier = rawSpecifier.trim();
+      const parsed = parseAffectedSpecifier(rawSpecifier);
+      assert.ok(parsed !== null);
+
+      const atIndex = specifier.lastIndexOf("@");
+      if (atIndex <= 0) {
+        assert.equal(parsed.name, specifier);
+        assert.equal(parsed.versionSpec, "*");
+      } else {
+        assert.equal(parsed.name, specifier.slice(0, atIndex));
+        assert.equal(parsed.versionSpec, specifier.slice(atIndex + 1));
+      }
+    }),
+    { numRuns: 300 },
+  );
+
+  fc.assert(
+    fc.property(SAFE_SEGMENT, (suffix) => {
+      const fallback = `/tmp/clawsec-suite/${suffix}`;
+      const resolved = resolveConfiguredPath(`\\$HOME/${suffix}`, fallback, {
+        label: "FUZZ_PATH",
+      });
+      assert.equal(resolved, path.normalize(fallback));
+    }),
+    { numRuns: 200 },
+  );
+}

skills/clawsec-suite/test/fuzz_properties.test.mjs

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+
+/**
+ * Property-based fuzzing checks for core advisory parsing/path helpers.
+ *
+ * Run: node skills/clawsec-suite/test/fuzz_properties.test.mjs
+ */
+
+import { runFuzzProperties } from "./fuzz_properties.js";
+
+try {
+  console.log("=== ClawSec Fast-Check Fuzz Properties ===\n");
+  runFuzzProperties();
+  console.log("=== Results: all fuzz properties passed ===");
+} catch (error) {
+  console.error("Fuzz property test failed:");
+  console.error(error);
+  process.exit(1);
+}

skills/clawsec-suite/test/fuzz_semver_scope_suppression.test.mjs

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+
+/**
+ * Property-based fuzz tests for semver matching, advisory scope, and suppression matching.
+ *
+ * Run: node skills/clawsec-suite/test/fuzz_semver_scope_suppression.test.mjs
+ */
+
+import assert from "node:assert/strict";
+import fc from "fast-check";
+import { advisoryAppliesToOpenclaw } from "../hooks/clawsec-advisory-guardian/lib/advisory_scope.mjs";
+import { isAdvisorySuppressed } from "../hooks/clawsec-advisory-guardian/lib/suppression.mjs";
+import { compareSemver, parseSemver, versionMatches } from "../hooks/clawsec-advisory-guardian/lib/version.mjs";
+
+const semverCoreArb = fc.tuple(
+  fc.integer({ min: 0, max: 999 }),
+  fc.integer({ min: 0, max: 999 }),
+  fc.integer({ min: 0, max: 999 }),
+);
+
+const semverArb = semverCoreArb.map(([major, minor, patch]) => `${major}.${minor}.${patch}`);
+const idArb = fc.string({ minLength: 1, maxLength: 24 });
+const skillArb = fc.string({ minLength: 1, maxLength: 24 });
+
+function runSemverProperties() {
+  fc.assert(
+    fc.property(semverCoreArb, ([major, minor, patch]) => {
+      const version = `v${major}.${minor}.${patch}`;
+      assert.deepEqual(parseSemver(version), [major, minor, patch]);
+    }),
+    { numRuns: 250 },
+  );
+
+  fc.assert(
+    fc.property(semverArb, semverArb, (left, right) => {
+      const leftVsRight = compareSemver(left, right);
+      const rightVsLeft = compareSemver(right, left);
+      assert.notEqual(leftVsRight, null);
+      assert.notEqual(rightVsLeft, null);
+      assert.equal(leftVsRight, -rightVsLeft);
+    }),
+    { numRuns: 250 },
+  );
+
+  fc.assert(
+    fc.property(semverArb, semverArb, (left, right) => {
+      const compared = compareSemver(left, right);
+      assert.notEqual(compared, null);
+
+      assert.equal(versionMatches(left, `>=${right}`), compared >= 0);
+      assert.equal(versionMatches(left, `<=${right}`), compared <= 0);
+      assert.equal(versionMatches(left, `>${right}`), compared > 0);
+      assert.equal(versionMatches(left, `<${right}`), compared < 0);
+      assert.equal(versionMatches(left, `=${right}`), compared === 0);
+    }),
+    { numRuns: 250 },
+  );
+}
+
+function runAdvisoryScopeProperties() {
+  fc.assert(
+    fc.property(fc.string(), (application) => {
+      const normalized = application.trim().toLowerCase();
+      const expected = normalized === "" || normalized === "openclaw" || normalized === "all";
+      assert.equal(advisoryAppliesToOpenclaw({ application }), expected);
+    }),
+    { numRuns: 250 },
+  );
+
+  fc.assert(
+    fc.property(fc.array(fc.string(), { maxLength: 8 }), (applications) => {
+      const normalized = applications
+        .map((entry) => entry.trim().toLowerCase())
+        .filter(Boolean);
+      const expected =
+        normalized.length === 0 || normalized.includes("openclaw") || normalized.includes("all");
+      assert.equal(advisoryAppliesToOpenclaw({ application: applications }), expected);
+    }),
+    { numRuns: 250 },
+  );
+
+  assert.equal(advisoryAppliesToOpenclaw({}), true);
+  assert.equal(advisoryAppliesToOpenclaw({ application: null }), true);
+}
+
+function runSuppressionProperties() {
+  fc.assert(
+    fc.property(idArb, skillArb, (id, skill) => {
+      const match = {
+        advisory: { id },
+        skill: { name: skill.toUpperCase() },
+      };
+      const suppressions = [
+        {
+          checkId: id,
+          skill: skill.toLowerCase(),
+          reason: "fuzz",
+          suppressedAt: "2026-02-25",
+        },
+      ];
+      assert.equal(isAdvisorySuppressed(match, suppressions), true);
+    }),
+    { numRuns: 250 },
+  );
+
+  fc.assert(
+    fc.property(idArb, idArb, skillArb, (targetId, otherId, skill) => {
+      const differentId = targetId === otherId ? `${otherId}-x` : otherId;
+      const match = {
+        advisory: { id: targetId },
+        skill: { name: skill },
+      };
+      const suppressions = [
+        {
+          checkId: differentId,
+          skill,
+          reason: "fuzz",
+          suppressedAt: "2026-02-25",
+        },
+      ];
+      assert.equal(isAdvisorySuppressed(match, suppressions), false);
+    }),
+    { numRuns: 250 },
+  );
+}
+
+try {
+  console.log("=== ClawSec Semver/Scope/Suppression Fuzz Properties ===\n");
+  runSemverProperties();
+  runAdvisoryScopeProperties();
+  runSuppressionProperties();
+  console.log("=== Results: all fuzz properties passed ===");
+} catch (error) {
+  console.error("Fuzz property test failed:");
+  console.error(error);
+  process.exit(1);
+}

skills/openclaw-audit-watchdog/test/suppression_config_fuzz.test.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+
+/**
+ * Property-based fuzz tests for openclaw suppression config gating behavior.
+ *
+ * Run: node skills/openclaw-audit-watchdog/test/suppression_config_fuzz.test.mjs
+ */
+
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import fc from "fast-check";
+import { loadSuppressionConfig } from "../scripts/load_suppression_config.mjs";
+
+const pipelineArb = fc.constantFrom("audit", "advisory", "watchdog");
+
+function makeValidConfig({ pipeline, includePipeline }) {
+  const enabledFor = includePipeline ? [pipeline.toUpperCase(), "other"] : ["other"];
+  return JSON.stringify({
+    enabledFor,
+    suppressions: [
+      {
+        checkId: "SCAN-001",
+        skill: "soul-guardian",
+        reason: "fuzz test",
+        suppressedAt: "2026-02-25",
+      },
+    ],
+  });
+}
+
+async function withTempConfig(content, fn) {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "watchdog-fuzz-"));
+  const configPath = path.join(tmpDir, "suppression.json");
+  await fs.writeFile(configPath, content, "utf8");
+  try {
+    await fn(configPath);
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+}
+
+async function withSilencedStderr(fn) {
+  const originalWrite = process.stderr.write;
+  process.stderr.write = () => true;
+  try {
+    return await fn();
+  } finally {
+    process.stderr.write = originalWrite;
+  }
+}
+
+async function runProperties() {
+  await fc.assert(
+    fc.asyncProperty(fc.string(), pipelineArb, async (rawPath, pipeline) => {
+      const result = await loadSuppressionConfig(rawPath, { enabled: false, pipeline });
+      assert.equal(result.source, "none");
+      assert.deepEqual(result.suppressions, []);
+    }),
+    { numRuns: 120 },
+  );
+
+  await fc.assert(
+    fc.asyncProperty(pipelineArb, fc.boolean(), async (pipeline, includePipeline) => {
+      const content = makeValidConfig({ pipeline, includePipeline });
+      await withTempConfig(content, async (configPath) => {
+        const result = await withSilencedStderr(() =>
+          loadSuppressionConfig(configPath, { enabled: true, pipeline }),
+        );
+        if (includePipeline) {
+          assert.equal(result.source, configPath);
+          assert.equal(result.suppressions.length, 1);
+          assert.equal(result.suppressions[0].checkId, "SCAN-001");
+        } else {
+          assert.equal(result.source, "none");
+          assert.deepEqual(result.suppressions, []);
+        }
+      });
+    }),
+    { numRuns: 80 },
+  );
+}
+
+try {
+  console.log("=== OpenClaw Suppression Config Fuzz Properties ===\n");
+  await runProperties();
+  console.log("=== Results: all fuzz properties passed ===");
+} catch (error) {
+  console.error("Fuzz property test failed:");
+  console.error(error);
+  process.exit(1);
+}