clawsec/scripts/ghsa-without-cve-feed.mjs

#!/usr/bin/env node

import { existsSync } from 'node:fs';
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

export const DEFAULT_REPOSITORIES = [
  'openclaw/openclaw',
  'qwibitai/nanoclaw',
  'softwarepub/hermes',
  'nousresearch/hermes-agent',
  'sipeed/picoclaw',
];

export const DEFAULT_STALE_AFTER_DAYS = 60;
export const FEED_VERSION = '0.1.0';

const PLATFORM_BY_REPOSITORY = new Map([
  ['openclaw/openclaw', 'openclaw'],
  ['qwibitai/nanoclaw', 'nanoclaw'],
  ['softwarepub/hermes', 'hermes'],
  ['nousresearch/hermes-agent', 'hermes'],
  ['sipeed/picoclaw', 'picoclaw'],
]);

const CWE_TYPE_BY_ID = new Map([
  ['CWE-22', 'path_traversal'],
  ['CWE-78', 'os_command_injection'],
  ['CWE-79', 'cross_site_scripting'],
  ['CWE-94', 'code_injection'],
  ['CWE-200', 'exposure_of_sensitive_information'],
  ['CWE-284', 'improper_access_control'],
  ['CWE-287', 'improper_authentication'],
  ['CWE-306', 'missing_authentication_for_critical_function'],
  ['CWE-352', 'cross_site_request_forgery'],
  ['CWE-400', 'uncontrolled_resource_consumption'],
  ['CWE-502', 'deserialization_of_untrusted_data'],
  ['CWE-862', 'missing_authorization'],
  ['CWE-863', 'incorrect_authorization'],
  ['CWE-918', 'server_side_request_forgery'],
]);

function cleanText(value) {
  return String(value ?? '')
    .replace(/\r/g, '')
    .replace(/```[\s\S]*?```/g, ' ')
    .replace(/`([^`]+)`/g, '$1')
    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
    .replace(/^#+\s+/gm, '')
    .replace(/[*_>]/g, '')
    .replace(/\s+/g, ' ')
    .trim();
}

function daysBetween(startIso, endIso) {
  const start = Date.parse(startIso);
  const end = Date.parse(endIso);
  if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) {
    return 0;
  }
  return Math.floor((end - start) / 86_400_000);
}

function toArray(value) {
  return Array.isArray(value) ? value : [];
}

function uniqueStrings(values) {
  return [...new Set(values.filter((value) => typeof value === 'string' && value.length > 0))];
}

export function inferPlatforms(repository) {
  const known = PLATFORM_BY_REPOSITORY.get(String(repository).toLowerCase());
  return known ? [known] : [];
}

function nextLinkFromHeader(linkHeader) {
  if (!linkHeader) {
    return null;
  }
  for (const part of linkHeader.split(',')) {
    const match = part.trim().match(/^<([^>]+)>;\s*rel="next"$/);
    if (match) {
      return match[1];
    }
  }
  return null;
}

function affectedFromVulnerabilities(advisory, platforms) {
  const affected = toArray(advisory.vulnerabilities).flatMap((vulnerability) => {
    const packageName = vulnerability?.package?.name;
    const versionRange = vulnerability?.vulnerable_version_range;
    if (!packageName) {
      return [];
    }
    return [`${packageName}@${versionRange || '*'}`];
  });

  if (affected.length > 0) {
    return uniqueStrings(affected);
  }

  return platforms.length > 0 ? platforms.map((platform) => `${platform}@*`) : [];
}

function patchedFromVulnerabilities(advisory) {
  return uniqueStrings(
    toArray(advisory.vulnerabilities).flatMap((vulnerability) => {
      const packageName = vulnerability?.package?.name;
      const patchedVersions = vulnerability?.patched_versions;
      if (!packageName || !patchedVersions) {
        return [];
      }
      return [`${packageName}@${patchedVersions}`];
    }),
  );
}

function githubAdvisoryUrl(advisory) {
  return advisory.html_url || advisory.url || `https://github.com/advisories/${advisory.ghsa_id}`;
}

function resolveCveId(advisory, cveIdByGhsa) {
  return advisory.cve_id || cveIdByGhsa.get(advisory.ghsa_id) || null;
}

export function normalizeGhsaAdvisory(
  advisory,
  {
    now,
    repository,
    staleAfterDays = DEFAULT_STALE_AFTER_DAYS,
    cveId = advisory.cve_id || null,
  },
) {
  const platforms = inferPlatforms(repository);
  const published = advisory.published_at || advisory.created_at || advisory.updated_at || now;
  const ageDays = daysBetween(published, now);
  const stale = !cveId && ageDays >= staleAfterDays;
  const status = cveId ? 'matured' : stale ? 'stale' : 'active';
  const cweIds = uniqueStrings(toArray(advisory.cwe_ids));
  const cvss = advisory.cvss || advisory.cvss_severities?.cvss_v3 || {};
  const ghsaUrl = githubAdvisoryUrl(advisory);
  const affected = affectedFromVulnerabilities(advisory, platforms);
  const patched = patchedFromVulnerabilities(advisory);
  const title = cleanText(advisory.summary) || advisory.ghsa_id;
  const description = cleanText(advisory.description) || title;

  return {
    id: advisory.ghsa_id,
    ghsa_id: advisory.ghsa_id,
    cve_id: cveId,
    status,
    stale,
    stale_after_days: staleAfterDays,
    severity: advisory.severity || 'medium',
    type: CWE_TYPE_BY_ID.get(cweIds[0]) || 'github_security_advisory',
    nvd_category_id: cweIds[0] || null,
    title,
    description,
    affected,
    patched,
    platforms,
    action: cveId
      ? `Track ${cveId} in the canonical CVE advisory feed and verify affected components.`
      : 'Review the GitHub Security Advisory and update affected components; no CVE is assigned yet.',
    published,
    updated: advisory.updated_at || published,
    references: uniqueStrings([ghsaUrl, cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null]),
    source: 'GitHub Security Advisory',
    repository,
    github_advisory_url: ghsaUrl,
    nvd_url: cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null,
    cvss_score: cvss.score ?? null,
    cvss_vector: cvss.vector_string ?? null,
    cwe_ids: cweIds,
    credits: uniqueStrings(toArray(advisory.credits).map((credit) => credit?.login)),
    aliases: uniqueStrings([advisory.ghsa_id, cveId]),
  };
}

function ghsaToCveMapFromNvdFeed(nvdFeed) {
  const map = new Map();
  for (const advisory of toArray(nvdFeed?.advisories)) {
    const cveId = advisory?.id;
    if (typeof cveId !== 'string' || !cveId.startsWith('CVE-')) {
      continue;
    }
    const references = toArray(advisory.references).join('\n');
    for (const match of references.matchAll(/GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}/gi)) {
      map.set(match[0], cveId);
    }
  }
  return map;
}

function equivalentAdvisories(left, right) {
  return JSON.stringify(left ?? []) === JSON.stringify(right ?? []);
}

function isCveId(value) {
  return typeof value === 'string' && /^CVE-\d{4}-\d{4,}$/i.test(value);
}

function ghsaIdentifier(entry) {
  if (typeof entry?.ghsa_id === 'string' && entry.ghsa_id.length > 0) {
    return entry.ghsa_id.toLowerCase();
  }
  if (/^GHSA-/i.test(String(entry?.id || ''))) {
    return String(entry.id).toLowerCase();
  }
  return null;
}

function refreshExistingEntry(entry, { now, staleAfterDays, cveIdByGhsa }) {
  const cveId = entry.cve_id || cveIdByGhsa.get(entry.ghsa_id || entry.id) || null;
  const ageDays = daysBetween(entry.published, now);
  const stale = !cveId && ageDays >= staleAfterDays;
  return {
    ...entry,
    cve_id: cveId,
    status: cveId ? 'matured' : stale ? 'stale' : 'active',
    stale,
    stale_after_days: staleAfterDays,
    references: uniqueStrings([
      ...toArray(entry.references),
      cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null,
    ]),
    nvd_url: cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null,
    aliases: uniqueStrings([...(entry.aliases || []), entry.ghsa_id || entry.id, cveId]),
  };
}

export function buildConsolidatedAdvisoryFeed({ canonicalFeed = {}, ghsaFeed = {}, now }) {
  const canonicalFeedEntries = toArray(canonicalFeed.advisories);
  const canonicalCveIds = new Set(canonicalFeedEntries.map((entry) => entry?.id).filter(isCveId));
  const replacementGhsaIds = new Set(toArray(ghsaFeed.advisories).map(ghsaIdentifier).filter(Boolean));
  const canonicalEntries = canonicalFeedEntries.filter((entry) => {
    const ghsaId = ghsaIdentifier(entry);
    if (!ghsaId) {
      return true;
    }
    if (entry?.cve_id && canonicalCveIds.has(entry.cve_id)) {
      return false;
    }
    return !replacementGhsaIds.has(ghsaId);
  });
  const ghsaEntries = toArray(ghsaFeed.advisories)
    .filter((entry) => !(entry?.cve_id && canonicalCveIds.has(entry.cve_id)))
    .map((entry) => ({
      ...entry,
      source_feed: 'ghsa-without-cve',
    }));

  const advisories = [...canonicalEntries, ...ghsaEntries].sort((a, b) => {
    const published = Date.parse(b.published || '') - Date.parse(a.published || '');
    if (Number.isFinite(published) && published !== 0) {
      return published;
    }
    return String(a.id || '').localeCompare(String(b.id || ''));
  });

  return {
    ...canonicalFeed,
    version: canonicalFeed.version || '1.0.0',
    updated: canonicalFeed.updated || now,
    description: canonicalFeed.description || 'Community-driven security advisory feed for ClawSec',
    advisories,
  };
}

export function buildGhsaWithoutCveFeed({
  fetched,
  existingFeed = {},
  nvdFeed = {},
  now,
  staleAfterDays = DEFAULT_STALE_AFTER_DAYS,
}) {
  const existingEntries = toArray(existingFeed.advisories);
  const existingIds = new Set(existingEntries.map((entry) => entry.ghsa_id || entry.id));
  const cveIdByGhsa = ghsaToCveMapFromNvdFeed(nvdFeed);
  const entriesById = new Map();

  for (const { repository, advisories } of fetched) {
    for (const advisory of advisories) {
      const ghsaId = advisory.ghsa_id;
      if (!ghsaId) {
        continue;
      }
      const cveId = resolveCveId(advisory, cveIdByGhsa);
      if (cveId && !existingIds.has(ghsaId)) {
        continue;
      }
      entriesById.set(
        ghsaId,
        normalizeGhsaAdvisory(advisory, {
          now,
          repository,
          staleAfterDays,
          cveId,
        }),
      );
    }
  }

  for (const entry of existingEntries) {
    const ghsaId = entry.ghsa_id || entry.id;
    if (!ghsaId || entriesById.has(ghsaId)) {
      continue;
    }
    entriesById.set(ghsaId, refreshExistingEntry(entry, { now, staleAfterDays, cveIdByGhsa }));
  }

  const advisories = [...entriesById.values()].sort((a, b) => {
    const published = Date.parse(b.published) - Date.parse(a.published);
    if (published !== 0) {
      return published;
    }
    return a.id.localeCompare(b.id);
  });

  const updated = equivalentAdvisories(advisories, existingEntries)
    ? existingFeed.updated || now
    : now;

  return {
    version: FEED_VERSION,
    updated,
    description:
      'Provisional ClawSec advisory feed for public GitHub Security Advisories that do not yet have CVE identifiers.',
    stale_after_days: staleAfterDays,
    semantics: {
      active: 'GHSA is published and has no CVE identifier yet.',
      matured: 'GHSA now has a CVE identifier and should be reconciled with the canonical CVE feed.',
      stale: 'GHSA is older than stale_after_days and still has no CVE identifier.',
    },
    sources: DEFAULT_REPOSITORIES.map((repository) => ({
      repository,
      platform: inferPlatforms(repository)[0] || 'unknown',
      url: `https://github.com/${repository}/security/advisories`,
    })),
    advisories,
  };
}

export async function fetchGitHubSecurityAdvisories(repository, { token } = {}) {
  const advisories = [];
  let url = `https://api.github.com/repos/${repository}/security-advisories?per_page=100`;
  const seenUrls = new Set();

  while (url) {
    if (seenUrls.has(url)) {
      throw new Error(`GitHub advisory pagination loop detected for ${repository}: ${url}`);
    }
    seenUrls.add(url);

    const response = await globalThis.fetch(url, {
      headers: {
        Accept: 'application/vnd.github+json',
        'User-Agent': 'clawsec-ghsa-without-cve-poller',
        'X-GitHub-Api-Version': '2022-11-28',
        ...(token ? { Authorization: `Bearer ${token}` } : {}),
      },
    });

    if (!response.ok) {
      const message = await response.text();
      throw new Error(
        `GitHub advisory fetch failed for ${repository}: HTTP ${response.status} ${message.slice(0, 200)}`,
      );
    }

    const pageItems = await response.json();
    advisories.push(...pageItems);
    if (!Array.isArray(pageItems)) {
      break;
    }
    url = nextLinkFromHeader(response.headers.get('link'));
  }
  return advisories;
}

async function readJsonIfExists(path, fallback) {
  if (!existsSync(path)) {
    return fallback;
  }
  return JSON.parse(await readFile(path, 'utf8'));
}

async function writeJson(path, value) {
  await mkdir(dirname(path), { recursive: true });
  await writeFile(`${path}.tmp`, `${JSON.stringify(value, null, 2)}\n`);
  await rename(`${path}.tmp`, path);
}

function parseArgs(argv) {
  const options = {
    output: 'advisories/ghsa-without-cve.json',
    consolidatedFeed: null,
    existingFeed: null,
    nvdFeed: 'advisories/feed.json',
    repositories: [...DEFAULT_REPOSITORIES],
    staleAfterDays: DEFAULT_STALE_AFTER_DAYS,
    token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '',
  };

  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    if (arg === '--output') {
      options.output = argv[++index];
    } else if (arg === '--consolidated-feed') {
      options.consolidatedFeed = argv[++index];
    } else if (arg === '--existing-feed') {
      options.existingFeed = argv[++index];
    } else if (arg === '--nvd-feed') {
      options.nvdFeed = argv[++index];
    } else if (arg === '--repo') {
      options.repositories.push(argv[++index]);
    } else if (arg === '--only-default-repos') {
      options.repositories = [...DEFAULT_REPOSITORIES];
    } else if (arg === '--stale-after-days') {
      options.staleAfterDays = Number.parseInt(argv[++index], 10);
    } else if (arg === '--help') {
      options.help = true;
    } else {
      throw new Error(`Unknown argument: ${arg}`);
    }
  }

  if (!Number.isInteger(options.staleAfterDays) || options.staleAfterDays < 1) {
    throw new Error('--stale-after-days must be a positive integer');
  }

  options.repositories = uniqueStrings(options.repositories.map((repo) => repo.toLowerCase()));
  options.existingFeed ||= options.output;
  return options;
}

function printHelp() {
  console.log(`Usage: node scripts/ghsa-without-cve-feed.mjs [options]

Options:
  --output PATH             Feed output path (default: advisories/ghsa-without-cve.json)
  --consolidated-feed PATH  Also merge active GHSA advisories into agent-facing feed PATH
  --existing-feed PATH      Existing provisional feed path (default: output path)
  --nvd-feed PATH           Canonical CVE feed path for GHSA-to-CVE reconciliation
  --repo OWNER/NAME         Additional repository to poll
  --only-default-repos      Reset repository list to built-in ClawSec sources
  --stale-after-days N      Mark GHSA-only advisories stale after N days (default: 60)
`);
}

async function main() {
  const options = parseArgs(process.argv.slice(2));
  if (options.help) {
    printHelp();
    return;
  }

  const now = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
  const fetched = [];
  for (const repository of options.repositories) {
    const advisories = await fetchGitHubSecurityAdvisories(repository, { token: options.token });
    console.log(`Fetched ${advisories.length} GitHub Security Advisories from ${repository}`);
    fetched.push({ repository, advisories });
  }

  const existingFeed = await readJsonIfExists(options.existingFeed, {});
  const nvdFeed = await readJsonIfExists(options.nvdFeed, { advisories: [] });
  const feed = buildGhsaWithoutCveFeed({
    fetched,
    existingFeed,
    nvdFeed,
    now,
    staleAfterDays: options.staleAfterDays,
  });

  await writeJson(options.output, feed);
  console.log(`Wrote ${feed.advisories.length} provisional GHSA advisories to ${options.output}`);

  if (options.consolidatedFeed) {
    const canonicalFeed = await readJsonIfExists(options.consolidatedFeed, {
      version: '1.0.0',
      advisories: [],
    });
    const consolidatedFeed = buildConsolidatedAdvisoryFeed({
      canonicalFeed,
      ghsaFeed: feed,
      now,
    });
    await writeJson(options.consolidatedFeed, consolidatedFeed);
    console.log(
      `Wrote ${consolidatedFeed.advisories.length} consolidated agent advisories to ${options.consolidatedFeed}`,
    );
  }

  console.log(
    `Status counts: ${JSON.stringify(
      feed.advisories.reduce((counts, advisory) => {
        counts[advisory.status] = (counts[advisory.status] || 0) + 1;
        return counts;
      }, {}),
    )}`,
  );
}

if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
  main().catch((error) => {
    console.error(error);
    process.exit(1);
  });
}