#!/usr/bin/env node import { existsSync } from 'node:fs'; import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; export const DEFAULT_REPOSITORIES = [ 'openclaw/openclaw', 'qwibitai/nanoclaw', 'softwarepub/hermes', 'nousresearch/hermes-agent', 'sipeed/picoclaw', ]; export const DEFAULT_STALE_AFTER_DAYS = 60; export const FEED_VERSION = '0.1.0'; const PLATFORM_BY_REPOSITORY = new Map([ ['openclaw/openclaw', 'openclaw'], ['qwibitai/nanoclaw', 'nanoclaw'], ['softwarepub/hermes', 'hermes'], ['nousresearch/hermes-agent', 'hermes'], ['sipeed/picoclaw', 'picoclaw'], ]); const CWE_TYPE_BY_ID = new Map([ ['CWE-22', 'path_traversal'], ['CWE-78', 'os_command_injection'], ['CWE-79', 'cross_site_scripting'], ['CWE-94', 'code_injection'], ['CWE-200', 'exposure_of_sensitive_information'], ['CWE-284', 'improper_access_control'], ['CWE-287', 'improper_authentication'], ['CWE-306', 'missing_authentication_for_critical_function'], ['CWE-352', 'cross_site_request_forgery'], ['CWE-400', 'uncontrolled_resource_consumption'], ['CWE-502', 'deserialization_of_untrusted_data'], ['CWE-862', 'missing_authorization'], ['CWE-863', 'incorrect_authorization'], ['CWE-918', 'server_side_request_forgery'], ]); function cleanText(value) { return String(value ?? '') .replace(/\r/g, '') .replace(/```[\s\S]*?```/g, ' ') .replace(/`([^`]+)`/g, '$1') .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') .replace(/^#+\s+/gm, '') .replace(/[*_>]/g, '') .replace(/\s+/g, ' ') .trim(); } function daysBetween(startIso, endIso) { const start = Date.parse(startIso); const end = Date.parse(endIso); if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { return 0; } return Math.floor((end - start) / 86_400_000); } function toArray(value) { return Array.isArray(value) ? value : []; } function uniqueStrings(values) { return [...new Set(values.filter((value) => typeof value === 'string' && value.length > 0))]; } export function inferPlatforms(repository) { const known = PLATFORM_BY_REPOSITORY.get(String(repository).toLowerCase()); return known ? [known] : []; } function nextLinkFromHeader(linkHeader) { if (!linkHeader) { return null; } for (const part of linkHeader.split(',')) { const match = part.trim().match(/^<([^>]+)>;\s*rel="next"$/); if (match) { return match[1]; } } return null; } function affectedFromVulnerabilities(advisory, platforms) { const affected = toArray(advisory.vulnerabilities).flatMap((vulnerability) => { const packageName = vulnerability?.package?.name; const versionRange = vulnerability?.vulnerable_version_range; if (!packageName) { return []; } return [`${packageName}@${versionRange || '*'}`]; }); if (affected.length > 0) { return uniqueStrings(affected); } return platforms.length > 0 ? platforms.map((platform) => `${platform}@*`) : []; } function patchedFromVulnerabilities(advisory) { return uniqueStrings( toArray(advisory.vulnerabilities).flatMap((vulnerability) => { const packageName = vulnerability?.package?.name; const patchedVersions = vulnerability?.patched_versions; if (!packageName || !patchedVersions) { return []; } return [`${packageName}@${patchedVersions}`]; }), ); } function githubAdvisoryUrl(advisory) { return advisory.html_url || advisory.url || `https://github.com/advisories/${advisory.ghsa_id}`; } function resolveCveId(advisory, cveIdByGhsa) { return advisory.cve_id || cveIdByGhsa.get(advisory.ghsa_id) || null; } export function normalizeGhsaAdvisory( advisory, { now, repository, staleAfterDays = DEFAULT_STALE_AFTER_DAYS, cveId = advisory.cve_id || null, }, ) { const platforms = inferPlatforms(repository); const published = advisory.published_at || advisory.created_at || advisory.updated_at || now; const ageDays = daysBetween(published, now); const stale = !cveId && ageDays >= staleAfterDays; const status = cveId ? 'matured' : stale ? 'stale' : 'active'; const cweIds = uniqueStrings(toArray(advisory.cwe_ids)); const cvss = advisory.cvss || advisory.cvss_severities?.cvss_v3 || {}; const ghsaUrl = githubAdvisoryUrl(advisory); const affected = affectedFromVulnerabilities(advisory, platforms); const patched = patchedFromVulnerabilities(advisory); const title = cleanText(advisory.summary) || advisory.ghsa_id; const description = cleanText(advisory.description) || title; return { id: advisory.ghsa_id, ghsa_id: advisory.ghsa_id, cve_id: cveId, status, stale, stale_after_days: staleAfterDays, severity: advisory.severity || 'medium', type: CWE_TYPE_BY_ID.get(cweIds[0]) || 'github_security_advisory', nvd_category_id: cweIds[0] || null, title, description, affected, patched, platforms, action: cveId ? `Track ${cveId} in the canonical CVE advisory feed and verify affected components.` : 'Review the GitHub Security Advisory and update affected components; no CVE is assigned yet.', published, updated: advisory.updated_at || published, references: uniqueStrings([ghsaUrl, cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null]), source: 'GitHub Security Advisory', repository, github_advisory_url: ghsaUrl, nvd_url: cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null, cvss_score: cvss.score ?? null, cvss_vector: cvss.vector_string ?? null, cwe_ids: cweIds, credits: uniqueStrings(toArray(advisory.credits).map((credit) => credit?.login)), aliases: uniqueStrings([advisory.ghsa_id, cveId]), }; } function ghsaToCveMapFromNvdFeed(nvdFeed) { const map = new Map(); for (const advisory of toArray(nvdFeed?.advisories)) { const cveId = advisory?.id; if (typeof cveId !== 'string' || !cveId.startsWith('CVE-')) { continue; } const references = toArray(advisory.references).join('\n'); for (const match of references.matchAll(/GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}/gi)) { map.set(match[0], cveId); } } return map; } function equivalentAdvisories(left, right) { return JSON.stringify(left ?? []) === JSON.stringify(right ?? []); } function isCveId(value) { return typeof value === 'string' && /^CVE-\d{4}-\d{4,}$/i.test(value); } function ghsaIdentifier(entry) { if (typeof entry?.ghsa_id === 'string' && entry.ghsa_id.length > 0) { return entry.ghsa_id.toLowerCase(); } if (/^GHSA-/i.test(String(entry?.id || ''))) { return String(entry.id).toLowerCase(); } return null; } function refreshExistingEntry(entry, { now, staleAfterDays, cveIdByGhsa }) { const cveId = entry.cve_id || cveIdByGhsa.get(entry.ghsa_id || entry.id) || null; const ageDays = daysBetween(entry.published, now); const stale = !cveId && ageDays >= staleAfterDays; return { ...entry, cve_id: cveId, status: cveId ? 'matured' : stale ? 'stale' : 'active', stale, stale_after_days: staleAfterDays, references: uniqueStrings([ ...toArray(entry.references), cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null, ]), nvd_url: cveId ? `https://nvd.nist.gov/vuln/detail/${cveId}` : null, aliases: uniqueStrings([...(entry.aliases || []), entry.ghsa_id || entry.id, cveId]), }; } export function buildConsolidatedAdvisoryFeed({ canonicalFeed = {}, ghsaFeed = {}, now }) { const canonicalFeedEntries = toArray(canonicalFeed.advisories); const canonicalCveIds = new Set(canonicalFeedEntries.map((entry) => entry?.id).filter(isCveId)); const replacementGhsaIds = new Set(toArray(ghsaFeed.advisories).map(ghsaIdentifier).filter(Boolean)); const canonicalEntries = canonicalFeedEntries.filter((entry) => { const ghsaId = ghsaIdentifier(entry); if (!ghsaId) { return true; } if (entry?.cve_id && canonicalCveIds.has(entry.cve_id)) { return false; } return !replacementGhsaIds.has(ghsaId); }); const ghsaEntries = toArray(ghsaFeed.advisories) .filter((entry) => !(entry?.cve_id && canonicalCveIds.has(entry.cve_id))) .map((entry) => ({ ...entry, source_feed: 'ghsa-without-cve', })); const advisories = [...canonicalEntries, ...ghsaEntries].sort((a, b) => { const published = Date.parse(b.published || '') - Date.parse(a.published || ''); if (Number.isFinite(published) && published !== 0) { return published; } return String(a.id || '').localeCompare(String(b.id || '')); }); return { ...canonicalFeed, version: canonicalFeed.version || '1.0.0', updated: canonicalFeed.updated || now, description: canonicalFeed.description || 'Community-driven security advisory feed for ClawSec', advisories, }; } export function buildGhsaWithoutCveFeed({ fetched, existingFeed = {}, nvdFeed = {}, now, staleAfterDays = DEFAULT_STALE_AFTER_DAYS, }) { const existingEntries = toArray(existingFeed.advisories); const existingIds = new Set(existingEntries.map((entry) => entry.ghsa_id || entry.id)); const cveIdByGhsa = ghsaToCveMapFromNvdFeed(nvdFeed); const entriesById = new Map(); for (const { repository, advisories } of fetched) { for (const advisory of advisories) { const ghsaId = advisory.ghsa_id; if (!ghsaId) { continue; } const cveId = resolveCveId(advisory, cveIdByGhsa); if (cveId && !existingIds.has(ghsaId)) { continue; } entriesById.set( ghsaId, normalizeGhsaAdvisory(advisory, { now, repository, staleAfterDays, cveId, }), ); } } for (const entry of existingEntries) { const ghsaId = entry.ghsa_id || entry.id; if (!ghsaId || entriesById.has(ghsaId)) { continue; } entriesById.set(ghsaId, refreshExistingEntry(entry, { now, staleAfterDays, cveIdByGhsa })); } const advisories = [...entriesById.values()].sort((a, b) => { const published = Date.parse(b.published) - Date.parse(a.published); if (published !== 0) { return published; } return a.id.localeCompare(b.id); }); const updated = equivalentAdvisories(advisories, existingEntries) ? existingFeed.updated || now : now; return { version: FEED_VERSION, updated, description: 'Provisional ClawSec advisory feed for public GitHub Security Advisories that do not yet have CVE identifiers.', stale_after_days: staleAfterDays, semantics: { active: 'GHSA is published and has no CVE identifier yet.', matured: 'GHSA now has a CVE identifier and should be reconciled with the canonical CVE feed.', stale: 'GHSA is older than stale_after_days and still has no CVE identifier.', }, sources: DEFAULT_REPOSITORIES.map((repository) => ({ repository, platform: inferPlatforms(repository)[0] || 'unknown', url: `https://github.com/${repository}/security/advisories`, })), advisories, }; } export async function fetchGitHubSecurityAdvisories(repository, { token } = {}) { const advisories = []; let url = `https://api.github.com/repos/${repository}/security-advisories?per_page=100`; const seenUrls = new Set(); while (url) { if (seenUrls.has(url)) { throw new Error(`GitHub advisory pagination loop detected for ${repository}: ${url}`); } seenUrls.add(url); const response = await globalThis.fetch(url, { headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'clawsec-ghsa-without-cve-poller', 'X-GitHub-Api-Version': '2022-11-28', ...(token ? { Authorization: `Bearer ${token}` } : {}), }, }); if (!response.ok) { const message = await response.text(); throw new Error( `GitHub advisory fetch failed for ${repository}: HTTP ${response.status} ${message.slice(0, 200)}`, ); } const pageItems = await response.json(); advisories.push(...pageItems); if (!Array.isArray(pageItems)) { break; } url = nextLinkFromHeader(response.headers.get('link')); } return advisories; } async function readJsonIfExists(path, fallback) { if (!existsSync(path)) { return fallback; } return JSON.parse(await readFile(path, 'utf8')); } async function writeJson(path, value) { await mkdir(dirname(path), { recursive: true }); await writeFile(`${path}.tmp`, `${JSON.stringify(value, null, 2)}\n`); await rename(`${path}.tmp`, path); } function parseArgs(argv) { const options = { output: 'advisories/ghsa-without-cve.json', consolidatedFeed: null, existingFeed: null, nvdFeed: 'advisories/feed.json', repositories: [...DEFAULT_REPOSITORIES], staleAfterDays: DEFAULT_STALE_AFTER_DAYS, token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '', }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--output') { options.output = argv[++index]; } else if (arg === '--consolidated-feed') { options.consolidatedFeed = argv[++index]; } else if (arg === '--existing-feed') { options.existingFeed = argv[++index]; } else if (arg === '--nvd-feed') { options.nvdFeed = argv[++index]; } else if (arg === '--repo') { options.repositories.push(argv[++index]); } else if (arg === '--only-default-repos') { options.repositories = [...DEFAULT_REPOSITORIES]; } else if (arg === '--stale-after-days') { options.staleAfterDays = Number.parseInt(argv[++index], 10); } else if (arg === '--help') { options.help = true; } else { throw new Error(`Unknown argument: ${arg}`); } } if (!Number.isInteger(options.staleAfterDays) || options.staleAfterDays < 1) { throw new Error('--stale-after-days must be a positive integer'); } options.repositories = uniqueStrings(options.repositories.map((repo) => repo.toLowerCase())); options.existingFeed ||= options.output; return options; } function printHelp() { console.log(`Usage: node scripts/ghsa-without-cve-feed.mjs [options] Options: --output PATH Feed output path (default: advisories/ghsa-without-cve.json) --consolidated-feed PATH Also merge active GHSA advisories into agent-facing feed PATH --existing-feed PATH Existing provisional feed path (default: output path) --nvd-feed PATH Canonical CVE feed path for GHSA-to-CVE reconciliation --repo OWNER/NAME Additional repository to poll --only-default-repos Reset repository list to built-in ClawSec sources --stale-after-days N Mark GHSA-only advisories stale after N days (default: 60) `); } async function main() { const options = parseArgs(process.argv.slice(2)); if (options.help) { printHelp(); return; } const now = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); const fetched = []; for (const repository of options.repositories) { const advisories = await fetchGitHubSecurityAdvisories(repository, { token: options.token }); console.log(`Fetched ${advisories.length} GitHub Security Advisories from ${repository}`); fetched.push({ repository, advisories }); } const existingFeed = await readJsonIfExists(options.existingFeed, {}); const nvdFeed = await readJsonIfExists(options.nvdFeed, { advisories: [] }); const feed = buildGhsaWithoutCveFeed({ fetched, existingFeed, nvdFeed, now, staleAfterDays: options.staleAfterDays, }); await writeJson(options.output, feed); console.log(`Wrote ${feed.advisories.length} provisional GHSA advisories to ${options.output}`); if (options.consolidatedFeed) { const canonicalFeed = await readJsonIfExists(options.consolidatedFeed, { version: '1.0.0', advisories: [], }); const consolidatedFeed = buildConsolidatedAdvisoryFeed({ canonicalFeed, ghsaFeed: feed, now, }); await writeJson(options.consolidatedFeed, consolidatedFeed); console.log( `Wrote ${consolidatedFeed.advisories.length} consolidated agent advisories to ${options.consolidatedFeed}`, ); } console.log( `Status counts: ${JSON.stringify( feed.advisories.reduce((counts, advisory) => { counts[advisory.status] = (counts[advisory.status] || 0) + 1; return counts; }, {}), )}`, ); } if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { main().catch((error) => { console.error(error); process.exit(1); }); }