diff --git a/.gitignore b/.gitignore index a3bffa5..4b2cd9e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ node_modules/ # Re-voice workspace tmp/ +data/ *.log diff --git a/site/red-team-shadow-dossiers/README.md b/site/red-team-shadow-dossiers/README.md index d6cb408..ce99045 100644 --- a/site/red-team-shadow-dossiers/README.md +++ b/site/red-team-shadow-dossiers/README.md @@ -35,6 +35,16 @@ Create `.env.local` from `.env.example`: - `VITE_ENABLE_ROAST` Defaults to `false`. When `true`, renders the Roast Generator UI which calls `POST /api/roast`. +## Private upload (internal) + +The server exposes a token-gated upload flow that generates a shadow dossier via the `revoice` pipeline: + +- Set server env: + - `PRIVATE_UPLOAD_TOKEN` (required) + - `PRIVATE_UPLOAD_STYLE` (optional, default `if.dave.v1.2`) + - `PRIVATE_UPLOAD_MAX_BYTES` (optional, default 25MB) +- Open: `/private/` + ## Staging behavior (important) This bundle is **pre-launch**. Dossier buttons are gated by `dossier.status`: diff --git a/site/red-team-shadow-dossiers/package-lock.json b/site/red-team-shadow-dossiers/package-lock.json index a83edd0..1486b33 100644 --- a/site/red-team-shadow-dossiers/package-lock.json +++ b/site/red-team-shadow-dossiers/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "dependencies": { "express": "^4.21.2", + "multer": "^2.0.2", "react": "^19.2.3", "react-dom": "^19.2.3" }, @@ -1204,6 +1205,12 @@ "node": ">= 0.6" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1293,6 +1300,23 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1352,6 +1376,21 @@ ], "license": "CC-BY-4.0" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -1961,12 +2000,51 @@ "node": ">= 0.6" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/multer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", + "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", + "object-assign": "^4.1.1", + "type-is": "^1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">= 10.16.0" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -2002,6 +2080,15 @@ "dev": true, "license": "MIT" }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -2173,6 +2260,20 @@ "node": ">=0.10.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/rollup": { "version": "4.54.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.54.0.tgz", @@ -2408,6 +2509,23 @@ "node": ">= 0.8" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", @@ -2447,6 +2565,12 @@ "node": ">= 0.6" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", @@ -2508,6 +2632,12 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -2601,6 +2731,15 @@ } } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/site/red-team-shadow-dossiers/package.json b/site/red-team-shadow-dossiers/package.json index 0cfb754..2e7c2f0 100644 --- a/site/red-team-shadow-dossiers/package.json +++ b/site/red-team-shadow-dossiers/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "express": "^4.21.2", + "multer": "^2.0.2", "react": "^19.2.3", "react-dom": "^19.2.3" }, diff --git a/site/red-team-shadow-dossiers/server/server.mjs b/site/red-team-shadow-dossiers/server/server.mjs index fdd2574..1d4ccbe 100644 --- a/site/red-team-shadow-dossiers/server/server.mjs +++ b/site/red-team-shadow-dossiers/server/server.mjs @@ -1,8 +1,11 @@ +import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import url from "node:url"; +import { spawn } from "node:child_process"; import express from "express"; +import multer from "multer"; const __filename = url.fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -11,6 +14,117 @@ const projectRoot = path.resolve(__dirname, ".."); const distDir = path.join(projectRoot, "dist"); const indexHtmlPath = path.join(distDir, "index.html"); +const privateUploadToken = process.env.PRIVATE_UPLOAD_TOKEN || ""; +const privateUploadMaxBytes = Number(process.env.PRIVATE_UPLOAD_MAX_BYTES || 25 * 1024 * 1024); +const privateUploadStyle = process.env.PRIVATE_UPLOAD_STYLE || "if.dave.v1.2"; +const revoiceRepoRoot = path.resolve(projectRoot, "..", ".."); + +function escapeHtml(value) { + return String(value || "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function ensureDir(dirPath) { + fs.mkdirSync(dirPath, { recursive: true }); +} + +function looksLikeUuid(value) { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); +} + +function jobJsonPath(jobsDir, jobId) { + return path.join(jobsDir, `${jobId}.json`); +} + +function readJob(jobsDir, jobId) { + const p = jobJsonPath(jobsDir, jobId); + if (!fs.existsSync(p)) return null; + try { + return JSON.parse(fs.readFileSync(p, "utf8")); + } catch { + return null; + } +} + +function writeJob(jobsDir, job) { + const p = jobJsonPath(jobsDir, job.id); + fs.writeFileSync(p, JSON.stringify(job, null, 2) + "\n", "utf8"); +} + +async function sha256File(filePath) { + return await new Promise((resolve, reject) => { + const h = crypto.createHash("sha256"); + const s = fs.createReadStream(filePath); + s.on("error", reject); + s.on("data", (chunk) => h.update(chunk)); + s.on("end", () => resolve(h.digest("hex"))); + }); +} + +function runProcess(command, args, { cwd, env }) { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => { + stdout += d.toString("utf8"); + if (stdout.length > 256_000) stdout = stdout.slice(-256_000); + }); + child.stderr.on("data", (d) => { + stderr += d.toString("utf8"); + if (stderr.length > 256_000) stderr = stderr.slice(-256_000); + }); + + child.on("error", (err) => { + const msg = err?.message ? String(err.message) : String(err); + resolve({ code: 127, stdout, stderr: (stderr ? `${stderr}\n` : "") + msg }); + }); + + child.on("close", (code) => resolve({ code: code ?? 0, stdout, stderr })); + }); +} + +async function generateShadowDossier({ inputPath, outputPath }) { + const revoiceModule = path.join(revoiceRepoRoot, "src", "revoice"); + if (!fs.existsSync(revoiceModule)) { + throw new Error(`Missing revoice pipeline at ${revoiceModule}`); + } + + const baseEnv = { + ...process.env, + PYTHONPATH: path.join(revoiceRepoRoot, "src"), + }; + + const gen = await runProcess( + "python3", + ["-m", "revoice", "generate", "--style", privateUploadStyle, "--input", inputPath, "--output", outputPath], + { cwd: revoiceRepoRoot, env: baseEnv } + ); + if (gen.code !== 0) { + throw new Error(`revoice generate failed (code ${gen.code}): ${gen.stderr || gen.stdout}`); + } + + const preflight = await runProcess( + "python3", + ["-m", "revoice", "preflight", "--style", privateUploadStyle, "--input", outputPath, "--source", inputPath], + { cwd: revoiceRepoRoot, env: baseEnv } + ); + const warnings = preflight.code === 0 ? "" : preflight.stderr || preflight.stdout; + if (preflight.code !== 0 && preflight.code !== 2) { + throw new Error(`revoice preflight failed (code ${preflight.code}): ${preflight.stderr || preflight.stdout}`); + } + return { warnings }; +} + function pickPhrases(input) { const text = String(input || "").replace(/\r\n?/g, "\n"); const lines = text @@ -52,6 +166,14 @@ function generateRoastText(content) { function main() { const port = Number(process.env.PORT || 8080); const app = express(); + const dataDir = path.join(projectRoot, "data"); + const uploadsDir = path.join(dataDir, "uploads"); + const outputsDir = path.join(dataDir, "outputs"); + const jobsDir = path.join(dataDir, "jobs"); + + ensureDir(uploadsDir); + ensureDir(outputsDir); + ensureDir(jobsDir); app.disable("x-powered-by"); app.use(express.json({ limit: "256kb" })); @@ -67,6 +189,192 @@ function main() { return res.status(200).json({ text: generateRoastText(content) }); }); + const privateUploadEnabled = Boolean(privateUploadToken.trim()); + const privateGuard = (req, res, next) => { + if (!privateUploadEnabled) return res.status(404).type("text/plain").send("Not found"); + if (req.params?.token !== privateUploadToken) return res.status(404).type("text/plain").send("Not found"); + return next(); + }; + + const upload = multer({ + storage: multer.diskStorage({ + destination: (_req, _file, cb) => cb(null, uploadsDir), + filename: (req, file, cb) => { + const id = crypto.randomUUID(); + req._jobId = id; + const ext = path.extname(file.originalname || "").slice(0, 12).toLowerCase(); + cb(null, `${id}${ext}`); + }, + }), + limits: { fileSize: privateUploadMaxBytes, files: 1 }, + }); + + app.get("/private/:token", privateGuard, (req, res) => { + const token = req.params.token; + res + .status(200) + .type("text/html; charset=utf-8") + .send( + [ + "", + "", + "", + "", + "Private Upload · Shadow Dossier", + "", + "", + "

Private dossier upload

", + `

Style: ${escapeHtml(privateUploadStyle)} · Max: ${escapeHtml( + String(privateUploadMaxBytes) + )} bytes

`, + "
", + "
", + "

", + "

", + "

Supported: PDF/MD/TXT. Output: Markdown shadow dossier.

", + "
", + "
", + "", + ].join("") + ); + }); + + app.get("/private/:token/job/:jobId", privateGuard, (req, res) => { + const jobId = String(req.params.jobId || ""); + if (!looksLikeUuid(jobId)) return res.status(404).type("text/plain").send("Not found"); + + const job = readJob(jobsDir, jobId); + if (!job) return res.status(404).type("text/plain").send("Not found"); + + const status = String(job.status || "unknown"); + const isDone = status === "done" || status === "done_with_warnings"; + const isError = status === "error"; + const token = req.params.token; + + const refresh = isDone || isError ? "" : ""; + + const downloadLink = isDone + ? `

Download shadow dossier

` + : ""; + + const sourceLink = job.sourcePath + ? `

Download source

` + : ""; + + const warnings = job.warnings ? `
${escapeHtml(job.warnings)}
` : ""; + const error = job.error ? `
${escapeHtml(job.error)}
` : ""; + + res + .status(200) + .type("text/html; charset=utf-8") + .send( + [ + "", + "", + "", + "", + refresh, + "Job · Shadow Dossier", + "", + "", + "

Shadow dossier job

", + `

Status: ${escapeHtml(status)}

`, + `

Job ID: ${escapeHtml(jobId)}

`, + job.originalFilename ? `

Source: ${escapeHtml(job.originalFilename)}

` : "", + job.sourceSha256 ? `

Source sha256: ${escapeHtml(job.sourceSha256)}

` : "", + job.outputSha256 ? `

Output sha256: ${escapeHtml(job.outputSha256)}

` : "", + downloadLink, + sourceLink, + warnings ? "

Warnings

" + warnings : "", + isError ? "

Error

" + error : "", + `

Back to upload

`, + "", + ].join("") + ); + }); + + app.get("/private/:token/download/:jobId", privateGuard, (req, res) => { + const jobId = String(req.params.jobId || ""); + if (!looksLikeUuid(jobId)) return res.status(404).type("text/plain").send("Not found"); + const job = readJob(jobsDir, jobId); + if (!job) return res.status(404).type("text/plain").send("Not found"); + if (!job.outputPath) return res.status(409).type("text/plain").send("Not ready"); + const abs = path.resolve(projectRoot, job.outputPath); + if (!abs.startsWith(outputsDir + path.sep)) return res.status(400).type("text/plain").send("Bad path"); + if (!fs.existsSync(abs)) return res.status(404).type("text/plain").send("Not found"); + + const baseName = (job.originalFilename || "dossier").replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60); + res.download(abs, `${baseName}.shadow.dave.md`); + }); + + app.get("/private/:token/source/:jobId", privateGuard, (req, res) => { + const jobId = String(req.params.jobId || ""); + if (!looksLikeUuid(jobId)) return res.status(404).type("text/plain").send("Not found"); + const job = readJob(jobsDir, jobId); + if (!job) return res.status(404).type("text/plain").send("Not found"); + if (!job.sourcePath) return res.status(404).type("text/plain").send("Not found"); + const abs = path.resolve(projectRoot, job.sourcePath); + if (!abs.startsWith(uploadsDir + path.sep)) return res.status(400).type("text/plain").send("Bad path"); + if (!fs.existsSync(abs)) return res.status(404).type("text/plain").send("Not found"); + const baseName = (job.originalFilename || "source").replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 80); + res.download(abs, baseName); + }); + + app.post("/api/private/:token/upload", privateGuard, upload.single("file"), async (req, res) => { + const jobId = req._jobId || crypto.randomUUID(); + const file = req.file; + if (!file?.path) return res.status(400).type("text/plain").send("Missing file"); + + const relSourcePath = path.relative(projectRoot, file.path); + const relOutputPath = path.join("data", "outputs", `${jobId}.shadow.dave.md`); + const absOutputPath = path.resolve(projectRoot, relOutputPath); + + const now = new Date().toISOString(); + const job = { + id: jobId, + status: "processing", + createdAt: now, + originalFilename: file.originalname || "", + sourcePath: relSourcePath, + outputPath: relOutputPath, + style: privateUploadStyle, + sourceBytes: Number(file.size || 0), + sourceSha256: "", + outputSha256: "", + warnings: "", + error: "", + }; + + try { + job.sourceSha256 = await sha256File(file.path); + } catch (e) { + job.status = "error"; + job.error = String(e?.message || e || "hash_failed"); + writeJob(jobsDir, job); + return res.status(500).type("text/plain").send("Failed to hash upload"); + } + + writeJob(jobsDir, job); + + void (async () => { + try { + const { warnings } = await generateShadowDossier({ inputPath: file.path, outputPath: absOutputPath }); + job.warnings = warnings ? warnings.trim() : ""; + job.outputSha256 = await sha256File(absOutputPath); + job.status = job.warnings ? "done_with_warnings" : "done"; + writeJob(jobsDir, job); + } catch (e) { + job.status = "error"; + job.error = String(e?.message || e || "generation_failed"); + writeJob(jobsDir, job); + } + })(); + + res.redirect(303, `/private/${encodeURIComponent(req.params.token)}/job/${encodeURIComponent(jobId)}`); + }); + if (fs.existsSync(distDir) && fs.existsSync(indexHtmlPath)) { app.use(express.static(distDir, { fallthrough: true })); @@ -90,4 +398,3 @@ function main() { } main(); -