diff --git a/packages/core/src/studio-api/routes/render.ts b/packages/core/src/studio-api/routes/render.ts index f21fc433..4948e8a1 100644 --- a/packages/core/src/studio-api/routes/render.ts +++ b/packages/core/src/studio-api/routes/render.ts @@ -216,6 +216,50 @@ export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void }); }); + // Upload a browser-rendered video (client-side renderer posts the blob here) + api.post("/projects/:id/renders/upload", async (c) => { + const project = await adapter.resolveProject(c.req.param("id")); + if (!project) return c.json({ error: "not found" }, 404); + + const rendersDir = adapter.rendersDir(project); + if (!existsSync(rendersDir)) mkdirSync(rendersDir, { recursive: true }); + + const body = await c.req.arrayBuffer(); + if (!body || body.byteLength === 0) { + return c.json({ error: "empty body" }, 400); + } + + const format = c.req.query("format") ?? "mp4"; + const now = new Date(); + const datePart = now.toISOString().slice(0, 10); + const timePart = now.toTimeString().slice(0, 8).replace(/:/g, "-"); + const jobId = `${project.id}_browser_${datePart}_${timePart}`; + const ext = format === "webm" ? ".webm" : ".mp4"; + const outputPath = join(rendersDir, `${jobId}${ext}`); + + const { writeFileSync: writeFile } = await import("node:fs"); + writeFile(outputPath, Buffer.from(body)); + writeFile( + outputPath.replace(/\.(mp4|webm)$/, ".meta.json"), + JSON.stringify({ + status: "complete", + source: "browser", + durationMs: Number(c.req.query("durationMs") ?? 0), + }), + ); + + // Register in job store so it shows in the renders list + renderJobs.set(jobId, { + id: jobId, + status: "complete", + progress: 100, + outputPath, + createdAt: Date.now(), + } as RenderJobState & { createdAt: number }); + + return c.json({ jobId, filename: `${jobId}${ext}`, size: body.byteLength }); + }); + // List renders api.get("/projects/:id/renders", async (c) => { const project = await adapter.resolveProject(c.req.param("id")); diff --git a/packages/renderer/package.json b/packages/renderer/package.json new file mode 100644 index 00000000..2670bc15 --- /dev/null +++ b/packages/renderer/package.json @@ -0,0 +1,52 @@ +{ + "name": "@hyperframes/renderer", + "version": "0.0.1", + "description": "Client-side video renderer for HyperFrames compositions", + "repository": { + "type": "git", + "url": "https://github.com/heygen-com/hyperframes", + "directory": "packages/renderer" + }, + "files": [ + "dist", + "README.md" + ], + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": { + "import": "./src/index.ts", + "types": "./src/index.ts" + } + }, + "publishConfig": { + "access": "public", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts" + }, + "scripts": { + "build": "tsc", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@zumer/snapdom": "^2.8.0", + "mediabunny": "^1.40.0" + }, + "devDependencies": { + "@types/jsdom": "^28.0.1", + "jsdom": "^29.0.2", + "typescript": "^5.0.0", + "vitest": "^3.2.4" + }, + "peerDependencies": { + "@hyperframes/core": "workspace:^" + } +} diff --git a/packages/renderer/public/turbo-worker.html b/packages/renderer/public/turbo-worker.html new file mode 100644 index 00000000..e64dc004 --- /dev/null +++ b/packages/renderer/public/turbo-worker.html @@ -0,0 +1,10 @@ + + + + + Turbo Worker + + + + + diff --git a/packages/renderer/scripts/compile-test.ts b/packages/renderer/scripts/compile-test.ts new file mode 100644 index 00000000..37843a98 --- /dev/null +++ b/packages/renderer/scripts/compile-test.ts @@ -0,0 +1,92 @@ +/** + * Compile a single test composition via the producer's htmlCompiler. + * Usage: npx tsx --esm packages/renderer/scripts/compile-test.ts + */ +import { writeFileSync, mkdirSync, cpSync } from "node:fs"; +import { resolve, join } from "node:path"; +import { compileForRender } from "../../producer/src/services/htmlCompiler.js"; +import { getVerifiedHyperframeRuntimeSource } from "../../producer/src/services/hyperframeRuntimeLoader.js"; + +const testName = process.argv[2]; +if (!testName) { + console.error("Usage: npx tsx packages/renderer/scripts/compile-test.ts "); + process.exit(1); +} + +const srcDir = resolve(`packages/producer/tests/${testName}/src`); +const htmlPath = join(srcDir, "index.html"); +const outputDir = resolve(`renders/parity-regression/${testName}`); +mkdirSync(outputDir, { recursive: true }); + +console.log(`Compiling ${testName}...`); + +// Copy all source assets to output dir so the composition can resolve relative paths +cpSync(srcDir, outputDir, { recursive: true, force: true }); + +const result = await compileForRender(srcDir, htmlPath, outputDir); + +// Inject the HyperFrames runtime IIFE and the __player → __hf bridge. +// The producer's file server normally serves these separately, +// but for client-side rendering we need them inline. +const runtimeSource = getVerifiedHyperframeRuntimeSource(); + +// Bridge script: maps window.__player (set by runtime) → window.__hf (engine protocol) +const bridgeScript = `(function() { + function getDeclaredDuration() { + var root = document.querySelector('[data-composition-id]'); + if (!root) return 0; + var d = Number(root.getAttribute('data-duration')); + return Number.isFinite(d) && d > 0 ? d : 0; + } + function bridge() { + var p = window.__player; + if (!p || typeof p.renderSeek !== "function" || typeof p.getDuration !== "function") { + return false; + } + window.__hf = { + get duration() { + var d = p.getDuration(); + return d > 0 ? d : getDeclaredDuration(); + }, + seek: function(t) { p.renderSeek(t); }, + }; + return true; + } + if (bridge()) return; + var iv = setInterval(function() { + if (bridge()) clearInterval(iv); + }, 50); +})();`; + +const runtimeTag = ``; +const bridgeTag = ``; +const htmlWithRuntime = result.html.replace("", `${runtimeTag}\n${bridgeTag}\n`); + +writeFileSync(join(outputDir, "compiled.html"), htmlWithRuntime); +writeFileSync( + join(outputDir, "compile-meta.json"), + JSON.stringify( + { + width: result.width, + height: result.height, + staticDuration: result.staticDuration, + videoCount: result.videos.length, + audioCount: result.audios.length, + htmlLength: result.html.length, + }, + null, + 2, + ), +); + +console.log( + JSON.stringify({ + ok: true, + width: result.width, + height: result.height, + staticDuration: result.staticDuration, + videos: result.videos.length, + audios: result.audios.length, + htmlLength: result.html.length, + }), +); diff --git a/packages/renderer/scripts/regression-parity.ts b/packages/renderer/scripts/regression-parity.ts new file mode 100644 index 00000000..77446c55 --- /dev/null +++ b/packages/renderer/scripts/regression-parity.ts @@ -0,0 +1,554 @@ +#!/usr/bin/env npx tsx +/** + * Regression Parity Harness + * + * Compiles test compositions via the server-side producer, then renders them + * via the client-side renderer in a real browser, then compares the output + * against golden reference videos using PSNR. + * + * Usage: + * npx tsx packages/renderer/scripts/regression-parity.ts [test-name...] + * npx tsx packages/renderer/scripts/regression-parity.ts chat vignelli-stacking + * npx tsx packages/renderer/scripts/regression-parity.ts # runs all tests + */ + +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, + rmSync, + symlinkSync, +} from "node:fs"; +import { resolve, join } from "node:path"; +import { execSync, spawnSync, spawn as spawnChild } from "node:child_process"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; + +// ── Config ────────────────────────────────────────────────────────────────── + +const PRODUCER_TESTS_DIR = resolve("packages/producer/tests"); +const RENDERER_DIST = resolve("packages/renderer/dist"); +const PARITY_OUTPUT_DIR = resolve("renders/parity-regression"); +const MIN_PSNR = 25; // dB — threshold for "visually similar" (30+ = near-identical) +const CHECKPOINTS = 10; // number of frames to compare +const SERVER_PORT = 4789; + +// ── Types ─────────────────────────────────────────────────────────────────── + +type TestMeta = { + name: string; + description: string; + tags: string[]; + minPsnr: number; + maxFrameFailures: number; + renderConfig: { fps: 24 | 30 | 60; format?: string }; +}; + +type TestSuite = { + id: string; + dir: string; + srcDir: string; + meta: TestMeta; + goldenVideo: string; +}; + +type CheckpointResult = { + time: number; + psnr: number; + passed: boolean; +}; + +type TestResult = { + suite: TestSuite; + passed: boolean; + compileOk: boolean; + renderOk: boolean; + checkpoints: CheckpointResult[]; + avgPsnr: number; + minPsnr: number; + failedFrames: number; + renderTimeMs: number; + error?: string; +}; + +// ── Discovery ─────────────────────────────────────────────────────────────── + +function discoverTests(filterNames: string[]): TestSuite[] { + const dirs = readdirSync(PRODUCER_TESTS_DIR, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .filter((name) => { + if (filterNames.length > 0) return filterNames.includes(name); + return true; + }); + + const suites: TestSuite[] = []; + for (const name of dirs) { + const dir = join(PRODUCER_TESTS_DIR, name); + const metaPath = join(dir, "meta.json"); + const srcIndex = join(dir, "src", "index.html"); + const goldenMp4 = join(dir, "output", "output.mp4"); + const goldenWebm = join(dir, "output", "output.webm"); + + if (!existsSync(metaPath) || !existsSync(srcIndex)) continue; + const goldenVideo = existsSync(goldenMp4) + ? goldenMp4 + : existsSync(goldenWebm) + ? goldenWebm + : ""; + if (!goldenVideo) continue; + + const meta: TestMeta = JSON.parse(readFileSync(metaPath, "utf-8")); + + // Skip tests tagged "slow" for initial parity — these are long compositions + // with video elements that need more setup + if (meta.tags.includes("slow") && filterNames.length === 0) continue; + + suites.push({ + id: name, + dir, + srcDir: join(dir, "src"), + meta, + goldenVideo, + }); + } + + return suites; +} + +// ── Step 1: Compile ───────────────────────────────────────────────────────── + +async function compileComposition(suite: TestSuite, _outputDir: string): Promise { + const compiledPath = join(_outputDir, "compiled.html"); + + // Use compile-test.ts which injects the HyperFrames runtime + __hf bridge + const result = spawnSync( + "npx", + ["tsx", resolve("packages/renderer/scripts/compile-test.ts"), suite.id], + { + cwd: resolve("."), + encoding: "utf-8", + timeout: 60_000, + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }, + ); + + if (result.status !== 0) { + throw new Error(`Compilation failed for ${suite.id}: ${result.stderr?.slice(-500)}`); + } + + if (!existsSync(compiledPath)) { + throw new Error(`Compiled HTML not created for ${suite.id}`); + } + + return compiledPath; +} + +// ── Upload server ────────────────────────────────────────────────────────── + +const UPLOAD_PORT = SERVER_PORT + 1; +const pendingUploads = new Map< + string, + { resolve: (path: string) => void; reject: (err: Error) => void; outputPath: string } +>(); + +function startUploadServer(): ReturnType { + const corsHeaders = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + }; + + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + // CORS preflight + if (req.method === "OPTIONS") { + res.writeHead(200, corsHeaders); + res.end(); + return; + } + + if (req.method === "POST" && req.url?.startsWith("/upload/")) { + const testId = req.url.slice("/upload/".length); + const pending = pendingUploads.get(testId); + if (!pending) { + res.writeHead(404, corsHeaders); + res.end("Unknown test"); + return; + } + + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + const data = Buffer.concat(chunks); + writeFileSync(pending.outputPath, data); + res.writeHead(200, corsHeaders); + res.end("OK"); + pending.resolve(pending.outputPath); + pendingUploads.delete(testId); + }); + return; + } + + res.writeHead(404, corsHeaders); + res.end("Not found"); + }); + + server.listen(UPLOAD_PORT); + return server; +} + +// ── Step 2: Render client-side ────────────────────────────────────────────── + +async function renderClientSide( + suite: TestSuite, + _compiledHtmlPath: string, + outputDir: string, +): Promise<{ videoPath: string; durationMs: number }> { + const metaJson = JSON.parse(readFileSync(join(outputDir, "compile-meta.json"), "utf-8")); + const width = metaJson.width || 1080; + const height = metaJson.height || 1920; + const fps = suite.meta.renderConfig.fps; + + const testPagePath = join(outputDir, "render-test.html"); + const videoOutputPath = join(outputDir, "client-render.mp4"); + + // The test page renders client-side, then POSTs the blob to our upload server + const testPage = ` +Parity Test: ${suite.id} + + +`; + + writeFileSync(testPagePath, testPage); + + // Register upload promise + const uploadPromise = new Promise((resolve, reject) => { + pendingUploads.set(suite.id, { resolve, reject, outputPath: videoOutputPath }); + }); + + // Open in browser — use spawn (async) so the upload server can process requests + execSync( + `agent-browser --session parity-${suite.id} open "http://localhost:${SERVER_PORT}/renders/parity-regression/${suite.id}/render-test.html"`, + { + encoding: "utf-8", + timeout: 10_000, + }, + ); + + // Wait for title to indicate completion — use spawn (async) so the event loop + // stays active for the upload HTTP server to process the POST from the browser + await new Promise((resolve) => { + const child = spawnChild( + "agent-browser", + [ + "--session", + `parity-${suite.id}`, + "wait", + "--fn", + "document.title.startsWith('DONE') || document.title.startsWith('ERROR')", + "--timeout", + "180000", + ], + { encoding: "utf-8", timeout: 190_000, stdio: ["ignore", "pipe", "pipe"] } as any, + ); + let out = ""; + child.stdout?.on("data", (d: Buffer) => { + out += d.toString(); + }); + child.stderr?.on("data", (d: Buffer) => { + out += d.toString(); + }); + child.on("close", () => resolve(out)); + }); + + const titleResult = spawnSync( + "agent-browser", + ["--session", `parity-${suite.id}`, "get", "title"], + { encoding: "utf-8", timeout: 5_000 }, + ); + const title = (titleResult.stdout || "").trim(); + + execSync(`agent-browser --session parity-${suite.id} close 2>/dev/null || true`, { + encoding: "utf-8", + timeout: 5_000, + }); + + if (title.startsWith("ERROR")) { + throw new Error(title); + } + + // Wait for upload to complete (render can take minutes for long compositions) + await Promise.race([ + uploadPromise, + new Promise((_, reject) => + setTimeout(() => reject(new Error("Upload timeout")), 300_000), + ), + ]); + + const durationMs = title.startsWith("DONE:") ? parseInt(title.slice(5)) : 0; + return { videoPath: videoOutputPath, durationMs }; +} + +// ── Step 3: PSNR comparison ───────────────────────────────────────────────── + +function psnrAtCheckpoint(videoA: string, videoB: string, timeSec: number): number { + const result = spawnSync( + "ffmpeg", + [ + "-ss", + String(timeSec), + "-i", + videoA, + "-ss", + String(timeSec), + "-i", + videoB, + "-frames:v", + "1", + "-lavfi", + "psnr=stats_file=/dev/null", + "-f", + "null", + "-", + ], + { encoding: "utf-8", timeout: 15_000 }, + ); + + const output = (result.stderr || "") + (result.stdout || ""); + const match = output.match(/average:(\d+\.?\d*|inf)/); + if (!match) return 0; + if (match[1] === "inf") return 100; // identical frames + return parseFloat(match[1]!); +} + +// ── Orchestrator ──────────────────────────────────────────────────────────── + +async function runTest(suite: TestSuite): Promise { + const testOutputDir = join(PARITY_OUTPUT_DIR, suite.id); + if (existsSync(testOutputDir)) rmSync(testOutputDir, { recursive: true }); + mkdirSync(testOutputDir, { recursive: true }); + + const result: TestResult = { + suite, + passed: false, + compileOk: false, + renderOk: false, + checkpoints: [], + avgPsnr: 0, + minPsnr: 0, + failedFrames: 0, + renderTimeMs: 0, + }; + + // Step 1: Compile + console.log(` [${suite.id}] Compiling...`); + let compiledPath: string; + try { + compiledPath = await compileComposition(suite, testOutputDir); + result.compileOk = true; + console.log(` [${suite.id}] ✓ Compiled`); + } catch (err) { + result.error = `Compile failed: ${err instanceof Error ? err.message : String(err)}`; + console.log(` [${suite.id}] ✗ ${result.error}`); + return result; + } + + // Step 2: Render via browser + console.log(` [${suite.id}] Rendering client-side...`); + const renderStart = Date.now(); + try { + const { videoPath, durationMs } = await renderClientSide(suite, compiledPath, testOutputDir); + result.renderTimeMs = durationMs || Date.now() - renderStart; + + if (!existsSync(videoPath)) { + result.error = "Client-side render produced no output file"; + console.log(` [${suite.id}] ✗ No output file`); + return result; + } + + result.renderOk = true; + console.log(` [${suite.id}] ✓ Rendered (${result.renderTimeMs}ms)`); + + // Step 3: PSNR comparison + console.log(` [${suite.id}] Comparing PSNR...`); + + const probeResult = spawnSync( + "ffprobe", + ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", videoPath], + { encoding: "utf-8", timeout: 10_000 }, + ); + const videoDuration = parseFloat((probeResult.stdout || "0").trim()) || 3; + + for (let i = 0; i < CHECKPOINTS; i++) { + const time = (videoDuration * i) / CHECKPOINTS; + const psnr = psnrAtCheckpoint(videoPath, suite.goldenVideo, time); + const passed = psnr >= (suite.meta.minPsnr || MIN_PSNR); + result.checkpoints.push({ time, psnr, passed }); + } + + const psnrValues = result.checkpoints.map((c) => c.psnr); + result.avgPsnr = psnrValues.reduce((a, b) => a + b, 0) / psnrValues.length; + result.minPsnr = Math.min(...psnrValues); + result.failedFrames = result.checkpoints.filter((c) => !c.passed).length; + result.passed = result.failedFrames <= (suite.meta.maxFrameFailures || 0); + + console.log( + ` [${suite.id}] ${result.passed ? "✓" : "✗"} PSNR avg=${result.avgPsnr.toFixed(1)}dB min=${result.minPsnr.toFixed(1)}dB failed=${result.failedFrames}/${CHECKPOINTS}`, + ); + } catch (err) { + result.error = `Render failed: ${err instanceof Error ? err.message : String(err)}`; + console.log(` [${suite.id}] ✗ ${result.error}`); + } + + return result; +} + +// ── Main ──────────────────────────────────────────────────────────────────── + +async function main() { + const filterNames = process.argv.slice(2); + const suites = discoverTests(filterNames); + + if (suites.length === 0) { + console.error("No test suites found."); + process.exit(1); + } + + console.log(`\n@hyperframes/renderer — Regression Parity Harness`); + console.log(`${"─".repeat(50)}`); + console.log(`Tests: ${suites.length}`); + console.log(`Checkpoints per test: ${CHECKPOINTS}`); + console.log(`Min PSNR threshold: ${MIN_PSNR} dB\n`); + + // Ensure bundles exist + if (!existsSync(join(RENDERER_DIST, "renderer.bundle.js"))) { + console.log("Building renderer bundles..."); + execSync( + "npx esbuild packages/renderer/src/index.ts --bundle --format=esm --outfile=packages/renderer/dist/renderer.bundle.js --external:@hyperframes/core --platform=browser", + { encoding: "utf-8" }, + ); + execSync( + "npx esbuild packages/renderer/src/encoding/worker.ts --bundle --format=esm --outfile=packages/renderer/dist/worker.bundle.js --platform=browser", + { encoding: "utf-8" }, + ); + } + + // Start static file server + upload server + mkdirSync(PARITY_OUTPUT_DIR, { recursive: true }); + + console.log(`Starting file server on port ${SERVER_PORT}...`); + // Write serve config to disable cleanUrls (which strips .html and causes 301 redirects) + const serveConfigPath = resolve("serve.json"); + writeFileSync( + serveConfigPath, + JSON.stringify({ + cleanUrls: false, + headers: [{ source: "**", headers: [{ key: "Access-Control-Allow-Origin", value: "*" }] }], + }), + ); + + const rootServer = spawnChild("npx", ["serve", "-l", String(SERVER_PORT), "-C", "."], { + cwd: resolve("."), + detached: true, + stdio: "ignore", + }); + rootServer.unref(); + + console.log(`Starting upload server on port ${UPLOAD_PORT}...`); + const uploadServer = startUploadServer(); + + // Wait for server to start + await new Promise((r) => setTimeout(r, 2000)); + + // Symlink renderer dist into a place the server can find + const distLink = join(PARITY_OUTPUT_DIR, "..", "dist"); + if (!existsSync(distLink)) { + try { + symlinkSync(RENDERER_DIST, distLink); + } catch {} + } + + const results: TestResult[] = []; + for (const suite of suites) { + const result = await runTest(suite); + results.push(result); + } + + // Summary + console.log(`\n${"═".repeat(60)}`); + console.log("REGRESSION PARITY RESULTS"); + console.log(`${"═".repeat(60)}\n`); + + let totalPassed = 0; + let totalFailed = 0; + + for (const r of results) { + const icon = r.passed ? "✓" : "✗"; + const psnrStr = r.renderOk + ? `avg=${r.avgPsnr.toFixed(1)}dB min=${r.minPsnr.toFixed(1)}dB` + : "N/A"; + console.log(` ${icon} ${r.suite.id.padEnd(30)} ${psnrStr.padEnd(30)} ${r.error || ""}`); + if (r.passed) totalPassed++; + else totalFailed++; + } + + console.log(`\n Passed: ${totalPassed}/${results.length}`); + console.log(` Failed: ${totalFailed}/${results.length}\n`); + + // Write JSON report + const reportPath = join(PARITY_OUTPUT_DIR, "parity-report.json"); + writeFileSync(reportPath, JSON.stringify(results, null, 2)); + console.log(`Report: ${reportPath}`); + + // Cleanup servers and temp files + try { + rootServer.kill(); + } catch {} + try { + uploadServer.close(); + } catch {} + try { + rmSync(resolve("serve.json")); + } catch {} + + process.exit(totalFailed > 0 ? 1 : 0); +} + +main().catch((err) => { + console.error("Fatal:", err); + process.exit(1); +}); diff --git a/packages/renderer/src/audio/mixer.test.ts b/packages/renderer/src/audio/mixer.test.ts new file mode 100644 index 00000000..74524a7a --- /dev/null +++ b/packages/renderer/src/audio/mixer.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect, vi } from "vitest"; +import type { AudioMixConfig } from "../types.js"; + +// Mock OfflineAudioContext for jsdom environment +if (typeof globalThis.OfflineAudioContext === "undefined") { + globalThis.OfflineAudioContext = vi + .fn() + .mockImplementation((channels: number, length: number, sampleRate: number) => ({ + createBufferSource: () => ({ + buffer: null, + connect: vi.fn(), + start: vi.fn(), + }), + createGain: () => ({ + gain: { value: 1 }, + connect: vi.fn(), + }), + destination: {}, + decodeAudioData: vi.fn().mockResolvedValue({ + duration: 1, + numberOfChannels: channels, + sampleRate, + getChannelData: () => new Float32Array(length), + }), + startRendering: vi.fn().mockResolvedValue({ + duration: length / sampleRate, + numberOfChannels: channels, + sampleRate, + length, + getChannelData: () => new Float32Array(length), + }), + })) as unknown as typeof OfflineAudioContext; +} + +// Dynamic import AFTER mock is set up +const { mixAudio } = await import("./mixer.js"); + +describe("mixAudio", () => { + it("returns silent audio when no sources provided", async () => { + const config: AudioMixConfig = { + duration: 1.0, + sampleRate: 44100, + channels: 2, + sources: [], + }; + const result = await mixAudio(config); + expect(result.sampleRate).toBe(44100); + expect(result.channels).toBe(2); + }); + + it("uses default sample rate and channels", async () => { + const result = await mixAudio({ duration: 0.5, sources: [] }); + expect(result.sampleRate).toBe(44100); + expect(result.channels).toBe(2); + }); +}); diff --git a/packages/renderer/src/audio/mixer.ts b/packages/renderer/src/audio/mixer.ts new file mode 100644 index 00000000..7ab202d6 --- /dev/null +++ b/packages/renderer/src/audio/mixer.ts @@ -0,0 +1,64 @@ +/** + * Audio Mixer + * + * Mixes audio sources using OfflineAudioContext. + * Decodes audio files, applies volume/timing offsets, + * and renders to a single AudioBuffer (PCM). + */ + +import type { AudioMixConfig, AudioMixResult } from "../types.js"; + +const DEFAULT_SAMPLE_RATE = 44100; +const DEFAULT_CHANNELS = 2; + +export async function mixAudio(config: AudioMixConfig): Promise { + const sampleRate = config.sampleRate ?? DEFAULT_SAMPLE_RATE; + const channels = config.channels ?? DEFAULT_CHANNELS; + const totalSamples = Math.ceil(config.duration * sampleRate); + + const offlineCtx = new OfflineAudioContext(channels, totalSamples, sampleRate); + + for (const source of config.sources) { + const arrayBuffer = await fetchAudioData(source.src); + const audioBuffer = await offlineCtx.decodeAudioData(arrayBuffer); + + const bufferSource = offlineCtx.createBufferSource(); + bufferSource.buffer = audioBuffer; + + // Apply volume + const gainNode = offlineCtx.createGain(); + gainNode.gain.value = source.volume ?? 1; + + bufferSource.connect(gainNode); + gainNode.connect(offlineCtx.destination); + + // Calculate timing + const mediaOffset = source.mediaOffset ?? 0; + const clipDuration = source.endTime - source.startTime; + + if (mediaOffset > 0) { + bufferSource.start(source.startTime, mediaOffset, clipDuration); + } else { + bufferSource.start(source.startTime, 0, clipDuration); + } + } + + const renderedBuffer = await offlineCtx.startRendering(); + + return { + buffer: renderedBuffer, + sampleRate, + channels, + }; +} + +async function fetchAudioData(src: string | Blob): Promise { + if (src instanceof Blob) { + return src.arrayBuffer(); + } + const response = await fetch(src); + if (!response.ok) { + throw new Error(`Failed to fetch audio: ${src} (${response.status})`); + } + return response.arrayBuffer(); +} diff --git a/packages/renderer/src/capture/iframe-pool.test.ts b/packages/renderer/src/capture/iframe-pool.test.ts new file mode 100644 index 00000000..0025bfaf --- /dev/null +++ b/packages/renderer/src/capture/iframe-pool.test.ts @@ -0,0 +1,20 @@ +import { describe, it, expect } from "vitest"; +import { calculateConcurrency } from "./iframe-pool.js"; + +describe("calculateConcurrency", () => { + it("returns at least 1", () => { + expect(calculateConcurrency(0)).toBe(1); + }); + + it("caps at 8", () => { + expect(calculateConcurrency(100)).toBeLessThanOrEqual(8); + }); + + it("leaves 1 core for encoding", () => { + expect(calculateConcurrency(4)).toBe(3); + }); + + it("handles single-core", () => { + expect(calculateConcurrency(1)).toBe(1); + }); +}); diff --git a/packages/renderer/src/capture/iframe-pool.ts b/packages/renderer/src/capture/iframe-pool.ts new file mode 100644 index 00000000..6d7958db --- /dev/null +++ b/packages/renderer/src/capture/iframe-pool.ts @@ -0,0 +1,86 @@ +/** + * Iframe Pool + * + * Manages N parallel hidden iframes, each loading the same composition. + * Each iframe captures a contiguous range of frames using the provided + * FrameSource for DOM-to-ImageBitmap conversion. + */ + +import type { FrameSource, HfMediaElement } from "../types.js"; +import { distributeFrames } from "../utils/timing.js"; + +export function calculateConcurrency(hardwareConcurrency: number): number { + return Math.max(1, Math.min(hardwareConcurrency - 1, 8)); +} + +export interface IframePoolConfig { + compositionUrl: string; + width: number; + height: number; + devicePixelRatio?: number; + concurrency: number; + createFrameSource: () => FrameSource; +} + +export interface CapturedFrame { + bitmap: ImageBitmap; + index: number; + time: number; +} + +export class IframePool { + private sources: FrameSource[] = []; + private config: IframePoolConfig | null = null; + + async init(config: IframePoolConfig): Promise<{ duration: number; media: HfMediaElement[] }> { + this.config = config; + + // Initialize all frame sources in parallel + const initPromises: Promise[] = []; + for (let i = 0; i < config.concurrency; i++) { + const source = config.createFrameSource(); + this.sources.push(source); + initPromises.push( + source.init({ + compositionUrl: config.compositionUrl, + width: config.width, + height: config.height, + devicePixelRatio: config.devicePixelRatio, + }), + ); + } + await Promise.all(initPromises); + + const first = this.sources[0]!; + return { duration: first.duration, media: first.media }; + } + + async captureAll( + frameTimes: number[], + onFrame: (frame: CapturedFrame) => void, + signal?: AbortSignal, + ): Promise { + const ranges = distributeFrames(frameTimes.length, this.sources.length); + + await Promise.all( + ranges.map(async (range, workerIdx) => { + const source = this.sources[workerIdx]!; + for (let i = range.start; i <= range.end; i++) { + if (signal?.aborted) return; + const time = frameTimes[i]!; + const bitmap = await source.capture(time); + onFrame({ bitmap, index: i, time }); + // Yield to event loop every frame so Chrome can process CDP, + // repaint UI, and handle user interactions during long renders + await new Promise((r) => setTimeout(r, 0)); + } + }), + ); + } + + async dispose(): Promise { + await Promise.all(this.sources.map((s) => s.dispose())); + this.sources = []; + this.config = null; + } +} diff --git a/packages/renderer/src/capture/turbo-pool.ts b/packages/renderer/src/capture/turbo-pool.ts new file mode 100644 index 00000000..bc46084d --- /dev/null +++ b/packages/renderer/src/capture/turbo-pool.ts @@ -0,0 +1,229 @@ +/** + * Turbo Pool — Multi-Tab Parallel Capture + * + * Opens N browser tabs (via window.open with noopener) to achieve + * true multi-process parallelism for SnapDOM frame capture. Each tab + * runs in its own Chromium renderer process with independent main thread. + * + * Communication via BroadcastChannel. Frames are PNG-encoded since + * BroadcastChannel doesn't support transferable ImageBitmaps. + */ + +import type { HfMediaElement } from "../types.js"; +import { distributeFrames } from "../utils/timing.js"; + +export interface TurboPoolConfig { + compositionUrl: string; + width: number; + height: number; + devicePixelRatio?: number; + concurrency: number; + /** URL to the turbo-worker.html page that loads the worker script */ + turboWorkerUrl: string; +} + +export interface TurboCapturedFrame { + bitmap: ImageBitmap; + index: number; +} + +interface WorkerState { + id: string; + window: Window | null; + ready: boolean; + done: boolean; + error?: string; +} + +export function calculateTurboConcurrency(): number { + const cores = typeof navigator !== "undefined" ? navigator.hardwareConcurrency : 4; + return Math.max(2, Math.min(Math.floor(cores / 2), 6)); +} + +export class TurboPool { + private workers: WorkerState[] = []; + private channel: BroadcastChannel | null = null; + private sessionId = crypto.randomUUID().slice(0, 8); + private config: TurboPoolConfig | null = null; + + get supported(): boolean { + return typeof BroadcastChannel !== "undefined" && typeof window?.open === "function"; + } + + async init(config: TurboPoolConfig): Promise<{ duration: number; media: HfMediaElement[] }> { + this.config = config; + const channelName = `hf-turbo-${this.sessionId}`; + this.channel = new BroadcastChannel(channelName); + + // Open worker tabs + const workerPromises: Promise[] = []; + + for (let i = 0; i < config.concurrency; i++) { + const workerId = `w${i}-${this.sessionId}`; + const workerUrl = `${config.turboWorkerUrl}?channel=${encodeURIComponent(channelName)}&workerId=${encodeURIComponent(workerId)}`; + + const win = window.open(workerUrl, "_blank", "width=1,height=1,left=-9999,top=-9999"); + + if (!win) { + // Popup blocked — clean up and signal failure + await this.dispose(); + throw new Error("POPUP_BLOCKED"); + } + + // Minimize the popup to reduce visual noise + try { + win.blur(); + } catch { + // cross-origin after navigation — expected + } + + const worker: WorkerState = { id: workerId, window: win, ready: false, done: false }; + this.workers.push(worker); + + // Wait for this worker to report ready + workerPromises.push( + new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`Worker ${workerId} init timeout`)), + 30_000, + ); + + const onMessage = (e: MessageEvent) => { + if (e.data?.workerId !== workerId) return; + if (e.data.type === "ready") { + worker.ready = true; + clearTimeout(timeout); + this.channel!.removeEventListener("message", onMessage); + resolve(); + } else if (e.data.type === "error") { + clearTimeout(timeout); + this.channel!.removeEventListener("message", onMessage); + reject(new Error(e.data.message)); + } + }; + this.channel!.addEventListener("message", onMessage); + }), + ); + + // Send init message to this worker + this.channel.postMessage({ + type: "init", + workerId, + compositionUrl: config.compositionUrl, + width: config.width, + height: config.height, + dpr: config.devicePixelRatio ?? 1, + }); + } + + // Wait for all workers to initialize + await Promise.all(workerPromises); + + // Get duration from the first worker's ready message + // (we need a separate query — workers report duration in their ready message) + // For now, init a temporary local source to get duration/media + const { SnapdomFrameSource } = await import("../sources/snapdom.js"); + const tempSource = new SnapdomFrameSource(); + await tempSource.init({ + compositionUrl: config.compositionUrl, + width: config.width, + height: config.height, + devicePixelRatio: config.devicePixelRatio, + }); + const duration = tempSource.duration; + const media = [...tempSource.media]; + await tempSource.dispose(); + + return { duration, media }; + } + + async captureAll( + frameTimes: number[], + onFrame: (frame: TurboCapturedFrame) => void, + signal?: AbortSignal, + ): Promise { + if (!this.channel || this.workers.length === 0) { + throw new Error("TurboPool not initialized"); + } + + const ranges = distributeFrames(frameTimes.length, this.workers.length); + const channel = this.channel; + let receivedCount = 0; + const totalExpected = frameTimes.length; + + return new Promise((resolve, reject) => { + const onAbort = () => { + channel.postMessage({ type: "abort" }); + cleanup(); + resolve(); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + + const cleanup = () => { + channel.removeEventListener("message", handleMessage); + signal?.removeEventListener("abort", onAbort); + }; + + const handleMessage = async (e: MessageEvent) => { + const msg = e.data; + if (!msg?.workerId) return; + + if (msg.type === "frame") { + try { + // Decode PNG ArrayBuffer → ImageBitmap + const blob = new Blob([msg.png], { type: "image/png" }); + const bitmap = await createImageBitmap(blob); + onFrame({ bitmap, index: msg.index }); + receivedCount++; + + if (receivedCount >= totalExpected) { + cleanup(); + resolve(); + } + } catch (err) { + cleanup(); + reject(err instanceof Error ? err : new Error(String(err))); + } + } else if (msg.type === "error") { + cleanup(); + reject(new Error(`Worker ${msg.workerId}: ${msg.message}`)); + } + }; + + channel.addEventListener("message", handleMessage); + + // Send capture assignments to each worker + for (let i = 0; i < ranges.length; i++) { + const range = ranges[i]!; + const worker = this.workers[i]!; + const frames: { index: number; time: number }[] = []; + for (let j = range.start; j <= range.end; j++) { + frames.push({ index: j, time: frameTimes[j]! }); + } + channel.postMessage({ + type: "capture", + workerId: worker.id, + frames, + }); + } + }); + } + + async dispose(): Promise { + if (this.channel) { + this.channel.postMessage({ type: "abort" }); + this.channel.close(); + this.channel = null; + } + // Close worker windows + for (const worker of this.workers) { + try { + worker.window?.close(); + } catch { + // noopener windows may not be closeable + } + } + this.workers = []; + this.config = null; + } +} diff --git a/packages/renderer/src/capture/turbo-worker.ts b/packages/renderer/src/capture/turbo-worker.ts new file mode 100644 index 00000000..5998575e --- /dev/null +++ b/packages/renderer/src/capture/turbo-worker.ts @@ -0,0 +1,141 @@ +/** + * Turbo Render — Worker Tab Entry Point + * + * This script runs in each worker tab opened by TurboPool. + * It listens on a BroadcastChannel for frame capture assignments, + * loads the composition via SnapdomFrameSource, captures frames, + * and sends PNG-encoded results back to the coordinator. + * + * Each worker tab runs in its own renderer process (thanks to + * `noopener`), giving true multi-process parallelism. + */ + +import { SnapdomFrameSource } from "../sources/snapdom.js"; + +interface InitMessage { + type: "init"; + workerId: string; + compositionUrl: string; + width: number; + height: number; + dpr: number; +} + +interface CaptureMessage { + type: "capture"; + workerId: string; + frames: { index: number; time: number }[]; +} + +interface AbortMessage { + type: "abort"; +} + +type CoordinatorMessage = InitMessage | CaptureMessage | AbortMessage; + +const CHANNEL_NAME = new URLSearchParams(location.search).get("channel"); +if (!CHANNEL_NAME) { + document.title = "ERROR: missing channel param"; + throw new Error("Turbo worker requires ?channel= query param"); +} + +const WORKER_ID = new URLSearchParams(location.search).get("workerId") ?? crypto.randomUUID(); +const channel = new BroadcastChannel(CHANNEL_NAME); +let source: SnapdomFrameSource | null = null; +let aborted = false; + +channel.onmessage = async (e: MessageEvent) => { + const msg = e.data; + + // Only handle messages addressed to this worker (or broadcast) + if ("workerId" in msg && msg.workerId !== WORKER_ID) return; + + if (msg.type === "abort") { + aborted = true; + await cleanup(); + return; + } + + if (msg.type === "init") { + try { + source = new SnapdomFrameSource(); + await source.init({ + compositionUrl: msg.compositionUrl, + width: msg.width, + height: msg.height, + devicePixelRatio: msg.dpr, + }); + channel.postMessage({ type: "ready", workerId: WORKER_ID, duration: source.duration }); + } catch (err) { + channel.postMessage({ + type: "error", + workerId: WORKER_ID, + message: err instanceof Error ? err.message : String(err), + }); + } + return; + } + + if (msg.type === "capture") { + if (!source) { + channel.postMessage({ + type: "error", + workerId: WORKER_ID, + message: "Not initialized", + }); + return; + } + + try { + for (const frame of msg.frames) { + if (aborted) break; + + const bitmap = await source.capture(frame.time); + + // Convert ImageBitmap → PNG ArrayBuffer (BroadcastChannel can't transfer ImageBitmap) + const canvas = document.createElement("canvas"); + canvas.width = bitmap.width; + canvas.height = bitmap.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(bitmap, 0, 0); + bitmap.close(); + + const blob = await new Promise((resolve) => + canvas.toBlob((b) => resolve(b!), "image/png"), + ); + const buffer = await blob.arrayBuffer(); + + channel.postMessage({ + type: "frame", + workerId: WORKER_ID, + index: frame.index, + png: buffer, + }); + + // Yield to keep the tab responsive + await new Promise((r) => setTimeout(r, 0)); + } + + channel.postMessage({ type: "done", workerId: WORKER_ID }); + } catch (err) { + channel.postMessage({ + type: "error", + workerId: WORKER_ID, + message: err instanceof Error ? err.message : String(err), + }); + } + return; + } +}; + +async function cleanup(): Promise { + if (source) { + await source.dispose(); + source = null; + } + channel.close(); + window.close(); +} + +// Signal that the worker tab script has loaded +document.title = `turbo-worker:${WORKER_ID}`; diff --git a/packages/renderer/src/capture/video-frame-injector.ts b/packages/renderer/src/capture/video-frame-injector.ts new file mode 100644 index 00000000..56fe0f75 --- /dev/null +++ b/packages/renderer/src/capture/video-frame-injector.ts @@ -0,0 +1,115 @@ +/** + * Video Frame Injector + * + * Seeks each