refactor: introduce esm runner

packages/core/src/index.ts

@@ -5,7 +5,7 @@ export {
 } from './lib/implementation/persist';
 export {
   executePlugins,
-  PluginOutputError,
+  PluginOutputMissingAuditError,
 } from './lib/implementation/execute-plugin';
 export { collect, CollectOptions } from './lib/implementation/collect';
 export { upload, UploadOptions } from './lib/upload';

packages/core/src/lib/implementation/execute-esm-runner.spec.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'vitest';
+import {
+  AuditOutputs,
+  EsmObserver,
+  auditOutputsSchema,
+} from '@code-pushup/models';
+import { executeEsmRunner } from './execute-esm-runner';
+
+describe('executeEsmRunner', () => {
+  it('should execute valid plugin config', async () => {
+    const autidOutputs = await executeEsmRunner({
+      observer: { next: console.log },
+      runner: (observer?: EsmObserver) =>
+        Promise.resolve([
+          { slug: 'mock-audit-slug', score: 0, value: 0 },
+        ] satisfies AuditOutputs),
+    });
+    expect(autidOutputs.audits[0]?.slug).toBe('mock-audit-slug');
+    expect(autidOutputs).toBe('mock-audit-slug');
+    expect(() => auditOutputsSchema.parse(autidOutputs)).not.toThrow();
+  });
+});

packages/core/src/lib/implementation/execute-esm-runner.ts

@@ -0,0 +1,25 @@
+import {
+  EsmObserver,
+  EsmRunnerConfig,
+  RunnerResult,
+  runnerResultSchema,
+} from '@code-pushup/models';
+import { calcDuration } from '@code-pushup/utils';
+
+export type EsmRunnerProcessConfig = {
+  runner: EsmRunnerConfig;
+  observer?: EsmObserver;
+};
+
+export function executeEsmRunner(
+  cfg: EsmRunnerProcessConfig,
+): Promise<RunnerResult> {
+  const { observer, runner } = cfg;
+  const date = new Date().toISOString();
+  const start = performance.now();
+
+  return runner(observer).then(result => {
+    const timings = { date, duration: calcDuration(start) };
+    return runnerResultSchema.parse({ result, ...timings });
+  });
+}

packages/core/src/lib/implementation/execute-plugin.spec.ts

@@ -1,5 +1,6 @@
 import { join } from 'path';
 import { describe, expect, it } from 'vitest';
+import { PluginConfig, pluginReportSchema } from '@code-pushup/models';
 import {
   AuditReport,
   PluginConfig,
@@ -9,6 +10,10 @@ import {
   auditReport,
   echoRunnerConfig,
   pluginConfig,
+} from '@code-pushup/models/testing';
+  auditReport,
+  outputFileToAuditOutputs,
+  pluginConfig,
 } from '@code-pushup/models/testing';
 import { DEFAULT_TESTING_CLI_OPTIONS } from '../../../test/constants';
 import { executePlugin, executePlugins } from './execute-plugin';
@@ -32,7 +37,7 @@ describe('executePlugin', () => {
   it('should execute valid plugin config', async () => {
     const pluginResult = await executePlugin(validPluginCfg);
     expect(pluginResult.audits[0]?.slug).toBe('mock-audit-slug');
-    expect(() => auditOutputsSchema.parse(pluginResult.audits)).not.toThrow();
+    expect(() => pluginReportSchema.parse(pluginResult.audits)).not.toThrow();
   });
 
   it('should throws with invalid plugin audits slug', async () => {
@@ -42,15 +47,15 @@ describe('executePlugin', () => {
     );
   });
 
-  it('should throw if invalid runnerOutput is produced', async () => {
-    const invalidAuditOutputs: AuditReport[] = [
-      { p: 42 } as unknown as AuditReport,
-    ];
-    const pluginCfg = pluginConfig([auditReport()]);
-    pluginCfg.runner = echoRunnerConfig(
-      invalidAuditOutputs,
-      join('tmp', 'out.json'),
-    );
+  it('should throw if invalid runnerOutput is produced with transform', async () => {
+    const pluginCfg: PluginConfig = {
+      ...validPluginCfg,
+      runner: {
+        ...validPluginCfg.runner,
+        outputFileToAuditResults: outputFileToAuditOutputs(),
+      },
+    };
+
     await expect(() => executePlugin(pluginCfg)).rejects.toThrow(
       /Plugin output of plugin .* is invalid./,
     );
@@ -60,19 +65,10 @@ describe('executePlugin', () => {
 describe('executePlugins', () => {
   it('should work with valid plugins', async () => {
     const plugins = [validPluginCfg, validPluginCfg2];
-    const pluginResult = await executePlugins(plugins, DEFAULT_OPTIONS);
 
-    expect(pluginResult[0]?.date.endsWith('Z')).toBeTruthy();
-    expect(pluginResult[0]?.duration).toBeTruthy();
+    const pluginResult = await executePlugins(plugins, DEFAULT_OPTIONS);
 
-    expect(pluginResult[0]?.audits[0]?.slug).toBe('mock-audit-slug');
-    expect(pluginResult[1]?.audits[0]?.slug).toBe('mock-audit-slug');
-    expect(() =>
-      auditOutputsSchema.parse(pluginResult[0]?.audits),
-    ).not.toThrow();
-    expect(() =>
-      auditOutputsSchema.parse(pluginResult[1]?.audits),
-    ).not.toThrow();
+    expect(() => pluginReportSchema.parse(pluginResult)).not.toThrow();
   });
 
   it('should throws with invalid plugins', async () => {

packages/core/src/lib/implementation/execute-plugin.ts

@@ -1,29 +1,23 @@
 import chalk from 'chalk';
-import { readFile } from 'fs/promises';
-import { join } from 'path';
 import {
+  AuditOutputs,
+  AuditReport,
   PluginConfig,
   PluginReport,
-  auditOutputsSchema,
+  RunnerResult,
+  auditReportSchema,
 } from '@code-pushup/models';
-import {
-  ProcessObserver,
-  executeProcess,
-  getProgressBar,
-} from '@code-pushup/utils';
+import { ProcessObserver, getProgressBar } from '@code-pushup/utils';
+import { executeRunner } from './execute-runner';
 
 /**
  * Error thrown when plugin output is invalid.
  */
-export class PluginOutputError extends Error {
-  constructor(pluginSlug: string, error?: Error) {
+export class PluginOutputMissingAuditError extends Error {
+  constructor(auditSlug: string, pluginSlug: string) {
     super(
-      `Plugin output of plugin with slug ${pluginSlug} is invalid. \n Error: ${error?.message}`,
+      `Audit metadata not found for slug ${auditSlug} from plugin ${pluginSlug}`,
     );
-    if (error) {
-      this.name = error.name;
-      this.stack = error.stack;
-    }
   }
 }
 
@@ -34,7 +28,7 @@ export class PluginOutputError extends Error {
  * @param pluginConfig - {@link ProcessConfig} object with runner and meta
  * @param observer - process {@link ProcessObserver}
  * @returns {Promise<AuditOutput[]>} - audit outputs from plugin runner
- * @throws {PluginOutputError} - if plugin runner output is invalid
+ * @throws {PluginOutputMissingAuditError} - if plugin runner output is invalid
  *
  * @example
  * // plugin execution
@@ -54,70 +48,41 @@ export async function executePlugin(
   observer?: ProcessObserver,
 ): Promise<PluginReport> {
   const {
-    slug,
-    title,
-    icon,
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    audits: _,
     description,
     docsUrl,
-    version,
-    packageName,
     groups,
+    ...pluginMeta
   } = pluginConfig;
-  const { args, command } = pluginConfig.runner;
-
-  const { duration, date } = await executeProcess({
-    command,
-    args,
+  const runnerResult: RunnerResult = await executeRunner(
+    pluginConfig.runner,
     observer,
-  });
+  );
+  const { audits: runnerAuditOutputs, ...executionMeta } = runnerResult;
 
-  try {
-    const processOutputPath = join(
-      process.cwd(),
-      pluginConfig.runner.outputFile,
-    );
-
-    // read process output from file system and parse it
-    const auditOutputs = auditOutputsSchema.parse(
-      JSON.parse((await readFile(processOutputPath)).toString()),
-    );
+  // read process output from file system and parse it
+  /*auditOutputsCorrelateWithPluginOutput(
+    runnerAuditOutputs,
+    pluginConfig.audits,
+  );*/
 
-    const audits = auditOutputs.map(auditOutput => {
-      const auditMetadata = pluginConfig.audits.find(
-        audit => audit.slug === auditOutput.slug,
-      );
-      if (!auditMetadata) {
-        throw new PluginOutputError(
-          slug,
-          new Error(
-            `Audit metadata not found for slug ${auditOutput.slug} from runner output`,
-          ),
-        );
-      }
-      return {
-        ...auditOutput,
-        ...auditMetadata,
-      };
+  // enrich `AuditOutputs` to `AuditReport`
+  const audits: AuditReport[] = runnerAuditOutputs.map(auditOutput => {
+    return auditReportSchema.parse({
+      ...auditOutput,
+      ...pluginConfig.audits.find(audit => audit.slug === auditOutput.slug),
     });
+  });
 
-    // @TODO consider just resting/spreading the values
-    return {
-      version,
-      packageName,
-      slug,
-      title,
-      icon,
-      date,
-      duration,
-      audits,
-      ...(description && { description }),
-      ...(docsUrl && { docsUrl }),
-      ...(groups && { groups }),
-    } satisfies PluginReport;
-  } catch (error) {
-    const e = error as Error;
-    throw new PluginOutputError(slug, e);
-  }
+  return {
+    ...pluginMeta,
+    ...executionMeta,
+    audits,
+    ...(description && { description }),
+    ...(docsUrl && { docsUrl }),
+    ...(groups && { groups }),
+  } satisfies PluginReport;
 }
 
 /**
@@ -165,3 +130,17 @@ export async function executePlugins(
 
   return pluginsResult;
 }
+
+function auditOutputsCorrelateWithPluginOutput(
+  auditOutputs: AuditOutputs,
+  pluginConfigAudits: PluginConfig['audits'],
+) {
+  auditOutputs.forEach(auditOutput => {
+    const auditMetadata = pluginConfigAudits.find(
+      audit => audit.slug === auditOutput.slug,
+    );
+    if (!auditMetadata) {
+      throw new Error(`Missing audit ${auditOutput.slug}.`);
+    }
+  });
+}

packages/core/src/lib/implementation/execute-runner.spec.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest';
+import { runnerResultSchema } from '@code-pushup/models';
+import {
+  auditReport,
+  echoRunnerConfig,
+  outputFileToAuditOutputs,
+} from '@code-pushup/models/testing';
+import { executeRunner } from './execute-runner';
+
+const validRunnerCfg = echoRunnerConfig([auditReport()], 'output.json');
+
+describe('executeRunner', () => {
+  it('should work with valid plugins', async () => {
+    const runnerResult = await executeRunner(validRunnerCfg);
+
+    // data sanity
+    expect(runnerResult.date.endsWith('Z')).toBeTruthy();
+    expect(runnerResult.duration).toBeTruthy();
+    expect(runnerResult.audits[0]?.slug).toBe('mock-audit-slug');
+
+    // schema validation
+    expect(() => runnerResultSchema.parse(runnerResult.audits)).not.toThrow();
+  });
+
+  it('should use transform if provided', async () => {
+    const runnerCfgWithTransform = {
+      ...validRunnerCfg,
+      outputFileToAuditResults: outputFileToAuditOutputs(),
+    };
+
+    const runnerResult = await executeRunner(runnerCfgWithTransform);
+
+    expect(runnerResult.audits[0]?.displayValue).toBe(
+      'transformed - mock-audit-slug',
+    );
+  });
+});

packages/core/src/lib/implementation/execute-runner.ts

@@ -0,0 +1,41 @@
+import { join } from 'path';
+import {
+  AuditOutputs,
+  RunnerConfig,
+  RunnerResult,
+  runnerResultSchema,
+} from '@code-pushup/models';
+import {
+  ProcessObserver,
+  executeProcess,
+  readJsonFile,
+} from '@code-pushup/utils';
+
+export async function executeRunner(
+  cfg: RunnerConfig,
+  observer?: ProcessObserver,
+): Promise<RunnerResult> {
+  const { args, command, outputFile, outputFileToAuditResults } = cfg;
+
+  const { duration, date } = await executeProcess({
+    command,
+    args,
+    observer,
+  });
+
+  // read process output from file system and parse it
+  let audits = await readJsonFile<AuditOutputs>(
+    join(process.cwd(), outputFile),
+  );
+
+  // transform unknownAuditOutputs to auditOutputs
+  if (outputFileToAuditResults) {
+    audits = outputFileToAuditResults(audits) as RunnerResult['audits'];
+  }
+
+  return runnerResultSchema.parse({
+    duration,
+    date,
+    audits,
+  });
+}