Skip to content

fix/core hooks #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adriencaccia merged 3 commits into from
Aug 3, 2023
Merged

fix/core hooks #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e04227b
test: refactor core mocks
adriencaccia Aug 1, 2023
571fa46
fix: make sure teardown is called only once after all the benchmarks ran
adriencaccia Aug 1, 2023
cd532fd
fix(tinybench-plugin): make sure beforeEach and afterEach are called …
adriencaccia Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/benchmark.js-plugin/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ async function runBenchmarks({
}
console.log(` ✔ Measured ${uri}`);
benchmarkCompletedListeners.forEach((listener) => listener());
teardownCore();
}
teardownCore();
console.log(`[CodSpeed] Done running ${benches.length} benches.`);
}
75 changes: 43 additions & 32 deletions packages/benchmark.js-plugin/tests/index.integ.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
import { mockDeep, mockReset } from "jest-mock-extended";
const mockCore = mockDeep<typeof Measurement>();
const mockCore = mockDeep<typeof core>();

import type { Measurement } from "@codspeed/core";
import * as core from "@codspeed/core";
import Benchmark from "benchmark";
import { withCodSpeed } from "..";
import { registerBenchmarks } from "./registerBenchmarks";
import { registerOtherBenchmarks } from "./registerOtherBenchmarks";

jest.mock("@codspeed/core", () => ({
...jest.requireActual("@codspeed/core"),
Measurement: mockCore,
}));
jest.mock("@codspeed/core", () => mockCore);

beforeEach(() => {
mockReset(mockCore);
Expand All @@ -23,7 +20,7 @@ const benchOptions: Benchmark.Options = {

describe("Benchmark", () => {
it("simple benchmark", async () => {
mockCore.isInstrumented.mockReturnValue(false);
mockCore.Measurement.isInstrumented.mockReturnValue(false);
const bench = withCodSpeed(
new Benchmark(
"RegExp",
Expand All @@ -37,11 +34,11 @@ describe("Benchmark", () => {
bench.on("complete", onComplete);
await bench.run();
expect(onComplete).toHaveBeenCalled();
expect(mockCore.startInstrumentation).not.toHaveBeenCalled();
expect(mockCore.stopInstrumentation).not.toHaveBeenCalled();
expect(mockCore.Measurement.startInstrumentation).not.toHaveBeenCalled();
expect(mockCore.Measurement.stopInstrumentation).not.toHaveBeenCalled();
});
it("check core methods are called", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);

const bench = withCodSpeed(
new Benchmark(
Expand All @@ -56,13 +53,13 @@ describe("Benchmark", () => {
bench.on("complete", onComplete);
await bench.run();
expect(onComplete).toHaveBeenCalled();
expect(mockCore.startInstrumentation).toHaveBeenCalled();
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled();
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/index.integ.test.ts::RegExpSingle"
);
});
it("check error handling", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);
const bench = withCodSpeed(
new Benchmark(
"throwing",
Expand All @@ -79,7 +76,7 @@ describe("Benchmark", () => {
async (instrumented) => {
const logSpy = jest.spyOn(console, "log");
const warnSpy = jest.spyOn(console, "warn");
mockCore.isInstrumented.mockReturnValue(instrumented);
mockCore.Measurement.isInstrumented.mockReturnValue(instrumented);
await withCodSpeed(
new Benchmark(
"RegExpSingle",
Expand Down Expand Up @@ -111,7 +108,7 @@ describe("Benchmark", () => {

describe("Benchmark.Suite", () => {
it("simple suite", async () => {
mockCore.isInstrumented.mockReturnValue(false);
mockCore.Measurement.isInstrumented.mockReturnValue(false);
const suite = withCodSpeed(new Benchmark.Suite());
suite.add(
"RegExp",
Expand All @@ -124,11 +121,11 @@ describe("Benchmark.Suite", () => {
suite.on("complete", onComplete);
await suite.run({ maxTime: 0.1, initCount: 1 });
expect(onComplete).toHaveBeenCalled();
expect(mockCore.startInstrumentation).not.toHaveBeenCalled();
expect(mockCore.stopInstrumentation).not.toHaveBeenCalled();
expect(mockCore.Measurement.startInstrumentation).not.toHaveBeenCalled();
expect(mockCore.Measurement.stopInstrumentation).not.toHaveBeenCalled();
});
it("check core methods are called", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);
const suite = withCodSpeed(new Benchmark.Suite()).add(
"RegExp",
function () {
Expand All @@ -139,13 +136,13 @@ describe("Benchmark.Suite", () => {
const onComplete = jest.fn();
suite.on("complete", onComplete);
await suite.run({ maxTime: 0.1, initCount: 1 });
expect(mockCore.startInstrumentation).toHaveBeenCalled();
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled();
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/index.integ.test.ts::RegExp"
);
});
it("check suite name is in the uri", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);
await withCodSpeed(new Benchmark.Suite("thesuite"))
.add(
"RegExp",
Expand All @@ -158,15 +155,15 @@ describe("Benchmark.Suite", () => {
/o/.test("Hello World!");
}, benchOptions)
.run();
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/index.integ.test.ts::thesuite::RegExp"
);
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/index.integ.test.ts::thesuite::unknown_1"
);
});
it("check error handling", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);
const bench = withCodSpeed(new Benchmark.Suite("thesuite")).add(
"throwing",
() => {
Expand All @@ -180,7 +177,7 @@ describe("Benchmark.Suite", () => {
async (instrumented) => {
const logSpy = jest.spyOn(console, "log");
const warnSpy = jest.spyOn(console, "warn");
mockCore.isInstrumented.mockReturnValue(instrumented);
mockCore.Measurement.isInstrumented.mockReturnValue(instrumented);
await withCodSpeed(new Benchmark.Suite("thesuite"))
.add(
"RegExp",
Expand Down Expand Up @@ -212,31 +209,45 @@ describe("Benchmark.Suite", () => {
}
);
it("check nested file path is in the uri when bench is registered in another file", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);
const suite = withCodSpeed(new Benchmark.Suite("thesuite"));
registerBenchmarks(suite);
const onComplete = jest.fn();
suite.on("complete", onComplete);
await suite.run({ maxTime: 0.1, initCount: 1 });
expect(mockCore.startInstrumentation).toHaveBeenCalled();
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled();
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/registerBenchmarks.ts::thesuite::RegExp"
);
});
it("check that benchmarks with same name have different URIs when registered in different files", async () => {
mockCore.isInstrumented.mockReturnValue(true);
mockCore.Measurement.isInstrumented.mockReturnValue(true);
const suite = withCodSpeed(new Benchmark.Suite("thesuite"));
registerBenchmarks(suite);
registerOtherBenchmarks(suite);
const onComplete = jest.fn();
suite.on("complete", onComplete);
await suite.run({ maxTime: 0.1, initCount: 1 });
expect(mockCore.startInstrumentation).toHaveBeenCalled();
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled();
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/registerBenchmarks.ts::thesuite::RegExp"
);
expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith(
"packages/benchmark.js-plugin/tests/registerOtherBenchmarks.ts::thesuite::RegExp"
);
});
it("should call setupCore and teardownCore only once after run()", async () => {
mockCore.Measurement.isInstrumented.mockReturnValue(true);
const suite = withCodSpeed(new Benchmark.Suite("thesuite"));
registerBenchmarks(suite);
registerOtherBenchmarks(suite);

expect(mockCore.setupCore).not.toHaveBeenCalled();
expect(mockCore.teardownCore).not.toHaveBeenCalled();

await suite.run({ maxTime: 0.1, initCount: 1 });

expect(mockCore.setupCore).toHaveBeenCalledTimes(1);
expect(mockCore.teardownCore).toHaveBeenCalledTimes(1);
});
});
35 changes: 16 additions & 19 deletions packages/tinybench-plugin/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,43 +44,40 @@ export function withCodSpeed(bench: Bench): Bench {
const rootCallingFile = getCallingFile();

bench.run = async () => {
setupCore();
console.log(`[CodSpeed] running with @codspeed/tinybench v${__VERSION__}`);
setupCore();
for (const task of bench.tasks) {
// run before hooks
if (task.opts.beforeAll != null) {
await task.opts.beforeAll.call(task);
}
if (task.opts.beforeEach != null) {
await task.opts.beforeEach.call(task);
}

// run the actual benchmark, with instrumentation
const uri = isCodSpeedBenchOptions(task.opts)
? task.opts.uri
: `${rootCallingFile}::${task.name}`;
await optimizeFunction(task.fn);

await task.opts.beforeAll?.call(task);

// run optimizations
await optimizeFunction(async () => {
await task.opts.beforeEach?.call(task);
await task.fn();
await task.opts.afterEach?.call(task);
});

// run instrumented benchmark
await task.opts.beforeEach?.call(task);
await (async function __codspeed_root_frame__() {
Measurement.startInstrumentation();
await task.fn();
Measurement.stopInstrumentation(uri);
})();
await task.opts.afterEach?.call(task);

// run after hooks
if (task.opts.afterEach != null) {
await task.opts.afterEach.call(task);
}
if (task.opts.afterAll != null) {
await task.opts.afterAll.call(task);
}
await task.opts.afterAll?.call(task);

// print results
console.log(` ✔ Measured ${uri}`);
}
teardownCore();
console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
return bench.tasks;
};
teardownCore();
return bench;
}

Expand Down
Loading