Categorize screenshots by engine, and require perfect match

sockmaster27 · Jan 3, 2025 · 6a3ca9d · 6a3ca9d
1 parent a1b8efc
commit 6a3ca9d
Show file tree

Hide file tree

Showing 62 changed files with 109 additions and 25 deletions.
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/hello-world-webgl-chromium.png b/tests/.generated-screenshots/test-sites.spec.ts/hello-world-webgl-chromium.png
diff --git a/.../test-sites.spec.ts/hello-world-webgl.png → ...tes.spec.ts/hello-world-webgl-firefox.png b/.../test-sites.spec.ts/hello-world-webgl.png → ...tes.spec.ts/hello-world-webgl-firefox.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/hello-world-webgpu-chromium.png b/tests/.generated-screenshots/test-sites.spec.ts/hello-world-webgpu-chromium.png
diff --git a/...ated-screenshots/test-sites.spec.ts/hello-world-webgpu-unsupported-chromium.png b/...ated-screenshots/test-sites.spec.ts/hello-world-webgpu-unsupported-chromium.png
diff --git a/...pec.ts/hello-world-webgpu-unsupported.png → ...ello-world-webgpu-unsupported-firefox.png b/...pec.ts/hello-world-webgpu-unsupported.png → ...ello-world-webgpu-unsupported-firefox.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/hello-world-webgpu.png b/tests/.generated-screenshots/test-sites.spec.ts/hello-world-webgpu.png
diff --git a/...enerated-screenshots/test-sites.spec.ts/landing-page-bubbles-webgl-chromium.png b/...enerated-screenshots/test-sites.spec.ts/landing-page-bubbles-webgl-chromium.png
diff --git a/...es.spec.ts/landing-page-bubbles-webgl.png → ...ts/landing-page-bubbles-webgl-firefox.png b/...es.spec.ts/landing-page-bubbles-webgl.png → ...ts/landing-page-bubbles-webgl-firefox.png
diff --git a/...nerated-screenshots/test-sites.spec.ts/landing-page-bubbles-webgpu-chromium.png b/...nerated-screenshots/test-sites.spec.ts/landing-page-bubbles-webgpu-chromium.png
diff --git a/...enshots/test-sites.spec.ts/landing-page-bubbles-webgpu-unsupported-chromium.png b/...enshots/test-sites.spec.ts/landing-page-bubbles-webgpu-unsupported-chromium.png
diff --git a/...nding-page-bubbles-webgpu-unsupported.png → ...ge-bubbles-webgpu-unsupported-firefox.png b/...nding-page-bubbles-webgpu-unsupported.png → ...ge-bubbles-webgpu-unsupported-firefox.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/landing-page-bubbles-webgpu.png b/tests/.generated-screenshots/test-sites.spec.ts/landing-page-bubbles-webgpu.png
diff --git a/.../.generated-screenshots/test-sites.spec.ts/landing-page-halo-webgl-chromium.png b/.../.generated-screenshots/test-sites.spec.ts/landing-page-halo-webgl-chromium.png
diff --git a/...sites.spec.ts/landing-page-halo-webgl.png → ...ec.ts/landing-page-halo-webgl-firefox.png b/...sites.spec.ts/landing-page-halo-webgl.png → ...ec.ts/landing-page-halo-webgl-firefox.png
diff --git a/....generated-screenshots/test-sites.spec.ts/landing-page-halo-webgpu-chromium.png b/....generated-screenshots/test-sites.spec.ts/landing-page-halo-webgpu-chromium.png
diff --git a/...creenshots/test-sites.spec.ts/landing-page-halo-webgpu-unsupported-chromium.png b/...creenshots/test-sites.spec.ts/landing-page-halo-webgpu-unsupported-chromium.png
diff --git a/.../landing-page-halo-webgpu-unsupported.png → ...-page-halo-webgpu-unsupported-firefox.png b/.../landing-page-halo-webgpu-unsupported.png → ...-page-halo-webgpu-unsupported-firefox.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/landing-page-halo-webgpu.png b/tests/.generated-screenshots/test-sites.spec.ts/landing-page-halo-webgpu.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/logo-webgl.png b/tests/.generated-screenshots/test-sites.spec.ts/logo-webgl.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/logo-webgpu-unsupported.png b/tests/.generated-screenshots/test-sites.spec.ts/logo-webgpu-unsupported.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/logo-webgpu.png b/tests/.generated-screenshots/test-sites.spec.ts/logo-webgpu.png
diff --git a/....generated-screenshots/test-sites.spec.ts/oversized-canvas-webgl-chromium-1.png b/....generated-screenshots/test-sites.spec.ts/oversized-canvas-webgl-chromium-1.png
diff --git a/...tes.spec.ts/oversized-canvas-webgpu-1.png → ....ts/oversized-canvas-webgl-chromium-2.png b/...tes.spec.ts/oversized-canvas-webgpu-1.png → ....ts/oversized-canvas-webgl-chromium-2.png
diff --git a/...ites.spec.ts/oversized-canvas-webgl-1.png → ...c.ts/oversized-canvas-webgl-firefox-1.png b/...ites.spec.ts/oversized-canvas-webgl-1.png → ...c.ts/oversized-canvas-webgl-firefox-1.png
diff --git a/...ites.spec.ts/oversized-canvas-webgl-2.png → ...c.ts/oversized-canvas-webgl-firefox-2.png b/...ites.spec.ts/oversized-canvas-webgl-2.png → ...c.ts/oversized-canvas-webgl-firefox-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/oversized-canvas-webgpu-2.png b/tests/.generated-screenshots/test-sites.spec.ts/oversized-canvas-webgpu-2.png
diff --git a/...generated-screenshots/test-sites.spec.ts/oversized-canvas-webgpu-chromium-1.png b/...generated-screenshots/test-sites.spec.ts/oversized-canvas-webgpu-chromium-1.png
diff --git a/...generated-screenshots/test-sites.spec.ts/oversized-canvas-webgpu-chromium-2.png b/...generated-screenshots/test-sites.spec.ts/oversized-canvas-webgpu-chromium-2.png
diff --git a/...reenshots/test-sites.spec.ts/oversized-canvas-webgpu-unsupported-chromium-1.png b/...reenshots/test-sites.spec.ts/oversized-canvas-webgpu-unsupported-chromium-1.png
diff --git a/...reenshots/test-sites.spec.ts/oversized-canvas-webgpu-unsupported-chromium-2.png b/...reenshots/test-sites.spec.ts/oversized-canvas-webgpu-unsupported-chromium-2.png
diff --git a/...oversized-canvas-webgpu-unsupported-1.png → ...d-canvas-webgpu-unsupported-firefox-1.png b/...oversized-canvas-webgpu-unsupported-1.png → ...d-canvas-webgpu-unsupported-firefox-1.png
diff --git a/...oversized-canvas-webgpu-unsupported-2.png → ...d-canvas-webgpu-unsupported-firefox-2.png b/...oversized-canvas-webgpu-unsupported-2.png → ...d-canvas-webgpu-unsupported-firefox-2.png
diff --git a/...s/test-sites.spec.ts/remount-webgpu-1.png → ...ites.spec.ts/remount-webgl-chromium-1.png b/...s/test-sites.spec.ts/remount-webgpu-1.png → ...ites.spec.ts/remount-webgl-chromium-1.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/remount-webgl-chromium-2.png b/tests/.generated-screenshots/test-sites.spec.ts/remount-webgl-chromium-2.png
diff --git a/...ts/test-sites.spec.ts/remount-webgl-1.png → ...sites.spec.ts/remount-webgl-firefox-1.png b/...ts/test-sites.spec.ts/remount-webgl-1.png → ...sites.spec.ts/remount-webgl-firefox-1.png
diff --git a/...ts/test-sites.spec.ts/remount-webgl-2.png → ...sites.spec.ts/remount-webgl-firefox-2.png b/...ts/test-sites.spec.ts/remount-webgl-2.png → ...sites.spec.ts/remount-webgl-firefox-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/remount-webgpu-2.png b/tests/.generated-screenshots/test-sites.spec.ts/remount-webgpu-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/remount-webgpu-chromium-1.png b/tests/.generated-screenshots/test-sites.spec.ts/remount-webgpu-chromium-1.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/remount-webgpu-chromium-2.png b/tests/.generated-screenshots/test-sites.spec.ts/remount-webgpu-chromium-2.png
diff --git a/...erated-screenshots/test-sites.spec.ts/remount-webgpu-unsupported-chromium-1.png b/...erated-screenshots/test-sites.spec.ts/remount-webgpu-unsupported-chromium-1.png
diff --git a/...erated-screenshots/test-sites.spec.ts/remount-webgpu-unsupported-chromium-2.png b/...erated-screenshots/test-sites.spec.ts/remount-webgpu-unsupported-chromium-2.png
diff --git a/....spec.ts/remount-webgpu-unsupported-1.png → .../remount-webgpu-unsupported-firefox-1.png b/....spec.ts/remount-webgpu-unsupported-1.png → .../remount-webgpu-unsupported-firefox-1.png
diff --git a/....spec.ts/remount-webgpu-unsupported-2.png → .../remount-webgpu-unsupported-firefox-2.png b/....spec.ts/remount-webgpu-unsupported-2.png → .../remount-webgpu-unsupported-firefox-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgl-chromium-1.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgl-chromium-1.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgl-chromium-2.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgl-chromium-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgl-chromium-3.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgl-chromium-3.png
diff --git a/...ots/test-sites.spec.ts/slider-webgl-1.png → ...-sites.spec.ts/slider-webgl-firefox-1.png b/...ots/test-sites.spec.ts/slider-webgl-1.png → ...-sites.spec.ts/slider-webgl-firefox-1.png
diff --git a/...ots/test-sites.spec.ts/slider-webgl-2.png → ...-sites.spec.ts/slider-webgl-firefox-2.png b/...ots/test-sites.spec.ts/slider-webgl-2.png → ...-sites.spec.ts/slider-webgl-firefox-2.png
diff --git a/...ots/test-sites.spec.ts/slider-webgl-3.png → ...-sites.spec.ts/slider-webgl-firefox-3.png b/...ots/test-sites.spec.ts/slider-webgl-3.png → ...-sites.spec.ts/slider-webgl-firefox-3.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-1.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-1.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-2.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-3.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-3.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-chromium-1.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-chromium-1.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-chromium-2.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-chromium-2.png
diff --git a/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-chromium-3.png b/tests/.generated-screenshots/test-sites.spec.ts/slider-webgpu-chromium-3.png
diff --git a/...nerated-screenshots/test-sites.spec.ts/slider-webgpu-unsupported-chromium-1.png b/...nerated-screenshots/test-sites.spec.ts/slider-webgpu-unsupported-chromium-1.png
diff --git a/...nerated-screenshots/test-sites.spec.ts/slider-webgpu-unsupported-chromium-2.png b/...nerated-screenshots/test-sites.spec.ts/slider-webgpu-unsupported-chromium-2.png
diff --git a/...nerated-screenshots/test-sites.spec.ts/slider-webgpu-unsupported-chromium-3.png b/...nerated-screenshots/test-sites.spec.ts/slider-webgpu-unsupported-chromium-3.png
diff --git a/...s.spec.ts/slider-webgpu-unsupported-1.png → ...s/slider-webgpu-unsupported-firefox-1.png b/...s.spec.ts/slider-webgpu-unsupported-1.png → ...s/slider-webgpu-unsupported-firefox-1.png
diff --git a/...s.spec.ts/slider-webgpu-unsupported-2.png → ...s/slider-webgpu-unsupported-firefox-2.png b/...s.spec.ts/slider-webgpu-unsupported-2.png → ...s/slider-webgpu-unsupported-firefox-2.png
diff --git a/...s.spec.ts/slider-webgpu-unsupported-3.png → ...s/slider-webgpu-unsupported-firefox-3.png b/...s.spec.ts/slider-webgpu-unsupported-3.png → ...s/slider-webgpu-unsupported-firefox-3.png
diff --git a/tests/test-sites.spec.ts b/tests/test-sites.spec.ts
@@ -12,35 +12,37 @@ test.beforeEach(async ({ page }) => {
  *
  * @param page
  * @param info
+ * @param browserName
  * @param name A name that identifies this test.
  * @param api The API used in the test, either `"webgl"` or `"webgpu"`.
  * @param number A number identifying the specific screeenshot within the test, if multiple are included.
  */
 async function assertScreenshot(
     page: Page,
     info: TestInfo,
+    browserName: string,
     name: string,
     api: "webgl" | "webgpu",
     number?: number,
 ) {
     const projectName = info.project.name;
-    const isMobile = info.project.use.isMobile ?? false;
-
-    const mobileString = isMobile ? "-mobile" : "";
-    const numberString = number !== undefined ? `-${number}` : "";
 
     const isWebGpu = api === "webgpu";
     const isWebGpuUnsupported = projectName.includes("<No WebGPU>");
     const unsupportedString =
         isWebGpu && isWebGpuUnsupported ? "-unsupported" : "";
 
-    const fileName = `${name}-${api}${unsupportedString}${mobileString}${numberString}.png`;
+    const engineString = `-${browserName}`;
+
+    const isMobile = info.project.use.isMobile ?? false;
+    const mobileString = isMobile ? "-mobile" : "";
+
+    const numberString = number !== undefined ? `-${number}` : "";
+
+    const fileName = `${name}-${api}${unsupportedString}${engineString}${mobileString}${numberString}.png`;
 
     await expect.soft(page).toHaveScreenshot(fileName, {
-        // Allow for slightly different rendering on different platforms,
-        // since Chrome and Firefox have slightly different text-rendering and inputs etc.
-        maxDiffPixelRatio: 0.01,
-        timeout: 10000,
+        threshold: 0.1,
     });
 }
 
@@ -56,67 +58,149 @@ builds.forEach(({ name, port }) => {
 
         apis.forEach(api => {
             test.describe(api, () => {
-                test("Hello world", async ({ page }, info) => {
+                test("Hello world", async ({ page, browserName }, info) => {
                     const pageName = "hello-world";
 
                     await page.goto(`/${pageName}/${api}`);
-                    await assertScreenshot(page, info, pageName, api);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                    );
                 });
 
-                test("Remounting canvas", async ({ page }, info) => {
+                test("Remounting canvas", async ({
+                    page,
+                    browserName,
+                }, info) => {
                     const pageName = "remount";
 
                     await page.goto(`/${pageName}/${api}`);
                     const show = page.getByLabel("Show");
                     await show.uncheck();
-                    await assertScreenshot(page, info, pageName, api, 1);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        1,
+                    );
                     for (let i = 0; i < 10; i++) {
                         await show.check();
                         await show.uncheck();
                     }
                     await show.check();
-                    await assertScreenshot(page, info, pageName, api, 2);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        2,
+                    );
                 });
 
-                test("Oversized canvas", async ({ page }, info) => {
+                test("Oversized canvas", async ({
+                    page,
+                    browserName,
+                }, info) => {
                     const pageName = "oversized-canvas";
 
                     await page.goto(`/${pageName}/${api}`);
-                    await assertScreenshot(page, info, pageName, api, 1);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        1,
+                    );
                     // Scroll to bottom-right corner
                     await page.evaluate(() =>
                         window.scrollBy(
                             document.body.scrollWidth,
                             document.body.scrollHeight,
                         ),
                     );
-                    await assertScreenshot(page, info, pageName, api, 2);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        2,
+                    );
                 });
 
-                test("Landing page with bubbles", async ({ page }, info) => {
+                test("Landing page with bubbles", async ({
+                    page,
+                    browserName,
+                }, info) => {
                     const pageName = "landing-page-bubbles";
 
                     await page.goto(`/${pageName}/${api}`);
-                    await assertScreenshot(page, info, pageName, api);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                    );
                 });
 
-                test("Landing page with a halo", async ({ page }, info) => {
+                test("Landing page with a halo", async ({
+                    page,
+                    browserName,
+                }, info) => {
                     const pageName = "landing-page-halo";
 
                     await page.goto(`/${pageName}/${api}`);
-                    await assertScreenshot(page, info, pageName, api);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                    );
                 });
 
-                test("Slider component", async ({ page }, info) => {
+                test("Slider component", async ({
+                    page,
+                    browserName,
+                }, info) => {
                     const pageName = "slider";
 
                     await page.goto(`/${pageName}/${api}`);
                     const slider = page.getByRole("slider");
-                    await assertScreenshot(page, info, pageName, api, 1);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        1,
+                    );
                     await slider.fill("1");
-                    await assertScreenshot(page, info, pageName, api, 2);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        2,
+                    );
                     await slider.fill("0");
-                    await assertScreenshot(page, info, pageName, api, 3);
+                    await assertScreenshot(
+                        page,
+                        info,
+                        browserName,
+                        pageName,
+                        api,
+                        3,
+                    );
                 });
             });
         });