🧪 Test Hub
testing: Demo target v3.0.0 · built-in demo site

DEMO-005 Visual comparison (screenshot vs baseline)

Edit Load test Duplicate

Pattern for image comparison with OpenCV + scikit-image. First run saves a baseline; later runs fail if the page looks too different and leave a red-marked diff image behind.

demoimagevisual version: 1.0 DEMO-005__visual_compare.py

Runs
3
Pass rate
100%
3 passed / 0 failed
Avg duration
2s
p95 2s
Estimate
2s
avg of last 10 judged runs
Flakiness
0.00
0 stable · 1 alternates
Current streak
3
passed

Last 3 results newest on the right — click a square to open that run

Duration per run point color = outcome; dashed = 7-run average; click a point to open the run

Step timing trends the same step compared across the last 3 runs — slowest steps first, one line each

How long it takes distribution of 3 runs — two humps mean two different behaviours hiding behind one average

fastest 1s · slowest 2s

Why this test failed grouped by message, last 90 days

No failures recorded. 👍

Outcomes by target version

What this test does plain language, derived from the code

  1. Open the app's page
  2. Wait until #name appears
  3. shot = ctx.artifacts_dir / "screenshots" / "current.png"
  4. shot.parent.mkdir(parents=True, exist_ok=True)
  5. page.screenshot(path=str(shot))
  6. baseline = Path(__file__).with_name(f"{ctx.test_id}__baseline.png")
  7. if not baseline.exists():
  8. baseline.write_bytes(shot.read_bytes())
  9. ctx.log(f"no baseline yet — saved this run as the baseline "
  10. f"({baseline.name}). Re-run to start comparing.")
  11. return
  12. current = cv2.imread(str(shot))
  13. expected = cv2.imread(str(baseline))
  14. assert current is not None and expected is not None, "could not read the images"
  15. if current.shape != expected.shape:
  16. raise AssertionError(
  17. f"page size changed: baseline {expected.shape[1]}x{expected.shape[0]}, "
  18. f"now {current.shape[1]}x{current.shape[0]}")
  19. grey_a = cv2.cvtColor(expected, cv2.COLOR_BGR2GRAY)
  20. grey_b = cv2.cvtColor(current, cv2.COLOR_BGR2GRAY)
  21. score, diff = ssim(grey_a, grey_b, full=True)
  22. changed = float(np.mean((diff < 0.9).astype(np.float32)))
  23. ctx.log(f"similarity {score:.4f} · {changed*100:.2f}% of pixels differ")
  24. heat = (255 - (diff * 255)).astype("uint8")
  25. overlay = current.copy()
  26. overlay[heat > 60] = (0, 0, 255) # mark differing areas in red
  27. cv2.imwrite(str(shot.with_name("02-diff.png")), overlay)
  28. assert score >= SSIM_MIN and changed <= DIFF_MAX, (
  29. f"the page looks different: similarity {score:.4f} (min {SSIM_MIN}), "
  30. f"{changed*100:.2f}% pixels changed (max {DIFF_MAX*100:.0f}%). "
  31. f"See 02-diff.png in this run's screenshots — red marks what moved.")
Show the code
"""Visual comparison: does the page still LOOK right?

Shows the pattern for screenshot comparison with the bundled image stack.
First run saves a baseline into the test's own folder; later runs compare
against it and fail if too many pixels moved.

Tune SSIM_MIN / DIFF_MAX to taste: anti-aliasing and clocks always differ a
little, so a threshold is normal. Delete the baseline PNG to re-bless it
after an intentional UI change.
"""
from pathlib import Path

SSIM_MIN = 0.97      # 1.0 = identical
DIFF_MAX = 0.02      # fraction of pixels allowed to differ


def run(page, ctx):
    page.goto(ctx.base_url)
    page.wait_for_selector("#name")

    shot = ctx.artifacts_dir / "screenshots" / "current.png"
    shot.parent.mkdir(parents=True, exist_ok=True)
    page.screenshot(path=str(shot))

    baseline = Path(__file__).with_name(f"{ctx.test_id}__baseline.png")
    if not baseline.exists():
        baseline.write_bytes(shot.read_bytes())
        ctx.log(f"no baseline yet — saved this run as the baseline "
                f"({baseline.name}). Re-run to start comparing.")
        return

    import cv2
    import numpy as np
    from skimage.metrics import structural_similarity as ssim

    current = cv2.imread(str(shot))
    expected = cv2.imread(str(baseline))
    assert current is not None and expected is not None, "could not read the images"

    if current.shape != expected.shape:
        raise AssertionError(
            f"page size changed: baseline {expected.shape[1]}x{expected.shape[0]}, "
            f"now {current.shape[1]}x{current.shape[0]}")

    grey_a = cv2.cvtColor(expected, cv2.COLOR_BGR2GRAY)
    grey_b = cv2.cvtColor(current, cv2.COLOR_BGR2GRAY)
    score, diff = ssim(grey_a, grey_b, full=True)
    changed = float(np.mean((diff < 0.9).astype(np.float32)))
    ctx.log(f"similarity {score:.4f} · {changed*100:.2f}% of pixels differ")

    # Always leave a human-readable diff image behind, pass or fail.
    heat = (255 - (diff * 255)).astype("uint8")
    overlay = current.copy()
    overlay[heat > 60] = (0, 0, 255)          # mark differing areas in red
    cv2.imwrite(str(shot.with_name("02-diff.png")), overlay)

    assert score >= SSIM_MIN and changed <= DIFF_MAX, (
        f"the page looks different: similarity {score:.4f} (min {SSIM_MIN}), "
        f"{changed*100:.2f}% pixels changed (max {DIFF_MAX*100:.0f}%). "
        f"See 02-diff.png in this run's screenshots — red marks what moved.")

All runs

RunStatusQueuedDuration VersionTriggerBatch
#455 passed 2026-09-25 00:25:05 2s 3.0.0 cli-adopted 8 tests (terminal)
#447 passed 2026-09-25 00:24:35 2s 3.0.0 cli-adopted 8 tests (terminal)
#439 passed 2026-09-25 00:24:06 1s 3.0.0 cli-adopted 8 tests (terminal)