Pattern for image comparison with OpenCV + scikit-image. First run saves a baseline; later runs fail if the page looks too different and leave a red-marked diff image behind.
demoimagevisual version: 1.0 DEMO-005__visual_compare.py
Runs
3
Pass rate
100%
3 passed / 0 failed
Avg duration
2s
p95 2s
Estimate
2s
avg of last 10 judged runs
Flakiness
0.00
0 stable · 1 alternates
Current streak
3
passed
Duration per run point color = outcome; dashed = 7-run average; click a point to open the run
Step timing trends the same step compared across the last 3 runs — slowest steps first, one line each
How long it takes distribution of 3 runs — two humps mean two different behaviours hiding behind one average
fastest 1s · slowest 2s
Why this test failed grouped by message, last 90 days
No failures recorded. 👍
Outcomes by target version
What this test does plain language, derived from the code
- Open the app's page
- Wait until #name appears
- shot = ctx.artifacts_dir / "screenshots" / "current.png"
- shot.parent.mkdir(parents=True, exist_ok=True)
- page.screenshot(path=str(shot))
- baseline = Path(__file__).with_name(f"{ctx.test_id}__baseline.png")
- if not baseline.exists():
- baseline.write_bytes(shot.read_bytes())
- ctx.log(f"no baseline yet — saved this run as the baseline "
- f"({baseline.name}). Re-run to start comparing.")
- return
- current = cv2.imread(str(shot))
- expected = cv2.imread(str(baseline))
- assert current is not None and expected is not None, "could not read the images"
- if current.shape != expected.shape:
- raise AssertionError(
- f"page size changed: baseline {expected.shape[1]}x{expected.shape[0]}, "
- f"now {current.shape[1]}x{current.shape[0]}")
- grey_a = cv2.cvtColor(expected, cv2.COLOR_BGR2GRAY)
- grey_b = cv2.cvtColor(current, cv2.COLOR_BGR2GRAY)
- score, diff = ssim(grey_a, grey_b, full=True)
- changed = float(np.mean((diff < 0.9).astype(np.float32)))
- ctx.log(f"similarity {score:.4f} · {changed*100:.2f}% of pixels differ")
- heat = (255 - (diff * 255)).astype("uint8")
- overlay = current.copy()
- overlay[heat > 60] = (0, 0, 255) # mark differing areas in red
- cv2.imwrite(str(shot.with_name("02-diff.png")), overlay)
- assert score >= SSIM_MIN and changed <= DIFF_MAX, (
- f"the page looks different: similarity {score:.4f} (min {SSIM_MIN}), "
- f"{changed*100:.2f}% pixels changed (max {DIFF_MAX*100:.0f}%). "
- f"See 02-diff.png in this run's screenshots — red marks what moved.")
Show the code
"""Visual comparison: does the page still LOOK right?
Shows the pattern for screenshot comparison with the bundled image stack.
First run saves a baseline into the test's own folder; later runs compare
against it and fail if too many pixels moved.
Tune SSIM_MIN / DIFF_MAX to taste: anti-aliasing and clocks always differ a
little, so a threshold is normal. Delete the baseline PNG to re-bless it
after an intentional UI change.
"""
from pathlib import Path
SSIM_MIN = 0.97 # 1.0 = identical
DIFF_MAX = 0.02 # fraction of pixels allowed to differ
def run(page, ctx):
page.goto(ctx.base_url)
page.wait_for_selector("#name")
shot = ctx.artifacts_dir / "screenshots" / "current.png"
shot.parent.mkdir(parents=True, exist_ok=True)
page.screenshot(path=str(shot))
baseline = Path(__file__).with_name(f"{ctx.test_id}__baseline.png")
if not baseline.exists():
baseline.write_bytes(shot.read_bytes())
ctx.log(f"no baseline yet — saved this run as the baseline "
f"({baseline.name}). Re-run to start comparing.")
return
import cv2
import numpy as np
from skimage.metrics import structural_similarity as ssim
current = cv2.imread(str(shot))
expected = cv2.imread(str(baseline))
assert current is not None and expected is not None, "could not read the images"
if current.shape != expected.shape:
raise AssertionError(
f"page size changed: baseline {expected.shape[1]}x{expected.shape[0]}, "
f"now {current.shape[1]}x{current.shape[0]}")
grey_a = cv2.cvtColor(expected, cv2.COLOR_BGR2GRAY)
grey_b = cv2.cvtColor(current, cv2.COLOR_BGR2GRAY)
score, diff = ssim(grey_a, grey_b, full=True)
changed = float(np.mean((diff < 0.9).astype(np.float32)))
ctx.log(f"similarity {score:.4f} · {changed*100:.2f}% of pixels differ")
# Always leave a human-readable diff image behind, pass or fail.
heat = (255 - (diff * 255)).astype("uint8")
overlay = current.copy()
overlay[heat > 60] = (0, 0, 255) # mark differing areas in red
cv2.imwrite(str(shot.with_name("02-diff.png")), overlay)
assert score >= SSIM_MIN and changed <= DIFF_MAX, (
f"the page looks different: similarity {score:.4f} (min {SSIM_MIN}), "
f"{changed*100:.2f}% pixels changed (max {DIFF_MAX*100:.0f}%). "
f"See 02-diff.png in this run's screenshots — red marks what moved.")
All runs
| Run | Status | Queued | Duration | Version | Trigger | Batch |
|---|---|---|---|---|---|---|
| #455 | passed | 2026-09-25 00:25:05 | 2s | 3.0.0 | cli-adopted | 8 tests (terminal) |
| #447 | passed | 2026-09-25 00:24:35 | 2s | 3.0.0 | cli-adopted | 8 tests (terminal) |
| #439 | passed | 2026-09-25 00:24:06 | 1s | 3.0.0 | cli-adopted | 8 tests (terminal) |