perf(scroll): build the strip's PIL image only when something reads it (#695)

* perf(timing): say which render-thread work a late frame followed The soak already says how often a moving frame reached the panel late, but not what the render thread was doing just before it. Vegas does two kinds of work there between frames -- building its strip (compose, extend) and, with live elements, patching changed pixels into it -- and deciding whether either is affordable needs their own numbers. - FrameTimingRecorder.note_op(kind, nbytes) tags the next presented frame. Totals gain op_frames, late_op_frames, op_freezes and op_bytes per kind; aggregate() still takes frames without ops. The file schema is unchanged. - Vegas tags compose and every strip extension (with the bytes it copied). - frame_soak prints an "after work" table: frames, late %, freezes and MB moved per kind, only when something tagged its work. - render_bench gains --strip-screens (Vegas-sized strips), --patch-bytes / --patch-every / --patch-where (in-place column writes, as a live element update does) and --extend-every-screens / --extend-width (append + trim on a fixed cadence that holds the strip's width). No runtime behaviour changes: this is the measurement gate for live Vegas elements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(changelog): note the frame-op attribution and bench modes Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * perf(scroll): build the strip's PIL image only when something reads it Every Vegas strip extension rebuilt ScrollHelper.cached_image from cached_array in full, twice (append, then trim), on the render thread: Image.fromarray is 1.7ms for an 8,000px strip and 3.8ms for 20,000px on a Pi 4 (measured on ledpi), about two thirds of an extension's render-thread cost. Nothing on the frame path reads the image's pixels; every frame is cut from the array. cached_image is now a property. append_content and drop_scrolled_prefix defer it; the first read builds it from the array it started with and keeps it only if the strip has not changed meanwhile, so a sync push racing an extension cannot leave a stale image cached. Assigning cached_image stores exactly what was assigned, as before. has_strip() says whether there is a strip without building its image; the helper's frame path, Vegas and the adapter's scroll-cache invalidation use it. The strip is also no longer held in memory twice. In Vegas the image is now built only by a multi-display sync push. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 14:25:08 +00:00 · 2026-09-30 20:50:34 -04:00
parent 77862b631b
commit 596809acc3
5 changed files with 322 additions and 29 deletions
@@ -0,0 +1,218 @@
+"""ScrollHelper builds the strip's PIL image only when something reads it.
+
+Vegas extends and trims one long strip on the render thread. Each of those used
+to rebuild ``cached_image`` from ``cached_array`` in full -- 1.7-3.8ms apiece on
+a Pi 4 for a Vegas-sized strip, twice per extension -- though nothing on the
+frame path reads the image's pixels. These tests pin that the frame path never
+builds it, that a read still gets the right pixels, and that the two threads
+which do read it (a multi-display sync push, the render thread) cannot leave a
+stale image behind.
+"""
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+from PIL import Image
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from src.common import scroll_helper as scroll_helper_module  # noqa: E402
+from src.common.scroll_helper import ScrollHelper  # noqa: E402
+
+W, H = 64, 16
+
+
+def _block(width, seed):
+    # frombytes, not fromarray: the no_fromarray fixture refuses the latter
+    # everywhere, and a test block is not the strip.
+    rng = np.random.default_rng(seed)
+    pixels = rng.integers(0, 255, (H, width, 3), dtype=np.uint8)
+    return Image.frombytes("RGB", (width, H), pixels.tobytes())
+
+
+def _helper(width=400):
+    helper = ScrollHelper(W, H)
+    helper.set_scrolling_image(_block(width, 0))
+    return helper
+
+
+@pytest.fixture
+def no_fromarray(monkeypatch):
+    """Fail if the helper builds a PIL image from its array."""
+    def refuse(*_a, **_k):
+        raise AssertionError("the strip's PIL image was built")
+    monkeypatch.setattr(scroll_helper_module.Image, "fromarray", refuse)
+
+
+def test_append_and_trim_do_not_build_the_image(no_fromarray):
+    helper = _helper()
+    helper.scroll_position = 300.0
+    assert helper.append_content([_block(200, 1)], item_gap=8)
+    assert helper.drop_scrolled_prefix(keep_before=W) > 0
+    assert helper.__dict__["_cached_image"] is None
+
+
+def test_the_frame_path_never_builds_the_image(no_fromarray):
+    helper = _helper()
+    helper.set_pixels_per_frame(2)
+    for i in range(400):
+        if helper.remaining_unscrolled() <= 2 * W:
+            helper.append_content([_block(150, i)], item_gap=8)
+            helper.drop_scrolled_prefix(keep_before=W)
+        helper.update_scroll_position()
+        frame = helper.get_visible_portion()
+        assert frame is not None and frame.size == (W, H)
+        helper.get_scroll_info()
+    assert not helper.is_scroll_complete()
+
+
+def test_reading_the_image_gives_the_strip_as_it_is():
+    helper = _helper()
+    helper.scroll_position = 250.0
+    helper.append_content([_block(120, 7)], item_gap=8)
+    helper.drop_scrolled_prefix(keep_before=W)
+    image = helper.cached_image
+    assert image.size == (helper.cached_array.shape[1], H)
+    assert np.array_equal(np.asarray(image), helper.cached_array)
+    assert helper.total_scroll_width == image.width
+
+
+def test_the_built_image_is_kept_until_the_strip_changes():
+    helper = _helper()
+    helper.append_content([_block(50, 1)], item_gap=0)
+    first = helper.cached_image
+    assert helper.cached_image is first
+    helper.append_content([_block(50, 2)], item_gap=0)
+    assert helper.cached_image is not first
+    assert helper.cached_image.width == first.width + 50
+
+
+def test_an_assigned_image_is_kept_exactly():
+    # Plugins and the multi-display follower assign cached_image themselves.
+    helper = _helper()
+    helper.append_content([_block(50, 1)], item_gap=0)
+    mine = _block(99, 3)
+    helper.cached_image = mine
+    assert helper.cached_image is mine
+    helper.cached_image = None
+    assert helper.cached_image is None
+
+
+def test_frames_are_the_same_as_with_an_eager_image():
+    lazy = _helper()
+    eager = _helper()
+    blocks = [_block(90, 10 + i) for i in range(6)]
+    for i, block in enumerate(blocks):
+        for helper in (lazy, eager):
+            helper.scroll_position = 60.0 * (i + 1)
+            helper.append_content([block], item_gap=8)
+            helper.drop_scrolled_prefix(keep_before=W)
+        eager.cached_image = Image.fromarray(eager.cached_array)   # the old way
+        for x in range(0, lazy.cached_array.shape[1] - W, 7):
+            lazy.scroll_position = eager.scroll_position = float(x)
+            assert lazy.get_visible_portion().tobytes() == \
+                eager.get_visible_portion().tobytes()
+
+
+def test_a_read_racing_an_extension_does_not_keep_a_stale_image(monkeypatch):
+    # The sync push reads the image on its own thread. If the render thread
+    # extends the strip while that read is building the image, the read gets
+    # the strip as it was, and the next read must not be handed it again.
+    helper = _helper()
+    helper.append_content([_block(40, 1)], item_gap=0)
+    real = Image.fromarray
+
+    def build_while_the_strip_changes(array, *a, **k):
+        monkeypatch.setattr(scroll_helper_module.Image, "fromarray", real)
+        helper.append_content([_block(40, 2)], item_gap=0)   # "render thread"
+        return real(array, *a, **k)
+
+    monkeypatch.setattr(scroll_helper_module.Image, "fromarray",
+                        build_while_the_strip_changes)
+    before_width = helper.cached_array.shape[1]
+    raced = helper.cached_image
+    assert raced.width == before_width
+    fresh = helper.cached_image
+    assert fresh.width == before_width + 40
+    assert np.array_equal(np.asarray(fresh), helper.cached_array)
+
+
+def test_clearing_the_cache_forgets_a_deferred_image():
+    helper = _helper()
+    helper.append_content([_block(40, 1)], item_gap=0)
+    helper.clear_cache()
+    assert helper.cached_image is None
+    assert not helper.has_strip()
+    assert helper.get_visible_portion() is None
+    assert helper.remaining_unscrolled() == 0
+
+
+def test_a_helper_with_no_strip_has_nothing_to_scroll():
+    helper = ScrollHelper(W, H)
+    assert not helper.has_strip()
+    helper.update_scroll_position()
+    assert helper.scroll_position == 0.0
+    assert helper.get_scroll_info()["cached_image_size"] is None
+
+
+def test_dropping_a_plugins_scroll_cache_does_not_build_it_first(no_fromarray):
+    # Vegas clears a plugin's own scroll cache on the render thread whenever
+    # the plugin updates; that must not build a deferred image to discard it.
+    from types import SimpleNamespace
+
+    from src.vegas_mode.plugin_adapter import PluginAdapter
+
+    helper = _helper()
+    helper.append_content([_block(40, 1)], item_gap=0)
+    dm = SimpleNamespace(width=W, height=H, image=Image.new("RGB", (W, H)))
+    adapter = PluginAdapter(dm)
+    assert adapter.invalidate_plugin_scroll_cache(
+        SimpleNamespace(scroll_helper=helper), "p")
+    assert helper.cached_array is None and not helper.has_strip()
+
+
+def test_vegas_extends_without_building_the_image(no_fromarray):
+    from src.vegas_mode.config import VegasModeConfig
+    from src.vegas_mode.render_pipeline import RenderPipeline
+
+    groups = [[("a", [_block(300, 1)])]] + [[(f"p{i}", [_block(300, i + 2)])]
+                                           for i in range(6)]
+
+    class Stream:
+        plugin_manager = type("PM", (), {"plugins": {}})()
+        plugin_adapter = None
+        i = 0
+
+        def get_grouped_content_for_composition(self):
+            return groups[0]
+
+        def get_active_plugin_ids(self):
+            return ["a"]
+
+        def take_next_group(self, count=None, offscreen_only=False):
+            self.i += 1
+            return groups[self.i] if self.i < len(groups) else []
+
+    class DM:
+        width, height = W, H
+
+        def __init__(self):
+            self.image = Image.new("RGB", (W, H))
+
+        def set_scrolling_state(self, *a):
+            pass
+
+        def update_display(self):
+            pass
+
+    p = RenderPipeline(VegasModeConfig(lead_in_width=0, continuous_scroll=True),
+                       DM(), Stream())
+    # compose pastes into a new image and converts that to the array once;
+    # it never needs fromarray either.
+    assert p.compose_scroll_content()
+    for _ in range(5):
+        p.scroll_helper.scroll_position += 250
+        assert p.extend_scroll_content()
+        assert p.render_frame()
+    assert p.scroll_helper.__dict__["_cached_image"] is None