From ec1aa34f40ee5003c7cd1689a86a7ca27b7feaa4 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:17:31 -0400 Subject: [PATCH] perf(scroll): extend and trim the Vegas strip in place Every strip extension rebuilt the whole strip (np.concatenate: 2-2.6 ms for a 10-14k px strip at 512x64 on a Pi 4) and every trim copied what was left (1.2-1.8 ms), on the render thread. With ~4 ms of slack per refresh, every extension frame on hdpi missed its refresh (5/5 in each soak run). The strip now lives in a buffer with spare room; cached_array is a view of its live columns. An append writes only the new columns (~0.2 ms), a trim only moves the view's start, and the one full copy happens when the buffer is reallocated (STRIP_SPARE_FACTOR 3: about once every two strip-lengths scrolled). A cached_array set from outside -- the multi-display follower's read-only one, create_scrolling_image's -- is never written through, and a new strip lets the old buffer go. last_copy_bytes says what was copied, and the Vegas frame-timing attribution reports that instead of the whole strip. test_scroll_helper_in_place.py: the buffer is reused and only new columns copied, trims copy nothing, reallocation when the room runs out, outside arrays untouched, and random appends/trims/patches/scrolling checked frame by frame against the old copying strip (mutation-checked). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++ src/common/scroll_helper.py | 79 +++++++++++++-- src/vegas_mode/render_pipeline.py | 16 ++- test/test_scroll_helper_in_place.py | 146 ++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 14 deletions(-) create mode 100644 test/test_scroll_helper_in_place.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ba3bec7..8ac8f7cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -418,6 +418,15 @@ read any of them: ### Scrolling +- A Vegas strip extension no longer costs a late frame. Appending the next + group rebuilt the whole strip (`np.concatenate`, 2-2.6ms for a 10-14k px + strip at 512x64 on a Pi 4) and trimming copied what was left (1.2-1.8ms), + so on hdpi every extension frame missed its refresh. The strip now lives in + a buffer with spare room (`ScrollHelper.STRIP_SPARE_FACTOR`): an append + writes only the new columns (~0.2ms), a trim only moves the start, and the + one full copy happens when the buffer is reallocated, about once every two + strip-lengths scrolled. A strip set from outside (the multi-display + follower's) is never written through. - A Vegas strip extension costs the render thread about a third of what it did. Appending the next group and trimming what has scrolled past each rebuilt the strip's PIL image from its numpy array in full diff --git a/src/common/scroll_helper.py b/src/common/scroll_helper.py index 0f740abd..db2d248f 100644 --- a/src/common/scroll_helper.py +++ b/src/common/scroll_helper.py @@ -114,6 +114,18 @@ class ScrollHelper: self.cached_image = None # see the property below self.cached_array: Optional[np.ndarray] = None # Numpy array cache for fast operations self.total_scroll_width = 0 + # An extended strip lives in a buffer with spare room after it, and + # cached_array is a view of the buffer's live columns: an append writes + # only the new columns, and a trim only moves the view's start. See + # append_content. _strip_view is the view this helper last made; a + # cached_array that is anything else was set from outside and is not + # written through. + self._strip_buffer: Optional[np.ndarray] = None + self._strip_view: Optional[np.ndarray] = None + self._strip_start = 0 + #: Bytes the last append_content / drop_scrolled_prefix copied, for + #: frame-timing attribution (src/common/frame_timing.py note_op). + self.last_copy_bytes = 0 # Pre-allocated buffer for output frame (reused to avoid allocations) self._frame_buffer: Optional[np.ndarray] = None @@ -249,6 +261,7 @@ class ScrollHelper: self.total_scroll_width = 0 self.cached_image = Image.new('RGB', (self.display_width, self.display_height), (0, 0, 0)) self.cached_array = np.array(self.cached_image) + self._forget_strip_buffer() self.scroll_position = 0.0 self.total_distance_scrolled = 0.0 self.scroll_complete = False @@ -284,6 +297,7 @@ class ScrollHelper: self.cached_image = full_image # Convert to numpy array for fast operations self.cached_array = np.array(full_image) + self._forget_strip_buffer() actual_image_width = full_image.width self.total_scroll_width = actual_image_width @@ -712,12 +726,12 @@ class ScrollHelper: addition.paste(img, (x, 0)) x += img.width + element_gap - # numpy concatenate, and no conversion back: the strip can be tens of - # thousands of columns wide and this runs on the render path. The PIL - # image is built from the array only if something reads it (see the - # cached_image property). - self.cached_array = np.concatenate( - (self.cached_array, np.array(addition)), axis=1) + # Written into the spare room after the strip, when there is some: + # the strip can be tens of thousands of columns wide and this runs on + # the render thread, where copying all of it (2-3 ms at 512x64 on a + # Pi 4) cost the frame after every extension. The PIL image is built + # from the array only if something reads it (see cached_image). + self.cached_array = self._extended_strip(np.asarray(addition)) self._defer_image() self.total_scroll_width = self.cached_array.shape[1] self.scroll_complete = False @@ -729,6 +743,42 @@ class ScrollHelper: ) return True + #: Room an extended strip's buffer is given, as a multiple of what it + #: holds when (re)allocated. Trims free columns at the front and appends + #: use them at the back, so with 3x the buffer is reallocated -- the one + #: full copy -- about once every two strip-lengths scrolled. + STRIP_SPARE_FACTOR = 3.0 + + def _extended_strip(self, addition: np.ndarray) -> np.ndarray: + """The strip with ``addition`` after it, written in place when it fits.""" + live = self.cached_array + assert live is not None + width = live.shape[1] + added = addition.shape[1] + buffer = self._strip_buffer + if (live is self._strip_view and buffer is not None + and self._strip_start + width + added <= buffer.shape[1]): + end = self._strip_start + width + buffer[:, end:end + added] = addition + self.last_copy_bytes = addition.nbytes + else: + total = width + added + buffer = np.empty((live.shape[0], max(total + 1, int(total * self.STRIP_SPARE_FACTOR))) + + live.shape[2:], dtype=live.dtype) + buffer[:, :width] = live + buffer[:, width:total] = addition + self._strip_buffer = buffer + self._strip_start = 0 + self.last_copy_bytes = live.nbytes + addition.nbytes + self._strip_view = buffer[:, self._strip_start:self._strip_start + width + added] + return self._strip_view + + def _forget_strip_buffer(self) -> None: + """A new strip replaces the extended one: let its buffer go.""" + self._strip_buffer = None + self._strip_view = None + self._strip_start = 0 + def drop_scrolled_prefix(self, keep_before: int = 0) -> int: """ Discard columns that have already scrolled past, to bound memory. @@ -766,9 +816,18 @@ class ScrollHelper: if cut <= 0: return 0 - # .copy() so the original buffer is released rather than kept alive by - # a numpy view. The PIL image is deferred, as in append_content. - self.cached_array = self.cached_array[:, cut:].copy() + if self.cached_array is self._strip_view: + # Only the view's start moves; the columns behind it are reused + # when the buffer is next reallocated (append_content). + self._strip_view = self.cached_array[:, cut:] + self._strip_start += cut + self.cached_array = self._strip_view + self.last_copy_bytes = 0 + else: + # Not a strip this helper extended: .copy() so the original + # buffer is released rather than kept alive by a view. + self.cached_array = self.cached_array[:, cut:].copy() + self.last_copy_bytes = self.cached_array.nbytes self._defer_image() self.total_scroll_width = self.cached_array.shape[1] self.scroll_position -= cut @@ -881,6 +940,7 @@ class ScrollHelper: # Convert to numpy array for fast operations (required for get_visible_portion) self.cached_array = np.array(image) + self._forget_strip_buffer() # Update scroll width self.total_scroll_width = image.width @@ -1138,6 +1198,7 @@ class ScrollHelper: """ self.cached_image = None self.cached_array = None + self._forget_strip_buffer() self.total_scroll_width = 0 self.scroll_position = 0.0 self.total_distance_scrolled = 0.0 diff --git a/src/vegas_mode/render_pipeline.py b/src/vegas_mode/render_pipeline.py index ba72cb97..861d5ca2 100644 --- a/src/vegas_mode/render_pipeline.py +++ b/src/vegas_mode/render_pipeline.py @@ -466,6 +466,11 @@ class RenderPipeline: if note is not None: note(kind, nbytes) + def _copied_bytes(self) -> int: + """Bytes the scroll helper's last append or trim copied (the strip, if it cannot say).""" + copied = getattr(self.scroll_helper, 'last_copy_bytes', None) + return int(copied) if isinstance(copied, int) else self._strip_nbytes() + def _strip_nbytes(self) -> int: array = self.scroll_helper.cached_array return int(array.nbytes) if array is not None else 0 @@ -660,7 +665,7 @@ class RenderPipeline: element_gap=0, ) if appended: - self._note_op('extend', self._strip_nbytes()) + self._note_op('extend', self._copied_bytes()) logger.info( "[%s] Appended deferred content: strip now %dpx, %dpx ahead", plugin_id, self.scroll_helper.total_scroll_width, @@ -770,7 +775,7 @@ class RenderPipeline: ) if not appended: return False - moved = self._strip_nbytes() + moved = self._copied_bytes() # Where each block starts, laid out as append_content does: a # separator before every block, or -- when there was no strip to @@ -789,9 +794,10 @@ class RenderPipeline: self._static_markers = tuple( (max(0, x - cut), pid) for x, pid in self._static_markers) self._forget_trimmed_records(cut) - # The append built the whole strip anew, and a trim copies what is - # left of it again: both land in the frame after this one. - self._note_op('extend', moved + (self._strip_nbytes() if cut else 0)) + # Both land in the frame after this one: the append's new columns + # (the whole strip when its buffer had to be reallocated), and a + # trim's copy, if it made one. + self._note_op('extend', moved + (self._copied_bytes() if cut else 0)) self._segments_in_scroll = [pid for pid, _ in grouped] self.stats['composition_count'] += 1 diff --git a/test/test_scroll_helper_in_place.py b/test/test_scroll_helper_in_place.py new file mode 100644 index 00000000..b1bf3858 --- /dev/null +++ b/test/test_scroll_helper_in_place.py @@ -0,0 +1,146 @@ +"""ScrollHelper extends and trims a strip in place (src/common/scroll_helper.py). + +Every extension of the Vegas strip used to rebuild it whole (np.concatenate) +and every trim copied what was left: 3.5-4.5 ms on the render thread at 512x64 +on a Pi 4, so the frame after each extension was late. The strip now lives in +a buffer with spare room: an append writes only the new columns, a trim only +moves the view's start, and a full copy happens only when the buffer is +reallocated. What a frame shows must not change at all. +""" +import random +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.common.scroll_helper import ScrollHelper # noqa: E402 + +W, H = 64, 16 + + +def _items(rng, n): + out = [] + for _ in range(n): + width = rng.randint(5, 60) + out.append(Image.frombytes("RGB", (width, H), + bytes(rng.randrange(256) for _ in range(width * H * 3)))) + return out + + +def _helper(): + helper = ScrollHelper(W, H) + helper.create_scrolling_image(_items(random.Random(1), 4), item_gap=3, lead_gap=0) + return helper + + +def _reference_append(strip, items, gap): + """What append_content used to do.""" + width = sum(i.width for i in items) + gap * len(items) + addition = Image.new("RGB", (width, H)) + x = 0 + for item in items: + x += gap + addition.paste(item, (x, 0)) + x += item.width + return np.concatenate((strip, np.array(addition)), axis=1) + + +def test_an_append_writes_into_the_buffer_and_copies_only_the_new_columns(): + helper = _helper() + helper.append_content(_items(random.Random(2), 2), item_gap=3) # allocates + buffer = helper._strip_buffer + before = helper.cached_array.shape[1] + items = _items(random.Random(3), 2) + helper.append_content(items, item_gap=3) + assert helper._strip_buffer is buffer + assert np.shares_memory(helper.cached_array, buffer) + added = helper.cached_array.shape[1] - before + assert helper.last_copy_bytes == added * H * 3 + + +def test_a_trim_copies_nothing(): + helper = _helper() + helper.append_content(_items(random.Random(2), 3), item_gap=3) + helper.scroll_position = 120.0 + cut = helper.drop_scrolled_prefix() + assert cut == 120 and helper.last_copy_bytes == 0 + assert np.shares_memory(helper.cached_array, helper._strip_buffer) + + +def test_the_buffer_is_reallocated_when_the_room_runs_out(): + helper = _helper() + helper.append_content(_items(random.Random(2), 1), item_gap=3) + first = helper._strip_buffer + rng = random.Random(4) + while helper._strip_buffer is first: + helper.append_content(_items(rng, 3), item_gap=3) + assert helper.last_copy_bytes == helper.cached_array.nbytes + assert helper._strip_start == 0 + + +def test_a_strip_set_from_outside_is_never_written_through(): + # The multi-display follower adopts a read-only array straight from an image. + helper = _helper() + helper.append_content(_items(random.Random(2), 1), item_gap=3) + adopted = np.asarray(Image.new("RGB", (300, H), (9, 9, 9))) + helper.cached_array = adopted + helper.total_scroll_width = 300 + helper.append_content(_items(random.Random(3), 1), item_gap=3) + assert not np.shares_memory(helper.cached_array, adopted) + assert (adopted == 9).all() + helper.cached_array = adopted + helper.scroll_position = 100.0 + helper.drop_scrolled_prefix() + assert not np.shares_memory(helper.cached_array, adopted) + + +def test_a_new_strip_lets_the_old_buffer_go(): + helper = _helper() + helper.append_content(_items(random.Random(2), 1), item_gap=3) + helper.create_scrolling_image(_items(random.Random(5), 2), item_gap=3, lead_gap=0) + assert helper._strip_buffer is None + helper.append_content(_items(random.Random(2), 1), item_gap=3) + helper.clear_cache() + assert helper._strip_buffer is None and helper._strip_view is None + + +@pytest.mark.parametrize("seed", range(12)) +def test_every_frame_matches_the_old_copying_strip(seed): + """Random appends, trims, scrolling and patches, against a strip kept the old way.""" + rng = random.Random(seed) + helper = _helper() + reference = helper.cached_array.copy() + for _ in range(60): + op = rng.random() + if op < 0.35: + items = _items(rng, rng.randint(1, 3)) + gap = rng.randint(0, 6) + helper.append_content(items, item_gap=gap) + reference = _reference_append(reference, items, gap) + elif op < 0.55: + keep = rng.randint(0, W) + before = helper.scroll_position + cut = helper.drop_scrolled_prefix(keep_before=keep) + reference = reference[:, cut:].copy() + assert helper.scroll_position == before - cut + elif op < 0.7 and helper.cached_array.shape[1] > 8: + x = rng.randrange(helper.cached_array.shape[1] - 4) + pixels = np.full((H, 4, 3), rng.randrange(256), dtype=np.uint8) + helper.patch_columns(x, pixels) + reference[:, x:x + 4] = pixels + else: + limit = max(0, helper.cached_array.shape[1] - W - 1) + helper.scroll_position = float(rng.randint(0, limit)) if limit else 0.0 + assert helper.cached_array.shape == reference.shape + assert (helper.cached_array == reference).all() + assert helper.total_scroll_width == reference.shape[1] + frame = np.asarray(helper.get_visible_portion()) + x = int(helper.scroll_position) + if x + W <= reference.shape[1]: + assert (frame == reference[:, x:x + W]).all() + # The lazily built image is the strip as it stands. + assert (np.asarray(helper.cached_image) == reference).all()