diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ba3bec7..8ac8f7cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -418,6 +418,15 @@ read any of them: ### Scrolling +- A Vegas strip extension no longer costs a late frame. Appending the next + group rebuilt the whole strip (`np.concatenate`, 2-2.6ms for a 10-14k px + strip at 512x64 on a Pi 4) and trimming copied what was left (1.2-1.8ms), + so on hdpi every extension frame missed its refresh. The strip now lives in + a buffer with spare room (`ScrollHelper.STRIP_SPARE_FACTOR`): an append + writes only the new columns (~0.2ms), a trim only moves the start, and the + one full copy happens when the buffer is reallocated, about once every two + strip-lengths scrolled. A strip set from outside (the multi-display + follower's) is never written through. - A Vegas strip extension costs the render thread about a third of what it did. Appending the next group and trimming what has scrolled past each rebuilt the strip's PIL image from its numpy array in full diff --git a/src/common/scroll_helper.py b/src/common/scroll_helper.py index 0f740abd..a0f2bac8 100644 --- a/src/common/scroll_helper.py +++ b/src/common/scroll_helper.py @@ -18,7 +18,7 @@ Features: import logging import math import time -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List, Tuple from PIL import Image import numpy as np @@ -29,6 +29,15 @@ import numpy as np FPS_LOG_INTERVAL = 5.0 +def _rgb_pixels(item) -> np.ndarray: + """An appended item's pixels as an RGB array, as pasting it would draw them.""" + if isinstance(item, np.ndarray): + return item + if item.mode != 'RGB': + item = item.convert('RGB') + return np.asarray(item) + + def frame_stats(frame_times: list) -> Dict[str, Any]: """Summary statistics over one window of frame durations (seconds). @@ -114,6 +123,18 @@ class ScrollHelper: self.cached_image = None # see the property below self.cached_array: Optional[np.ndarray] = None # Numpy array cache for fast operations self.total_scroll_width = 0 + # An extended strip lives in a buffer with spare room after it, and + # cached_array is a view of the buffer's live columns: an append writes + # only the new columns, and a trim only moves the view's start. See + # append_content. _strip_view is the view this helper last made; a + # cached_array that is anything else was set from outside and is not + # written through. + self._strip_buffer: Optional[np.ndarray] = None + self._strip_view: Optional[np.ndarray] = None + self._strip_start = 0 + #: Bytes the last append_content / drop_scrolled_prefix copied, for + #: frame-timing attribution (src/common/frame_timing.py note_op). + self.last_copy_bytes = 0 # Pre-allocated buffer for output frame (reused to avoid allocations) self._frame_buffer: Optional[np.ndarray] = None @@ -249,6 +270,7 @@ class ScrollHelper: self.total_scroll_width = 0 self.cached_image = Image.new('RGB', (self.display_width, self.display_height), (0, 0, 0)) self.cached_array = np.array(self.cached_image) + self._forget_strip_buffer() self.scroll_position = 0.0 self.total_distance_scrolled = 0.0 self.scroll_complete = False @@ -284,6 +306,7 @@ class ScrollHelper: self.cached_image = full_image # Convert to numpy array for fast operations self.cached_array = np.array(full_image) + self._forget_strip_buffer() actual_image_width = full_image.width self.total_scroll_width = actual_image_width @@ -680,7 +703,10 @@ class ScrollHelper: strip also defers completion, which is the intent. Args: - content_items: Images to append, in order + content_items: Images to append, in order. An item may instead be + its pixels already as an RGB array (``np.asarray`` of an RGB + image), so a caller can do that conversion off the render + thread (Vegas prepares its blocks with the group). item_gap: Gap between appended items, and between the existing content and the first appended item element_gap: Extra gap after each item, mirroring @@ -695,40 +721,100 @@ class ScrollHelper: if self.cached_array is None or not self.has_strip(): # Nothing to extend yet — this is just the first build. self.create_scrolling_image( - content_items, item_gap=item_gap, element_gap=element_gap, lead_gap=0) + [Image.fromarray(item) if isinstance(item, np.ndarray) else item + for item in content_items], + item_gap=item_gap, element_gap=element_gap, lead_gap=0) return True gap = max(0, item_gap) - addition_width = ( - sum(img.width for img in content_items) - + gap * len(content_items) # one leading gap per item - + element_gap * len(content_items) - ) - - addition = Image.new('RGB', (addition_width, self.display_height), (0, 0, 0)) + pieces = [] x = 0 - for img in content_items: + for item in content_items: x += gap # separate from whatever precedes - addition.paste(img, (x, 0)) - x += img.width + element_gap + pixels = _rgb_pixels(item) + pieces.append((x, pixels)) + x += pixels.shape[1] + element_gap + addition_width = x - # numpy concatenate, and no conversion back: the strip can be tens of - # thousands of columns wide and this runs on the render path. The PIL - # image is built from the array only if something reads it (see the - # cached_image property). - self.cached_array = np.concatenate( - (self.cached_array, np.array(addition)), axis=1) + # Each item is written straight into the spare room after the strip, + # when there is some: the strip can be tens of thousands of columns + # wide and this runs on the render thread, where copying all of it + # (2-3 ms at 512x64 on a Pi 4) -- or even laying the items out in an + # image of their own first (another 4-5 ms) -- cost the frame after + # every extension. The PIL image is built from the array only if + # something reads it (see cached_image). + self.cached_array = self._extended_strip(pieces, addition_width) self._defer_image() self.total_scroll_width = self.cached_array.shape[1] self.scroll_complete = False - self.logger.info( + # Debug: this runs on the render thread, and the caller (Vegas) logs + # each extension itself. + self.logger.debug( "Appended %d item(s) (%dpx) to scroll strip: now %dpx, position %.0f", len(content_items), addition_width, self.total_scroll_width, self.scroll_position ) return True + #: Room an extended strip's buffer is given, as a multiple of what it + #: holds when (re)allocated. Trims free columns at the front and appends + #: use them at the back, so with 3x the buffer is reallocated -- the one + #: full copy -- about once every two strip-lengths scrolled. + STRIP_SPARE_FACTOR = 3.0 + + def _extended_strip(self, pieces: List[Tuple[int, np.ndarray]], added: int) -> np.ndarray: + """The strip with ``added`` black columns after it, ``pieces`` drawn in. + + Each piece is ``(x, pixels)``, x counted from the old strip's end; + written in place when the buffer has the room. + """ + live = self.cached_array + if live is None: + # append_content builds a first strip itself and never comes here. + raise RuntimeError("no strip to extend") + width = live.shape[1] + buffer = self._strip_buffer + if (live is self._strip_view and buffer is not None + and self._strip_start + width + added <= buffer.shape[1]): + end = self._strip_start + width + self.last_copy_bytes = 0 + else: + total = width + added + buffer = np.empty((live.shape[0], max(total + 1, int(total * self.STRIP_SPARE_FACTOR))) + + live.shape[2:], dtype=live.dtype) + buffer[:, :width] = live + self._strip_buffer = buffer + self._strip_start = 0 + end = width + self.last_copy_bytes = live.nbytes + rows = buffer.shape[0] + region = buffer[:, end:end + added] + # Black only where no piece lands -- the gaps, and below a short + # piece: blanking the whole region first cost as much again as + # writing the pieces (1.8 ms at 512x64 on a Pi 4). + covered = 0 + for x, pixels in pieces: + pixels = pixels[:rows] + cols = pixels.shape[1] + if x > covered: + region[:, covered:x] = 0 + region[:pixels.shape[0], x:x + cols] = pixels + if pixels.shape[0] < rows: + region[pixels.shape[0]:, x:x + cols] = 0 + covered = max(covered, x + cols) + if covered < added: + region[:, covered:] = 0 + self.last_copy_bytes += region.nbytes + self._strip_view = buffer[:, self._strip_start:self._strip_start + width + added] + return self._strip_view + + def _forget_strip_buffer(self) -> None: + """A new strip replaces the extended one: let its buffer go.""" + self._strip_buffer = None + self._strip_view = None + self._strip_start = 0 + def drop_scrolled_prefix(self, keep_before: int = 0) -> int: """ Discard columns that have already scrolled past, to bound memory. @@ -766,9 +852,18 @@ class ScrollHelper: if cut <= 0: return 0 - # .copy() so the original buffer is released rather than kept alive by - # a numpy view. The PIL image is deferred, as in append_content. - self.cached_array = self.cached_array[:, cut:].copy() + if self.cached_array is self._strip_view: + # Only the view's start moves; the columns behind it are reused + # when the buffer is next reallocated (append_content). + self._strip_view = self.cached_array[:, cut:] + self._strip_start += cut + self.cached_array = self._strip_view + self.last_copy_bytes = 0 + else: + # Not a strip this helper extended: .copy() so the original + # buffer is released rather than kept alive by a view. + self.cached_array = self.cached_array[:, cut:].copy() + self.last_copy_bytes = self.cached_array.nbytes self._defer_image() self.total_scroll_width = self.cached_array.shape[1] self.scroll_position -= cut @@ -881,6 +976,7 @@ class ScrollHelper: # Convert to numpy array for fast operations (required for get_visible_portion) self.cached_array = np.array(image) + self._forget_strip_buffer() # Update scroll width self.total_scroll_width = image.width @@ -1138,6 +1234,7 @@ class ScrollHelper: """ self.cached_image = None self.cached_array = None + self._forget_strip_buffer() self.total_scroll_width = 0 self.scroll_position = 0.0 self.total_distance_scrolled = 0.0 diff --git a/src/vegas_mode/live_worker.py b/src/vegas_mode/live_worker.py index 4dfece55..61c4628a 100644 --- a/src/vegas_mode/live_worker.py +++ b/src/vegas_mode/live_worker.py @@ -369,6 +369,7 @@ class VegasWorker(threading.Thread): member = p.stream_manager.fetch_group_member( job.pending.pop(0), offscreen_only=True) if member is not None: + p.prepare_group_member(member) job.group.append(member) if not job.pending: self._group_job = None diff --git a/src/vegas_mode/render_pipeline.py b/src/vegas_mode/render_pipeline.py index ba72cb97..44cd6e90 100644 --- a/src/vegas_mode/render_pipeline.py +++ b/src/vegas_mode/render_pipeline.py @@ -13,6 +13,7 @@ import threading from collections import deque from contextlib import nullcontext from typing import Optional, List, Any, Dict, Deque, Tuple +import numpy as np from PIL import Image from src.common.scroll_config import solve_crisp @@ -77,6 +78,19 @@ def join_plugin_rows( return block, layout +class PreparedBlock: + """One plugin's block, joined and turned into pixels ahead of the strip.""" + + __slots__ = ('images', 'block', 'layout', 'pixels') + + def __init__(self, images: List[Image.Image], config: VegasModeConfig) -> None: + # Held so the id() it is filed under cannot be reused while it waits. + self.images = images + self.block, self.layout = join_plugin_rows(images, config) + block = self.block if self.block.mode == 'RGB' else self.block.convert('RGB') + self.pixels = np.asarray(block) + + class RenderPipeline: """ High-performance render pipeline for Vegas scroll mode. @@ -115,6 +129,17 @@ class RenderPipeline: # without __init__ (tests). _static_markers: Tuple[Tuple[int, str], ...] = () + # Blocks joined off the render thread by whichever thread fetched the + # group (prepare_group_member): id(images) -> PreparedBlock. Laying a + # plugin's rows out and turning the block into pixels cost the frame + # after every extension ~35 ms on a Pi 4 when done there. Only producer + # threads add entries and only extend_scroll_content takes them; a + # reset drops the lot. Made on first use, so pipelines built without + # __init__ (tests) work too. + _prepared_blocks: Optional[Dict[int, 'PreparedBlock']] = None + #: Blocks kept waiting at most; a group is a handful of plugins. + PREPARED_BLOCKS_MAX = 64 + # Live elements in the strip (see "live element records" below). Replaced, # never mutated, like _static_markers, and class-level for the same reason. _elements: Tuple[ElementRecord, ...] = () @@ -466,6 +491,11 @@ class RenderPipeline: if note is not None: note(kind, nbytes) + def _copied_bytes(self) -> int: + """Bytes the scroll helper's last append or trim copied (the strip, if it cannot say).""" + copied = getattr(self.scroll_helper, 'last_copy_bytes', None) + return int(copied) if isinstance(copied, int) else self._strip_nbytes() + def _strip_nbytes(self) -> int: array = self.scroll_helper.cached_array return int(array.nbytes) if array is not None else 0 @@ -593,6 +623,8 @@ class RenderPipeline: try: with gate.yielding() if gate is not None else nullcontext(): group = self.stream_manager.take_next_group(offscreen_only=True) + for member in group or (): + self.prepare_group_member(member) except Exception: logger.exception("Background prefetch failed") group = [] @@ -660,7 +692,7 @@ class RenderPipeline: element_gap=0, ) if appended: - self._note_op('extend', self._strip_nbytes()) + self._note_op('extend', self._copied_bytes()) logger.info( "[%s] Appended deferred content: strip now %dpx, %dpx ahead", plugin_id, self.scroll_helper.total_scroll_width, @@ -672,6 +704,35 @@ class RenderPipeline: """Whether any canvas-bound plugins are still queued.""" return bool(self._deferred_queue) + def prepare_group_member(self, member) -> None: + """Join one fetched ``(plugin_id, images)`` ahead of the strip. + + Called by the thread that fetched it (the prefetch thread or the live + worker, under the render gate), so the extension that appends it only + has to copy its pixels into the strip. Never raises: a member left + unprepared is joined at the extension instead, as before. + """ + try: + images = member[1] + if not images: + return + blocks = self._prepared_blocks + if blocks is None: + blocks = self._prepared_blocks = {} + if len(blocks) >= self.PREPARED_BLOCKS_MAX: + blocks.clear() # left by groups that were never appended + blocks[id(images)] = PreparedBlock(images, self.config) + except Exception: # pylint: disable=broad-except + logger.debug("Could not prepare a Vegas block ahead", exc_info=True) + + def _take_prepared_block(self, images: List[Image.Image]) -> Optional[PreparedBlock]: + """The block prepared for exactly these images, if there is one.""" + blocks = self._prepared_blocks + if not blocks: + return None + prepared = blocks.pop(id(images), None) + return prepared if prepared is not None and prepared.images is images else None + def _claim_prepared_group(self): """Take the prefetched group, if one is ready.""" with self._prefetch_lock: @@ -714,6 +775,8 @@ class RenderPipeline: for pid, images in grouped: if is_static is not None and is_static(pid): statics.append((sum(1 for _p, imgs in content if imgs), pid)) + if images: + self._take_prepared_block(images) # never appended else: content.append((pid, images)) grouped = content @@ -752,10 +815,18 @@ class RenderPipeline: blocks = [] layouts = [] + items = [] total_rows = 0 for _plugin_id, images in grouped: total_rows += len(images) - block, layout = self._join_plugin_rows_with_layout(images) + prepared = self._take_prepared_block(images) + if prepared is not None: + block, layout = prepared.block, prepared.layout + items.append(prepared.pixels) + else: + # Fetched inline, or by a thread that did not prepare it. + block, layout = self._join_plugin_rows_with_layout(images) + items.append(block) blocks.append(block) layouts.append(layout) @@ -764,13 +835,13 @@ class RenderPipeline: # append_content is about to build a strip from scratch. self._reset_records() appended = self.scroll_helper.append_content( - content_items=blocks, + content_items=items, item_gap=self.config.separator_width, element_gap=0, ) if not appended: return False - moved = self._strip_nbytes() + moved = self._copied_bytes() # Where each block starts, laid out as append_content does: a # separator before every block, or -- when there was no strip to @@ -789,9 +860,10 @@ class RenderPipeline: self._static_markers = tuple( (max(0, x - cut), pid) for x, pid in self._static_markers) self._forget_trimmed_records(cut) - # The append built the whole strip anew, and a trim copies what is - # left of it again: both land in the frame after this one. - self._note_op('extend', moved + (self._strip_nbytes() if cut else 0)) + # Both land in the frame after this one: the append's new columns + # (the whole strip when its buffer had to be reallocated), and a + # trim's copy, if it made one. + self._note_op('extend', moved + (self._copied_bytes() if cut else 0)) self._segments_in_scroll = [pid for pid, _ in grouped] self.stats['composition_count'] += 1 @@ -1404,6 +1476,7 @@ class RenderPipeline: self._prefetch_generation += 1 self._prepared_group = None self._deferred_queue = [] + self._prepared_blocks = None self._static_markers = () self._stop_live_worker() self._reset_records() diff --git a/test/test_scroll_helper_in_place.py b/test/test_scroll_helper_in_place.py new file mode 100644 index 00000000..b1bf3858 --- /dev/null +++ b/test/test_scroll_helper_in_place.py @@ -0,0 +1,146 @@ +"""ScrollHelper extends and trims a strip in place (src/common/scroll_helper.py). + +Every extension of the Vegas strip used to rebuild it whole (np.concatenate) +and every trim copied what was left: 3.5-4.5 ms on the render thread at 512x64 +on a Pi 4, so the frame after each extension was late. The strip now lives in +a buffer with spare room: an append writes only the new columns, a trim only +moves the view's start, and a full copy happens only when the buffer is +reallocated. What a frame shows must not change at all. +""" +import random +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.common.scroll_helper import ScrollHelper # noqa: E402 + +W, H = 64, 16 + + +def _items(rng, n): + out = [] + for _ in range(n): + width = rng.randint(5, 60) + out.append(Image.frombytes("RGB", (width, H), + bytes(rng.randrange(256) for _ in range(width * H * 3)))) + return out + + +def _helper(): + helper = ScrollHelper(W, H) + helper.create_scrolling_image(_items(random.Random(1), 4), item_gap=3, lead_gap=0) + return helper + + +def _reference_append(strip, items, gap): + """What append_content used to do.""" + width = sum(i.width for i in items) + gap * len(items) + addition = Image.new("RGB", (width, H)) + x = 0 + for item in items: + x += gap + addition.paste(item, (x, 0)) + x += item.width + return np.concatenate((strip, np.array(addition)), axis=1) + + +def test_an_append_writes_into_the_buffer_and_copies_only_the_new_columns(): + helper = _helper() + helper.append_content(_items(random.Random(2), 2), item_gap=3) # allocates + buffer = helper._strip_buffer + before = helper.cached_array.shape[1] + items = _items(random.Random(3), 2) + helper.append_content(items, item_gap=3) + assert helper._strip_buffer is buffer + assert np.shares_memory(helper.cached_array, buffer) + added = helper.cached_array.shape[1] - before + assert helper.last_copy_bytes == added * H * 3 + + +def test_a_trim_copies_nothing(): + helper = _helper() + helper.append_content(_items(random.Random(2), 3), item_gap=3) + helper.scroll_position = 120.0 + cut = helper.drop_scrolled_prefix() + assert cut == 120 and helper.last_copy_bytes == 0 + assert np.shares_memory(helper.cached_array, helper._strip_buffer) + + +def test_the_buffer_is_reallocated_when_the_room_runs_out(): + helper = _helper() + helper.append_content(_items(random.Random(2), 1), item_gap=3) + first = helper._strip_buffer + rng = random.Random(4) + while helper._strip_buffer is first: + helper.append_content(_items(rng, 3), item_gap=3) + assert helper.last_copy_bytes == helper.cached_array.nbytes + assert helper._strip_start == 0 + + +def test_a_strip_set_from_outside_is_never_written_through(): + # The multi-display follower adopts a read-only array straight from an image. + helper = _helper() + helper.append_content(_items(random.Random(2), 1), item_gap=3) + adopted = np.asarray(Image.new("RGB", (300, H), (9, 9, 9))) + helper.cached_array = adopted + helper.total_scroll_width = 300 + helper.append_content(_items(random.Random(3), 1), item_gap=3) + assert not np.shares_memory(helper.cached_array, adopted) + assert (adopted == 9).all() + helper.cached_array = adopted + helper.scroll_position = 100.0 + helper.drop_scrolled_prefix() + assert not np.shares_memory(helper.cached_array, adopted) + + +def test_a_new_strip_lets_the_old_buffer_go(): + helper = _helper() + helper.append_content(_items(random.Random(2), 1), item_gap=3) + helper.create_scrolling_image(_items(random.Random(5), 2), item_gap=3, lead_gap=0) + assert helper._strip_buffer is None + helper.append_content(_items(random.Random(2), 1), item_gap=3) + helper.clear_cache() + assert helper._strip_buffer is None and helper._strip_view is None + + +@pytest.mark.parametrize("seed", range(12)) +def test_every_frame_matches_the_old_copying_strip(seed): + """Random appends, trims, scrolling and patches, against a strip kept the old way.""" + rng = random.Random(seed) + helper = _helper() + reference = helper.cached_array.copy() + for _ in range(60): + op = rng.random() + if op < 0.35: + items = _items(rng, rng.randint(1, 3)) + gap = rng.randint(0, 6) + helper.append_content(items, item_gap=gap) + reference = _reference_append(reference, items, gap) + elif op < 0.55: + keep = rng.randint(0, W) + before = helper.scroll_position + cut = helper.drop_scrolled_prefix(keep_before=keep) + reference = reference[:, cut:].copy() + assert helper.scroll_position == before - cut + elif op < 0.7 and helper.cached_array.shape[1] > 8: + x = rng.randrange(helper.cached_array.shape[1] - 4) + pixels = np.full((H, 4, 3), rng.randrange(256), dtype=np.uint8) + helper.patch_columns(x, pixels) + reference[:, x:x + 4] = pixels + else: + limit = max(0, helper.cached_array.shape[1] - W - 1) + helper.scroll_position = float(rng.randint(0, limit)) if limit else 0.0 + assert helper.cached_array.shape == reference.shape + assert (helper.cached_array == reference).all() + assert helper.total_scroll_width == reference.shape[1] + frame = np.asarray(helper.get_visible_portion()) + x = int(helper.scroll_position) + if x + W <= reference.shape[1]: + assert (frame == reference[:, x:x + W]).all() + # The lazily built image is the strip as it stands. + assert (np.asarray(helper.cached_image) == reference).all() diff --git a/test/test_vegas_live_property.py b/test/test_vegas_live_property.py index 86e90e24..75fdcfdf 100644 --- a/test/test_vegas_live_property.py +++ b/test/test_vegas_live_property.py @@ -91,6 +91,16 @@ class _DM: def _pipeline(groups): p = RenderPipeline(VegasModeConfig(continuous_scroll=True, lead_in_width=0, separator_width=12), _DM(), _Stream(groups)) + start_prefetch = p.start_prefetch + + def prefetch_now(): + # Done before the next extension, so both twins append the same + # groups the same way (prepared ahead) whatever the thread timing. + start_prefetch() + if p._prefetch_thread is not None: + p._prefetch_thread.join(5) + + p.start_prefetch = prefetch_now assert p.compose_scroll_content() return p diff --git a/test/test_vegas_live_worker.py b/test/test_vegas_live_worker.py index 0abb6fd8..1cffc60e 100644 --- a/test/test_vegas_live_worker.py +++ b/test/test_vegas_live_worker.py @@ -93,7 +93,8 @@ def _pipeline(gate=True, **cfg): display_width=W, display_height=H, frame_interval=0.01, config=VegasModeConfig(**cfg), display_manager=SimpleNamespace(render_gate=_Gate() if gate else None), - stream_manager=_Stream(adapter), _prefetch_thread=None) + stream_manager=_Stream(adapter), _prefetch_thread=None, prepared=[]) + p.prepare_group_member = p.prepared.append return p, adapter @@ -364,6 +365,9 @@ def test_a_group_is_fetched_a_member_at_a_time_and_published(): assert p._prepared_group is None worker._run(worker._pick(NOW)) assert p._prepared_group == [("a", ["img-a"]), ("b", ["img-b"]), ("c", ["img-c"])] + # Each member was laid out for the strip here, as it arrived, not by the + # render thread at the extension. + assert p.prepared == p._prepared_group assert p.stream_manager.plans == [None] assert all(offscreen for _pid, offscreen in p.stream_manager.fetched) assert worker._pick(NOW) is None # the slot is full diff --git a/test/test_vegas_prepared_blocks.py b/test/test_vegas_prepared_blocks.py new file mode 100644 index 00000000..f9df9289 --- /dev/null +++ b/test/test_vegas_prepared_blocks.py @@ -0,0 +1,249 @@ +"""A group's blocks are laid out by the thread that fetched it, not at the extension. + +Joining a plugin's rows (measuring the separation between them) and turning +the block into pixels cost the frame after every extension ~35 ms on a Pi 4, +against ~3.75 ms of slack. The prefetch thread and the live-element worker now +do it as each member arrives (RenderPipeline.prepare_group_member), and the +extension only writes those pixels into the strip (ScrollHelper.append_content +takes arrays). The strip a viewer sees must be exactly what it was. +""" +import random +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image, ImageDraw + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.common.scroll_helper import ScrollHelper # noqa: E402 +from src.vegas_mode import elements # noqa: E402 +from src.vegas_mode import render_pipeline # noqa: E402 +from src.vegas_mode.config import VegasModeConfig # noqa: E402 +from src.vegas_mode.elements import ElementMeta # noqa: E402 +from src.vegas_mode.render_pipeline import RenderPipeline # noqa: E402 + +W, H = 96, 24 + + +def _row(width, seed, live_key=None): + rng = np.random.default_rng(seed) + pixels = np.zeros((H, width, 3), dtype=np.uint8) + # Margins of different sizes, so the measured separation matters. + left, right = seed % 5, width - 1 - (seed % 7) + pixels[2:H - 2, left:right] = rng.integers(20, 255, (H - 4, right - left, 3), dtype=np.uint8) + image = Image.frombytes("RGB", (width, H), pixels.tobytes()) + if live_key is None: + return image + pinned, array = elements.pin_element(image, 8) + return elements.tag(pinned, ElementMeta("p", live_key, 0, elements.pixel_digest(array), 0.0, 0.0)) + + +def _groups(seed, n=8): + rng = random.Random(seed) + groups = [] + for g in range(n): + group = [] + for p in range(rng.randint(1, 3)): + rows = [_row(rng.randint(12, 60), rng.randint(0, 10**6), + f"g{g}p{p}r{r}" if rng.random() < 0.5 else None) + for r in range(rng.randint(1, 4))] + group.append((f"pl{g}{p}", rows)) + groups.append(group) + return groups + + +class _Stream: + def __init__(self, groups, statics=()): + self.groups = groups + self.statics = set(statics) + self.plugin_manager = type("PM", (), {"plugins": {}})() + self.plugin_adapter = None + self.i = 0 + + def get_grouped_content_for_composition(self): + return self.groups[0] + + def get_active_plugin_ids(self): + return [] + + def take_next_group(self, count=None, offscreen_only=False): + self.i += 1 + return self.groups[self.i] if self.i < len(self.groups) else [] + + def is_static_plugin(self, plugin_id): + return plugin_id in self.statics + + +class _DM: + width, height = W, H + + def __init__(self): + self.image = Image.new("RGB", (W, H)) + + def set_scrolling_state(self, *a): + pass + + def update_display(self): + pass + + +def _pipeline(groups, prepared, statics=()): + p = RenderPipeline(VegasModeConfig(continuous_scroll=True, lead_in_width=0, + separator_width=12), _DM(), _Stream(groups, statics)) + if prepared: + # The prefetch thread, run to completion before each extension. + def prefetch_now(): + p._prepared_group = p.stream_manager.take_next_group(offscreen_only=True) + for member in p._prepared_group: + p.prepare_group_member(member) + p.start_prefetch = prefetch_now + else: + p.start_prefetch = lambda: None # every extension fetches inline + assert p.compose_scroll_content() + return p + + +def _run(p, steps=6): + for _ in range(steps): + p.scroll_helper.scroll_position = max( + p.scroll_helper.scroll_position, p.scroll_helper.total_scroll_width - 2 * W) + if not p.extend_scroll_content(): + break + return p + + +@pytest.mark.parametrize("seed", range(6)) +def test_the_strip_is_the_same_prepared_ahead_or_joined_at_the_extension(seed): + inline = _run(_pipeline(_groups(seed), prepared=False)) + ahead = _run(_pipeline(_groups(seed), prepared=True)) + assert np.array_equal(ahead.scroll_helper.cached_array, inline.scroll_helper.cached_array) + assert ahead._strip_origin == inline._strip_origin + assert [(r.key, r.abs_x, r.width) for r in ahead.live_records()] == \ + [(r.key, r.abs_x, r.width) for r in inline.live_records()] + assert ahead.live_records() + + +def test_a_prepared_extension_joins_nothing_on_the_render_thread(monkeypatch): + p = _pipeline(_groups(1), prepared=True) + p.start_prefetch() # the group, prepared ahead + p.start_prefetch = lambda: None # and not the one after it + + def joined_here(*_a, **_k): + raise AssertionError("the extension laid a block out itself") + + monkeypatch.setattr(render_pipeline, "join_plugin_rows", joined_here) + monkeypatch.setattr(render_pipeline, "separation_gap", joined_here) + monkeypatch.setattr(Image.Image, "paste", joined_here) + assert p.extend_scroll_content() + assert not p._prepared_blocks # each block used once, then let go + + +def test_a_static_plugins_block_is_let_go_unused(): + groups = _groups(2) + static = groups[1][0][0] + p = _pipeline(groups, prepared=True, statics=[static]) + p.start_prefetch() + p.start_prefetch = lambda: None + assert p.extend_scroll_content() + assert not p._prepared_blocks + assert any(pid == static for _x, pid in p._static_markers) + + +def test_blocks_prepared_for_other_images_are_not_used(): + groups = _groups(3) + p = _pipeline(groups, prepared=False) + images = groups[1][0][1] + p.prepare_group_member(("x", list(images))) # equal, not the same list + assert p._take_prepared_block(images) is None + # Filed under these images' id() -- as when a list dies and its id is + # reused -- but made from others. + p._prepared_blocks[id(images)] = render_pipeline.PreparedBlock( + [_row(20, 99)], p.config) + assert p._take_prepared_block(images) is None + + +def test_a_reset_drops_prepared_blocks(): + p = _pipeline(_groups(4), prepared=True) + p.start_prefetch() + assert p._prepared_blocks + p.reset() + assert not p._prepared_blocks + + +def test_preparing_never_raises(caplog): + p = _pipeline(_groups(5), prepared=False) + p.prepare_group_member(("broken", [object()])) + p.prepare_group_member(("empty", [])) + p.prepare_group_member(("deferred", None)) + assert not p._prepared_blocks + + +def test_the_waiting_blocks_are_bounded(): + p = _pipeline(_groups(6), prepared=False) + for n in range(RenderPipeline.PREPARED_BLOCKS_MAX * 2): + p.prepare_group_member((f"p{n}", [_row(20, n)])) + assert len(p._prepared_blocks) <= RenderPipeline.PREPARED_BLOCKS_MAX + + +# -- ScrollHelper.append_content with arrays ---------------------------------------- + + +def _strip(): + helper = ScrollHelper(W, H) + helper.create_scrolling_image([_row(40, k) for k in range(4)], item_gap=3, lead_gap=0) + return helper + + +@pytest.mark.parametrize("in_place", [False, True]) +def test_appending_pixels_draws_exactly_what_appending_images_does(in_place): + items = [_row(30, 7), + _row(25, 8).convert("RGBA"), # converted, as paste does + Image.new("RGB", (20, H - 6), (200, 40, 40)), # short: black beneath + Image.new("RGB", (15, H + 6), (40, 200, 40))] # tall: cut at the strip + as_images, as_pixels = _strip(), _strip() + if in_place: + for helper in (as_images, as_pixels): + helper.append_content([_row(10, 1)], item_gap=2) # a buffer with room + as_images.append_content(items, item_gap=5, element_gap=2) + as_pixels.append_content([np.asarray(i.convert("RGB")) for i in items], + item_gap=5, element_gap=2) + assert np.array_equal(as_images.cached_array, as_pixels.cached_array) + assert as_images.total_scroll_width == as_pixels.total_scroll_width + + +def test_reference_paste_layout_is_unchanged(): + """Gaps black, short items padded below, tall ones cut: what paste did.""" + items = [Image.new("RGB", (20, H - 6), (200, 40, 40)), + Image.new("RGBA", (15, H + 6), (40, 200, 40, 9))] + helper = _strip() + before = helper.cached_array.copy() + helper.append_content(items, item_gap=5, element_gap=2) + addition = Image.new("RGB", (5 + 20 + 2 + 5 + 15 + 2, H)) + addition.paste(items[0], (5, 0)) + addition.paste(items[1], (5 + 20 + 2 + 5, 0)) + expected = np.concatenate((before, np.asarray(addition)), axis=1) + assert np.array_equal(helper.cached_array, expected) + + +def test_a_first_build_from_pixels_matches_one_from_images(): + items = [_row(30, 3), _row(40, 4)] + a, b = ScrollHelper(W, H), ScrollHelper(W, H) + a.append_content(items, item_gap=4) + b.append_content([np.asarray(i) for i in items], item_gap=4) + assert np.array_equal(a.cached_array, b.cached_array) + + +def test_gaps_and_short_rows_are_black_whatever_the_spare_room_held(): + """Only uncovered columns are blanked, so they must all be.""" + items = [Image.new("RGB", (20, H - 6), (200, 40, 40)), _row(30, 5)] + clean, dirty = _strip(), _strip() + for helper in (clean, dirty): + helper.append_content([_row(10, 1)], item_gap=2) # a buffer with room + end = dirty._strip_start + dirty.cached_array.shape[1] + dirty._strip_buffer[:, end:] = 255 # what an old strip left + for helper in (clean, dirty): + helper.append_content([np.asarray(i) for i in items], item_gap=5, element_gap=3) + assert np.array_equal(dirty.cached_array, clean.cached_array) + assert not dirty.cached_array[:, -3:].any() # the trailing element gap