fix(store): serialize concurrent updates per plugin, check cleanup results

CodeRabbit review on #405 flagged two things in
_reinstall_with_rollback, both verified against current code:

- Real race: the web UI runs Flask with threaded=True and there's a
  single update route, so two overlapping requests for the same
  plugin_id (double-click, two tabs) can interleave. The loser could
  rename the winner's in-progress install aside mid-download, deleting
  its own rollback safety net — worse than the bug this function
  exists to fix. Added a lazy per-plugin_id lock dict (mirrors the
  plugin_manager per-plugin lock pattern) held for the whole function.
- _safe_remove_directory's return value was ignored at both call
  sites. Stale-aside cleanup failure now aborts cleanly instead of
  falling through to a rename that would fail anyway with a less
  useful error; post-success backup-removal failure now logs instead
  of failing silently (still returns True — the update itself
  succeeded, and the next update self-heals the leftover aside).

Left the third nitpick (test_stale_aside_from_previous_crash_is_cleared)
addressed by asserting the stale dir is actually gone and that
install_plugin was reached, rather than just the end-to-end result.

Added a concurrency regression test asserting install_plugin never
runs for the same plugin_id while another call is in flight.
This commit is contained in:
Chuck
2026-07-13 09:11:21 -04:00
parent 641296990d
commit 782731052d
2 changed files with 124 additions and 36 deletions
+36 -2
View File
@@ -142,9 +142,28 @@ class PluginStoreManager:
# then get the result from the warm cache (double-checked locking). # then get the result from the warm cache (double-checked locking).
self._registry_fetch_lock = threading.Lock() self._registry_fetch_lock = threading.Lock()
# Per-plugin locks for _reinstall_with_rollback: the web UI runs
# Flask with threaded=True, so two overlapping requests for the
# same plugin_id (double-click, two browser tabs) would otherwise
# both rename the same directory aside — one succeeds, and the
# loser can end up renaming the winner's in-progress install aside
# mid-download, stealing its own rollback safety net. Keyed by
# plugin_id so unrelated plugins still update concurrently.
self._reinstall_locks: Dict[str, threading.Lock] = {}
self._reinstall_locks_guard = threading.Lock()
# Ensure plugins directory exists # Ensure plugins directory exists
self.plugins_dir.mkdir(exist_ok=True) self.plugins_dir.mkdir(exist_ok=True)
def _get_reinstall_lock(self, plugin_id: str) -> threading.Lock:
"""Lazily create (or fetch) the per-plugin reinstall lock."""
with self._reinstall_locks_guard:
lock = self._reinstall_locks.get(plugin_id)
if lock is None:
lock = threading.Lock()
self._reinstall_locks[plugin_id] = lock
return lock
def _record_cache_backoff(self, cache_dict: Dict, cache_key: str, def _record_cache_backoff(self, cache_dict: Dict, cache_key: str,
cache_timeout: int, payload: Any) -> None: cache_timeout: int, payload: Any) -> None:
"""Bump a cache entry's timestamp so subsequent lookups hit the """Bump a cache entry's timestamp so subsequent lookups hit the
@@ -2277,12 +2296,23 @@ class PluginStoreManager:
The aside name embeds '.standalone-backup-' so plugin discovery The aside name embeds '.standalone-backup-' so plugin discovery
(plugin_manager._scan_directory_for_plugins) ignores it even though (plugin_manager._scan_directory_for_plugins) ignores it even though
it still contains a manifest.json. it still contains a manifest.json.
Held for the whole operation under a per-plugin_id lock: two
overlapping requests for the same plugin (double-click, two
browser tabs — the web UI runs Flask with threaded=True) must not
interleave their renames, or the second could steal the first's
rollback safety net mid-install. Other plugin_ids are unaffected.
""" """
with self._get_reinstall_lock(plugin_id):
backup_path = plugin_path.with_name( backup_path = plugin_path.with_name(
f"{plugin_path.name}.standalone-backup-migrating") f"{plugin_path.name}.standalone-backup-migrating")
# A stale aside from a previous crash would block the rename # A stale aside from a previous crash would block the rename
if backup_path.exists(): if backup_path.exists():
self._safe_remove_directory(backup_path) if not self._safe_remove_directory(backup_path):
self.logger.error(
f"Could not clear stale backup for {plugin_id} at "
f"{backup_path}; leaving old install in place")
return False
try: try:
plugin_path.rename(backup_path) plugin_path.rename(backup_path)
except OSError as e: except OSError as e:
@@ -2297,7 +2327,11 @@ class PluginStoreManager:
installed = False installed = False
if installed: if installed:
self._safe_remove_directory(backup_path) if not self._safe_remove_directory(backup_path):
self.logger.warning(
f"Update of {plugin_id} succeeded but the old backup "
f"at {backup_path} could not be removed; it will be "
f"cleared on the next update")
return True return True
# Install failed (bad network, registry error...) — put the old # Install failed (bad network, registry error...) — put the old
+55 -1
View File
@@ -10,6 +10,8 @@ plugins from one update pass.
import json import json
import os import os
import sys import sys
import threading
import time
from pathlib import Path from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
@@ -95,10 +97,62 @@ class TestReinstallWithRollback:
stale = plugin_dir.parent / f"{PLUGIN_ID}.standalone-backup-migrating" stale = plugin_dir.parent / f"{PLUGIN_ID}.standalone-backup-migrating"
stale.mkdir() stale.mkdir()
(stale / "old.txt").write_text("stale") (stale / "old.txt").write_text("stale")
with patch.object(mgr, "install_plugin", return_value=False): with patch.object(mgr, "install_plugin", return_value=False) as mock_install:
ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir) ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir)
# The reinstall itself still fails (mocked) and the old install is
# restored, but the stale aside must not have survived — otherwise
# it would have blocked this run's own rename (or a future one).
assert not stale.exists()
mock_install.assert_called_once_with(PLUGIN_ID)
assert ok is False assert ok is False
assert plugin_dir.exists() assert plugin_dir.exists()
assert "old version marker" in (plugin_dir / "manager.py").read_text()
def test_concurrent_updates_for_same_plugin_are_serialized(self, store):
"""Two overlapping requests for the same plugin_id (double-click,
two browser tabs the web UI runs Flask with threaded=True) must
not interleave: the loser must wait for the winner to finish
rather than renaming the winner's in-progress install aside and
stealing its rollback safety net."""
mgr, plugin_dir = store
active = 0
max_active = 0
guard = threading.Lock()
def fake_install(plugin_id):
nonlocal active, max_active
with guard:
active += 1
max_active = max(max_active, active)
time.sleep(0.05)
plugin_dir.mkdir(exist_ok=True)
(plugin_dir / "manager.py").write_text("# new version\n")
(plugin_dir / "manifest.json").write_text(json.dumps(
{"id": PLUGIN_ID, "name": "Rollback Test", "version": "2.0.0"}))
with guard:
active -= 1
return True
results = []
def worker():
results.append(mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir))
with patch.object(mgr, "install_plugin", side_effect=fake_install):
threads = [threading.Thread(target=worker) for _ in range(2)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=5)
assert max_active == 1, "install_plugin ran concurrently for the same plugin_id"
assert results == [True, True]
assert plugin_dir.exists()
assert "new version" in (plugin_dir / "manager.py").read_text()
leftovers = [p for p in plugin_dir.parent.iterdir()
if "standalone-backup" in p.name]
assert leftovers == []
def test_aside_name_is_invisible_to_discovery(self, store, tmp_path): def test_aside_name_is_invisible_to_discovery(self, store, tmp_path):
"""The aside still contains a manifest.json — discovery must skip it """The aside still contains a manifest.json — discovery must skip it