Merge audit16-core/autoupdate into the September 16 core audit fixes

web_interface/auto_update.py

@@ -263,6 +263,56 @@ def start_setup_if_needed(was_enabled, config):
     return 'Could not restart the display to finish automatic update setup; restart it from the Overview tab.'
 
 
+#: Top-level folders of separately installed plugins. Edits there are not the
+#: core's local changes: the Plugin Store updates plugins in place, including
+#: the bundled ones committed under plugin-repos/, and the pull's --autostash
+#: carries those edits across and reapplies them.
+SEPARATE_INSTALL_DIRS = ('plugins', 'plugin-repos')
+
+
+def local_changes(project_dir, run=None, timeout=30):
+    """Tracked files edited in the checkout, as both code updates count them.
+
+    The one definition shared by the automatic update's preflight and
+    ``perform_core_update`` (Update Code), so the preflight's promise that
+    nothing will be stashed holds for the pull that follows it.
+
+    * Mode-only changes do not count (``core.fileMode=false``): installers
+      chmod tracked scripts, and --autostash carries modes across the pull.
+    * Paths under ``SEPARATE_INSTALL_DIRS`` do not count. The match is on
+      the leading folder, not a substring, so
+      ``web_interface/static/v3/js/plugins/x.js`` is still a core edit.
+
+    Returns the changed paths, or None when git could not tell. Raises what
+    ``run`` raises (e.g. ``subprocess.TimeoutExpired``).
+    """
+    run = run or subprocess.run
+    result = run(['git', '-c', 'core.fileMode=false', 'status', '--porcelain',
+                  '--untracked-files=no', '-z'],
+                 cwd=str(project_dir), capture_output=True, text=True, timeout=timeout)
+    if result.returncode != 0:
+        return None
+    changed = []
+    entries = iter((result.stdout or '').split('\0'))
+    for entry in entries:
+        if len(entry) < 4:
+            continue
+        paths = [entry[3:]]
+        if entry[0] in 'RC' or entry[1] in 'RC':
+            paths.append(next(entries, ''))  # -z puts a rename's source next
+        if any(p and p.split('/', 1)[0] not in SEPARATE_INSTALL_DIRS for p in paths):
+            changed.append(paths[0])
+    return changed
+
+
+def describe_local_changes(changed):
+    """Why an automatic update refused to touch a checkout with local edits."""
+    example = f' (for example {changed[0]})' if changed else ''
+    return (f'{len(changed or ()) or "Some"} tracked file(s) in the LEDMatrix folder were edited '
+            f'locally{example}. Automatic updates will not stash your changes; '
+            'commit or revert them, or update manually with Update Code.')
+
+
 def _load_verifier(path):
     spec = importlib.util.spec_from_file_location('ledmatrix_auto_update_verifier', str(path))
     module = importlib.util.module_from_spec(spec)
@@ -346,11 +396,16 @@ class AutoUpdater:
             core = {'outcome': 'error', 'message': f'The LEDMatrix update failed unexpectedly: {e}'}
 
         deferred = core['outcome'] == 'verifying'
-        updated, failed = ([], []) if deferred else self._update_plugins()
+        # A core whose rollback failed is in an unknown state: as when the
+        # health check reports rollback_failed, plugins are left alone and
+        # nothing is restarted onto it.
+        stranded = core['outcome'] == 'rollback_failed'
+        updated, failed = ([], []) if deferred or stranded else self._update_plugins()
         self._store_run(state, core, updated, failed)
         logger.info("Automatic update: core %s (%s); plugins updated=%s failed=%s%s",
                     core['outcome'], core['message'], updated, failed,
-                    '; plugins wait for the health check' if deferred else '')
+                    '; plugins wait for the health check' if deferred
+                    else '; plugins left alone' if stranded else '')
 
         # Code restarts belong to the health check; this only covers plugins.
         if updated:
@@ -424,17 +479,11 @@ class AutoUpdater:
             return 'blocked', ('The current branch has no upstream to update from (or HEAD is detached). '
                                'Use Update Code once, or Tools -> Switch branch.'), {}
 
-        # Mode-only changes are ignored: older installers chmod tracked
-        # scripts, and --autostash in the pull carries those across. Plugin
-        # folders are separate installs, as in perform_core_update.
-        status = self._git('-c', 'core.fileMode=false', 'status', '--porcelain', '--untracked-files=no')
-        changed = [line[3:] for line in status.stdout.splitlines()
-                   if line.strip() and 'plugins/' not in line and 'plugin-repos/' not in line]
-        if status.returncode != 0 or changed:
-            example = f' (for example {changed[0]})' if changed else ''
-            return 'blocked', (f'{len(changed) or "Some"} tracked file(s) in the LEDMatrix folder were edited '
-                               f'locally{example}. Automatic updates will not stash your changes; '
-                               'commit or revert them, or update manually with Update Code.'), {}
+        # The same predicate perform_core_update refuses on when called from
+        # here, so what passes this check is never stashed by the pull.
+        changed = local_changes(self.project_root, run=self.run_command)
+        if changed is None or changed:
+            return 'blocked', describe_local_changes(changed), {}
 
         free = self.disk_free(str(self.project_root))
         if free < MIN_FREE_BYTES:
@@ -479,11 +528,15 @@ class AutoUpdater:
             return {'outcome': 'error', 'message': f'Could not prepare the update health check: {e}.'}
 
         try:
-            core = self.core_update()
+            # Refuse rather than stash: edits made since the preflight are the
+            # user's, and nothing would ever restore a stash taken here.
+            core = self.core_update(stash_local_changes=False)
         except Exception as e:
             logger.exception("perform_core_update raised")
             core = {'status': 'error', 'message': f'Update failed: {e}'}
         new_head = self._git('rev-parse', 'HEAD').stdout.strip()
+        if core.get('local_changes') is not None and new_head == old_head:
+            return {'outcome': 'blocked', 'message': describe_local_changes(core['local_changes'])}
         pending = {
             'status': 'pending',
             'old_head': old_head,

web_interface/blueprints/api_v3/system.py

@@ -228,14 +228,24 @@ def _sudo_hint_for(text):
 
 _core_update_lock = threading.Lock()
 
+#: The core's own requirement files, installed after a pull that changes them.
+#: scripts/fix_perms/safe_pip_install.sh must accept every one (it refuses
+#: anything it does not list), and scripts/utils/auto_update_verify.py
+#: reinstalls the same files when it rolls an update back.
+CORE_REQUIREMENT_FILES = ('requirements.txt', 'web_interface/requirements.txt')
 
-def perform_core_update():
+
+def perform_core_update(stash_local_changes=True):
     """Pull the latest LEDMatrix code and sync its dependencies.
 
     Shared by the Overview "Update Code" button and the weekly automatic
     updater (web_interface/auto_update.py), so both take exactly the same
     path. Returns the JSON-able payload the button has always received:
     ``status``, ``message`` and ``restart_required``.
+
+    Update Code stashes local edits before pulling. The automatic updater
+    passes ``stash_local_changes=False``: then local edits make this return
+    an error carrying ``local_changes`` (the edited paths) without pulling.
     """
     # The button and the scheduler can fire together; two pulls racing
     # over one checkout (and one stash) is how local changes get lost.
@@ -243,12 +253,12 @@ def perform_core_update():
         return {'status': 'error', 'restart_required': False,
                 'message': 'An update is already in progress; try again shortly.'}
     try:
-        return _perform_core_update_locked()
+        return _perform_core_update_locked(stash_local_changes)
     finally:
         _core_update_lock.release()
 
 
-def _perform_core_update_locked():
+def _perform_core_update_locked(stash_local_changes=True):
     project_dir = str(PROJECT_ROOT)
 
     # Decide how to pull BEFORE stashing. If this checkout cannot be
@@ -259,37 +269,35 @@ def _perform_core_update_locked():
         logger.warning("git pull not attempted: %s", pull_error)
         return {'status': 'error', 'message': pull_error, 'restart_required': False}
 
-    # Check if there are local changes that need to be stashed
-    # Exclude plugins directory - plugins are separate repos and shouldn't be stashed with base project
-    # Use --untracked-files=no to skip untracked files check (much faster with symlinked plugins)
+    # Local changes, counted exactly as the automatic update's preflight
+    # counts them (auto_update.local_changes): mode-only changes and the
+    # plugin folders don't count, and the pull's --autostash carries those
+    # across and reapplies them.
+    from web_interface import auto_update
     try:
-        status_result = subprocess.run(
-            ['git', 'status', '--porcelain', '--untracked-files=no'],
-            capture_output=True,
-            text=True,
-            timeout=30,
-            cwd=project_dir
-        )
-        # Filter out any changes in plugins directory - plugins are separate repositories
-        # Git status format: XY filename (where X is status of index, Y is status of work tree)
-        status_lines = [line for line in status_result.stdout.strip().split('\n')
-                       if line.strip() and 'plugins/' not in line]
-        has_changes = bool('\n'.join(status_lines).strip())
-    except subprocess.TimeoutExpired:
-        # If status check times out, assume there might be changes and proceed
-        # This is safer than failing the update
-        has_changes = True
-        status_result = type('obj', (object,), {'stdout': '', 'stderr': 'Status check timed out'})()
+        changed = auto_update.local_changes(project_dir)
+    except (subprocess.SubprocessError, OSError) as status_err:
+        logger.warning("git status failed before pull: %s", status_err)
+        changed = None
+    # When git cannot say, assume there are changes rather than pull over them.
+    has_changes = changed is None or bool(changed)
+
+    if has_changes and not stash_local_changes:
+        # The automatic updater: it promised not to stash, and nothing would
+        # ever restore a stash taken on its behalf.
+        return {'status': 'error', 'restart_required': False, 'dependency_failures': [],
+                'local_changes': list(changed or []),
+                'message': auto_update.describe_local_changes(changed)}
 
     stash_info = ""
 
-    # Stash local changes if they exist (excluding plugins)
-    # Plugins are separate repositories and shouldn't be stashed with base project updates
+    # Stash local changes if they exist. The plugin folders are left out:
+    # plugins are separate installs, and --autostash carries their edits.
     if has_changes:
         try:
-            # Use pathspec to exclude plugins directory from stash
             stash_result = subprocess.run(
-                ['git', 'stash', 'push', '-m', 'LEDMatrix auto-stash before update', '--', ':!plugins'],
+                ['git', 'stash', 'push', '-m', 'LEDMatrix auto-stash before update', '--',
+                 *(f':!{folder}' for folder in auto_update.SEPARATE_INSTALL_DIRS)],
                 capture_output=True,
                 text=True,
                 timeout=30,
@@ -369,7 +377,7 @@ def _perform_core_update_locked():
                     ['git', 'diff', '--name-only', f'{old_head}..{new_head}'],
                     capture_output=True, text=True, timeout=15, cwd=project_dir)
                 changed = set(diff.stdout.split()) if diff.returncode == 0 else set()
-                for rel in ('requirements.txt', 'web_interface/requirements.txt'):
+                for rel in CORE_REQUIREMENT_FILES:
                     req_path = PROJECT_ROOT / rel
                     if rel not in changed or not req_path.exists():
                         continue