From 0ab2980ff400ace146fe2e48f6f26cfed5537d0c Mon Sep 17 00:00:00 2001 From: Canina Engineering Date: Sun, 25 Oct 2026 08:16:26 +0000 Subject: [PATCH] fix(editorial): crash-safe upgrade with durable phase, previous metadata and recovery --- infra/hermes-editorial/installer.py | 94 +++++++-- .../hermes-editorial/tests/test_installer.py | 187 ++++++++++++++++++ 2 files changed, 268 insertions(+), 13 deletions(-) diff --git a/infra/hermes-editorial/installer.py b/infra/hermes-editorial/installer.py index a192a709..8527c03c 100644 --- a/infra/hermes-editorial/installer.py +++ b/infra/hermes-editorial/installer.py @@ -65,6 +65,50 @@ def bundle(): return result +def recover(journal, paths): + if not journal.exists(): + return + try: + state = json.loads(journal.read_text()) + except Exception: + raise ValueError('invalid journal json') + if state.get('phase') != 'applying': + return + if (set(state.get('before', {})) != ALLOWED or + set(state.get('installed', {})) != ALLOWED): + raise ValueError('invalid in-flight journal allowlist') + if state.get('previous_installed') is not None and set(state['previous_installed']) != ALLOWED: + raise ValueError('invalid in-flight journal previous_installed allowlist') + if state.get('previous_before') is not None and set(state['previous_before']) != ALLOWED: + raise ValueError('invalid in-flight journal previous_before allowlist') + for r, p in paths.items(): + curr = p.read_bytes() if p.exists() else None + curr_d = digest(curr) if curr is not None else None + if curr_d == state['installed'][r]: + continue + if state.get('previous_installed') is not None: + if curr_d == state['previous_installed'][r]: + continue + else: + old = state['before'][r] + old_data = base64.b64decode(old['data']) if old else None + if curr == old_data: + continue + raise ValueError(f'interrupted transaction conflict: {r}') + restore_target = state.get('previous_before') or state['before'] + restore(paths, restore_target) + if state.get('previous_journal'): + atomic(journal, base64.b64decode(state['previous_journal'])) + elif state.get('previous_installed'): + atomic(journal, json.dumps({ + 'phase': 'installed', + 'before': state['before'], + 'installed': state['previous_installed'] + }, sort_keys=True).encode()) + else: + journal.unlink() + + def execute(action, home): if not home.is_absolute() or not home.is_dir(): raise ValueError('--target-home must be an existing absolute directory') @@ -75,18 +119,23 @@ def execute(action, home): bad = [r for r, p in paths.items() if not p.exists() or p.read_bytes() != files[r]] if bad: raise ValueError('verification mismatch: ' + ', '.join(bad)) + if journal.exists(): + jstate = json.loads(journal.read_text()) + if jstate.get('phase', 'installed') != 'installed': + raise ValueError('verification refused: in-flight transaction requires recovery') return {'status': 'configured_only', 'files': len(files), 'workflow_verified': False} lock = safe(home, 'editorial-v4/.installer/lock') lock.parent.mkdir(parents=True, exist_ok=True) with lock.open('a') as stream: fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + recover(journal, paths) if action == 'rollback': if not journal.exists(): return {'status': 'nothing_to_rollback'} state = json.loads(journal.read_text()) if set(state['before']) != ALLOWED or set(state['installed']) != ALLOWED: raise ValueError('invalid rollback allowlist') - # Refuse external edits rather than destroying another writer\'s work. + # Refuse external edits rather than destroying another writer's work. for r, p in paths.items(): current = p.read_bytes() if p.exists() else None before = state['before'][r] @@ -103,10 +152,12 @@ def execute(action, home): previous_state = None if previous_journal is not None: previous_state = json.loads(previous_journal) - if (set(previous_state) != {'before', 'installed'} or + if (set(previous_state) not in ({'before', 'installed'}, {'phase', 'before', 'installed'}) or set(previous_state['before']) != ALLOWED or set(previous_state['installed']) != ALLOWED): raise ValueError('invalid upgrade journal allowlist') + if previous_state.get('phase', 'installed') != 'installed': + raise ValueError('invalid upgrade journal phase') for r, p in paths.items(): old = previous_state['before'][r] if old is not None: @@ -118,22 +169,39 @@ def execute(action, home): # advance. Never recover partial writes or overwrite live drift. if not p.exists() or digest(p.read_bytes()) != previous_state['installed'][r]: raise ValueError(f'existing transaction conflict: {r}') - before = {r: {'data': base64.b64encode(p.read_bytes()).decode(), - 'mode': p.stat().st_mode & 0o777} if p.exists() else None - for r, p in paths.items()} - state = {'before': previous_state['before'] if previous_state else before, - 'installed': {r: digest(v) for r, v in files.items()}} - atomic(journal, json.dumps(state, sort_keys=True).encode()) + current_snapshot = {r: {'data': base64.b64encode(p.read_bytes()).decode(), + 'mode': p.stat().st_mode & 0o777} if p.exists() else None + for r, p in paths.items()} + original_before = previous_state['before'] if previous_state else current_snapshot + incoming_installed = {r: digest(v) for r, v in files.items()} + in_flight_state = { + 'phase': 'applying', + 'before': original_before, + 'installed': incoming_installed, + 'previous_installed': previous_state['installed'] if previous_state else None, + 'previous_before': current_snapshot, + 'previous_journal': base64.b64encode(previous_journal).decode() if previous_journal else None, + } + atomic(journal, json.dumps(in_flight_state, sort_keys=True).encode()) try: for r in changes: atomic(paths[r], files[r]) except BaseException: - restore(paths, before) - if previous_journal is None: - journal.unlink() - else: - atomic(journal, previous_journal) + try: + restore(paths, current_snapshot) + if previous_journal is None: + journal.unlink() + else: + atomic(journal, previous_journal) + except BaseException: + pass raise + committed_state = { + 'phase': 'installed', + 'before': original_before, + 'installed': incoming_installed, + } + atomic(journal, json.dumps(committed_state, sort_keys=True).encode()) return {'status': 'installed_config_only', 'files': len(files)} diff --git a/infra/hermes-editorial/tests/test_installer.py b/infra/hermes-editorial/tests/test_installer.py index 80cbcbae..4c422d8d 100644 --- a/infra/hermes-editorial/tests/test_installer.py +++ b/infra/hermes-editorial/tests/test_installer.py @@ -158,5 +158,192 @@ class InstallerTest(unittest.TestCase): self.assertNotEqual(self.run_cli('rollback').returncode, 0) self.assertEqual(path.read_text(), 'external change') + def test_interrupted_upgrade_multi_file_safe_recovery_and_install(self): + import hashlib + path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md' + path_pmo.parent.mkdir(parents=True, exist_ok=True) + path_pmo.write_text('original owned baseline') + path_pmo.chmod(0o600) + self.assertEqual(self.run_cli('install').returncode, 0) + + # Verified previous installed state modifying status.json and pmo SKILL.md + path_status = self.home / 'editorial-v4/status.json' + v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}' + v1_pmo = 'reviewed pmo v1 content' + path_status.write_text(v1_status) + path_pmo.write_text(v1_pmo) + + journal = self.home / 'editorial-v4/.installer/transaction.json' + state = json.loads(journal.read_text()) + rel_status = str(path_status.relative_to(self.home)) + rel_pmo = str(path_pmo.relative_to(self.home)) + state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest() + state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest() + journal.write_text(json.dumps(state)) + + # Crash midway during install: status.json is written, but process dies before pmo SKILL.md + crash_code = f""" +import os, sys, importlib.util, pathlib +spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +real_atomic = m.atomic +def crash_atomic(p, data): + res = real_atomic(p, data) + if str(p).endswith('editorial-v4/status.json'): + os._exit(42) + return res +m.atomic = crash_atomic +m.execute('install', pathlib.Path('{self.home}')) +""" + proc = subprocess.run([sys.executable, '-c', crash_code]) + self.assertEqual(proc.returncode, 42) + + # Confirm mixed state on disk: status.json updated to incoming, pmo SKILL.md still v1 + self.assertNotEqual(path_status.read_text(), v1_status) + self.assertEqual(path_pmo.read_text(), v1_pmo) + + # Rerun install: must safely recover from interrupted transaction and finish upgrade + res = self.run_cli('install') + self.assertEqual(res.returncode, 0, f"install recovery failed: {res.stderr}") + self.assertEqual(self.run_cli('verify').returncode, 0) + + # Rollback should still restore original owned baseline + self.assertEqual(self.run_cli('rollback').returncode, 0) + self.assertEqual(path_pmo.read_text(), 'original owned baseline') + self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600) + + def test_interrupted_upgrade_multi_file_rollback_restores_original(self): + import hashlib + path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md' + path_pmo.parent.mkdir(parents=True, exist_ok=True) + path_pmo.write_text('original owned baseline') + path_pmo.chmod(0o600) + self.assertEqual(self.run_cli('install').returncode, 0) + + path_status = self.home / 'editorial-v4/status.json' + v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}' + v1_pmo = 'reviewed pmo v1 content' + path_status.write_text(v1_status) + path_pmo.write_text(v1_pmo) + + journal = self.home / 'editorial-v4/.installer/transaction.json' + state = json.loads(journal.read_text()) + rel_status = str(path_status.relative_to(self.home)) + rel_pmo = str(path_pmo.relative_to(self.home)) + state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest() + state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest() + journal.write_text(json.dumps(state)) + + crash_code = f""" +import os, sys, importlib.util, pathlib +spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +real_atomic = m.atomic +def crash_atomic(p, data): + res = real_atomic(p, data) + if str(p).endswith('editorial-v4/status.json'): + os._exit(42) + return res +m.atomic = crash_atomic +m.execute('install', pathlib.Path('{self.home}')) +""" + proc = subprocess.run([sys.executable, '-c', crash_code]) + self.assertEqual(proc.returncode, 42) + + # Rollback on interrupted state must safely recover and restore original owned baseline + res = self.run_cli('rollback') + self.assertEqual(res.returncode, 0, f"rollback recovery failed: {res.stderr}") + self.assertEqual(path_pmo.read_text(), 'original owned baseline') + self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600) + self.assertFalse(journal.exists()) + + def test_interrupted_upgrade_rejects_external_drift(self): + import hashlib + self.assertEqual(self.run_cli('install').returncode, 0) + + path_status = self.home / 'editorial-v4/status.json' + path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md' + v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}' + v1_pmo = 'reviewed pmo v1 content' + path_status.write_text(v1_status) + path_pmo.write_text(v1_pmo) + + journal = self.home / 'editorial-v4/.installer/transaction.json' + state = json.loads(journal.read_text()) + rel_status = str(path_status.relative_to(self.home)) + rel_pmo = str(path_pmo.relative_to(self.home)) + state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest() + state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest() + journal.write_text(json.dumps(state)) + + crash_code = f""" +import os, sys, importlib.util, pathlib +spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +real_atomic = m.atomic +def crash_atomic(p, data): + res = real_atomic(p, data) + if str(p).endswith('editorial-v4/status.json'): + os._exit(42) + return res +m.atomic = crash_atomic +m.execute('install', pathlib.Path('{self.home}')) +""" + proc = subprocess.run([sys.executable, '-c', crash_code]) + self.assertEqual(proc.returncode, 42) + + # External drift on path_pmo + path_pmo.write_text('tampered external edit') + + res_install = self.run_cli('install') + self.assertNotEqual(res_install.returncode, 0) + self.assertEqual(path_pmo.read_text(), 'tampered external edit') + + res_rollback = self.run_cli('rollback') + self.assertNotEqual(res_rollback.returncode, 0) + self.assertEqual(path_pmo.read_text(), 'tampered external edit') + + def test_failed_upgrade_restoration_failure_preserves_journal(self): + import hashlib, importlib.util + from unittest.mock import patch + spec = importlib.util.spec_from_file_location('installer', SCRIPT) + installer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(installer) + installer.execute('install', self.home) + + path = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md' + path.write_text('previous reviewed version') + journal = self.home / 'editorial-v4/.installer/transaction.json' + state = json.loads(journal.read_text()) + rel = str(path.relative_to(self.home)) + state['installed'][rel] = hashlib.sha256(path.read_bytes()).hexdigest() + journal.write_text(json.dumps(state)) + + real_atomic = installer.atomic + fail_target = path + def fail_pmo(dest, data): + if dest == fail_target: + raise OSError('injected write failure on pmo') + return real_atomic(dest, data) + + def fail_restore(paths, before): + raise OSError('injected restoration failure') + + with patch.object(installer, 'atomic', side_effect=fail_pmo): + with patch.object(installer, 'restore', side_effect=fail_restore): + with self.assertRaisesRegex(OSError, 'injected write failure on pmo'): + installer.execute('install', self.home) + + # Journal must not be lost or unlinked, and must retain previous rollback metadata + self.assertTrue(journal.exists()) + jdata = json.loads(journal.read_text()) + self.assertIn('before', jdata) + # Previous installed metadata must be preserved + prev = jdata.get('previous_installed') or jdata.get('installed') + self.assertEqual(prev[rel], hashlib.sha256(b'previous reviewed version').hexdigest()) + if __name__ == '__main__': unittest.main()