fix(editorial): crash-safe upgrade with durable phase, previous metadata and recovery

This commit is contained in:
Canina Engineering authored and devops committed 2026-10-25 08:19:44 +00:00
1 parent fd14c05f3a
commit 0ab2980ff4
2 files changed
+262 -7

No files matched your search

+75 -7
View File
@@ -65,6 +65,50 @@ def bundle():
return result
def recover(journal, paths):
if not journal.exists():
return
try:
state = json.loads(journal.read_text())
except Exception:
raise ValueError('invalid journal json')
if state.get('phase') != 'applying':
return
if (set(state.get('before', {})) != ALLOWED or
set(state.get('installed', {})) != ALLOWED):
raise ValueError('invalid in-flight journal allowlist')
if state.get('previous_installed') is not None and set(state['previous_installed']) != ALLOWED:
raise ValueError('invalid in-flight journal previous_installed allowlist')
if state.get('previous_before') is not None and set(state['previous_before']) != ALLOWED:
raise ValueError('invalid in-flight journal previous_before allowlist')
for r, p in paths.items():
curr = p.read_bytes() if p.exists() else None
curr_d = digest(curr) if curr is not None else None
if curr_d == state['installed'][r]:
continue
if state.get('previous_installed') is not None:
if curr_d == state['previous_installed'][r]:
continue
else:
old = state['before'][r]
old_data = base64.b64decode(old['data']) if old else None
if curr == old_data:
continue
raise ValueError(f'interrupted transaction conflict: {r}')
restore_target = state.get('previous_before') or state['before']
restore(paths, restore_target)
if state.get('previous_journal'):
atomic(journal, base64.b64decode(state['previous_journal']))
elif state.get('previous_installed'):
atomic(journal, json.dumps({
'phase': 'installed',
'before': state['before'],
'installed': state['previous_installed']
}, sort_keys=True).encode())
else:
journal.unlink()
def execute(action, home):
if not home.is_absolute() or not home.is_dir():
raise ValueError('--target-home must be an existing absolute directory')
@@ -75,18 +119,23 @@ def execute(action, home):
bad = [r for r, p in paths.items() if not p.exists() or p.read_bytes() != files[r]]
if bad:
raise ValueError('verification mismatch: ' + ', '.join(bad))
if journal.exists():
jstate = json.loads(journal.read_text())
if jstate.get('phase', 'installed') != 'installed':
raise ValueError('verification refused: in-flight transaction requires recovery')
return {'status': 'configured_only', 'files': len(files), 'workflow_verified': False}
lock = safe(home, 'editorial-v4/.installer/lock')
lock.parent.mkdir(parents=True, exist_ok=True)
with lock.open('a') as stream:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
recover(journal, paths)
if action == 'rollback':
if not journal.exists():
return {'status': 'nothing_to_rollback'}
state = json.loads(journal.read_text())
if set(state['before']) != ALLOWED or set(state['installed']) != ALLOWED:
raise ValueError('invalid rollback allowlist')
# Refuse external edits rather than destroying another writer\'s work.
# Refuse external edits rather than destroying another writer's work.
for r, p in paths.items():
current = p.read_bytes() if p.exists() else None
before = state['before'][r]
@@ -103,10 +152,12 @@ def execute(action, home):
previous_state = None
if previous_journal is not None:
previous_state = json.loads(previous_journal)
if (set(previous_state) != {'before', 'installed'} or
if (set(previous_state) not in ({'before', 'installed'}, {'phase', 'before', 'installed'}) or
set(previous_state['before']) != ALLOWED or
set(previous_state['installed']) != ALLOWED):
raise ValueError('invalid upgrade journal allowlist')
if previous_state.get('phase', 'installed') != 'installed':
raise ValueError('invalid upgrade journal phase')
for r, p in paths.items():
old = previous_state['before'][r]
if old is not None:
@@ -118,22 +169,39 @@ def execute(action, home):
# advance. Never recover partial writes or overwrite live drift.
if not p.exists() or digest(p.read_bytes()) != previous_state['installed'][r]:
raise ValueError(f'existing transaction conflict: {r}')
before = {r: {'data': base64.b64encode(p.read_bytes()).decode(),
current_snapshot = {r: {'data': base64.b64encode(p.read_bytes()).decode(),
'mode': p.stat().st_mode & 0o777} if p.exists() else None
for r, p in paths.items()}
state = {'before': previous_state['before'] if previous_state else before,
'installed': {r: digest(v) for r, v in files.items()}}
atomic(journal, json.dumps(state, sort_keys=True).encode())
original_before = previous_state['before'] if previous_state else current_snapshot
incoming_installed = {r: digest(v) for r, v in files.items()}
in_flight_state = {
'phase': 'applying',
'before': original_before,
'installed': incoming_installed,
'previous_installed': previous_state['installed'] if previous_state else None,
'previous_before': current_snapshot,
'previous_journal': base64.b64encode(previous_journal).decode() if previous_journal else None,
}
atomic(journal, json.dumps(in_flight_state, sort_keys=True).encode())
try:
for r in changes:
atomic(paths[r], files[r])
except BaseException:
restore(paths, before)
try:
restore(paths, current_snapshot)
if previous_journal is None:
journal.unlink()
else:
atomic(journal, previous_journal)
except BaseException:
pass
raise
committed_state = {
'phase': 'installed',
'before': original_before,
'installed': incoming_installed,
}
atomic(journal, json.dumps(committed_state, sort_keys=True).encode())
return {'status': 'installed_config_only', 'files': len(files)}
@@ -158,5 +158,192 @@ class InstallerTest(unittest.TestCase):
self.assertNotEqual(self.run_cli('rollback').returncode, 0)
self.assertEqual(path.read_text(), 'external change')
def test_interrupted_upgrade_multi_file_safe_recovery_and_install(self):
import hashlib
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
path_pmo.parent.mkdir(parents=True, exist_ok=True)
path_pmo.write_text('original owned baseline')
path_pmo.chmod(0o600)
self.assertEqual(self.run_cli('install').returncode, 0)
# Verified previous installed state modifying status.json and pmo SKILL.md
path_status = self.home / 'editorial-v4/status.json'
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
v1_pmo = 'reviewed pmo v1 content'
path_status.write_text(v1_status)
path_pmo.write_text(v1_pmo)
journal = self.home / 'editorial-v4/.installer/transaction.json'
state = json.loads(journal.read_text())
rel_status = str(path_status.relative_to(self.home))
rel_pmo = str(path_pmo.relative_to(self.home))
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
journal.write_text(json.dumps(state))
# Crash midway during install: status.json is written, but process dies before pmo SKILL.md
crash_code = f"""
import os, sys, importlib.util, pathlib
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
real_atomic = m.atomic
def crash_atomic(p, data):
res = real_atomic(p, data)
if str(p).endswith('editorial-v4/status.json'):
os._exit(42)
return res
m.atomic = crash_atomic
m.execute('install', pathlib.Path('{self.home}'))
"""
proc = subprocess.run([sys.executable, '-c', crash_code])
self.assertEqual(proc.returncode, 42)
# Confirm mixed state on disk: status.json updated to incoming, pmo SKILL.md still v1
self.assertNotEqual(path_status.read_text(), v1_status)
self.assertEqual(path_pmo.read_text(), v1_pmo)
# Rerun install: must safely recover from interrupted transaction and finish upgrade
res = self.run_cli('install')
self.assertEqual(res.returncode, 0, f"install recovery failed: {res.stderr}")
self.assertEqual(self.run_cli('verify').returncode, 0)
# Rollback should still restore original owned baseline
self.assertEqual(self.run_cli('rollback').returncode, 0)
self.assertEqual(path_pmo.read_text(), 'original owned baseline')
self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600)
def test_interrupted_upgrade_multi_file_rollback_restores_original(self):
import hashlib
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
path_pmo.parent.mkdir(parents=True, exist_ok=True)
path_pmo.write_text('original owned baseline')
path_pmo.chmod(0o600)
self.assertEqual(self.run_cli('install').returncode, 0)
path_status = self.home / 'editorial-v4/status.json'
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
v1_pmo = 'reviewed pmo v1 content'
path_status.write_text(v1_status)
path_pmo.write_text(v1_pmo)
journal = self.home / 'editorial-v4/.installer/transaction.json'
state = json.loads(journal.read_text())
rel_status = str(path_status.relative_to(self.home))
rel_pmo = str(path_pmo.relative_to(self.home))
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
journal.write_text(json.dumps(state))
crash_code = f"""
import os, sys, importlib.util, pathlib
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
real_atomic = m.atomic
def crash_atomic(p, data):
res = real_atomic(p, data)
if str(p).endswith('editorial-v4/status.json'):
os._exit(42)
return res
m.atomic = crash_atomic
m.execute('install', pathlib.Path('{self.home}'))
"""
proc = subprocess.run([sys.executable, '-c', crash_code])
self.assertEqual(proc.returncode, 42)
# Rollback on interrupted state must safely recover and restore original owned baseline
res = self.run_cli('rollback')
self.assertEqual(res.returncode, 0, f"rollback recovery failed: {res.stderr}")
self.assertEqual(path_pmo.read_text(), 'original owned baseline')
self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600)
self.assertFalse(journal.exists())
def test_interrupted_upgrade_rejects_external_drift(self):
import hashlib
self.assertEqual(self.run_cli('install').returncode, 0)
path_status = self.home / 'editorial-v4/status.json'
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
v1_pmo = 'reviewed pmo v1 content'
path_status.write_text(v1_status)
path_pmo.write_text(v1_pmo)
journal = self.home / 'editorial-v4/.installer/transaction.json'
state = json.loads(journal.read_text())
rel_status = str(path_status.relative_to(self.home))
rel_pmo = str(path_pmo.relative_to(self.home))
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
journal.write_text(json.dumps(state))
crash_code = f"""
import os, sys, importlib.util, pathlib
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
real_atomic = m.atomic
def crash_atomic(p, data):
res = real_atomic(p, data)
if str(p).endswith('editorial-v4/status.json'):
os._exit(42)
return res
m.atomic = crash_atomic
m.execute('install', pathlib.Path('{self.home}'))
"""
proc = subprocess.run([sys.executable, '-c', crash_code])
self.assertEqual(proc.returncode, 42)
# External drift on path_pmo
path_pmo.write_text('tampered external edit')
res_install = self.run_cli('install')
self.assertNotEqual(res_install.returncode, 0)
self.assertEqual(path_pmo.read_text(), 'tampered external edit')
res_rollback = self.run_cli('rollback')
self.assertNotEqual(res_rollback.returncode, 0)
self.assertEqual(path_pmo.read_text(), 'tampered external edit')
def test_failed_upgrade_restoration_failure_preserves_journal(self):
import hashlib, importlib.util
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('installer', SCRIPT)
installer = importlib.util.module_from_spec(spec)
spec.loader.exec_module(installer)
installer.execute('install', self.home)
path = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
path.write_text('previous reviewed version')
journal = self.home / 'editorial-v4/.installer/transaction.json'
state = json.loads(journal.read_text())
rel = str(path.relative_to(self.home))
state['installed'][rel] = hashlib.sha256(path.read_bytes()).hexdigest()
journal.write_text(json.dumps(state))
real_atomic = installer.atomic
fail_target = path
def fail_pmo(dest, data):
if dest == fail_target:
raise OSError('injected write failure on pmo')
return real_atomic(dest, data)
def fail_restore(paths, before):
raise OSError('injected restoration failure')
with patch.object(installer, 'atomic', side_effect=fail_pmo):
with patch.object(installer, 'restore', side_effect=fail_restore):
with self.assertRaisesRegex(OSError, 'injected write failure on pmo'):
installer.execute('install', self.home)
# Journal must not be lost or unlinked, and must retain previous rollback metadata
self.assertTrue(journal.exists())
jdata = json.loads(journal.read_text())
self.assertIn('before', jdata)
# Previous installed metadata must be preserved
prev = jdata.get('previous_installed') or jdata.get('installed')
self.assertEqual(prev[rel], hashlib.sha256(b'previous reviewed version').hexdigest())
if __name__ == '__main__':
unittest.main()