fix(editorial): crash-safe upgrade with durable phase, previous metadata and recovery
This commit is contained in:
1 parent
fd14c05f3a
commit
0ab2980ff4
2 files changed
+268
-13
No files matched your search
@@ -65,6 +65,50 @@ def bundle():
|
||||
return result
|
||||
|
||||
|
||||
def recover(journal, paths):
|
||||
if not journal.exists():
|
||||
return
|
||||
try:
|
||||
state = json.loads(journal.read_text())
|
||||
except Exception:
|
||||
raise ValueError('invalid journal json')
|
||||
if state.get('phase') != 'applying':
|
||||
return
|
||||
if (set(state.get('before', {})) != ALLOWED or
|
||||
set(state.get('installed', {})) != ALLOWED):
|
||||
raise ValueError('invalid in-flight journal allowlist')
|
||||
if state.get('previous_installed') is not None and set(state['previous_installed']) != ALLOWED:
|
||||
raise ValueError('invalid in-flight journal previous_installed allowlist')
|
||||
if state.get('previous_before') is not None and set(state['previous_before']) != ALLOWED:
|
||||
raise ValueError('invalid in-flight journal previous_before allowlist')
|
||||
for r, p in paths.items():
|
||||
curr = p.read_bytes() if p.exists() else None
|
||||
curr_d = digest(curr) if curr is not None else None
|
||||
if curr_d == state['installed'][r]:
|
||||
continue
|
||||
if state.get('previous_installed') is not None:
|
||||
if curr_d == state['previous_installed'][r]:
|
||||
continue
|
||||
else:
|
||||
old = state['before'][r]
|
||||
old_data = base64.b64decode(old['data']) if old else None
|
||||
if curr == old_data:
|
||||
continue
|
||||
raise ValueError(f'interrupted transaction conflict: {r}')
|
||||
restore_target = state.get('previous_before') or state['before']
|
||||
restore(paths, restore_target)
|
||||
if state.get('previous_journal'):
|
||||
atomic(journal, base64.b64decode(state['previous_journal']))
|
||||
elif state.get('previous_installed'):
|
||||
atomic(journal, json.dumps({
|
||||
'phase': 'installed',
|
||||
'before': state['before'],
|
||||
'installed': state['previous_installed']
|
||||
}, sort_keys=True).encode())
|
||||
else:
|
||||
journal.unlink()
|
||||
|
||||
|
||||
def execute(action, home):
|
||||
if not home.is_absolute() or not home.is_dir():
|
||||
raise ValueError('--target-home must be an existing absolute directory')
|
||||
@@ -75,18 +119,23 @@ def execute(action, home):
|
||||
bad = [r for r, p in paths.items() if not p.exists() or p.read_bytes() != files[r]]
|
||||
if bad:
|
||||
raise ValueError('verification mismatch: ' + ', '.join(bad))
|
||||
if journal.exists():
|
||||
jstate = json.loads(journal.read_text())
|
||||
if jstate.get('phase', 'installed') != 'installed':
|
||||
raise ValueError('verification refused: in-flight transaction requires recovery')
|
||||
return {'status': 'configured_only', 'files': len(files), 'workflow_verified': False}
|
||||
lock = safe(home, 'editorial-v4/.installer/lock')
|
||||
lock.parent.mkdir(parents=True, exist_ok=True)
|
||||
with lock.open('a') as stream:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
recover(journal, paths)
|
||||
if action == 'rollback':
|
||||
if not journal.exists():
|
||||
return {'status': 'nothing_to_rollback'}
|
||||
state = json.loads(journal.read_text())
|
||||
if set(state['before']) != ALLOWED or set(state['installed']) != ALLOWED:
|
||||
raise ValueError('invalid rollback allowlist')
|
||||
# Refuse external edits rather than destroying another writer\'s work.
|
||||
# Refuse external edits rather than destroying another writer's work.
|
||||
for r, p in paths.items():
|
||||
current = p.read_bytes() if p.exists() else None
|
||||
before = state['before'][r]
|
||||
@@ -103,10 +152,12 @@ def execute(action, home):
|
||||
previous_state = None
|
||||
if previous_journal is not None:
|
||||
previous_state = json.loads(previous_journal)
|
||||
if (set(previous_state) != {'before', 'installed'} or
|
||||
if (set(previous_state) not in ({'before', 'installed'}, {'phase', 'before', 'installed'}) or
|
||||
set(previous_state['before']) != ALLOWED or
|
||||
set(previous_state['installed']) != ALLOWED):
|
||||
raise ValueError('invalid upgrade journal allowlist')
|
||||
if previous_state.get('phase', 'installed') != 'installed':
|
||||
raise ValueError('invalid upgrade journal phase')
|
||||
for r, p in paths.items():
|
||||
old = previous_state['before'][r]
|
||||
if old is not None:
|
||||
@@ -118,22 +169,39 @@ def execute(action, home):
|
||||
# advance. Never recover partial writes or overwrite live drift.
|
||||
if not p.exists() or digest(p.read_bytes()) != previous_state['installed'][r]:
|
||||
raise ValueError(f'existing transaction conflict: {r}')
|
||||
before = {r: {'data': base64.b64encode(p.read_bytes()).decode(),
|
||||
'mode': p.stat().st_mode & 0o777} if p.exists() else None
|
||||
for r, p in paths.items()}
|
||||
state = {'before': previous_state['before'] if previous_state else before,
|
||||
'installed': {r: digest(v) for r, v in files.items()}}
|
||||
atomic(journal, json.dumps(state, sort_keys=True).encode())
|
||||
current_snapshot = {r: {'data': base64.b64encode(p.read_bytes()).decode(),
|
||||
'mode': p.stat().st_mode & 0o777} if p.exists() else None
|
||||
for r, p in paths.items()}
|
||||
original_before = previous_state['before'] if previous_state else current_snapshot
|
||||
incoming_installed = {r: digest(v) for r, v in files.items()}
|
||||
in_flight_state = {
|
||||
'phase': 'applying',
|
||||
'before': original_before,
|
||||
'installed': incoming_installed,
|
||||
'previous_installed': previous_state['installed'] if previous_state else None,
|
||||
'previous_before': current_snapshot,
|
||||
'previous_journal': base64.b64encode(previous_journal).decode() if previous_journal else None,
|
||||
}
|
||||
atomic(journal, json.dumps(in_flight_state, sort_keys=True).encode())
|
||||
try:
|
||||
for r in changes:
|
||||
atomic(paths[r], files[r])
|
||||
except BaseException:
|
||||
restore(paths, before)
|
||||
if previous_journal is None:
|
||||
journal.unlink()
|
||||
else:
|
||||
atomic(journal, previous_journal)
|
||||
try:
|
||||
restore(paths, current_snapshot)
|
||||
if previous_journal is None:
|
||||
journal.unlink()
|
||||
else:
|
||||
atomic(journal, previous_journal)
|
||||
except BaseException:
|
||||
pass
|
||||
raise
|
||||
committed_state = {
|
||||
'phase': 'installed',
|
||||
'before': original_before,
|
||||
'installed': incoming_installed,
|
||||
}
|
||||
atomic(journal, json.dumps(committed_state, sort_keys=True).encode())
|
||||
return {'status': 'installed_config_only', 'files': len(files)}
|
||||
|
||||
|
||||
|
||||
@@ -158,5 +158,192 @@ class InstallerTest(unittest.TestCase):
|
||||
self.assertNotEqual(self.run_cli('rollback').returncode, 0)
|
||||
self.assertEqual(path.read_text(), 'external change')
|
||||
|
||||
def test_interrupted_upgrade_multi_file_safe_recovery_and_install(self):
|
||||
import hashlib
|
||||
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||
path_pmo.parent.mkdir(parents=True, exist_ok=True)
|
||||
path_pmo.write_text('original owned baseline')
|
||||
path_pmo.chmod(0o600)
|
||||
self.assertEqual(self.run_cli('install').returncode, 0)
|
||||
|
||||
# Verified previous installed state modifying status.json and pmo SKILL.md
|
||||
path_status = self.home / 'editorial-v4/status.json'
|
||||
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
|
||||
v1_pmo = 'reviewed pmo v1 content'
|
||||
path_status.write_text(v1_status)
|
||||
path_pmo.write_text(v1_pmo)
|
||||
|
||||
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||
state = json.loads(journal.read_text())
|
||||
rel_status = str(path_status.relative_to(self.home))
|
||||
rel_pmo = str(path_pmo.relative_to(self.home))
|
||||
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
|
||||
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
|
||||
journal.write_text(json.dumps(state))
|
||||
|
||||
# Crash midway during install: status.json is written, but process dies before pmo SKILL.md
|
||||
crash_code = f"""
|
||||
import os, sys, importlib.util, pathlib
|
||||
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
real_atomic = m.atomic
|
||||
def crash_atomic(p, data):
|
||||
res = real_atomic(p, data)
|
||||
if str(p).endswith('editorial-v4/status.json'):
|
||||
os._exit(42)
|
||||
return res
|
||||
m.atomic = crash_atomic
|
||||
m.execute('install', pathlib.Path('{self.home}'))
|
||||
"""
|
||||
proc = subprocess.run([sys.executable, '-c', crash_code])
|
||||
self.assertEqual(proc.returncode, 42)
|
||||
|
||||
# Confirm mixed state on disk: status.json updated to incoming, pmo SKILL.md still v1
|
||||
self.assertNotEqual(path_status.read_text(), v1_status)
|
||||
self.assertEqual(path_pmo.read_text(), v1_pmo)
|
||||
|
||||
# Rerun install: must safely recover from interrupted transaction and finish upgrade
|
||||
res = self.run_cli('install')
|
||||
self.assertEqual(res.returncode, 0, f"install recovery failed: {res.stderr}")
|
||||
self.assertEqual(self.run_cli('verify').returncode, 0)
|
||||
|
||||
# Rollback should still restore original owned baseline
|
||||
self.assertEqual(self.run_cli('rollback').returncode, 0)
|
||||
self.assertEqual(path_pmo.read_text(), 'original owned baseline')
|
||||
self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600)
|
||||
|
||||
def test_interrupted_upgrade_multi_file_rollback_restores_original(self):
|
||||
import hashlib
|
||||
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||
path_pmo.parent.mkdir(parents=True, exist_ok=True)
|
||||
path_pmo.write_text('original owned baseline')
|
||||
path_pmo.chmod(0o600)
|
||||
self.assertEqual(self.run_cli('install').returncode, 0)
|
||||
|
||||
path_status = self.home / 'editorial-v4/status.json'
|
||||
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
|
||||
v1_pmo = 'reviewed pmo v1 content'
|
||||
path_status.write_text(v1_status)
|
||||
path_pmo.write_text(v1_pmo)
|
||||
|
||||
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||
state = json.loads(journal.read_text())
|
||||
rel_status = str(path_status.relative_to(self.home))
|
||||
rel_pmo = str(path_pmo.relative_to(self.home))
|
||||
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
|
||||
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
|
||||
journal.write_text(json.dumps(state))
|
||||
|
||||
crash_code = f"""
|
||||
import os, sys, importlib.util, pathlib
|
||||
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
real_atomic = m.atomic
|
||||
def crash_atomic(p, data):
|
||||
res = real_atomic(p, data)
|
||||
if str(p).endswith('editorial-v4/status.json'):
|
||||
os._exit(42)
|
||||
return res
|
||||
m.atomic = crash_atomic
|
||||
m.execute('install', pathlib.Path('{self.home}'))
|
||||
"""
|
||||
proc = subprocess.run([sys.executable, '-c', crash_code])
|
||||
self.assertEqual(proc.returncode, 42)
|
||||
|
||||
# Rollback on interrupted state must safely recover and restore original owned baseline
|
||||
res = self.run_cli('rollback')
|
||||
self.assertEqual(res.returncode, 0, f"rollback recovery failed: {res.stderr}")
|
||||
self.assertEqual(path_pmo.read_text(), 'original owned baseline')
|
||||
self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600)
|
||||
self.assertFalse(journal.exists())
|
||||
|
||||
def test_interrupted_upgrade_rejects_external_drift(self):
|
||||
import hashlib
|
||||
self.assertEqual(self.run_cli('install').returncode, 0)
|
||||
|
||||
path_status = self.home / 'editorial-v4/status.json'
|
||||
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
|
||||
v1_pmo = 'reviewed pmo v1 content'
|
||||
path_status.write_text(v1_status)
|
||||
path_pmo.write_text(v1_pmo)
|
||||
|
||||
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||
state = json.loads(journal.read_text())
|
||||
rel_status = str(path_status.relative_to(self.home))
|
||||
rel_pmo = str(path_pmo.relative_to(self.home))
|
||||
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
|
||||
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
|
||||
journal.write_text(json.dumps(state))
|
||||
|
||||
crash_code = f"""
|
||||
import os, sys, importlib.util, pathlib
|
||||
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
real_atomic = m.atomic
|
||||
def crash_atomic(p, data):
|
||||
res = real_atomic(p, data)
|
||||
if str(p).endswith('editorial-v4/status.json'):
|
||||
os._exit(42)
|
||||
return res
|
||||
m.atomic = crash_atomic
|
||||
m.execute('install', pathlib.Path('{self.home}'))
|
||||
"""
|
||||
proc = subprocess.run([sys.executable, '-c', crash_code])
|
||||
self.assertEqual(proc.returncode, 42)
|
||||
|
||||
# External drift on path_pmo
|
||||
path_pmo.write_text('tampered external edit')
|
||||
|
||||
res_install = self.run_cli('install')
|
||||
self.assertNotEqual(res_install.returncode, 0)
|
||||
self.assertEqual(path_pmo.read_text(), 'tampered external edit')
|
||||
|
||||
res_rollback = self.run_cli('rollback')
|
||||
self.assertNotEqual(res_rollback.returncode, 0)
|
||||
self.assertEqual(path_pmo.read_text(), 'tampered external edit')
|
||||
|
||||
def test_failed_upgrade_restoration_failure_preserves_journal(self):
|
||||
import hashlib, importlib.util
|
||||
from unittest.mock import patch
|
||||
spec = importlib.util.spec_from_file_location('installer', SCRIPT)
|
||||
installer = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(installer)
|
||||
installer.execute('install', self.home)
|
||||
|
||||
path = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||
path.write_text('previous reviewed version')
|
||||
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||
state = json.loads(journal.read_text())
|
||||
rel = str(path.relative_to(self.home))
|
||||
state['installed'][rel] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
journal.write_text(json.dumps(state))
|
||||
|
||||
real_atomic = installer.atomic
|
||||
fail_target = path
|
||||
def fail_pmo(dest, data):
|
||||
if dest == fail_target:
|
||||
raise OSError('injected write failure on pmo')
|
||||
return real_atomic(dest, data)
|
||||
|
||||
def fail_restore(paths, before):
|
||||
raise OSError('injected restoration failure')
|
||||
|
||||
with patch.object(installer, 'atomic', side_effect=fail_pmo):
|
||||
with patch.object(installer, 'restore', side_effect=fail_restore):
|
||||
with self.assertRaisesRegex(OSError, 'injected write failure on pmo'):
|
||||
installer.execute('install', self.home)
|
||||
|
||||
# Journal must not be lost or unlinked, and must retain previous rollback metadata
|
||||
self.assertTrue(journal.exists())
|
||||
jdata = json.loads(journal.read_text())
|
||||
self.assertIn('before', jdata)
|
||||
# Previous installed metadata must be preserved
|
||||
prev = jdata.get('previous_installed') or jdata.get('installed')
|
||||
self.assertEqual(prev[rel], hashlib.sha256(b'previous reviewed version').hexdigest())
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user