fix(editorial): crash-safe upgrade with durable phase, previous metadata and recovery
This commit is contained in:
1 parent
fd14c05f3a
commit
0ab2980ff4
2 files changed
+268
-13
No files matched your search
@@ -65,6 +65,50 @@ def bundle():
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def recover(journal, paths):
|
||||||
|
if not journal.exists():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
state = json.loads(journal.read_text())
|
||||||
|
except Exception:
|
||||||
|
raise ValueError('invalid journal json')
|
||||||
|
if state.get('phase') != 'applying':
|
||||||
|
return
|
||||||
|
if (set(state.get('before', {})) != ALLOWED or
|
||||||
|
set(state.get('installed', {})) != ALLOWED):
|
||||||
|
raise ValueError('invalid in-flight journal allowlist')
|
||||||
|
if state.get('previous_installed') is not None and set(state['previous_installed']) != ALLOWED:
|
||||||
|
raise ValueError('invalid in-flight journal previous_installed allowlist')
|
||||||
|
if state.get('previous_before') is not None and set(state['previous_before']) != ALLOWED:
|
||||||
|
raise ValueError('invalid in-flight journal previous_before allowlist')
|
||||||
|
for r, p in paths.items():
|
||||||
|
curr = p.read_bytes() if p.exists() else None
|
||||||
|
curr_d = digest(curr) if curr is not None else None
|
||||||
|
if curr_d == state['installed'][r]:
|
||||||
|
continue
|
||||||
|
if state.get('previous_installed') is not None:
|
||||||
|
if curr_d == state['previous_installed'][r]:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
old = state['before'][r]
|
||||||
|
old_data = base64.b64decode(old['data']) if old else None
|
||||||
|
if curr == old_data:
|
||||||
|
continue
|
||||||
|
raise ValueError(f'interrupted transaction conflict: {r}')
|
||||||
|
restore_target = state.get('previous_before') or state['before']
|
||||||
|
restore(paths, restore_target)
|
||||||
|
if state.get('previous_journal'):
|
||||||
|
atomic(journal, base64.b64decode(state['previous_journal']))
|
||||||
|
elif state.get('previous_installed'):
|
||||||
|
atomic(journal, json.dumps({
|
||||||
|
'phase': 'installed',
|
||||||
|
'before': state['before'],
|
||||||
|
'installed': state['previous_installed']
|
||||||
|
}, sort_keys=True).encode())
|
||||||
|
else:
|
||||||
|
journal.unlink()
|
||||||
|
|
||||||
|
|
||||||
def execute(action, home):
|
def execute(action, home):
|
||||||
if not home.is_absolute() or not home.is_dir():
|
if not home.is_absolute() or not home.is_dir():
|
||||||
raise ValueError('--target-home must be an existing absolute directory')
|
raise ValueError('--target-home must be an existing absolute directory')
|
||||||
@@ -75,18 +119,23 @@ def execute(action, home):
|
|||||||
bad = [r for r, p in paths.items() if not p.exists() or p.read_bytes() != files[r]]
|
bad = [r for r, p in paths.items() if not p.exists() or p.read_bytes() != files[r]]
|
||||||
if bad:
|
if bad:
|
||||||
raise ValueError('verification mismatch: ' + ', '.join(bad))
|
raise ValueError('verification mismatch: ' + ', '.join(bad))
|
||||||
|
if journal.exists():
|
||||||
|
jstate = json.loads(journal.read_text())
|
||||||
|
if jstate.get('phase', 'installed') != 'installed':
|
||||||
|
raise ValueError('verification refused: in-flight transaction requires recovery')
|
||||||
return {'status': 'configured_only', 'files': len(files), 'workflow_verified': False}
|
return {'status': 'configured_only', 'files': len(files), 'workflow_verified': False}
|
||||||
lock = safe(home, 'editorial-v4/.installer/lock')
|
lock = safe(home, 'editorial-v4/.installer/lock')
|
||||||
lock.parent.mkdir(parents=True, exist_ok=True)
|
lock.parent.mkdir(parents=True, exist_ok=True)
|
||||||
with lock.open('a') as stream:
|
with lock.open('a') as stream:
|
||||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
recover(journal, paths)
|
||||||
if action == 'rollback':
|
if action == 'rollback':
|
||||||
if not journal.exists():
|
if not journal.exists():
|
||||||
return {'status': 'nothing_to_rollback'}
|
return {'status': 'nothing_to_rollback'}
|
||||||
state = json.loads(journal.read_text())
|
state = json.loads(journal.read_text())
|
||||||
if set(state['before']) != ALLOWED or set(state['installed']) != ALLOWED:
|
if set(state['before']) != ALLOWED or set(state['installed']) != ALLOWED:
|
||||||
raise ValueError('invalid rollback allowlist')
|
raise ValueError('invalid rollback allowlist')
|
||||||
# Refuse external edits rather than destroying another writer\'s work.
|
# Refuse external edits rather than destroying another writer's work.
|
||||||
for r, p in paths.items():
|
for r, p in paths.items():
|
||||||
current = p.read_bytes() if p.exists() else None
|
current = p.read_bytes() if p.exists() else None
|
||||||
before = state['before'][r]
|
before = state['before'][r]
|
||||||
@@ -103,10 +152,12 @@ def execute(action, home):
|
|||||||
previous_state = None
|
previous_state = None
|
||||||
if previous_journal is not None:
|
if previous_journal is not None:
|
||||||
previous_state = json.loads(previous_journal)
|
previous_state = json.loads(previous_journal)
|
||||||
if (set(previous_state) != {'before', 'installed'} or
|
if (set(previous_state) not in ({'before', 'installed'}, {'phase', 'before', 'installed'}) or
|
||||||
set(previous_state['before']) != ALLOWED or
|
set(previous_state['before']) != ALLOWED or
|
||||||
set(previous_state['installed']) != ALLOWED):
|
set(previous_state['installed']) != ALLOWED):
|
||||||
raise ValueError('invalid upgrade journal allowlist')
|
raise ValueError('invalid upgrade journal allowlist')
|
||||||
|
if previous_state.get('phase', 'installed') != 'installed':
|
||||||
|
raise ValueError('invalid upgrade journal phase')
|
||||||
for r, p in paths.items():
|
for r, p in paths.items():
|
||||||
old = previous_state['before'][r]
|
old = previous_state['before'][r]
|
||||||
if old is not None:
|
if old is not None:
|
||||||
@@ -118,22 +169,39 @@ def execute(action, home):
|
|||||||
# advance. Never recover partial writes or overwrite live drift.
|
# advance. Never recover partial writes or overwrite live drift.
|
||||||
if not p.exists() or digest(p.read_bytes()) != previous_state['installed'][r]:
|
if not p.exists() or digest(p.read_bytes()) != previous_state['installed'][r]:
|
||||||
raise ValueError(f'existing transaction conflict: {r}')
|
raise ValueError(f'existing transaction conflict: {r}')
|
||||||
before = {r: {'data': base64.b64encode(p.read_bytes()).decode(),
|
current_snapshot = {r: {'data': base64.b64encode(p.read_bytes()).decode(),
|
||||||
'mode': p.stat().st_mode & 0o777} if p.exists() else None
|
'mode': p.stat().st_mode & 0o777} if p.exists() else None
|
||||||
for r, p in paths.items()}
|
for r, p in paths.items()}
|
||||||
state = {'before': previous_state['before'] if previous_state else before,
|
original_before = previous_state['before'] if previous_state else current_snapshot
|
||||||
'installed': {r: digest(v) for r, v in files.items()}}
|
incoming_installed = {r: digest(v) for r, v in files.items()}
|
||||||
atomic(journal, json.dumps(state, sort_keys=True).encode())
|
in_flight_state = {
|
||||||
|
'phase': 'applying',
|
||||||
|
'before': original_before,
|
||||||
|
'installed': incoming_installed,
|
||||||
|
'previous_installed': previous_state['installed'] if previous_state else None,
|
||||||
|
'previous_before': current_snapshot,
|
||||||
|
'previous_journal': base64.b64encode(previous_journal).decode() if previous_journal else None,
|
||||||
|
}
|
||||||
|
atomic(journal, json.dumps(in_flight_state, sort_keys=True).encode())
|
||||||
try:
|
try:
|
||||||
for r in changes:
|
for r in changes:
|
||||||
atomic(paths[r], files[r])
|
atomic(paths[r], files[r])
|
||||||
except BaseException:
|
except BaseException:
|
||||||
restore(paths, before)
|
try:
|
||||||
if previous_journal is None:
|
restore(paths, current_snapshot)
|
||||||
journal.unlink()
|
if previous_journal is None:
|
||||||
else:
|
journal.unlink()
|
||||||
atomic(journal, previous_journal)
|
else:
|
||||||
|
atomic(journal, previous_journal)
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
raise
|
raise
|
||||||
|
committed_state = {
|
||||||
|
'phase': 'installed',
|
||||||
|
'before': original_before,
|
||||||
|
'installed': incoming_installed,
|
||||||
|
}
|
||||||
|
atomic(journal, json.dumps(committed_state, sort_keys=True).encode())
|
||||||
return {'status': 'installed_config_only', 'files': len(files)}
|
return {'status': 'installed_config_only', 'files': len(files)}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -158,5 +158,192 @@ class InstallerTest(unittest.TestCase):
|
|||||||
self.assertNotEqual(self.run_cli('rollback').returncode, 0)
|
self.assertNotEqual(self.run_cli('rollback').returncode, 0)
|
||||||
self.assertEqual(path.read_text(), 'external change')
|
self.assertEqual(path.read_text(), 'external change')
|
||||||
|
|
||||||
|
def test_interrupted_upgrade_multi_file_safe_recovery_and_install(self):
|
||||||
|
import hashlib
|
||||||
|
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||||
|
path_pmo.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path_pmo.write_text('original owned baseline')
|
||||||
|
path_pmo.chmod(0o600)
|
||||||
|
self.assertEqual(self.run_cli('install').returncode, 0)
|
||||||
|
|
||||||
|
# Verified previous installed state modifying status.json and pmo SKILL.md
|
||||||
|
path_status = self.home / 'editorial-v4/status.json'
|
||||||
|
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
|
||||||
|
v1_pmo = 'reviewed pmo v1 content'
|
||||||
|
path_status.write_text(v1_status)
|
||||||
|
path_pmo.write_text(v1_pmo)
|
||||||
|
|
||||||
|
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||||
|
state = json.loads(journal.read_text())
|
||||||
|
rel_status = str(path_status.relative_to(self.home))
|
||||||
|
rel_pmo = str(path_pmo.relative_to(self.home))
|
||||||
|
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
|
||||||
|
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
|
||||||
|
journal.write_text(json.dumps(state))
|
||||||
|
|
||||||
|
# Crash midway during install: status.json is written, but process dies before pmo SKILL.md
|
||||||
|
crash_code = f"""
|
||||||
|
import os, sys, importlib.util, pathlib
|
||||||
|
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
|
||||||
|
m = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(m)
|
||||||
|
real_atomic = m.atomic
|
||||||
|
def crash_atomic(p, data):
|
||||||
|
res = real_atomic(p, data)
|
||||||
|
if str(p).endswith('editorial-v4/status.json'):
|
||||||
|
os._exit(42)
|
||||||
|
return res
|
||||||
|
m.atomic = crash_atomic
|
||||||
|
m.execute('install', pathlib.Path('{self.home}'))
|
||||||
|
"""
|
||||||
|
proc = subprocess.run([sys.executable, '-c', crash_code])
|
||||||
|
self.assertEqual(proc.returncode, 42)
|
||||||
|
|
||||||
|
# Confirm mixed state on disk: status.json updated to incoming, pmo SKILL.md still v1
|
||||||
|
self.assertNotEqual(path_status.read_text(), v1_status)
|
||||||
|
self.assertEqual(path_pmo.read_text(), v1_pmo)
|
||||||
|
|
||||||
|
# Rerun install: must safely recover from interrupted transaction and finish upgrade
|
||||||
|
res = self.run_cli('install')
|
||||||
|
self.assertEqual(res.returncode, 0, f"install recovery failed: {res.stderr}")
|
||||||
|
self.assertEqual(self.run_cli('verify').returncode, 0)
|
||||||
|
|
||||||
|
# Rollback should still restore original owned baseline
|
||||||
|
self.assertEqual(self.run_cli('rollback').returncode, 0)
|
||||||
|
self.assertEqual(path_pmo.read_text(), 'original owned baseline')
|
||||||
|
self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600)
|
||||||
|
|
||||||
|
def test_interrupted_upgrade_multi_file_rollback_restores_original(self):
|
||||||
|
import hashlib
|
||||||
|
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||||
|
path_pmo.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path_pmo.write_text('original owned baseline')
|
||||||
|
path_pmo.chmod(0o600)
|
||||||
|
self.assertEqual(self.run_cli('install').returncode, 0)
|
||||||
|
|
||||||
|
path_status = self.home / 'editorial-v4/status.json'
|
||||||
|
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
|
||||||
|
v1_pmo = 'reviewed pmo v1 content'
|
||||||
|
path_status.write_text(v1_status)
|
||||||
|
path_pmo.write_text(v1_pmo)
|
||||||
|
|
||||||
|
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||||
|
state = json.loads(journal.read_text())
|
||||||
|
rel_status = str(path_status.relative_to(self.home))
|
||||||
|
rel_pmo = str(path_pmo.relative_to(self.home))
|
||||||
|
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
|
||||||
|
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
|
||||||
|
journal.write_text(json.dumps(state))
|
||||||
|
|
||||||
|
crash_code = f"""
|
||||||
|
import os, sys, importlib.util, pathlib
|
||||||
|
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
|
||||||
|
m = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(m)
|
||||||
|
real_atomic = m.atomic
|
||||||
|
def crash_atomic(p, data):
|
||||||
|
res = real_atomic(p, data)
|
||||||
|
if str(p).endswith('editorial-v4/status.json'):
|
||||||
|
os._exit(42)
|
||||||
|
return res
|
||||||
|
m.atomic = crash_atomic
|
||||||
|
m.execute('install', pathlib.Path('{self.home}'))
|
||||||
|
"""
|
||||||
|
proc = subprocess.run([sys.executable, '-c', crash_code])
|
||||||
|
self.assertEqual(proc.returncode, 42)
|
||||||
|
|
||||||
|
# Rollback on interrupted state must safely recover and restore original owned baseline
|
||||||
|
res = self.run_cli('rollback')
|
||||||
|
self.assertEqual(res.returncode, 0, f"rollback recovery failed: {res.stderr}")
|
||||||
|
self.assertEqual(path_pmo.read_text(), 'original owned baseline')
|
||||||
|
self.assertEqual(path_pmo.stat().st_mode & 0o777, 0o600)
|
||||||
|
self.assertFalse(journal.exists())
|
||||||
|
|
||||||
|
def test_interrupted_upgrade_rejects_external_drift(self):
|
||||||
|
import hashlib
|
||||||
|
self.assertEqual(self.run_cli('install').returncode, 0)
|
||||||
|
|
||||||
|
path_status = self.home / 'editorial-v4/status.json'
|
||||||
|
path_pmo = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||||
|
v1_status = '{"stage": "blocked", "cron_activation_allowed": false, "v": "1"}'
|
||||||
|
v1_pmo = 'reviewed pmo v1 content'
|
||||||
|
path_status.write_text(v1_status)
|
||||||
|
path_pmo.write_text(v1_pmo)
|
||||||
|
|
||||||
|
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||||
|
state = json.loads(journal.read_text())
|
||||||
|
rel_status = str(path_status.relative_to(self.home))
|
||||||
|
rel_pmo = str(path_pmo.relative_to(self.home))
|
||||||
|
state['installed'][rel_status] = hashlib.sha256(v1_status.encode()).hexdigest()
|
||||||
|
state['installed'][rel_pmo] = hashlib.sha256(v1_pmo.encode()).hexdigest()
|
||||||
|
journal.write_text(json.dumps(state))
|
||||||
|
|
||||||
|
crash_code = f"""
|
||||||
|
import os, sys, importlib.util, pathlib
|
||||||
|
spec = importlib.util.spec_from_file_location('installer', '{SCRIPT}')
|
||||||
|
m = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(m)
|
||||||
|
real_atomic = m.atomic
|
||||||
|
def crash_atomic(p, data):
|
||||||
|
res = real_atomic(p, data)
|
||||||
|
if str(p).endswith('editorial-v4/status.json'):
|
||||||
|
os._exit(42)
|
||||||
|
return res
|
||||||
|
m.atomic = crash_atomic
|
||||||
|
m.execute('install', pathlib.Path('{self.home}'))
|
||||||
|
"""
|
||||||
|
proc = subprocess.run([sys.executable, '-c', crash_code])
|
||||||
|
self.assertEqual(proc.returncode, 42)
|
||||||
|
|
||||||
|
# External drift on path_pmo
|
||||||
|
path_pmo.write_text('tampered external edit')
|
||||||
|
|
||||||
|
res_install = self.run_cli('install')
|
||||||
|
self.assertNotEqual(res_install.returncode, 0)
|
||||||
|
self.assertEqual(path_pmo.read_text(), 'tampered external edit')
|
||||||
|
|
||||||
|
res_rollback = self.run_cli('rollback')
|
||||||
|
self.assertNotEqual(res_rollback.returncode, 0)
|
||||||
|
self.assertEqual(path_pmo.read_text(), 'tampered external edit')
|
||||||
|
|
||||||
|
def test_failed_upgrade_restoration_failure_preserves_journal(self):
|
||||||
|
import hashlib, importlib.util
|
||||||
|
from unittest.mock import patch
|
||||||
|
spec = importlib.util.spec_from_file_location('installer', SCRIPT)
|
||||||
|
installer = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(installer)
|
||||||
|
installer.execute('install', self.home)
|
||||||
|
|
||||||
|
path = self.home / 'profiles/pmo/skills/canina-editorial-v4/SKILL.md'
|
||||||
|
path.write_text('previous reviewed version')
|
||||||
|
journal = self.home / 'editorial-v4/.installer/transaction.json'
|
||||||
|
state = json.loads(journal.read_text())
|
||||||
|
rel = str(path.relative_to(self.home))
|
||||||
|
state['installed'][rel] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
journal.write_text(json.dumps(state))
|
||||||
|
|
||||||
|
real_atomic = installer.atomic
|
||||||
|
fail_target = path
|
||||||
|
def fail_pmo(dest, data):
|
||||||
|
if dest == fail_target:
|
||||||
|
raise OSError('injected write failure on pmo')
|
||||||
|
return real_atomic(dest, data)
|
||||||
|
|
||||||
|
def fail_restore(paths, before):
|
||||||
|
raise OSError('injected restoration failure')
|
||||||
|
|
||||||
|
with patch.object(installer, 'atomic', side_effect=fail_pmo):
|
||||||
|
with patch.object(installer, 'restore', side_effect=fail_restore):
|
||||||
|
with self.assertRaisesRegex(OSError, 'injected write failure on pmo'):
|
||||||
|
installer.execute('install', self.home)
|
||||||
|
|
||||||
|
# Journal must not be lost or unlinked, and must retain previous rollback metadata
|
||||||
|
self.assertTrue(journal.exists())
|
||||||
|
jdata = json.loads(journal.read_text())
|
||||||
|
self.assertIn('before', jdata)
|
||||||
|
# Previous installed metadata must be preserved
|
||||||
|
prev = jdata.get('previous_installed') or jdata.get('installed')
|
||||||
|
self.assertEqual(prev[rel], hashlib.sha256(b'previous reviewed version').hexdigest())
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
Reference in new issue
Block a user