Skip to content

Commit 0111ffb

Browse files
Merge pull request #139 from RetroReversing/feat/persistent-deep-permutation
2 parents 96922cf + 54650a4 commit 0111ffb

16 files changed

Lines changed: 806 additions & 21 deletions

‎ROADMAP.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -893,3 +893,9 @@ agent briefs and deep-search output. Planned: compiler-specific diagnostic flags
893893
opt-in diagnostic build command for imported projects (their ordinary build diagnostics remain available).
894894
Reuse discovery batches fingerprint names/cache reads and persists a separate cheap-discovery cursor; at most
895895
half the remaining shared reuse time goes to that phase, leaving time for wider matching.
896+
897+
Project settings now offer a resumable deep permutation pass across unmatched functions in ROM and imported
898+
projects. Each function's completed bounded turn is persisted with its source/build scope and budget. Unfinished
899+
turns resume after restart; changed inputs become eligible again. Busy functions and candidates without scored C
900+
wait without being marked complete. The pass uses the same local/random searches, claims and judges as a single
901+
function, with a visible progress bar and stop button. No paid helper is launched.

‎src/decompnet/app/deep_permuter.py‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ def __init__(self, net, minutes, max_variants, log, fn=None):
4949
self.baseline_measured = False
5050
self.phase = 'local repairs'
5151
self.measuring_baseline = False
52+
self.record_improvement = lambda score: None
53+
self.recorded_best = None
5254

5355
def check(self):
5456
self.log.check_cancel()
@@ -67,6 +69,9 @@ def percent(score):
6769
'best_score': self.best, 'best_percent': percent(self.best), 'improvements': self.improvements}
6870

6971
def report(self):
72+
if self.best is not None and (self.recorded_best is None or self.best < self.recorded_best):
73+
self.record_improvement(self.best)
74+
self.recorded_best = self.best
7075
self.log.progress({'phase': self.phase, 'tested': self.tested, 'skipped': self.skipped,
7176
**self.metrics(), 'variant_limit': self.limit, 'rejected': self.rejected,
7277
'seconds_left': max(0, round(self.deadline-time.monotonic()))})
@@ -121,7 +126,13 @@ def _score(self, agent, proj, fn, source, best, fork):
121126
def run(net, slug, name, minutes, max_variants, log):
122127
"""Visit the supported local repairs, then spend the remaining budget on random permutations."""
123128
proj, fn = validate(net, slug, name, minutes, max_variants)
129+
sweeps = getattr(net, 'deep_sweeps', None)
130+
if sweeps:
131+
from ..candidate_scope import key
132+
build_scope = key(net, proj, fn)
124133
search = Search(net, minutes, max_variants, log, fn)
134+
if sweeps:
135+
search.record_improvement = lambda score: sweeps.improvement(fn, score)
125136
search.report()
126137
state = State()
127138
agent, _ = net.local_agent('permuter')
@@ -145,17 +156,29 @@ def run(net, slug, name, minutes, max_variants, log):
145156
reason = 'Matched'
146157
else:
147158
best = baseline
148-
source = fn['best_source']
149-
rows = getattr(net, '_last_diffs', {}).get((agent.id, fn['id']), ([], ''))[0]
150-
choices = draft_repairs.variants(source, fn['name'], proj['arch'], fn['asm'] or '', flags, limit=None)
151-
choices += stack_padding_repairs.grow(source, fn['name'], proj['arch'], rows)
152-
choices += [s for _,s in permuter_local.variants(source, fn['name'])]
153-
for candidate in dict.fromkeys(choices):
154-
score = search._score(agent, proj, fn, candidate, best, fork)
155-
if score is not None and score < best:
156-
best = score
157-
if score == 0:
158-
reason = 'Matched'
159+
seen = {fn['best_source']}
160+
repair_fn = fn
161+
while best > 0:
162+
search.check()
163+
source = repair_fn['best_source']
164+
rows = getattr(net, '_last_diffs', {}).get((agent.id, repair_fn['id']), ([], ''))[0]
165+
choices = draft_repairs.variants(source, repair_fn['name'], proj['arch'], repair_fn['asm'] or '', flags, limit=None)
166+
choices += stack_padding_repairs.grow(source, repair_fn['name'], proj['arch'], rows)
167+
choices += [s for _,s in permuter_local.variants(source, repair_fn['name'])]
168+
for candidate in dict.fromkeys(choices):
169+
if candidate in seen:
170+
continue
171+
seen.add(candidate)
172+
score = search._score(agent, proj, repair_fn, candidate, best, fork)
173+
if score == 0:
174+
reason = 'Matched'
175+
if score is not None and score < best:
176+
best = score
177+
# Chain repairs from the saved winner and its new diff; keep the same
178+
# deadline/compile budget and avoid retesting candidates from earlier rounds.
179+
repair_fn = net._function(name, slug)
180+
break
181+
else:
159182
break
160183
finally:
161184
search._let_go(agent, fn)
@@ -179,5 +202,8 @@ def run(net, slug, name, minutes, max_variants, log):
179202
reason = str(error)
180203
search.report()
181204
log(f'{reason}: {search.tested} candidates tested, {search.skipped} cached misses skipped, {search.rejected} invalid calls skipped. Verified improvements are saved immediately.')
182-
return {'function': name, 'reason': reason, 'tested': search.tested, 'skipped': search.skipped,
183-
'rejected': search.rejected, 'matched': net._function(name, slug)['status'] == 'matched', **search.metrics()}
205+
result = {'function': name, 'reason': reason, 'tested': search.tested, 'skipped': search.skipped,
206+
'rejected': search.rejected, 'matched': net._function(name, slug)['status'] == 'matched', **search.metrics()}
207+
if sweeps:
208+
sweeps.completed(slug, fn, minutes, max_variants, result, build_scope)
209+
return result

‎src/decompnet/app/deep_sweep.py‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
"""Durable project-wide deep passes; each candidate gets a bounded turn through the normal judges."""
2+
import hashlib
3+
import json
4+
import threading
5+
import time
6+
7+
from .. import candidate_scope, scoring
8+
from ..network import NetworkError
9+
from . import agent_tokens, deep_permuter
10+
11+
POLL_SECONDS = 15
12+
SCHEMA = '''CREATE TABLE IF NOT EXISTS deep_permutation_passes(
13+
function_id INTEGER PRIMARY KEY, scope TEXT NOT NULL, state TEXT NOT NULL,
14+
minutes INTEGER NOT NULL, max_variants INTEGER NOT NULL, completed_at REAL,
15+
result TEXT NOT NULL DEFAULT '{}', error TEXT NOT NULL DEFAULT '')'''
16+
17+
18+
def signature(net, proj, fn, minutes, variants):
19+
return hashlib.sha256(repr(('deep-pass-v1', agent_tokens.target_scope(net, fn, ''),
20+
fn['best_source'], minutes, variants)).encode()).hexdigest()
21+
22+
23+
class Sweeps:
24+
def __init__(self, net, jobs):
25+
self.net, self.jobs = net, jobs
26+
self.active = {}
27+
self.lock = threading.RLock()
28+
with net.store.tx() as db:
29+
db.execute(SCHEMA)
30+
db.execute('''CREATE TABLE IF NOT EXISTS deep_permutation_improvements(
31+
function_id INTEGER PRIMARY KEY, starting_score INTEGER NOT NULL,
32+
starting_percent REAL, new_score INTEGER NOT NULL,
33+
new_percent REAL, updated_at REAL NOT NULL)''')
34+
35+
def config(self, slug):
36+
proj = self.net.project_row(slug)
37+
row = self.net.store.one('SELECT value FROM settings WHERE key=?', f'deep-sweep:{proj["id"]}')
38+
return json.loads(row['value']) if row else {'enabled': False, 'minutes': 5, 'max_variants': 10000}
39+
40+
def save(self, slug, cfg):
41+
proj = self.net.project_row(slug)
42+
with self.net.store.tx() as db:
43+
db.execute('INSERT OR REPLACE INTO settings(key,value) VALUES(?,?)',
44+
(f'deep-sweep:{proj["id"]}', json.dumps(cfg)))
45+
46+
def history(self, slug, fn):
47+
row = self.net.store.one('SELECT * FROM deep_permutation_passes WHERE function_id=?', fn['id'])
48+
if not row:
49+
return {'state': 'not tried', 'completed': False}
50+
result = dict(row)
51+
result['stale'] = row['scope'] != signature(self.net, self.net.project_row(slug), fn,
52+
row['minutes'], row['max_variants'])
53+
result['completed'] = row['state'] == 'completed' and not result['stale']
54+
result['result'] = json.loads(row['result'])
55+
return result
56+
57+
def improvement(self, before, score):
58+
"""Keep only scores measured by this search, including wins before cancellation."""
59+
start = before['best_score']
60+
if score is None or start is None or score >= start:
61+
return
62+
def percent(value):
63+
if value == 0:
64+
return 100.0
65+
if not before['n_insns']:
66+
return None
67+
return max(0.0, min(99.9 if value else 100.0,
68+
100.0 * (1 - value / max(1, before['n_insns'] * 100))))
69+
with self.net.store.tx() as db:
70+
db.execute('''INSERT INTO deep_permutation_improvements
71+
(function_id,starting_score,starting_percent,new_score,new_percent,updated_at)
72+
VALUES(?,?,?,?,?,?) ON CONFLICT(function_id) DO UPDATE SET
73+
new_score=excluded.new_score,new_percent=excluded.new_percent,updated_at=excluded.updated_at
74+
WHERE excluded.new_score < deep_permutation_improvements.new_score''',
75+
(before['id'], start, percent(start), score, percent(score), time.time()))
76+
77+
def improvements(self, project_id):
78+
rows = self.net.store.all('''SELECT i.*, f.name AS function FROM deep_permutation_improvements i
79+
JOIN functions f ON f.id=i.function_id WHERE f.project_id=? ORDER BY i.updated_at DESC,f.id''', project_id)
80+
return [dict(row) | {'starting_percent_text': scoring.percent_text(row['starting_percent']),
81+
'new_percent_text': scoring.percent_text(row['new_percent'])} for row in rows]
82+
83+
def status(self, slug):
84+
proj = self.net.project_row(slug)
85+
counts = self.net.store.one("SELECT COUNT(*) total, SUM(best_source IS NOT NULL AND best_score IS NOT NULL) ready "
86+
"FROM functions WHERE project_id=? AND status='open'", proj['id'])
87+
completed = self.net.store.one("SELECT COUNT(*) n FROM deep_permutation_passes d JOIN functions f ON f.id=d.function_id "
88+
"WHERE f.project_id=? AND d.state='completed'", proj['id'])['n']
89+
errors = self.net.store.one("SELECT COUNT(*) n FROM deep_permutation_passes d JOIN functions f ON f.id=d.function_id WHERE f.project_id=? AND d.state='error'", proj['id'])['n']
90+
improvements = self.improvements(proj['id'])
91+
with self.lock:
92+
current = dict(self.active.get(slug, {}))
93+
job = self.jobs.jobs.get(current.get('job'), {})
94+
return self.config(slug) | {'total': counts['total'], 'ready': counts['ready'] or 0,
95+
'improved': len(improvements), 'improvements': improvements,
96+
'completed': completed, 'errors': errors, 'state': job.get('state', 'idle'), **current,
97+
'progress': job.get('progress', {}), 'error': job.get('error')}
98+
99+
def start(self, slug, minutes=5, max_variants=10000, repeat=False):
100+
if (isinstance(minutes, bool) or not isinstance(minutes, int) or not 1 <= minutes <= 120 or
101+
isinstance(max_variants, bool) or not isinstance(max_variants, int) or not 1 <= max_variants <= 1000000):
102+
raise NetworkError('Choose 1–120 minutes and 1–1,000,000 variants per function.')
103+
self.net.project_row(slug)
104+
if slug in self.net.links:
105+
raise NetworkError('Run the deep pass on the room host’s project.')
106+
with self.lock:
107+
previous = self.active.get(slug, {})
108+
if self.jobs.jobs.get(previous.get('job'), {}).get('state') == 'running':
109+
raise NetworkError('A deep pass is already running for this project. Stop it before starting another.')
110+
if repeat:
111+
with self.net.store.tx() as db:
112+
db.execute("DELETE FROM deep_permutation_passes WHERE function_id IN "
113+
"(SELECT id FROM functions WHERE project_id=? AND status='open')",
114+
self.net.project_row(slug)['id'])
115+
self.save(slug, {'enabled': True, 'minutes': minutes, 'max_variants': max_variants})
116+
job = self.jobs.start('deep-permutation-pass', lambda log: self.run(slug, minutes, max_variants, log),
117+
cancel_message='Deep pass stopped. Verified improvements and completed passes are saved.')
118+
self.active[slug] = {'job': job}
119+
return self.status(slug)
120+
121+
def stop(self, slug):
122+
with self.lock:
123+
cfg = self.config(slug)
124+
self.save(slug, cfg | {'enabled': False})
125+
job = self.active.get(slug, {}).get('job')
126+
if job:
127+
self.jobs.cancel(job)
128+
return self.status(slug)
129+
130+
def resume(self):
131+
# An interrupted function is retried; only completed bounded turns are skipped.
132+
for proj in self.net.store.all('SELECT slug FROM projects'):
133+
cfg = self.config(proj['slug'])
134+
if cfg['enabled']:
135+
try:
136+
self.start(proj['slug'], cfg['minutes'], cfg['max_variants'])
137+
except NetworkError as error:
138+
self.save(proj['slug'], cfg | {'enabled': False, 'resume_error': str(error)})
139+
self.jobs.logger.warning('Deep pass could not resume: %s', error)
140+
141+
def shutdown(self):
142+
# Preserve enabled intent for the next launch, but let current searches release claims.
143+
with self.lock:
144+
for current in self.active.values():
145+
self.jobs.cancel(current['job'])
146+
147+
def completed(self, slug, before, minutes, variants, result, build_scope):
148+
self.improvement(before, result.get('best_score'))
149+
fresh = self.net._function(before['name'], slug)
150+
proj = self.net.project_row(slug)
151+
stable = build_scope == candidate_scope.key(self.net, proj, before)
152+
self.record(fresh, signature(self.net, proj, fresh, minutes, variants),
153+
'completed' if stable else 'pending', minutes, variants, result)
154+
155+
def record(self, fn, scope, state, minutes, variants, result=None, error=''):
156+
with self.net.store.tx() as db:
157+
db.execute('INSERT OR REPLACE INTO deep_permutation_passes '
158+
'(function_id,scope,state,minutes,max_variants,completed_at,result,error) VALUES(?,?,?,?,?,?,?,?)',
159+
(fn['id'], scope, state, minutes, variants, time.time() if state == 'completed' else None,
160+
json.dumps(result or {}), error))
161+
162+
def run(self, slug, minutes, variants, log):
163+
"""Finish one turn per unchanged candidate; wait for claims or usable C without marking them done."""
164+
visited = tested = matched = 0
165+
while True:
166+
log.check_cancel()
167+
proj = self.net.project_row(slug)
168+
pending = waiting = 0
169+
rows = self.net.store.all("SELECT * FROM functions WHERE project_id=? AND status='open' ORDER BY size,id", proj['id'])
170+
for row in rows:
171+
log.check_cancel()
172+
fn = self.net._function(row['name'], slug)
173+
if fn['status'] != 'open':
174+
continue
175+
if not fn['best_source'] or fn['best_score'] is None:
176+
waiting += 1
177+
continue
178+
scope = signature(self.net, proj, fn, minutes, variants)
179+
previous = self.net.store.one('SELECT * FROM deep_permutation_passes WHERE function_id=?', fn['id'])
180+
if previous and previous['scope'] == scope and previous['state'] in ('completed', 'error'):
181+
continue
182+
pending += 1
183+
if self.net.store.one('SELECT 1 FROM claims WHERE function_id=? AND expires_at>?', fn['id'], time.time()):
184+
waiting += 1
185+
continue
186+
build_scope = candidate_scope.key(self.net, proj, fn)
187+
self.record(fn, scope, 'running', minutes, variants)
188+
log(f'Deep pass: {fn["name"]} · up to {minutes} minutes / {variants:,} variants')
189+
total = len(rows)
190+
def progress(value, name=fn['name'], done=visited, wins=matched, total=total, blocked=waiting):
191+
log.progress({'function': name, 'functions_tested': done, 'matches': wins,
192+
'total_unmatched': total, 'waiting': blocked} | value)
193+
def function_log(line):
194+
log(line)
195+
function_log.check_cancel = log.check_cancel
196+
function_log.progress = progress
197+
try:
198+
result = deep_permuter.run(self.net, slug, fn['name'], minutes, variants, function_log)
199+
# Also records test backends and older search providers that do not publish history.
200+
# The actual single-function search uses the same completion gate.
201+
self.completed(slug, fn, minutes, variants, result, build_scope)
202+
visited += 1
203+
tested += result['tested']
204+
matched += int(result['matched'])
205+
except NetworkError as error:
206+
# Claim races are temporary. A bad unchanged candidate stays visible as an error,
207+
# not completed, and a changed source/header or an explicit repeat retries it.
208+
busy = self.net.store.one('SELECT 1 FROM claims WHERE function_id=? AND expires_at>?', fn['id'], time.time())
209+
self.record(fn, scope, 'pending' if busy or 'claim' in str(error).lower() or str(error).startswith('This function is working') else 'error', minutes, variants, error=str(error))
210+
log(f'{fn["name"]}: {error}')
211+
except BaseException:
212+
self.record(fn, scope, 'pending', minutes, variants)
213+
raise
214+
progress({'phase': 'moving to next function', 'tested': tested, 'functions_tested': visited, 'matches': matched})
215+
if not pending and not waiting:
216+
self.save(slug, self.config(slug) | {'enabled': False})
217+
log('Deep pass finished. Completed bounded passes are saved; this is not an exhaustive search of all C.')
218+
return {'functions_tested': visited, 'tested': tested, 'matched': matched}
219+
log.progress({'phase': 'waiting for unclaimed functions or scored C', 'functions_tested': visited,
220+
'matches': matched, 'waiting': waiting, 'total_unmatched': len(rows)})
221+
# Cancellation remains responsive while another worker owns the remaining functions.
222+
for _ in range(POLL_SECONDS):
223+
log.check_cancel()
224+
time.sleep(1)

0 commit comments

Comments
 (0)