"""Run with Python 3: python measure.py.txt DOWNLOAD_DIR OUTPUT_DIR. DOWNLOAD_DIR must contain the public archive-index.json and ZIPs for Nov 13–19. The analyst's clock is the node's poll time, not a miner's block header time. """ import csv import hashlib import io import json from bisect import bisect_right from collections import Counter from datetime import date, datetime, timezone from pathlib import Path from statistics import quantiles import sys import zipfile DAYS = [date(2026, 11, day).isoformat() for day in range(13, 20)] PROTOCOL = 'lw-confirmations-2026-09-27-v2' FIELDS = ['start_height', 'start_hash', 'start_poll_utc', 'target_height', 'target_hash', 'target_poll_utc', 'status', 'elapsed_seconds'] def digest(raw): return hashlib.sha256(raw).hexdigest() def rows(raw): return csv.DictReader(io.StringIO(raw.decode('utf-8'))) def utc(seconds): return datetime.fromtimestamp(seconds, timezone.utc).isoformat().replace('+00:00', 'Z') def read_exports(source): index = json.loads((source / 'archive-index.json').read_text(encoding='utf-8-sig')) entries = {row['day']: row for row in index['daily']} polls, inputs = {}, [] for day in DAYS: entry = entries[day] raw = (source / f'{day}.zip').read_bytes() if digest(raw) != entry['sha256'] or len(raw) != entry['bytes']: raise ValueError(f'Archive index mismatch for {day}') with zipfile.ZipFile(io.BytesIO(raw)) as archive: manifest = json.loads(archive.read('manifest.json')) if manifest['coverage_day'] != day or manifest['protocol'] != PROTOCOL: raise ValueError(f'Archive protocol mismatch for {day}') data = {name: archive.read(name) for name in ('polls.csv', 'cohort.csv')} if any(digest(value) != manifest['files'][name] for name, value in data.items()): raise ValueError(f'Archive member mismatch for {day}') coverage = 0 for row in rows(data['polls.csv']): at = int(row['at']) covered = int(row['covered_seconds']) if at in polls or not manifest['coverage_start'] <= at < manifest['coverage_end']: raise ValueError(f'Duplicate or out-of-window poll for {day}') polls[at] = (row['state'], covered) coverage += covered if coverage != entry['covered_seconds'] or coverage != manifest['covered_seconds']: raise ValueError(f'Archive coverage mismatch for {day}') inputs.append({'day': day, 'sha256': entry['sha256'], 'covered_seconds': coverage}) return polls, inputs, manifest, data['cohort.csv'] def block_observations(cohort, polls): blocks = {} matches, unusable_matches = 0, 0 first_day = int(datetime.fromisoformat(DAYS[0]).replace(tzinfo=timezone.utc).timestamp()) end = first_day + len(DAYS) * 86400 for row in rows(cohort): if row['state'] != 'confirmed': continue at = int(row['confirmed_at']) if not first_day <= at < end: continue if polls.get(at, ('missing', 0))[0] != 'collecting': unusable_matches += 1 continue height = int(row['block_height']) observation = (row['block_hash'], at) if blocks.setdefault(height, observation) != observation: raise ValueError(f'A height mapped to multiple observations at {height}') matches += 1 ordered = sorted(blocks) times = [blocks[height][1] for height in ordered] if any(new <= old for old, new in zip(times, times[1:])): raise ValueError('Block heights and poll times must be strictly increasing') return blocks, matches, unusable_matches def continuity_breaks(polls): breaks = [] previous = None for at, (state, covered) in sorted(polls.items()): if state != 'collecting' or previous is None or at - previous != covered or not 0 < covered <= 30: breaks.append(at) previous = at return breaks def measure_windows(blocks, breaks): windows = [] tip = max(blocks) for height, (start_hash, start_at) in sorted(blocks.items()): target = height + 11 end_hash, end_at = blocks.get(target, ('', None)) if target > tip: status = 'right_edge' elif end_at is None: status = 'missing_target' elif bisect_right(breaks, end_at) > bisect_right(breaks, start_at): status = 'polling_gap' elif any(intermediate not in blocks for intermediate in range(height + 1, target)): status = 'missing_intermediate' else: status = 'measured' windows.append(dict(zip(FIELDS, (height, start_hash, utc(start_at), target, end_hash, utc(end_at) if end_at else '', status, end_at - start_at if status == 'measured' else '')))) return windows def build(source, destination): polls, inputs, manifest, cohort = read_exports(source) blocks, matches, unusable_matches = block_observations(cohort, polls) windows = measure_windows(blocks, continuity_breaks(polls)) durations = [row['elapsed_seconds'] for row in windows if row['status'] == 'measured'] cuts = quantiles(durations, n=100, method='inclusive') report = { 'method': 'lw-deposit-depth-20261120-v1', 'utc_start_inclusive': DAYS[0], 'utc_end_exclusive': '2026-11-20', 'cohort_snapshot_at_utc': utc(manifest['snapshot_at']), 'source_archives': inputs, 'polls': len(polls), 'covered_seconds': sum(item['covered_seconds'] for item in inputs), 'scheduled_seconds': len(DAYS) * 86400, 'confirmed_transaction_matches': matches, 'confirmed_matches_without_collecting_poll': unusable_matches, 'identified_start_heights': len(blocks), 'status_counts': dict(Counter(row['status'] for row in windows)), 'minimum_seconds': min(durations), 'p10_seconds': round(cuts[9], 1), 'median_seconds': round(cuts[49], 1), 'p90_seconds': round(cuts[89], 1), 'p95_seconds': round(cuts[94], 1), 'maximum_seconds': max(durations), 'strictly_over': {str(limit): sum(value > limit for value in durations) for limit in (1200, 1800, 2400, 2700, 3600)}, 'shortest_window': min((row for row in windows if row['status'] == 'measured'), key=lambda row: row['elapsed_seconds']), 'longest_window': max((row for row in windows if row['status'] == 'measured'), key=lambda row: row['elapsed_seconds']), } destination.mkdir(parents=True, exist_ok=True) with (destination / 'windows.csv').open('w', encoding='utf-8', newline='') as output: writer = csv.DictWriter(output, fieldnames=FIELDS) writer.writeheader() writer.writerows(windows) report['windows_sha256'] = digest((destination / 'windows.csv').read_bytes()) (destination / 'report.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') return report if __name__ == '__main__': if len(sys.argv) != 3: raise SystemExit(__doc__) print(json.dumps(build(Path(sys.argv[1]), Path(sys.argv[2])), indent=2))