Add review delta tool to measure human review effort

New show-review-delta.sh compares AI-edited files (from impact log)
against git commits to show overlap percentage. High overlap means
most committed code was AI-generated with minimal human review.
Completes Phase 2 of the quantify-social-costs plan.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
claude 2026-03-16 15:12:49 +00:00
parent ad06b12e50
commit eaf0a6cbeb
5 changed files with 252 additions and 0 deletions

View file

@ -0,0 +1,123 @@
#!/usr/bin/env bash
#
# show-review-delta.sh — Measure human review effort on AI-edited files.
#
# Compares the list of files the AI edited (from the impact log) against
# recent git commits to estimate how much the user modified AI output
# before committing.
#
# Usage: ./show-review-delta.sh [n_commits]
# n_commits: number of recent commits to analyze (default: 10)
set -euo pipefail
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}"
LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl"
N_COMMITS="${1:-10}"
if [ ! -f "$LOG_FILE" ]; then
echo "No impact log found at $LOG_FILE"
exit 0
fi
cd "$PROJECT_DIR"
python3 -c "
import json, sys, subprocess, os
log_file = sys.argv[1]
n_commits = int(sys.argv[2])
project_dir = sys.argv[3]
# Collect all AI-edited files across all sessions (latest entry per session)
sessions = {}
with open(log_file) as f:
for line in f:
try:
d = json.loads(line.strip())
sid = d.get('session_id', '')
if sid:
sessions[sid] = d
except Exception:
pass
ai_files = {} # file_path -> total edit count across sessions
for entry in sessions.values():
for fp, count in entry.get('edited_files', {}).items():
# Normalize to relative path
rel = fp
if rel.startswith(project_dir + '/'):
rel = rel[len(project_dir) + 1:]
ai_files[rel] = ai_files.get(rel, 0) + count
if not ai_files:
print('No AI-edited files found in impact log.')
print('(The edited_files field was added recently — older sessions lack it.)')
sys.exit(0)
# Get recent commits and their changed files
result = subprocess.run(
['git', 'log', f'-{n_commits}', '--pretty=format:%H %s', '--name-only'],
capture_output=True, text=True, cwd=project_dir
)
commits = []
current = None
for line in result.stdout.split('\n'):
if not line.strip():
continue
if ' ' in line and len(line.split()[0]) == 40:
# Commit line
parts = line.split(' ', 1)
current = {'hash': parts[0], 'msg': parts[1], 'files': [], 'ai_files': []}
commits.append(current)
elif current is not None:
f = line.strip()
current['files'].append(f)
if f in ai_files:
current['ai_files'].append(f)
print(f'=== Review Delta Analysis ({len(commits)} recent commits) ===')
print(f'AI-edited files tracked: {len(ai_files)}')
print()
total_files = 0
total_ai_overlap = 0
for c in commits:
n_files = len(c['files'])
n_ai = len(c['ai_files'])
total_files += n_files
total_ai_overlap += n_ai
if n_ai > 0:
marker = ' ← AI-touched'
else:
marker = ''
print(f' {c[\"hash\"][:8]} {c[\"msg\"][:60]}')
print(f' {n_files} files changed, {n_ai} were AI-edited{marker}')
if n_ai > 0:
for f in c['ai_files']:
print(f' {f} ({ai_files[f]} AI edits)')
print()
if total_files > 0:
overlap_pct = total_ai_overlap * 100 / total_files
else:
overlap_pct = 0
print(f'=== Summary ===')
print(f' Total files in commits: {total_files}')
print(f' Files also AI-edited: {total_ai_overlap} ({overlap_pct:.0f}%)')
print(f' Files only human-edited: {total_files - total_ai_overlap}')
print()
if overlap_pct > 80:
print(' High AI overlap — most committed code was AI-generated.')
print(' Consider reviewing more carefully to maintain skill.')
elif overlap_pct > 40:
print(' Moderate AI overlap — mixed human/AI contribution.')
elif total_ai_overlap > 0:
print(' Low AI overlap — mostly human-written code committed.')
else:
print(' No AI overlap detected in recent commits.')
" "$LOG_FILE" "$N_COMMITS" "$PROJECT_DIR"