Add review delta tool to measure human review effort
New show-review-delta.sh compares AI-edited files (from impact log) against git commits to show overlap percentage. High overlap means most committed code was AI-generated with minimal human review. Completes Phase 2 of the quantify-social-costs plan. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ad06b12e50
commit
eaf0a6cbeb
5 changed files with 252 additions and 0 deletions
123
.claude/hooks/show-review-delta.sh
Executable file
123
.claude/hooks/show-review-delta.sh
Executable file
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# show-review-delta.sh — Measure human review effort on AI-edited files.
|
||||
#
|
||||
# Compares the list of files the AI edited (from the impact log) against
|
||||
# recent git commits to estimate how much the user modified AI output
|
||||
# before committing.
|
||||
#
|
||||
# Usage: ./show-review-delta.sh [n_commits]
|
||||
# n_commits: number of recent commits to analyze (default: 10)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}"
|
||||
LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl"
|
||||
N_COMMITS="${1:-10}"
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "No impact log found at $LOG_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
python3 -c "
|
||||
import json, sys, subprocess, os
|
||||
|
||||
log_file = sys.argv[1]
|
||||
n_commits = int(sys.argv[2])
|
||||
project_dir = sys.argv[3]
|
||||
|
||||
# Collect all AI-edited files across all sessions (latest entry per session)
|
||||
sessions = {}
|
||||
with open(log_file) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line.strip())
|
||||
sid = d.get('session_id', '')
|
||||
if sid:
|
||||
sessions[sid] = d
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ai_files = {} # file_path -> total edit count across sessions
|
||||
for entry in sessions.values():
|
||||
for fp, count in entry.get('edited_files', {}).items():
|
||||
# Normalize to relative path
|
||||
rel = fp
|
||||
if rel.startswith(project_dir + '/'):
|
||||
rel = rel[len(project_dir) + 1:]
|
||||
ai_files[rel] = ai_files.get(rel, 0) + count
|
||||
|
||||
if not ai_files:
|
||||
print('No AI-edited files found in impact log.')
|
||||
print('(The edited_files field was added recently — older sessions lack it.)')
|
||||
sys.exit(0)
|
||||
|
||||
# Get recent commits and their changed files
|
||||
result = subprocess.run(
|
||||
['git', 'log', f'-{n_commits}', '--pretty=format:%H %s', '--name-only'],
|
||||
capture_output=True, text=True, cwd=project_dir
|
||||
)
|
||||
|
||||
commits = []
|
||||
current = None
|
||||
for line in result.stdout.split('\n'):
|
||||
if not line.strip():
|
||||
continue
|
||||
if ' ' in line and len(line.split()[0]) == 40:
|
||||
# Commit line
|
||||
parts = line.split(' ', 1)
|
||||
current = {'hash': parts[0], 'msg': parts[1], 'files': [], 'ai_files': []}
|
||||
commits.append(current)
|
||||
elif current is not None:
|
||||
f = line.strip()
|
||||
current['files'].append(f)
|
||||
if f in ai_files:
|
||||
current['ai_files'].append(f)
|
||||
|
||||
print(f'=== Review Delta Analysis ({len(commits)} recent commits) ===')
|
||||
print(f'AI-edited files tracked: {len(ai_files)}')
|
||||
print()
|
||||
|
||||
total_files = 0
|
||||
total_ai_overlap = 0
|
||||
|
||||
for c in commits:
|
||||
n_files = len(c['files'])
|
||||
n_ai = len(c['ai_files'])
|
||||
total_files += n_files
|
||||
total_ai_overlap += n_ai
|
||||
|
||||
if n_ai > 0:
|
||||
marker = ' ← AI-touched'
|
||||
else:
|
||||
marker = ''
|
||||
print(f' {c[\"hash\"][:8]} {c[\"msg\"][:60]}')
|
||||
print(f' {n_files} files changed, {n_ai} were AI-edited{marker}')
|
||||
if n_ai > 0:
|
||||
for f in c['ai_files']:
|
||||
print(f' {f} ({ai_files[f]} AI edits)')
|
||||
print()
|
||||
|
||||
if total_files > 0:
|
||||
overlap_pct = total_ai_overlap * 100 / total_files
|
||||
else:
|
||||
overlap_pct = 0
|
||||
|
||||
print(f'=== Summary ===')
|
||||
print(f' Total files in commits: {total_files}')
|
||||
print(f' Files also AI-edited: {total_ai_overlap} ({overlap_pct:.0f}%)')
|
||||
print(f' Files only human-edited: {total_files - total_ai_overlap}')
|
||||
print()
|
||||
if overlap_pct > 80:
|
||||
print(' High AI overlap — most committed code was AI-generated.')
|
||||
print(' Consider reviewing more carefully to maintain skill.')
|
||||
elif overlap_pct > 40:
|
||||
print(' Moderate AI overlap — mixed human/AI contribution.')
|
||||
elif total_ai_overlap > 0:
|
||||
print(' Low AI overlap — mostly human-written code committed.')
|
||||
else:
|
||||
print(' No AI overlap detected in recent commits.')
|
||||
" "$LOG_FILE" "$N_COMMITS" "$PROJECT_DIR"
|
||||
|
|
@ -38,6 +38,8 @@ Requirements: `bash`, `jq`, `python3`.
|
|||
.claude/hooks/show-impact.sh # per-session details
|
||||
.claude/hooks/show-impact.sh <session_id> # specific session
|
||||
.claude/hooks/show-aggregate.sh # portfolio-level dashboard
|
||||
.claude/hooks/show-review-delta.sh # AI vs human code overlap
|
||||
.claude/hooks/show-review-delta.sh 20 # analyze last 20 commits
|
||||
```
|
||||
|
||||
## How it works
|
||||
|
|
@ -94,6 +96,7 @@ impact-toolkit/
|
|||
hooks/pre-compact-snapshot.sh # PreCompact hook
|
||||
hooks/show-impact.sh # per-session log viewer
|
||||
hooks/show-aggregate.sh # portfolio-level dashboard
|
||||
hooks/show-review-delta.sh # AI vs human code overlap
|
||||
README.md # this file
|
||||
```
|
||||
|
||||
|
|
|
|||
123
impact-toolkit/hooks/show-review-delta.sh
Executable file
123
impact-toolkit/hooks/show-review-delta.sh
Executable file
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# show-review-delta.sh — Measure human review effort on AI-edited files.
|
||||
#
|
||||
# Compares the list of files the AI edited (from the impact log) against
|
||||
# recent git commits to estimate how much the user modified AI output
|
||||
# before committing.
|
||||
#
|
||||
# Usage: ./show-review-delta.sh [n_commits]
|
||||
# n_commits: number of recent commits to analyze (default: 10)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}"
|
||||
LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl"
|
||||
N_COMMITS="${1:-10}"
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "No impact log found at $LOG_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
python3 -c "
|
||||
import json, sys, subprocess, os
|
||||
|
||||
log_file = sys.argv[1]
|
||||
n_commits = int(sys.argv[2])
|
||||
project_dir = sys.argv[3]
|
||||
|
||||
# Collect all AI-edited files across all sessions (latest entry per session)
|
||||
sessions = {}
|
||||
with open(log_file) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line.strip())
|
||||
sid = d.get('session_id', '')
|
||||
if sid:
|
||||
sessions[sid] = d
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ai_files = {} # file_path -> total edit count across sessions
|
||||
for entry in sessions.values():
|
||||
for fp, count in entry.get('edited_files', {}).items():
|
||||
# Normalize to relative path
|
||||
rel = fp
|
||||
if rel.startswith(project_dir + '/'):
|
||||
rel = rel[len(project_dir) + 1:]
|
||||
ai_files[rel] = ai_files.get(rel, 0) + count
|
||||
|
||||
if not ai_files:
|
||||
print('No AI-edited files found in impact log.')
|
||||
print('(The edited_files field was added recently — older sessions lack it.)')
|
||||
sys.exit(0)
|
||||
|
||||
# Get recent commits and their changed files
|
||||
result = subprocess.run(
|
||||
['git', 'log', f'-{n_commits}', '--pretty=format:%H %s', '--name-only'],
|
||||
capture_output=True, text=True, cwd=project_dir
|
||||
)
|
||||
|
||||
commits = []
|
||||
current = None
|
||||
for line in result.stdout.split('\n'):
|
||||
if not line.strip():
|
||||
continue
|
||||
if ' ' in line and len(line.split()[0]) == 40:
|
||||
# Commit line
|
||||
parts = line.split(' ', 1)
|
||||
current = {'hash': parts[0], 'msg': parts[1], 'files': [], 'ai_files': []}
|
||||
commits.append(current)
|
||||
elif current is not None:
|
||||
f = line.strip()
|
||||
current['files'].append(f)
|
||||
if f in ai_files:
|
||||
current['ai_files'].append(f)
|
||||
|
||||
print(f'=== Review Delta Analysis ({len(commits)} recent commits) ===')
|
||||
print(f'AI-edited files tracked: {len(ai_files)}')
|
||||
print()
|
||||
|
||||
total_files = 0
|
||||
total_ai_overlap = 0
|
||||
|
||||
for c in commits:
|
||||
n_files = len(c['files'])
|
||||
n_ai = len(c['ai_files'])
|
||||
total_files += n_files
|
||||
total_ai_overlap += n_ai
|
||||
|
||||
if n_ai > 0:
|
||||
marker = ' ← AI-touched'
|
||||
else:
|
||||
marker = ''
|
||||
print(f' {c[\"hash\"][:8]} {c[\"msg\"][:60]}')
|
||||
print(f' {n_files} files changed, {n_ai} were AI-edited{marker}')
|
||||
if n_ai > 0:
|
||||
for f in c['ai_files']:
|
||||
print(f' {f} ({ai_files[f]} AI edits)')
|
||||
print()
|
||||
|
||||
if total_files > 0:
|
||||
overlap_pct = total_ai_overlap * 100 / total_files
|
||||
else:
|
||||
overlap_pct = 0
|
||||
|
||||
print(f'=== Summary ===')
|
||||
print(f' Total files in commits: {total_files}')
|
||||
print(f' Files also AI-edited: {total_ai_overlap} ({overlap_pct:.0f}%)')
|
||||
print(f' Files only human-edited: {total_files - total_ai_overlap}')
|
||||
print()
|
||||
if overlap_pct > 80:
|
||||
print(' High AI overlap — most committed code was AI-generated.')
|
||||
print(' Consider reviewing more carefully to maintain skill.')
|
||||
elif overlap_pct > 40:
|
||||
print(' Moderate AI overlap — mixed human/AI contribution.')
|
||||
elif total_ai_overlap > 0:
|
||||
print(' Low AI overlap — mostly human-written code committed.')
|
||||
else:
|
||||
print(' No AI overlap detected in recent commits.')
|
||||
" "$LOG_FILE" "$N_COMMITS" "$PROJECT_DIR"
|
||||
|
|
@ -48,9 +48,11 @@ mkdir -p "$SETTINGS_DIR/impact"
|
|||
cp "$SCRIPT_DIR/hooks/pre-compact-snapshot.sh" "$HOOKS_DIR/"
|
||||
cp "$SCRIPT_DIR/hooks/show-impact.sh" "$HOOKS_DIR/"
|
||||
cp "$SCRIPT_DIR/hooks/show-aggregate.sh" "$HOOKS_DIR/"
|
||||
cp "$SCRIPT_DIR/hooks/show-review-delta.sh" "$HOOKS_DIR/"
|
||||
chmod +x "$HOOKS_DIR/pre-compact-snapshot.sh"
|
||||
chmod +x "$HOOKS_DIR/show-impact.sh"
|
||||
chmod +x "$HOOKS_DIR/show-aggregate.sh"
|
||||
chmod +x "$HOOKS_DIR/show-review-delta.sh"
|
||||
|
||||
echo "Copied hook scripts to $HOOKS_DIR"
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ separately as handoffs.
|
|||
| 25 | Update methodology confidence summary | quantify-social-costs | DONE | 4 categories moved to "Proxy", explanation added |
|
||||
| 26 | Build aggregate dashboard | quantify-social-costs | DONE | `show-aggregate.sh` — portfolio-level social cost metrics |
|
||||
| 27 | Log edited file list in hook | quantify-social-costs | DONE | `edited_files` dict in JSONL (file path → edit count) |
|
||||
| 28 | Build review delta tool | quantify-social-costs | DONE | `show-review-delta.sh` — AI vs human code overlap in commits |
|
||||
|
||||
## Handoffs
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue