From eaf0a6cbeb0943b6d58743f54afbb1e2211c2f21 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 16 Mar 2026 15:12:49 +0000 Subject: [PATCH] Add review delta tool to measure human review effort New show-review-delta.sh compares AI-edited files (from impact log) against git commits to show overlap percentage. High overlap means most committed code was AI-generated with minimal human review. Completes Phase 2 of the quantify-social-costs plan. Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/hooks/show-review-delta.sh | 123 ++++++++++++++++++++++ impact-toolkit/README.md | 3 + impact-toolkit/hooks/show-review-delta.sh | 123 ++++++++++++++++++++++ impact-toolkit/install.sh | 2 + tasks/README.md | 1 + 5 files changed, 252 insertions(+) create mode 100755 .claude/hooks/show-review-delta.sh create mode 100755 impact-toolkit/hooks/show-review-delta.sh diff --git a/.claude/hooks/show-review-delta.sh b/.claude/hooks/show-review-delta.sh new file mode 100755 index 0000000..a854e88 --- /dev/null +++ b/.claude/hooks/show-review-delta.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# show-review-delta.sh — Measure human review effort on AI-edited files. +# +# Compares the list of files the AI edited (from the impact log) against +# recent git commits to estimate how much the user modified AI output +# before committing. +# +# Usage: ./show-review-delta.sh [n_commits] +# n_commits: number of recent commits to analyze (default: 10) + +set -euo pipefail + +PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}" +LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl" +N_COMMITS="${1:-10}" + +if [ ! -f "$LOG_FILE" ]; then + echo "No impact log found at $LOG_FILE" + exit 0 +fi + +cd "$PROJECT_DIR" + +python3 -c " +import json, sys, subprocess, os + +log_file = sys.argv[1] +n_commits = int(sys.argv[2]) +project_dir = sys.argv[3] + +# Collect all AI-edited files across all sessions (latest entry per session) +sessions = {} +with open(log_file) as f: + for line in f: + try: + d = json.loads(line.strip()) + sid = d.get('session_id', '') + if sid: + sessions[sid] = d + except Exception: + pass + +ai_files = {} # file_path -> total edit count across sessions +for entry in sessions.values(): + for fp, count in entry.get('edited_files', {}).items(): + # Normalize to relative path + rel = fp + if rel.startswith(project_dir + '/'): + rel = rel[len(project_dir) + 1:] + ai_files[rel] = ai_files.get(rel, 0) + count + +if not ai_files: + print('No AI-edited files found in impact log.') + print('(The edited_files field was added recently — older sessions lack it.)') + sys.exit(0) + +# Get recent commits and their changed files +result = subprocess.run( + ['git', 'log', f'-{n_commits}', '--pretty=format:%H %s', '--name-only'], + capture_output=True, text=True, cwd=project_dir +) + +commits = [] +current = None +for line in result.stdout.split('\n'): + if not line.strip(): + continue + if ' ' in line and len(line.split()[0]) == 40: + # Commit line + parts = line.split(' ', 1) + current = {'hash': parts[0], 'msg': parts[1], 'files': [], 'ai_files': []} + commits.append(current) + elif current is not None: + f = line.strip() + current['files'].append(f) + if f in ai_files: + current['ai_files'].append(f) + +print(f'=== Review Delta Analysis ({len(commits)} recent commits) ===') +print(f'AI-edited files tracked: {len(ai_files)}') +print() + +total_files = 0 +total_ai_overlap = 0 + +for c in commits: + n_files = len(c['files']) + n_ai = len(c['ai_files']) + total_files += n_files + total_ai_overlap += n_ai + + if n_ai > 0: + marker = ' ← AI-touched' + else: + marker = '' + print(f' {c[\"hash\"][:8]} {c[\"msg\"][:60]}') + print(f' {n_files} files changed, {n_ai} were AI-edited{marker}') + if n_ai > 0: + for f in c['ai_files']: + print(f' {f} ({ai_files[f]} AI edits)') + print() + +if total_files > 0: + overlap_pct = total_ai_overlap * 100 / total_files +else: + overlap_pct = 0 + +print(f'=== Summary ===') +print(f' Total files in commits: {total_files}') +print(f' Files also AI-edited: {total_ai_overlap} ({overlap_pct:.0f}%)') +print(f' Files only human-edited: {total_files - total_ai_overlap}') +print() +if overlap_pct > 80: + print(' High AI overlap — most committed code was AI-generated.') + print(' Consider reviewing more carefully to maintain skill.') +elif overlap_pct > 40: + print(' Moderate AI overlap — mixed human/AI contribution.') +elif total_ai_overlap > 0: + print(' Low AI overlap — mostly human-written code committed.') +else: + print(' No AI overlap detected in recent commits.') +" "$LOG_FILE" "$N_COMMITS" "$PROJECT_DIR" diff --git a/impact-toolkit/README.md b/impact-toolkit/README.md index 9f058b1..3474a43 100644 --- a/impact-toolkit/README.md +++ b/impact-toolkit/README.md @@ -38,6 +38,8 @@ Requirements: `bash`, `jq`, `python3`. .claude/hooks/show-impact.sh # per-session details .claude/hooks/show-impact.sh # specific session .claude/hooks/show-aggregate.sh # portfolio-level dashboard +.claude/hooks/show-review-delta.sh # AI vs human code overlap +.claude/hooks/show-review-delta.sh 20 # analyze last 20 commits ``` ## How it works @@ -94,6 +96,7 @@ impact-toolkit/ hooks/pre-compact-snapshot.sh # PreCompact hook hooks/show-impact.sh # per-session log viewer hooks/show-aggregate.sh # portfolio-level dashboard + hooks/show-review-delta.sh # AI vs human code overlap README.md # this file ``` diff --git a/impact-toolkit/hooks/show-review-delta.sh b/impact-toolkit/hooks/show-review-delta.sh new file mode 100755 index 0000000..a854e88 --- /dev/null +++ b/impact-toolkit/hooks/show-review-delta.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# show-review-delta.sh — Measure human review effort on AI-edited files. +# +# Compares the list of files the AI edited (from the impact log) against +# recent git commits to estimate how much the user modified AI output +# before committing. +# +# Usage: ./show-review-delta.sh [n_commits] +# n_commits: number of recent commits to analyze (default: 10) + +set -euo pipefail + +PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}" +LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl" +N_COMMITS="${1:-10}" + +if [ ! -f "$LOG_FILE" ]; then + echo "No impact log found at $LOG_FILE" + exit 0 +fi + +cd "$PROJECT_DIR" + +python3 -c " +import json, sys, subprocess, os + +log_file = sys.argv[1] +n_commits = int(sys.argv[2]) +project_dir = sys.argv[3] + +# Collect all AI-edited files across all sessions (latest entry per session) +sessions = {} +with open(log_file) as f: + for line in f: + try: + d = json.loads(line.strip()) + sid = d.get('session_id', '') + if sid: + sessions[sid] = d + except Exception: + pass + +ai_files = {} # file_path -> total edit count across sessions +for entry in sessions.values(): + for fp, count in entry.get('edited_files', {}).items(): + # Normalize to relative path + rel = fp + if rel.startswith(project_dir + '/'): + rel = rel[len(project_dir) + 1:] + ai_files[rel] = ai_files.get(rel, 0) + count + +if not ai_files: + print('No AI-edited files found in impact log.') + print('(The edited_files field was added recently — older sessions lack it.)') + sys.exit(0) + +# Get recent commits and their changed files +result = subprocess.run( + ['git', 'log', f'-{n_commits}', '--pretty=format:%H %s', '--name-only'], + capture_output=True, text=True, cwd=project_dir +) + +commits = [] +current = None +for line in result.stdout.split('\n'): + if not line.strip(): + continue + if ' ' in line and len(line.split()[0]) == 40: + # Commit line + parts = line.split(' ', 1) + current = {'hash': parts[0], 'msg': parts[1], 'files': [], 'ai_files': []} + commits.append(current) + elif current is not None: + f = line.strip() + current['files'].append(f) + if f in ai_files: + current['ai_files'].append(f) + +print(f'=== Review Delta Analysis ({len(commits)} recent commits) ===') +print(f'AI-edited files tracked: {len(ai_files)}') +print() + +total_files = 0 +total_ai_overlap = 0 + +for c in commits: + n_files = len(c['files']) + n_ai = len(c['ai_files']) + total_files += n_files + total_ai_overlap += n_ai + + if n_ai > 0: + marker = ' ← AI-touched' + else: + marker = '' + print(f' {c[\"hash\"][:8]} {c[\"msg\"][:60]}') + print(f' {n_files} files changed, {n_ai} were AI-edited{marker}') + if n_ai > 0: + for f in c['ai_files']: + print(f' {f} ({ai_files[f]} AI edits)') + print() + +if total_files > 0: + overlap_pct = total_ai_overlap * 100 / total_files +else: + overlap_pct = 0 + +print(f'=== Summary ===') +print(f' Total files in commits: {total_files}') +print(f' Files also AI-edited: {total_ai_overlap} ({overlap_pct:.0f}%)') +print(f' Files only human-edited: {total_files - total_ai_overlap}') +print() +if overlap_pct > 80: + print(' High AI overlap — most committed code was AI-generated.') + print(' Consider reviewing more carefully to maintain skill.') +elif overlap_pct > 40: + print(' Moderate AI overlap — mixed human/AI contribution.') +elif total_ai_overlap > 0: + print(' Low AI overlap — mostly human-written code committed.') +else: + print(' No AI overlap detected in recent commits.') +" "$LOG_FILE" "$N_COMMITS" "$PROJECT_DIR" diff --git a/impact-toolkit/install.sh b/impact-toolkit/install.sh index 3097653..1d10b31 100755 --- a/impact-toolkit/install.sh +++ b/impact-toolkit/install.sh @@ -48,9 +48,11 @@ mkdir -p "$SETTINGS_DIR/impact" cp "$SCRIPT_DIR/hooks/pre-compact-snapshot.sh" "$HOOKS_DIR/" cp "$SCRIPT_DIR/hooks/show-impact.sh" "$HOOKS_DIR/" cp "$SCRIPT_DIR/hooks/show-aggregate.sh" "$HOOKS_DIR/" +cp "$SCRIPT_DIR/hooks/show-review-delta.sh" "$HOOKS_DIR/" chmod +x "$HOOKS_DIR/pre-compact-snapshot.sh" chmod +x "$HOOKS_DIR/show-impact.sh" chmod +x "$HOOKS_DIR/show-aggregate.sh" +chmod +x "$HOOKS_DIR/show-review-delta.sh" echo "Copied hook scripts to $HOOKS_DIR" diff --git a/tasks/README.md b/tasks/README.md index 14d4289..1d50a95 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -38,6 +38,7 @@ separately as handoffs. | 25 | Update methodology confidence summary | quantify-social-costs | DONE | 4 categories moved to "Proxy", explanation added | | 26 | Build aggregate dashboard | quantify-social-costs | DONE | `show-aggregate.sh` — portfolio-level social cost metrics | | 27 | Log edited file list in hook | quantify-social-costs | DONE | `edited_files` dict in JSONL (file path → edit count) | +| 28 | Build review delta tool | quantify-social-costs | DONE | `show-review-delta.sh` — AI vs human code overlap in commits | ## Handoffs