Add aggregate dashboard for portfolio-level social cost metrics
New show-aggregate.sh script computes cross-session metrics: monoculture index, spend concentration by provider, automation profile distribution, code quality signals, and data pollution risk summary. Integrated into toolkit installer and README. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
1b8f9a165e
commit
60eca18c85
5 changed files with 261 additions and 3 deletions
126
.claude/hooks/show-aggregate.sh
Executable file
126
.claude/hooks/show-aggregate.sh
Executable file
|
|
@ -0,0 +1,126 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# show-aggregate.sh — Portfolio-level impact metrics across all sessions.
|
||||
#
|
||||
# Reads impact-log.jsonl and computes aggregate social cost proxies:
|
||||
# monoculture index, spend concentration, automation profile, quality signals.
|
||||
#
|
||||
# Usage: ./show-aggregate.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}"
|
||||
LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl"
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "No impact log found at $LOG_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 -c "
|
||||
import json, sys
|
||||
from collections import defaultdict
|
||||
|
||||
sessions = {} # session_id -> latest entry (deduplicate)
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line.strip())
|
||||
sid = d.get('session_id', '')
|
||||
if sid:
|
||||
sessions[sid] = d
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not sessions:
|
||||
print('No sessions found.')
|
||||
sys.exit(0)
|
||||
|
||||
entries = list(sessions.values())
|
||||
n = len(entries)
|
||||
|
||||
# --- Environmental totals ---
|
||||
total_energy = sum(e.get('energy_wh', 0) for e in entries)
|
||||
total_co2 = sum(e.get('co2_g', 0) for e in entries)
|
||||
total_cost_cents = sum(e.get('cost_cents', 0) for e in entries)
|
||||
total_output = sum(e.get('output_tokens', 0) for e in entries)
|
||||
total_input = sum(e.get('cumulative_input_tokens', 0) for e in entries)
|
||||
|
||||
print(f'=== Aggregate Impact ({n} sessions) ===')
|
||||
print()
|
||||
print('--- Environmental & Financial ---')
|
||||
print(f' Total energy: ~{total_energy} Wh')
|
||||
print(f' Total CO2: ~{total_co2}g')
|
||||
print(f' Total cost: ~\${total_cost_cents / 100:.2f}')
|
||||
print(f' Total tokens: ~{total_input:,} input, ~{total_output:,} output')
|
||||
print()
|
||||
|
||||
# --- Monoculture index ---
|
||||
model_counts = defaultdict(int)
|
||||
model_spend = defaultdict(int)
|
||||
for e in entries:
|
||||
mid = e.get('model_id', '') or 'unknown'
|
||||
model_counts[mid] += 1
|
||||
model_spend[mid] += e.get('cost_cents', 0)
|
||||
|
||||
print('--- Provider Concentration ---')
|
||||
for model, count in sorted(model_counts.items(), key=lambda x: -x[1]):
|
||||
pct = count * 100 / n
|
||||
spend = model_spend[model] / 100
|
||||
print(f' {model}: {count}/{n} sessions ({pct:.0f}%), \${spend:.2f} spend')
|
||||
|
||||
if len(model_counts) == 1:
|
||||
print(' Monoculture index: 1.0 (single provider — maximum concentration)')
|
||||
else:
|
||||
dominant = max(model_counts.values())
|
||||
print(f' Monoculture index: {dominant / n:.2f} (dominant provider share)')
|
||||
print()
|
||||
|
||||
# --- Automation profile ---
|
||||
auto_ratios = [e.get('automation_ratio_pm', 0) for e in entries if e.get('automation_ratio_pm') is not None]
|
||||
if auto_ratios:
|
||||
avg_auto = sum(auto_ratios) / len(auto_ratios) / 10 # convert permille to %
|
||||
min_auto = min(auto_ratios) / 10
|
||||
max_auto = max(auto_ratios) / 10
|
||||
|
||||
# Categorize sessions
|
||||
low = sum(1 for r in auto_ratios if r < 500) # <50%
|
||||
med = sum(1 for r in auto_ratios if 500 <= r < 800) # 50-80%
|
||||
high = sum(1 for r in auto_ratios if r >= 800) # >80%
|
||||
|
||||
print('--- Automation Profile (Deskilling Risk) ---')
|
||||
print(f' Average automation ratio: {avg_auto:.1f}%')
|
||||
print(f' Range: {min_auto:.1f}% - {max_auto:.1f}%')
|
||||
print(f' Low risk (<50%): {low} sessions')
|
||||
print(f' Medium (50-80%): {med} sessions')
|
||||
print(f' High risk (>80%): {high} sessions')
|
||||
print()
|
||||
|
||||
# --- Code quality signals ---
|
||||
total_test_pass = sum(e.get('test_passes', 0) for e in entries)
|
||||
total_test_fail = sum(e.get('test_failures', 0) for e in entries)
|
||||
total_edits = sum(e.get('total_file_edits', 0) for e in entries)
|
||||
total_files = sum(e.get('unique_files_edited', 0) for e in entries)
|
||||
sessions_with_tests = sum(1 for e in entries if e.get('test_passes', 0) + e.get('test_failures', 0) > 0)
|
||||
|
||||
print('--- Code Quality Signals ---')
|
||||
print(f' Total file edits: {total_edits} across {total_files} unique files')
|
||||
if total_files > 0:
|
||||
print(f' Average churn: {total_edits / total_files:.1f} edits/file')
|
||||
if sessions_with_tests > 0:
|
||||
print(f' Test results: {total_test_pass} passed, {total_test_fail} failed ({sessions_with_tests} sessions with tests)')
|
||||
if total_test_pass + total_test_fail > 0:
|
||||
fail_rate = total_test_fail * 100 / (total_test_pass + total_test_fail)
|
||||
print(f' Test failure rate: {fail_rate:.0f}%')
|
||||
else:
|
||||
print(f' Tests: none detected in any session')
|
||||
print()
|
||||
|
||||
# --- Data pollution risk ---
|
||||
push_sessions = sum(1 for e in entries if e.get('has_public_push', 0))
|
||||
print('--- Data Pollution Risk ---')
|
||||
print(f' Sessions with public push: {push_sessions}/{n} ({push_sessions * 100 / n:.0f}%)')
|
||||
print(f' Tokens pushed publicly: ~{sum(e.get(\"output_tokens\", 0) for e in entries if e.get(\"has_public_push\", 0)):,}')
|
||||
print()
|
||||
" "$LOG_FILE"
|
||||
|
|
@ -35,8 +35,9 @@ Requirements: `bash`, `jq`, `python3`.
|
|||
## View results
|
||||
|
||||
```bash
|
||||
.claude/hooks/show-impact.sh # all sessions
|
||||
.claude/hooks/show-impact.sh # per-session details
|
||||
.claude/hooks/show-impact.sh <session_id> # specific session
|
||||
.claude/hooks/show-aggregate.sh # portfolio-level dashboard
|
||||
```
|
||||
|
||||
## How it works
|
||||
|
|
@ -91,7 +92,8 @@ accompanying methodology covers additional dimensions in depth.
|
|||
impact-toolkit/
|
||||
install.sh # installer
|
||||
hooks/pre-compact-snapshot.sh # PreCompact hook
|
||||
hooks/show-impact.sh # log viewer
|
||||
hooks/show-impact.sh # per-session log viewer
|
||||
hooks/show-aggregate.sh # portfolio-level dashboard
|
||||
README.md # this file
|
||||
```
|
||||
|
||||
|
|
|
|||
126
impact-toolkit/hooks/show-aggregate.sh
Executable file
126
impact-toolkit/hooks/show-aggregate.sh
Executable file
|
|
@ -0,0 +1,126 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# show-aggregate.sh — Portfolio-level impact metrics across all sessions.
|
||||
#
|
||||
# Reads impact-log.jsonl and computes aggregate social cost proxies:
|
||||
# monoculture index, spend concentration, automation profile, quality signals.
|
||||
#
|
||||
# Usage: ./show-aggregate.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}"
|
||||
LOG_FILE="$PROJECT_DIR/.claude/impact/impact-log.jsonl"
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "No impact log found at $LOG_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 -c "
|
||||
import json, sys
|
||||
from collections import defaultdict
|
||||
|
||||
sessions = {} # session_id -> latest entry (deduplicate)
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line.strip())
|
||||
sid = d.get('session_id', '')
|
||||
if sid:
|
||||
sessions[sid] = d
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not sessions:
|
||||
print('No sessions found.')
|
||||
sys.exit(0)
|
||||
|
||||
entries = list(sessions.values())
|
||||
n = len(entries)
|
||||
|
||||
# --- Environmental totals ---
|
||||
total_energy = sum(e.get('energy_wh', 0) for e in entries)
|
||||
total_co2 = sum(e.get('co2_g', 0) for e in entries)
|
||||
total_cost_cents = sum(e.get('cost_cents', 0) for e in entries)
|
||||
total_output = sum(e.get('output_tokens', 0) for e in entries)
|
||||
total_input = sum(e.get('cumulative_input_tokens', 0) for e in entries)
|
||||
|
||||
print(f'=== Aggregate Impact ({n} sessions) ===')
|
||||
print()
|
||||
print('--- Environmental & Financial ---')
|
||||
print(f' Total energy: ~{total_energy} Wh')
|
||||
print(f' Total CO2: ~{total_co2}g')
|
||||
print(f' Total cost: ~\${total_cost_cents / 100:.2f}')
|
||||
print(f' Total tokens: ~{total_input:,} input, ~{total_output:,} output')
|
||||
print()
|
||||
|
||||
# --- Monoculture index ---
|
||||
model_counts = defaultdict(int)
|
||||
model_spend = defaultdict(int)
|
||||
for e in entries:
|
||||
mid = e.get('model_id', '') or 'unknown'
|
||||
model_counts[mid] += 1
|
||||
model_spend[mid] += e.get('cost_cents', 0)
|
||||
|
||||
print('--- Provider Concentration ---')
|
||||
for model, count in sorted(model_counts.items(), key=lambda x: -x[1]):
|
||||
pct = count * 100 / n
|
||||
spend = model_spend[model] / 100
|
||||
print(f' {model}: {count}/{n} sessions ({pct:.0f}%), \${spend:.2f} spend')
|
||||
|
||||
if len(model_counts) == 1:
|
||||
print(' Monoculture index: 1.0 (single provider — maximum concentration)')
|
||||
else:
|
||||
dominant = max(model_counts.values())
|
||||
print(f' Monoculture index: {dominant / n:.2f} (dominant provider share)')
|
||||
print()
|
||||
|
||||
# --- Automation profile ---
|
||||
auto_ratios = [e.get('automation_ratio_pm', 0) for e in entries if e.get('automation_ratio_pm') is not None]
|
||||
if auto_ratios:
|
||||
avg_auto = sum(auto_ratios) / len(auto_ratios) / 10 # convert permille to %
|
||||
min_auto = min(auto_ratios) / 10
|
||||
max_auto = max(auto_ratios) / 10
|
||||
|
||||
# Categorize sessions
|
||||
low = sum(1 for r in auto_ratios if r < 500) # <50%
|
||||
med = sum(1 for r in auto_ratios if 500 <= r < 800) # 50-80%
|
||||
high = sum(1 for r in auto_ratios if r >= 800) # >80%
|
||||
|
||||
print('--- Automation Profile (Deskilling Risk) ---')
|
||||
print(f' Average automation ratio: {avg_auto:.1f}%')
|
||||
print(f' Range: {min_auto:.1f}% - {max_auto:.1f}%')
|
||||
print(f' Low risk (<50%): {low} sessions')
|
||||
print(f' Medium (50-80%): {med} sessions')
|
||||
print(f' High risk (>80%): {high} sessions')
|
||||
print()
|
||||
|
||||
# --- Code quality signals ---
|
||||
total_test_pass = sum(e.get('test_passes', 0) for e in entries)
|
||||
total_test_fail = sum(e.get('test_failures', 0) for e in entries)
|
||||
total_edits = sum(e.get('total_file_edits', 0) for e in entries)
|
||||
total_files = sum(e.get('unique_files_edited', 0) for e in entries)
|
||||
sessions_with_tests = sum(1 for e in entries if e.get('test_passes', 0) + e.get('test_failures', 0) > 0)
|
||||
|
||||
print('--- Code Quality Signals ---')
|
||||
print(f' Total file edits: {total_edits} across {total_files} unique files')
|
||||
if total_files > 0:
|
||||
print(f' Average churn: {total_edits / total_files:.1f} edits/file')
|
||||
if sessions_with_tests > 0:
|
||||
print(f' Test results: {total_test_pass} passed, {total_test_fail} failed ({sessions_with_tests} sessions with tests)')
|
||||
if total_test_pass + total_test_fail > 0:
|
||||
fail_rate = total_test_fail * 100 / (total_test_pass + total_test_fail)
|
||||
print(f' Test failure rate: {fail_rate:.0f}%')
|
||||
else:
|
||||
print(f' Tests: none detected in any session')
|
||||
print()
|
||||
|
||||
# --- Data pollution risk ---
|
||||
push_sessions = sum(1 for e in entries if e.get('has_public_push', 0))
|
||||
print('--- Data Pollution Risk ---')
|
||||
print(f' Sessions with public push: {push_sessions}/{n} ({push_sessions * 100 / n:.0f}%)')
|
||||
print(f' Tokens pushed publicly: ~{sum(e.get(\"output_tokens\", 0) for e in entries if e.get(\"has_public_push\", 0)):,}')
|
||||
print()
|
||||
" "$LOG_FILE"
|
||||
|
|
@ -47,8 +47,10 @@ mkdir -p "$SETTINGS_DIR/impact"
|
|||
# Copy hook scripts
|
||||
cp "$SCRIPT_DIR/hooks/pre-compact-snapshot.sh" "$HOOKS_DIR/"
|
||||
cp "$SCRIPT_DIR/hooks/show-impact.sh" "$HOOKS_DIR/"
|
||||
cp "$SCRIPT_DIR/hooks/show-aggregate.sh" "$HOOKS_DIR/"
|
||||
chmod +x "$HOOKS_DIR/pre-compact-snapshot.sh"
|
||||
chmod +x "$HOOKS_DIR/show-impact.sh"
|
||||
chmod +x "$HOOKS_DIR/show-aggregate.sh"
|
||||
|
||||
echo "Copied hook scripts to $HOOKS_DIR"
|
||||
|
||||
|
|
@ -80,4 +82,5 @@ echo "Installation complete."
|
|||
echo "Impact metrics will be logged to $SETTINGS_DIR/impact/impact-log.jsonl"
|
||||
echo "on each context compaction."
|
||||
echo ""
|
||||
echo "To view accumulated impact: $HOOKS_DIR/show-impact.sh"
|
||||
echo "To view per-session impact: $HOOKS_DIR/show-impact.sh"
|
||||
echo "To view aggregate dashboard: $HOOKS_DIR/show-aggregate.sh"
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ separately as handoffs.
|
|||
| 23 | Add public push flag to hook | quantify-social-costs | DONE | `has_public_push` flag in JSONL log |
|
||||
| 24 | Update show-impact.sh for new fields | quantify-social-costs | DONE | Social cost proxies displayed in impact viewer |
|
||||
| 25 | Update methodology confidence summary | quantify-social-costs | DONE | 4 categories moved to "Proxy", explanation added |
|
||||
| 26 | Build aggregate dashboard | quantify-social-costs | DONE | `show-aggregate.sh` — portfolio-level social cost metrics |
|
||||
|
||||
## Handoffs
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue