#!/usr/bin/env python3
"""
Comprehensive Data Analysis & Report Generator
Analyzes the 375-point EM parameter sweep and produces:
1. Full statistical summary (text)
2. Interactive HTML visualization dashboard
"""
import os
import sys
import numpy as np
import pandas as pd
from datetime import datetime
from collections import Counter
import json
# ── Config ──────────────────────────────────────────────────────────
MAGIC_NUMBERS = [2, 8, 20, 28, 50, 82, 126]
SWEEP_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "sweep_results")
RESULTS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results")
os.makedirs(RESULTS_DIR, exist_ok=True)
def load_data(csv_path=None):
if csv_path is None:
csvs = sorted([f for f in os.listdir(SWEEP_DIR) if f.startswith("em_direct_sweep") and f.endswith(".csv")])
if not csvs:
print("No sweep CSVs found"); sys.exit(1)
csv_path = os.path.join(SWEEP_DIR, csvs[-1])
print(f"Auto-selected: {csvs[-1]}")
df = pd.read_csv(csv_path)
print(f"Loaded {len(df)} points")
return df, os.path.basename(csv_path)
# ═══════════════════════════════════════════════════════════════════
# PART 1: Comprehensive Text Report
# ═══════════════════════════════════════════════════════════════════
def generate_text_report(df, source_name):
R = []
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
R.append("=" * 78)
R.append(" COMPREHENSIVE PARAMETER SWEEP ANALYSIS — RESONANCE ENGINE")
R.append("=" * 78)
R.append(f"Source: {source_name}")
R.append(f"Generated: {ts}")
R.append(f"Points: {len(df)}")
R.append(f"Duration: {df['timestamp'].iloc[0]} → {df['timestamp'].iloc[-1]}")
R.append("")
# ── Section 1: Global Statistics ──
R.append("─" * 78)
R.append("1. GLOBAL STATISTICS")
R.append("─" * 78)
numeric_cols = ['omega', 'khra_amp', 'gixx_amp', 'coherence', 'asymmetry',
'vorticity_mean', 'gpu_temp_c', 'gpu_power_w', 'cycle']
R.append(f" {'Column':<18} {'Min':>12} {'Max':>12} {'Mean':>12} {'Std':>12} {'Median':>12}")
for col in numeric_cols:
v = df[col]
R.append(f" {col:<18} {v.min():>12.6f} {v.max():>12.6f} {v.mean():>12.6f} {v.std():>12.6f} {v.median():>12.6f}")
# ── Section 2: Parameter Space Coverage ──
R.append("")
R.append("─" * 78)
R.append("2. PARAMETER SPACE COVERAGE")
R.append("─" * 78)
omega_vals = sorted(df['omega'].unique())
khra_vals = sorted(df['khra_amp'].unique())
gixx_vals = sorted(df['gixx_amp'].unique())
R.append(f" Omega: {len(omega_vals)} values: {[round(x,1) for x in omega_vals]}")
R.append(f" Khra_amp: {len(khra_vals)} values: {[round(x,3) for x in khra_vals]}")
R.append(f" Gixx_amp: {len(gixx_vals)} values: {[round(x,4) for x in gixx_vals]}")
R.append(f" Grid: {len(omega_vals)} × {len(khra_vals)} × {len(gixx_vals)} = {len(omega_vals)*len(khra_vals)*len(gixx_vals)} (actual: {len(df)})")
# ── Section 3: Coherence Analysis ──
R.append("")
R.append("─" * 78)
R.append("3. COHERENCE ANALYSIS")
R.append("─" * 78)
# Top 10 coherence values
top10 = df.nlargest(10, 'coherence')
R.append(" Top 10 coherence measurements:")
R.append(f" {'Rank':>4} {'Ω':>5} {'K':>6} {'G':>7} {'Coh':>10} {'Asym':>8} {'Vort':>10}")
for rank, (_, row) in enumerate(top10.iterrows(), 1):
R.append(f" {rank:4d} {row['omega']:5.1f} {row['khra_amp']:6.3f} {row['gixx_amp']:7.4f} "
f"{row['coherence']:10.6f} {row['asymmetry']:8.4f} {row['vorticity_mean']:10.6f}")
# Bottom 10
bot10 = df.nsmallest(10, 'coherence')
R.append("\n Bottom 10 coherence measurements:")
R.append(f" {'Rank':>4} {'Ω':>5} {'K':>6} {'G':>7} {'Coh':>10} {'Asym':>8} {'Vort':>10}")
for rank, (_, row) in enumerate(bot10.iterrows(), 1):
R.append(f" {rank:4d} {row['omega']:5.1f} {row['khra_amp']:6.3f} {row['gixx_amp']:7.4f} "
f"{row['coherence']:10.6f} {row['asymmetry']:8.4f} {row['vorticity_mean']:10.6f}")
# ── Section 4: Per-Omega Breakdown ──
R.append("")
R.append("─" * 78)
R.append("4. PER-OMEGA BREAKDOWN")
R.append("─" * 78)
R.append(f" {'Ω':>5} {'Coh_min':>10} {'Coh_max':>10} {'Coh_mean':>10} {'Coh_std':>10} "
f"{'Asym_mean':>10} {'Vort_mean':>10} {'Best K':>7} {'Best G':>8}")
for omega in omega_vals:
g = df[df['omega'] == omega]
best = g.loc[g['coherence'].idxmax()]
R.append(f" {omega:5.1f} {g['coherence'].min():10.6f} {g['coherence'].max():10.6f} "
f"{g['coherence'].mean():10.6f} {g['coherence'].std():10.6f} "
f"{g['asymmetry'].mean():10.4f} {g['vorticity_mean'].mean():10.6f} "
f"{best['khra_amp']:7.3f} {best['gixx_amp']:8.4f}")
# ── Section 5: Per-Khra Breakdown ──
R.append("")
R.append("─" * 78)
R.append("5. PER-KHRA BREAKDOWN")
R.append("─" * 78)
R.append(f" {'K':>6} {'Coh_min':>10} {'Coh_max':>10} {'Coh_mean':>10} {'Coh_std':>10} {'Best Ω':>6} {'Best G':>8}")
for khra in khra_vals:
g = df[df['khra_amp'] == khra]
best = g.loc[g['coherence'].idxmax()]
R.append(f" {khra:6.3f} {g['coherence'].min():10.6f} {g['coherence'].max():10.6f} "
f"{g['coherence'].mean():10.6f} {g['coherence'].std():10.6f} "
f"{best['omega']:6.1f} {best['gixx_amp']:8.4f}")
# ── Section 6: Per-Gixx Breakdown ──
R.append("")
R.append("─" * 78)
R.append("6. PER-GIXX BREAKDOWN")
R.append("─" * 78)
R.append(f" {'G':>7} {'Coh_min':>10} {'Coh_max':>10} {'Coh_mean':>10} {'Coh_std':>10} {'Best Ω':>6} {'Best K':>7}")
for gixx in gixx_vals:
g = df[df['gixx_amp'] == gixx]
best = g.loc[g['coherence'].idxmax()]
R.append(f" {gixx:7.4f} {g['coherence'].min():10.6f} {g['coherence'].max():10.6f} "
f"{g['coherence'].mean():10.6f} {g['coherence'].std():10.6f} "
f"{best['omega']:6.1f} {best['khra_amp']:7.3f}")
# ── Section 7: Correlation Matrix ──
R.append("")
R.append("─" * 78)
R.append("7. CORRELATION MATRIX")
R.append("─" * 78)
corr_cols = ['omega', 'khra_amp', 'gixx_amp', 'coherence', 'asymmetry', 'vorticity_mean']
corr = df[corr_cols].corr()
R.append(f" {'':>14}" + "".join(f"{c:>14}" for c in corr_cols))
for row_name in corr_cols:
vals = "".join(f"{corr.loc[row_name, c]:14.4f}" for c in corr_cols)
R.append(f" {row_name:>14}{vals}")
# ── Section 8: Coherence Sensitivity ──
R.append("")
R.append("─" * 78)
R.append("8. PARAMETER SENSITIVITY (effect on coherence)")
R.append("─" * 78)
# Omega sensitivity: variance of mean coherence across omega
omega_means = df.groupby('omega')['coherence'].mean()
khra_means = df.groupby('khra_amp')['coherence'].mean()
gixx_means = df.groupby('gixx_amp')['coherence'].mean()
omega_range = omega_means.max() - omega_means.min()
khra_range = khra_means.max() - khra_means.min()
gixx_range = gixx_means.max() - gixx_means.min()
total_range = omega_range + khra_range + gixx_range
R.append(f" Omega effect: range={omega_range:.6f} ({100*omega_range/total_range:.1f}% of total)")
R.append(f" Khra effect: range={khra_range:.6f} ({100*khra_range/total_range:.1f}% of total)")
R.append(f" Gixx effect: range={gixx_range:.6f} ({100*gixx_range/total_range:.1f}% of total)")
R.append(f" Most influential: {'omega' if omega_range >= max(khra_range, gixx_range) else 'khra_amp' if khra_range >= gixx_range else 'gixx_amp'}")
# Per-omega sensitivity to khra and gixx
R.append(f"\n Per-omega sensitivity (coherence std when varying K,G):")
R.append(f" {'Ω':>5} {'Std(coh)':>10} {'Sensitivity':>12}")
for omega in omega_vals:
g = df[df['omega'] == omega]
s = g['coherence'].std()
bar = "█" * int(s * 10000)
R.append(f" {omega:5.1f} {s:10.6f} {bar}")
# ── Section 9: Thermal & Power Profile ──
R.append("")
R.append("─" * 78)
R.append("9. THERMAL & POWER PROFILE")
R.append("─" * 78)
R.append(f" GPU Temperature:")
R.append(f" Min: {df['gpu_temp_c'].min():.0f}°C Max: {df['gpu_temp_c'].max():.0f}°C Mean: {df['gpu_temp_c'].mean():.1f}°C")
R.append(f" Points above 60°C: {(df['gpu_temp_c'] > 60).sum()} ({100*(df['gpu_temp_c'] > 60).mean():.1f}%)")
R.append(f" GPU Power:")
R.append(f" Min: {df['gpu_power_w'].min():.1f}W Max: {df['gpu_power_w'].max():.1f}W Mean: {df['gpu_power_w'].mean():.1f}W")
R.append(f" High power (>250W): {(df['gpu_power_w'] > 250).sum()} ({100*(df['gpu_power_w'] > 250).mean():.1f}%)")
R.append(f" Idle (<100W): {(df['gpu_power_w'] < 100).sum()} ({100*(df['gpu_power_w'] < 100).mean():.1f}%)")
# Temperature by omega
R.append(f"\n Temperature by omega:")
R.append(f" {'Ω':>5} {'Temp_mean':>10} {'Power_mean':>11}")
for omega in omega_vals:
g = df[df['omega'] == omega]
R.append(f" {omega:5.1f} {g['gpu_temp_c'].mean():10.1f} {g['gpu_power_w'].mean():11.1f}")
# ── Section 10: Asymmetry & Vorticity Analysis ──
R.append("")
R.append("─" * 78)
R.append("10. ASYMMETRY & VORTICITY ANALYSIS")
R.append("─" * 78)
R.append(f" Asymmetry: min={df['asymmetry'].min():.4f} max={df['asymmetry'].max():.4f} mean={df['asymmetry'].mean():.4f}")
R.append(f" Vorticity: min={df['vorticity_mean'].min():.6f} max={df['vorticity_mean'].max():.6f} mean={df['vorticity_mean'].mean():.6f}")
# Best asymmetry (lowest = most symmetric)
best_sym = df.nsmallest(5, 'asymmetry')
R.append(f"\n Most symmetric configurations (lowest asymmetry):")
for _, row in best_sym.iterrows():
R.append(f" Ω={row['omega']:.1f} K={row['khra_amp']:.3f} G={row['gixx_amp']:.4f} "
f"Asym={row['asymmetry']:.4f} Coh={row['coherence']:.6f}")
# Highest vorticity
high_vort = df.nlargest(5, 'vorticity_mean')
R.append(f"\n Highest vorticity configurations:")
for _, row in high_vort.iterrows():
R.append(f" Ω={row['omega']:.1f} K={row['khra_amp']:.3f} G={row['gixx_amp']:.4f} "
f"Vort={row['vorticity_mean']:.6f} Coh={row['coherence']:.6f}")
# Coherence-Asymmetry relationship
coh_asym_corr = df['coherence'].corr(df['asymmetry'])
coh_vort_corr = df['coherence'].corr(df['vorticity_mean'])
asym_vort_corr = df['asymmetry'].corr(df['vorticity_mean'])
R.append(f"\n Cross-correlations:")
R.append(f" Coherence ↔ Asymmetry: {coh_asym_corr:+.4f}")
R.append(f" Coherence ↔ Vorticity: {coh_vort_corr:+.4f}")
R.append(f" Asymmetry ↔ Vorticity: {asym_vort_corr:+.4f}")
# ── Section 11: Optimal Operating Regions ──
R.append("")
R.append("─" * 78)
R.append("11. OPTIMAL OPERATING REGIONS")
R.append("─" * 78)
# Multi-objective: high coherence + low asymmetry
df_copy = df.copy()
df_copy['score'] = (df_copy['coherence'] - df_copy['coherence'].min()) / (df_copy['coherence'].max() - df_copy['coherence'].min()) - \
0.5 * (df_copy['asymmetry'] - df_copy['asymmetry'].min()) / (df_copy['asymmetry'].max() - df_copy['asymmetry'].min())
best_multi = df_copy.nlargest(10, 'score')
R.append(f" Top 10 by composite score (high coherence + low asymmetry):")
R.append(f" {'Ω':>5} {'K':>6} {'G':>7} {'Coh':>10} {'Asym':>8} {'Vort':>10} {'Score':>8}")
for _, row in best_multi.iterrows():
R.append(f" {row['omega']:5.1f} {row['khra_amp']:6.3f} {row['gixx_amp']:7.4f} "
f"{row['coherence']:10.6f} {row['asymmetry']:8.4f} {row['vorticity_mean']:10.6f} {row['score']:8.4f}")
# Recommend optimal settings
best_overall = best_multi.iloc[0]
R.append(f"\n ★ RECOMMENDED OPERATING POINT:")
R.append(f" Ω = {best_overall['omega']:.1f}")
R.append(f" khra_amp = {best_overall['khra_amp']:.3f}")
R.append(f" gixx_amp = {best_overall['gixx_amp']:.4f}")
R.append(f" Expected coherence: {best_overall['coherence']:.6f}")
R.append(f" Expected asymmetry: {best_overall['asymmetry']:.4f}")
# ── Section 12: Data Quality Assessment ──
R.append("")
R.append("─" * 78)
R.append("12. DATA QUALITY ASSESSMENT")
R.append("─" * 78)
# Check for duplicate telemetry (stale reads)
# Count consecutive identical coherence values
coh_vals = df['coherence'].values
stale_runs = []
run_len = 1
for i in range(1, len(coh_vals)):
if coh_vals[i] == coh_vals[i-1]:
run_len += 1
else:
if run_len > 1:
stale_runs.append(run_len)
run_len = 1
if run_len > 1:
stale_runs.append(run_len)
total_stale = sum(stale_runs)
R.append(f" Consecutive identical readings: {len(stale_runs)} runs")
R.append(f" Total stale points: {total_stale}/{len(df)} ({100*total_stale/len(df):.1f}%)")
if stale_runs:
R.append(f" Longest stale run: {max(stale_runs)} points")
R.append(f" Mean stale run length: {np.mean(stale_runs):.1f}")
# Unique coherence values
n_unique = df['coherence'].nunique()
R.append(f" Unique coherence values: {n_unique}/{len(df)} ({100*n_unique/len(df):.1f}%)")
# Check cycle progression
cycles = df['cycle'].values
cycle_gaps = np.diff(cycles)
R.append(f"\n Cycle progression:")
R.append(f" Start cycle: {int(cycles[0])}")
R.append(f" End cycle: {int(cycles[-1])}")
R.append(f" Total cycles: {int(cycles[-1] - cycles[0])}")
R.append(f" Mean gap: {cycle_gaps.mean():.0f} cycles/point")
R.append(f" Backwards jumps: {(cycle_gaps < 0).sum()}")
R.append("")
R.append("=" * 78)
R.append("END OF REPORT")
R.append("=" * 78)
return "\n".join(R)
# ═══════════════════════════════════════════════════════════════════
# PART 2: Interactive HTML Dashboard
# ═══════════════════════════════════════════════════════════════════
def generate_html_report(df, source_name):
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
omega_vals = sorted(df['omega'].unique())
khra_vals = sorted(df['khra_amp'].unique())
gixx_vals = sorted(df['gixx_amp'].unique())
# Prepare data for JS
# 1. Per-omega stats
omega_stats = []
for omega in omega_vals:
g = df[df['omega'] == omega]
best = g.loc[g['coherence'].idxmax()]
omega_stats.append({
'omega': omega,
'coh_min': round(g['coherence'].min(), 6),
'coh_max': round(g['coherence'].max(), 6),
'coh_mean': round(g['coherence'].mean(), 6),
'coh_std': round(g['coherence'].std(), 6),
'asym_mean': round(g['asymmetry'].mean(), 4),
'vort_mean': round(g['vorticity_mean'].mean(), 6),
'temp_mean': round(g['gpu_temp_c'].mean(), 1),
'power_mean': round(g['gpu_power_w'].mean(), 1),
'best_k': round(best['khra_amp'], 3),
'best_g': round(best['gixx_amp'], 4),
})
# 2. Heatmap data: omega × khra → max coherence (across gixx)
heatmap_ok = []
for omega in omega_vals:
row = []
for khra in khra_vals:
g = df[(df['omega'] == omega) & (df['khra_amp'] == khra)]
row.append(round(g['coherence'].max(), 6))
heatmap_ok.append(row)
# 3. Heatmap: omega × gixx → max coherence (across khra)
heatmap_og = []
for omega in omega_vals:
row = []
for gixx in gixx_vals:
g = df[(df['omega'] == omega) & (df['gixx_amp'] == gixx)]
row.append(round(g['coherence'].max(), 6))
heatmap_og.append(row)
# 4. All data points for scatter
scatter_data = []
for _, row in df.iterrows():
scatter_data.append({
'o': round(row['omega'], 1),
'k': round(row['khra_amp'], 3),
'g': round(row['gixx_amp'], 4),
'c': round(row['coherence'], 6),
'a': round(row['asymmetry'], 4),
'v': round(row['vorticity_mean'], 6),
't': int(row['gpu_temp_c']),
'p': round(row['gpu_power_w'], 1),
})
# 5. Coherence distribution histogram
coh_vals = df['coherence'].values
hist_bins = 30
hist_counts, hist_edges = np.histogram(coh_vals, bins=hist_bins)
hist_centers = [round(0.5*(hist_edges[i] + hist_edges[i+1]), 6) for i in range(hist_bins)]
# 6. Top configurations
top20 = df.nlargest(20, 'coherence')
top_configs = []
for _, row in top20.iterrows():
top_configs.append({
'omega': round(row['omega'], 1),
'khra': round(row['khra_amp'], 3),
'gixx': round(row['gixx_amp'], 4),
'coh': round(row['coherence'], 6),
'asym': round(row['asymmetry'], 4),
'vort': round(row['vorticity_mean'], 6),
})
# 7. Nuclear magic analysis data
resolution = df['coherence'].std() * 0.1
if resolution < 1e-6:
resolution = 1e-4
mode_counts = {}
for omega in omega_vals:
g = df[df['omega'] == omega]
coh_sorted = np.sort(g['coherence'].values)
modes = [coh_sorted[0]]
for c in coh_sorted[1:]:
if c - modes[-1] > resolution:
modes.append(c)
mode_counts[round(omega, 1)] = len(modes)
html = f"""
Resonance Engine — Sweep Analysis Dashboard
⚛ Resonance Engine — Parameter Sweep Dashboard
{source_name} • {len(df)} points • {ts}
Key Metrics
Peak Coherence
{df['coherence'].max():.6f}
Mean Coherence
{df['coherence'].mean():.6f}
Coherence Range
{df['coherence'].max()-df['coherence'].min():.6f}
Best Omega
{top_configs[0]['omega']:.1f}
Magic Matches
{sum(1 for n in mode_counts.values() if n in MAGIC_NUMBERS)}/15
Coherence vs Omega
Coherence Distribution
Heatmap: Ω × Khra → Peak Coherence
Heatmap: Ω × Gixx → Peak Coherence
Mode Count vs Nuclear Magic Numbers
Asymmetry & Vorticity vs Omega
Thermal & Power Profile
★ Recommended Operating Point
Omega (Ω)
{top_configs[0]['omega']:.1f}
Khra Amp
{top_configs[0]['khra']:.3f}
Gixx Amp
{top_configs[0]['gixx']:.4f}
Coherence
{top_configs[0]['coh']:.6f}
Asymmetry
{top_configs[0]['asym']:.4f}
Vorticity
{top_configs[0]['vort']:.6f}
Top 20 Configurations
| # | Ω | K | G | Coherence | Asymmetry | Vorticity |
Per-Omega Summary
| Ω | Coh Min | Coh Max | Coh Mean | Coh Std | Asym Mean | Vort Mean | Best K | Best G |
"""
return html
# ═══════════════════════════════════════════════════════════════════
# Main
# ═══════════════════════════════════════════════════════════════════
def main():
csv_path = sys.argv[1] if len(sys.argv) > 1 else None
df, source = load_data(csv_path)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
# Generate text report
print("Generating text report...")
text_report = generate_text_report(df, source)
text_path = os.path.join(RESULTS_DIR, f"sweep_analysis_{timestamp}.txt")
with open(text_path, 'w', encoding='utf-8') as f:
f.write(text_report)
print(f" Saved: {text_path}")
# Generate HTML dashboard
print("Generating HTML dashboard...")
html_report = generate_html_report(df, source)
html_path = os.path.join(RESULTS_DIR, f"sweep_dashboard_{timestamp}.html")
with open(html_path, 'w', encoding='utf-8') as f:
f.write(html_report)
print(f" Saved: {html_path}")
# Print summary to stdout
print("\n" + text_report)
if __name__ == "__main__":
main()