Scruff AI
143 lines
4.6 KiB
Python
143 lines
4.6 KiB
Python
"""_univariate_horizon_sweep.py
|
|
|
|
For each candidate variable, compute correlation with |fwd_return| and
|
|
sign(fwd_return) across a horizon band. No lattice, no Ridge, no z-score
|
|
rolling. Just raw Pearson and Spearman on cleaned BTC minute data.
|
|
|
|
The point: find the horizon BAND each variable lives in, not a single number.
|
|
"""
|
|
from __future__ import annotations
|
|
import glob
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import pandas as pd
|
|
from scipy.stats import pearsonr, spearmanr
|
|
|
|
DATA_GLOB = "/mnt/d/PaperTrader/research/hl_data/minutes/202603*"
|
|
|
|
VARIABLES = [
|
|
"trade_count",
|
|
"taker_buy_usd",
|
|
"taker_sell_usd",
|
|
"large_print_cnt",
|
|
"wallet_entropy",
|
|
"signed_flow_usd", # for sanity / comparison
|
|
"vwap_drift", # for sanity / comparison
|
|
]
|
|
|
|
HORIZONS = [5, 15, 30, 60, 120, 240, 480, 720, 1440, 2880]
|
|
|
|
|
|
def load_btc() -> pd.DataFrame:
|
|
dfs = []
|
|
for d in sorted(glob.glob(DATA_GLOB)):
|
|
for f in sorted(glob.glob(f"{d}/*.parquet")):
|
|
try:
|
|
df = pd.read_parquet(f)
|
|
except Exception:
|
|
continue
|
|
if "coin" in df.columns:
|
|
df = df[df["coin"] == "BTC"]
|
|
if len(df):
|
|
dfs.append(df)
|
|
df = pd.concat(dfs, ignore_index=True)
|
|
df = df.sort_values("minute").drop_duplicates("minute").reset_index(drop=True)
|
|
return df
|
|
|
|
|
|
def add_forward_returns(df: pd.DataFrame) -> pd.DataFrame:
|
|
p = df["mid_price"].astype(float).values
|
|
for h in HORIZONS:
|
|
future = pd.Series(p).shift(-h).values
|
|
fwd = (future - p) / p
|
|
df[f"fwd_{h}"] = fwd
|
|
df[f"abs_fwd_{h}"] = np.abs(fwd)
|
|
return df
|
|
|
|
|
|
def corr_table(df: pd.DataFrame) -> pd.DataFrame:
|
|
"""For each (variable, horizon) compute Pearson and Spearman vs
|
|
|fwd_return| and sign(fwd_return). Only on rows where both are finite."""
|
|
rows = []
|
|
for v in VARIABLES:
|
|
if v not in df.columns:
|
|
print(f" WARN: {v} missing from data")
|
|
continue
|
|
x_full = df[v].astype(float)
|
|
for h in HORIZONS:
|
|
y_abs = df[f"abs_fwd_{h}"]
|
|
y_sgn = np.sign(df[f"fwd_{h}"])
|
|
mask = x_full.notna() & y_abs.notna() & np.isfinite(x_full) & np.isfinite(y_abs)
|
|
x = x_full[mask].values
|
|
ya = y_abs[mask].values
|
|
ys = y_sgn[mask].values
|
|
if len(x) < 1000:
|
|
continue
|
|
try:
|
|
p_abs, _ = pearsonr(x, ya)
|
|
s_abs, _ = spearmanr(x, ya)
|
|
p_sgn, _ = pearsonr(x, ys)
|
|
s_sgn, _ = spearmanr(x, ys)
|
|
except Exception:
|
|
continue
|
|
rows.append({
|
|
"variable": v,
|
|
"h_min": h,
|
|
"n": len(x),
|
|
"pearson_abs": p_abs,
|
|
"spearman_abs": s_abs,
|
|
"pearson_sign": p_sgn,
|
|
"spearman_sign": s_sgn,
|
|
})
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def main():
|
|
print("loading BTC minutes from", DATA_GLOB)
|
|
df = load_btc()
|
|
print(f"loaded {len(df)} unique minutes")
|
|
print("columns:", list(df.columns))
|
|
df = add_forward_returns(df)
|
|
|
|
print("\n=== sample variable stats ===")
|
|
for v in VARIABLES:
|
|
if v in df.columns:
|
|
s = df[v].astype(float)
|
|
n_zero = int((s == 0).sum())
|
|
print(f" {v:<20} mean={s.mean():+.4g} std={s.std():+.4g} "
|
|
f"min={s.min():+.4g} max={s.max():+.4g} zeros={n_zero}/{len(s)}")
|
|
|
|
tbl = corr_table(df)
|
|
pd.set_option("display.width", 200)
|
|
pd.set_option("display.max_rows", None)
|
|
pd.set_option("display.float_format", "{:+.4f}".format)
|
|
|
|
print("\n=== |fwd_return| correlations (Pearson) ===")
|
|
pivot_p = tbl.pivot(index="variable", columns="h_min", values="pearson_abs")
|
|
pivot_p = pivot_p.reindex(VARIABLES)
|
|
print(pivot_p.to_string())
|
|
|
|
print("\n=== |fwd_return| correlations (Spearman) ===")
|
|
pivot_s = tbl.pivot(index="variable", columns="h_min", values="spearman_abs")
|
|
pivot_s = pivot_s.reindex(VARIABLES)
|
|
print(pivot_s.to_string())
|
|
|
|
print("\n=== sign(fwd_return) correlations (Pearson) ===")
|
|
pivot_sgn = tbl.pivot(index="variable", columns="h_min", values="pearson_sign")
|
|
pivot_sgn = pivot_sgn.reindex(VARIABLES)
|
|
print(pivot_sgn.to_string())
|
|
|
|
print("\n=== sign(fwd_return) correlations (Spearman) ===")
|
|
pivot_sgn_s = tbl.pivot(index="variable", columns="h_min", values="spearman_sign")
|
|
pivot_sgn_s = pivot_sgn_s.reindex(VARIABLES)
|
|
print(pivot_sgn_s.to_string())
|
|
|
|
# Save raw table
|
|
out = Path("/mnt/d/Resonance_Engine/traj/univariate_sweep.csv")
|
|
tbl.to_csv(out, index=False)
|
|
print(f"\nsaved: {out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|