Scruff AI
354 lines
12 KiB
Python
354 lines
12 KiB
Python
"""overnight_v4_clean.py — clean-baseline trajectory battery, randomized, n=3 repeats.
|
|
|
|
Waits for the lattice to return to engineered clean baseline (asym 12.1-12.6
|
|
sustained for SETTLE_CONFIRM_S seconds), then measures true sigma on the
|
|
clean field, then runs the per-variable trajectory experiment THREE times.
|
|
Each repetition uses a different random variable order so we can separate
|
|
variable-driven differences from time-drift contamination.
|
|
|
|
Why three orderings:
|
|
v3 result was suggestive (asym range 23.7..33.0 across vars) but the field
|
|
was decaying from v2 residue during the run, so first-injected variables
|
|
saw a different field state than last-injected. If the same per-variable
|
|
ordering of effects holds across all 3 random shuffles, it's the variable.
|
|
If the ordering tracks injection position, it's drift.
|
|
|
|
Output:
|
|
/mnt/d/Resonance_Engine/traj/overnight_<runid>/
|
|
meta.json
|
|
baseline_clean.parquet (the verified-clean baseline samples)
|
|
baseline_clean_stats.json
|
|
rep_0/{var}.parquet, {var}_pulses.parquet
|
|
rep_1/...
|
|
rep_2/...
|
|
decay_<var>_<rep>.parquet (telemetry during 60s after each var run, for decay-shape analysis)
|
|
progress.log
|
|
"""
|
|
from __future__ import annotations
|
|
import glob, json, threading, time
|
|
from collections import deque
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pyarrow.parquet as pq
|
|
import zmq
|
|
|
|
# ---- config ----
|
|
DATA_DAY_WSL = "/mnt/d/PaperTrader/research/hl_data/minutes/20260601"
|
|
COIN = "BTC"
|
|
|
|
# Stable-platform detection (not band-detection - lattice may be parked at a
|
|
# higher-energy plateau after accumulated forcing; what matters is steadiness)
|
|
STABILITY_PROBE_S = 1800 # 30 min window for stability check
|
|
STABILITY_ASYM_STD = 0.6 # asym std must be below this
|
|
STABILITY_ASYM_SLOPE = 0.005 # mean drift per second must be below this
|
|
MAX_WAIT_S = 6 * 3600 # hard cap: 6 hours waiting for stability
|
|
|
|
# Clean baseline measurement
|
|
BASELINE_MEASURE_S = 600 # 10 min once verified clean
|
|
|
|
# Trajectory rep
|
|
N_REPEATS = 3
|
|
WINDOW_MINUTES = 60
|
|
PULSE_SPACING_S = 2.0
|
|
SETTLE_BETWEEN_VARS_S = 60 # 1 min between var-runs in same rep
|
|
SETTLE_BETWEEN_REPS_S = 300 # 5 min between reps (helps decay between reps)
|
|
PRE_RECORD_S = 5
|
|
POST_RECORD_S = 60 # extra long post-record to capture decay shape
|
|
|
|
STR_CAP = 0.3
|
|
INJECT_X = 512.0
|
|
INJECT_Y = 512.0
|
|
INJECT_SIG = 32.0
|
|
|
|
TEL_ADDR = "tcp://127.0.0.1:5556"
|
|
CMD_ADDR = "tcp://127.0.0.1:5557"
|
|
|
|
VARS = [
|
|
"mid_price",
|
|
"signed_flow_usd",
|
|
"trade_count",
|
|
"large_print_cnt",
|
|
"wallet_entropy",
|
|
"vwap_drift",
|
|
]
|
|
|
|
CHANNELS = ["asymmetry", "coherence", "stress_xx", "stress_yy", "stress_xy",
|
|
"vorticity_mean", "vel_mean", "vel_max", "vel_var"]
|
|
|
|
RUN_ID = time.strftime("%Y%m%dT%H%M%S")
|
|
OUT_DIR = Path(f"/mnt/d/Resonance_Engine/traj/overnight_{RUN_ID}")
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
LOG_PATH = OUT_DIR / "progress.log"
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%Y-%m-%dT%H:%M:%S')}] {msg}"
|
|
print(line, flush=True)
|
|
with LOG_PATH.open("a", encoding="utf-8") as fh:
|
|
fh.write(line + "\n")
|
|
|
|
|
|
# ---- telemetry sub ----
|
|
class TelSub:
|
|
def __init__(self, addr):
|
|
self.ctx = zmq.Context()
|
|
self.sub = self.ctx.socket(zmq.SUB)
|
|
self.sub.setsockopt_string(zmq.SUBSCRIBE, "")
|
|
self.sub.connect(addr)
|
|
self.lock = threading.Lock()
|
|
self.history = deque(maxlen=200000) # ~6h at 10 Hz
|
|
self.stop = False
|
|
self.thread = threading.Thread(target=self._run, daemon=True)
|
|
self.thread.start()
|
|
|
|
def _run(self):
|
|
while not self.stop:
|
|
try:
|
|
raw = self.sub.recv_string(flags=0)
|
|
msg = json.loads(raw)
|
|
msg["_recv_wall"] = time.time()
|
|
with self.lock:
|
|
self.history.append(msg)
|
|
except Exception as e:
|
|
log(f"[tel] {e}")
|
|
time.sleep(0.1)
|
|
|
|
def latest(self):
|
|
with self.lock:
|
|
return dict(self.history[-1]) if self.history else None
|
|
|
|
def since(self, t0):
|
|
with self.lock:
|
|
return [dict(m) for m in self.history if m["_recv_wall"] >= t0]
|
|
|
|
def reset_marker(self):
|
|
return time.time()
|
|
|
|
def close(self):
|
|
self.stop = True
|
|
self.sub.close()
|
|
self.ctx.term()
|
|
|
|
|
|
# ---- data ----
|
|
def load_btc_day() -> pd.DataFrame:
|
|
files = sorted(glob.glob(str(Path(DATA_DAY_WSL) / "*.parquet")))
|
|
dfs = [pq.read_table(f).to_pandas() for f in files]
|
|
df = pd.concat(dfs).sort_values("minute").reset_index(drop=True)
|
|
return df[df.coin == COIN].sort_values("minute").reset_index(drop=True)
|
|
|
|
|
|
def pick_window(df: pd.DataFrame) -> pd.DataFrame:
|
|
z = np.zeros(len(df))
|
|
for v in VARS:
|
|
s = df[v]
|
|
sd = float(s.std())
|
|
if sd == 0:
|
|
continue
|
|
z += ((s - float(s.mean())) / sd).abs().values
|
|
roll = pd.Series(z).rolling(WINDOW_MINUTES).sum()
|
|
end_idx = int(roll.idxmax())
|
|
start_idx = end_idx - WINDOW_MINUTES + 1
|
|
return df.iloc[start_idx:end_idx + 1].reset_index(drop=True)
|
|
|
|
|
|
def encode_strengths(window: pd.DataFrame, full: pd.DataFrame, var: str) -> np.ndarray:
|
|
s_full = full[var]
|
|
mu, sd = float(s_full.mean()), float(s_full.std())
|
|
if sd == 0:
|
|
return np.zeros(len(window))
|
|
z = (window[var].values - mu) / sd
|
|
return (np.clip(z, -3.0, 3.0) / 3.0) * STR_CAP
|
|
|
|
|
|
# ---- main ----
|
|
def main():
|
|
log(f"=== overnight_v4 run_id={RUN_ID} ===")
|
|
log(f"out_dir={OUT_DIR}")
|
|
|
|
df_day = load_btc_day()
|
|
window = pick_window(df_day)
|
|
log(f"window minutes {int(window.minute.iloc[0])}..{int(window.minute.iloc[-1])}")
|
|
window.to_parquet(OUT_DIR / "window.parquet")
|
|
|
|
strengths = {v: encode_strengths(window, df_day, v) for v in VARS}
|
|
|
|
meta = {
|
|
"run_id": RUN_ID,
|
|
"params": {
|
|
"stability_probe_s": STABILITY_PROBE_S,
|
|
"stability_asym_std": STABILITY_ASYM_STD,
|
|
"stability_asym_slope": STABILITY_ASYM_SLOPE,
|
|
"max_wait_s": MAX_WAIT_S,
|
|
"baseline_measure_s": BASELINE_MEASURE_S,
|
|
"n_repeats": N_REPEATS,
|
|
"window_minutes": WINDOW_MINUTES,
|
|
"pulse_spacing_s": PULSE_SPACING_S,
|
|
"settle_between_vars_s": SETTLE_BETWEEN_VARS_S,
|
|
"settle_between_reps_s": SETTLE_BETWEEN_REPS_S,
|
|
"pre_record_s": PRE_RECORD_S,
|
|
"post_record_s": POST_RECORD_S,
|
|
"str_cap": STR_CAP,
|
|
"inject_x": INJECT_X,
|
|
"inject_y": INJECT_Y,
|
|
"inject_sigma": INJECT_SIG,
|
|
"coin": COIN,
|
|
"vars": VARS,
|
|
},
|
|
"wall_start": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
|
}
|
|
(OUT_DIR / "meta.json").write_text(json.dumps(meta, indent=2))
|
|
|
|
# ---- ZMQ ----
|
|
tel = TelSub(TEL_ADDR)
|
|
ctx = zmq.Context()
|
|
pub = ctx.socket(zmq.PUB)
|
|
pub.connect(CMD_ADDR)
|
|
time.sleep(0.7)
|
|
# wait for telemetry
|
|
t0 = time.time()
|
|
while tel.latest() is None and time.time() - t0 < 30:
|
|
time.sleep(0.2)
|
|
if tel.latest() is None:
|
|
log("FATAL: no telemetry on 5556")
|
|
return
|
|
|
|
# ---- phase 1: wait for STABLE field (not band - lattice may be at plateau) ----
|
|
log(f"phase 1: waiting for stable field "
|
|
f"(asym std < {STABILITY_ASYM_STD} and |slope| < {STABILITY_ASYM_SLOPE}/s "
|
|
f"over {STABILITY_PROBE_S}s window)")
|
|
wait_start = time.time()
|
|
last_status = 0
|
|
while True:
|
|
now = time.time()
|
|
if now - wait_start > MAX_WAIT_S:
|
|
log(f"MAX_WAIT_S exceeded; proceeding on best-available field")
|
|
break
|
|
|
|
probe = tel.since(now - STABILITY_PROBE_S)
|
|
if len(probe) < 60:
|
|
time.sleep(10)
|
|
continue
|
|
ts = np.array([m["_recv_wall"] for m in probe])
|
|
a = np.array([m["asymmetry"] for m in probe])
|
|
a_std = float(a.std())
|
|
a_mean = float(a.mean())
|
|
# linear slope (Δ per second)
|
|
rel_ts = ts - ts[0]
|
|
if rel_ts[-1] > 0:
|
|
slope = float(np.polyfit(rel_ts, a, 1)[0])
|
|
else:
|
|
slope = 0.0
|
|
|
|
stable = (a_std < STABILITY_ASYM_STD) and (abs(slope) < STABILITY_ASYM_SLOPE)
|
|
if stable:
|
|
log(f"FIELD STABLE — asym mean={a_mean:.3f} std={a_std:.3f} "
|
|
f"slope={slope:+.5f}/s (waited {now-wait_start:.0f}s)")
|
|
break
|
|
if now - last_status > 60:
|
|
log(f" not stable yet asym mean={a_mean:.3f} std={a_std:.3f} "
|
|
f"slope={slope:+.5f}/s (waited {now-wait_start:.0f}s)")
|
|
last_status = now
|
|
time.sleep(10)
|
|
|
|
# ---- phase 2: measure clean baseline sigma ----
|
|
log(f"phase 2: measuring clean baseline for {BASELINE_MEASURE_S}s")
|
|
bm_start = time.time()
|
|
time.sleep(BASELINE_MEASURE_S)
|
|
base_samples = tel.since(bm_start)
|
|
log(f" collected {len(base_samples)} samples")
|
|
df_base = pd.DataFrame(base_samples)
|
|
df_base.to_parquet(OUT_DIR / "baseline_clean.parquet")
|
|
sigma = {}
|
|
means = {}
|
|
for ch in CHANNELS:
|
|
if ch in df_base.columns:
|
|
sigma[ch] = float(df_base[ch].std())
|
|
means[ch] = float(df_base[ch].mean())
|
|
(OUT_DIR / "baseline_clean_stats.json").write_text(json.dumps({
|
|
"n": len(base_samples),
|
|
"mean": means,
|
|
"sigma": sigma,
|
|
"start_cycle": int(base_samples[0]["cycle"]) if base_samples else -1,
|
|
"end_cycle": int(base_samples[-1]["cycle"]) if base_samples else -1,
|
|
}, indent=2))
|
|
log(f" clean sigma: asym={sigma.get('asymmetry'):.4f} "
|
|
f"coh={sigma.get('coherence'):.5f} sxy={sigma.get('stress_xy'):.6f}")
|
|
|
|
# ---- phase 3: 3 randomized-order trajectory reps ----
|
|
rng = np.random.default_rng(seed=int(time.time()))
|
|
for rep in range(N_REPEATS):
|
|
rep_dir = OUT_DIR / f"rep_{rep}"
|
|
rep_dir.mkdir(exist_ok=True)
|
|
order = VARS.copy()
|
|
rng.shuffle(order)
|
|
log(f"=== rep {rep+1}/{N_REPEATS} order: {order} ===")
|
|
(rep_dir / "order.json").write_text(json.dumps(order))
|
|
|
|
for vi, var in enumerate(order):
|
|
strn_seq = strengths[var]
|
|
log(f" rep{rep} var {vi+1}/{len(order)}: {var}")
|
|
run_start = time.time()
|
|
|
|
# pre-record then pulse sequence
|
|
time.sleep(PRE_RECORD_S)
|
|
pulse_recs = []
|
|
for i, strn in enumerate(strn_seq):
|
|
cmd = {
|
|
"cmd": "inject_density",
|
|
"x": INJECT_X, "y": INJECT_Y, "sigma": INJECT_SIG,
|
|
"strength": float(strn),
|
|
}
|
|
wall = time.time()
|
|
pub.send_string(json.dumps(cmd))
|
|
pulse_recs.append({
|
|
"i": i, "wall_ts": wall,
|
|
"rel_t": wall - run_start,
|
|
"minute": int(window.minute.iloc[i]),
|
|
"raw_value": float(window[var].iloc[i]),
|
|
"strength": float(strn),
|
|
})
|
|
target = run_start + PRE_RECORD_S + (i + 1) * PULSE_SPACING_S
|
|
sleep_for = max(0.0, target - time.time())
|
|
time.sleep(sleep_for)
|
|
# post-record
|
|
time.sleep(POST_RECORD_S)
|
|
run_end = time.time()
|
|
|
|
trace = tel.since(run_start)
|
|
if not trace:
|
|
log(f" no telemetry captured?!")
|
|
continue
|
|
df_t = pd.DataFrame(trace)
|
|
df_t["rel_t"] = df_t["_recv_wall"] - run_start
|
|
df_t["var"] = var
|
|
df_t["rep"] = rep
|
|
df_t["order_pos"] = vi
|
|
df_t.to_parquet(rep_dir / f"{var}.parquet")
|
|
pd.DataFrame(pulse_recs).to_parquet(rep_dir / f"{var}_pulses.parquet")
|
|
log(f" saved trace: {len(df_t)} samples over {df_t.rel_t.max():.1f}s, "
|
|
f"asym range [{df_t.asymmetry.min():.2f}, {df_t.asymmetry.max():.2f}]")
|
|
|
|
# inter-var settle
|
|
if vi < len(order) - 1:
|
|
time.sleep(SETTLE_BETWEEN_VARS_S)
|
|
|
|
# between-rep settle
|
|
if rep < N_REPEATS - 1:
|
|
log(f" inter-rep settle {SETTLE_BETWEEN_REPS_S}s")
|
|
time.sleep(SETTLE_BETWEEN_REPS_S)
|
|
|
|
pub.close()
|
|
ctx.term()
|
|
tel.close()
|
|
|
|
meta["wall_end"] = time.strftime("%Y-%m-%dT%H:%M:%S")
|
|
(OUT_DIR / "meta.json").write_text(json.dumps(meta, indent=2))
|
|
log("=== overnight_v4 DONE ===")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|