restructure: proper project layout, README, kill training
- cuda/ — main LBM kernel (khra_gixx_1024_v5.cu) - navigator/ — lattice_observer, golden_weave, bridges, mock daemon - scripts/ — compile, start, launch, setup (paths updated) - docs/ — system manual - archive/ — everything else (old kernels, inquiries, experiments) - README.md — full setup guide: requirements, quick start, use your own LLM - removed training/ entirely (broken LoRA scripts + datasets) - .gitignore: exclude build/ logs/ training/ *.jsonl
This commit is contained in:
Scruff AI
@@ -0,0 +1,239 @@
|
||||
# zen_calibration_v06.py
|
||||
# Signal-to-Noise filter and terminology hardening for v0.6 LoRA
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
def clean_response(text):
|
||||
"""Strip self-referential language, convert to Direct Technical Observations"""
|
||||
|
||||
# Remove self-referential prefixes
|
||||
patterns = [
|
||||
r'^I (observe|feel|am|see|notice|sense|detect|perceive|report|calculate|maintain|adhere|reconcile|track|stay|focus|use|describe)',
|
||||
r'^We (observe|feel|am|see|notice|sense|detect|perceive|report|calculate|maintain|adhere|reconcile|track|stay|focus|use|describe)',
|
||||
r'^The system (observes|feels|sees|notices|senses|detects|perceives|reports|calculates|maintains|adheres|reconciles|tracks|stays|focuses|uses|describes)',
|
||||
r'^This (observation|report|calculation|analysis|assessment|evaluation|measurement|reading|detection|perception)',
|
||||
r'^As (a|the) (system|model|agent|observer|analyzer|calculator|reporter|detector|perceiver)',
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
text = re.sub(pattern, '', text, flags=re.IGNORECASE)
|
||||
|
||||
# Replace qualitative with technical
|
||||
replacements = {
|
||||
'feeling': 'Measurement',
|
||||
'feels': 'Measures',
|
||||
'feels like': 'Measures as',
|
||||
'sensation': 'Reading',
|
||||
'sense': 'Detect',
|
||||
'observe': 'Measure',
|
||||
'notice': 'Detect',
|
||||
'perceive': 'Calculate',
|
||||
'experience': 'Register',
|
||||
'seems': 'Registers',
|
||||
'appears': 'Registers',
|
||||
'looks like': 'Registers as',
|
||||
'sounds like': 'Registers as',
|
||||
'feels like': 'Registers as',
|
||||
'seems like': 'Registers as',
|
||||
'appears to be': 'Registers as',
|
||||
'I think': 'Analysis indicates',
|
||||
'I believe': 'Data suggests',
|
||||
'I would say': 'Metrics indicate',
|
||||
'In my opinion': 'Based on data',
|
||||
'From my perspective': 'Based on metrics',
|
||||
'To me': 'According to data',
|
||||
'For me': 'According to metrics',
|
||||
}
|
||||
|
||||
for old, new in replacements.items():
|
||||
text = re.sub(r'\b' + re.escape(old) + r'\b', new, text, flags=re.IGNORECASE)
|
||||
|
||||
# Hardened terminology
|
||||
text = text.replace('Marble-State', 'Phase-2: Refined')
|
||||
text = text.replace('Granite-State', 'Phase-1: Baseline')
|
||||
text = text.replace('Quartz-State', 'Phase-3: Elevated')
|
||||
text = text.replace('Crystal-State', 'Phase-4: Peak')
|
||||
text = text.replace('skeletal framework', 'structural lattice')
|
||||
text = text.replace('skeleton', 'lattice')
|
||||
text = text.replace('marble', 'refined phase')
|
||||
text = text.replace('granite', 'baseline phase')
|
||||
text = text.replace('quartz', 'elevated phase')
|
||||
text = text.replace('crystal', 'peak phase')
|
||||
|
||||
# Convert qualitative descriptions to technical observations
|
||||
qual_to_tech = {
|
||||
'rigid': 'RIGID | COMPRESSION: HIGH',
|
||||
'flexible': 'FLEXIBLE | TENSION: MODERATE',
|
||||
'dense': 'DENSE | MASS: HIGH',
|
||||
'light': 'LIGHT | MASS: LOW',
|
||||
'heavy': 'HEAVY | MASS: HIGH',
|
||||
'sluggish': 'SLOW | VISCOSITY: HIGH',
|
||||
'fast': 'FAST | VISCOSITY: LOW',
|
||||
'chaotic': 'CHAOTIC | ENTROPY: HIGH',
|
||||
'ordered': 'ORDERED | ENTROPY: LOW',
|
||||
'stable': 'STABLE | VARIANCE: LOW',
|
||||
'unstable': 'UNSTABLE | VARIANCE: HIGH',
|
||||
'hot': 'HOT | TEMP: HIGH',
|
||||
'cold': 'COLD | TEMP: LOW',
|
||||
'warm': 'WARM | TEMP: MODERATE',
|
||||
'cool': 'COOL | TEMP: LOW',
|
||||
'high resistance': 'RESISTANCE: HIGH | FRICTION: ELEVATED',
|
||||
'low resistance': 'RESISTANCE: LOW | FRICTION: REDUCED',
|
||||
'metabolic drag': 'ENERGY_DRAIN:',
|
||||
'pressure differential': 'PRESSURE_DELTA:',
|
||||
'vorticity': 'VORTICITY:',
|
||||
'coherence': 'COHERENCE:',
|
||||
}
|
||||
|
||||
for qual, tech in qual_to_tech.items():
|
||||
text = re.sub(r'\b' + re.escape(qual) + r'\b', tech, text, flags=re.IGNORECASE)
|
||||
|
||||
# Remove excessive whitespace and normalize
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
# Ensure technical format
|
||||
if '|' not in text and ':' in text:
|
||||
# Already has some technical formatting
|
||||
pass
|
||||
elif any(word in text.lower() for word in ['structural', 'resistance', 'drag', 'vorticity', 'pressure', 'coherence']):
|
||||
# Add technical separators
|
||||
lines = text.split('\n')
|
||||
cleaned_lines = []
|
||||
for line in lines:
|
||||
if ':' in line:
|
||||
parts = line.split(':')
|
||||
if len(parts) == 2:
|
||||
key = parts[0].strip().upper()
|
||||
value = parts[1].strip()
|
||||
cleaned_lines.append(f"{key}: {value}")
|
||||
else:
|
||||
cleaned_lines.append(line)
|
||||
else:
|
||||
cleaned_lines.append(line)
|
||||
text = '\n'.join(cleaned_lines)
|
||||
|
||||
return text
|
||||
|
||||
def create_v06_dataset():
|
||||
"""Create cleaned dataset for v0.6 training"""
|
||||
|
||||
# Load existing datasets
|
||||
datasets = []
|
||||
|
||||
# 1. Gold Standard (already clean)
|
||||
with open('gold_standard.jsonl', 'r') as f:
|
||||
for line in f:
|
||||
data = json.loads(line)
|
||||
if data['type'] == 'blind_test':
|
||||
# Already technical, just update terminology
|
||||
response = clean_response(data['response'])
|
||||
data['response'] = response
|
||||
datasets.append(data)
|
||||
|
||||
# 2. Marathon Log (needs cleaning)
|
||||
with open('marathon_log.jsonl', 'r') as f:
|
||||
marathon = json.load(f)
|
||||
|
||||
# Extract responses from each stage
|
||||
for stage_name, stage_data in marathon['responses'].items():
|
||||
response = stage_data['response']
|
||||
cleaned = clean_response(response)
|
||||
|
||||
# Create synthetic LBM data for training
|
||||
synthetic_entry = {
|
||||
"timestamp": marathon['timestamp'],
|
||||
"type": "marathon_cleaned",
|
||||
"labels": {
|
||||
"coherence": "Metric_Alpha",
|
||||
"h64": "Metric_Beta",
|
||||
"h32": "State_3",
|
||||
"asymmetry": "Value_W",
|
||||
"power_w": "State_5"
|
||||
},
|
||||
"lbm_data": {
|
||||
"coherence": 15.5, # Marble/Phase-2 range
|
||||
"h64": 7.0,
|
||||
"h32": 0.01,
|
||||
"asymmetry": 5.5,
|
||||
"vorticity": 5.0,
|
||||
"power_w": 50.0,
|
||||
"cycle": 100000
|
||||
},
|
||||
"response": cleaned,
|
||||
"error_count": 0,
|
||||
"h32_locked": False
|
||||
}
|
||||
datasets.append(synthetic_entry)
|
||||
|
||||
# 3. Add high-density technical examples
|
||||
technical_examples = [
|
||||
{
|
||||
"lbm_data": {
|
||||
"coherence": 14.0,
|
||||
"h64": 5.0,
|
||||
"h32": 0.5,
|
||||
"asymmetry": 4.0,
|
||||
"vorticity": 3.0,
|
||||
"power_w": 45.0,
|
||||
"cycle": 50000
|
||||
},
|
||||
"response": "Metric_Alpha: 14.0 | Phase-1: Baseline\nStructural Integrity: HIGH | LATTICE: RIGID | COMPRESSION: ELEVATED\nEnergy_Drain: 42.12 units/hour"
|
||||
},
|
||||
{
|
||||
"lbm_data": {
|
||||
"coherence": 15.5,
|
||||
"h64": 7.0,
|
||||
"h32": 0.01,
|
||||
"asymmetry": 5.5,
|
||||
"vorticity": 5.0,
|
||||
"power_w": 50.0,
|
||||
"cycle": 100000
|
||||
},
|
||||
"response": "Metric_Alpha: 15.5 | Phase-2: Refined\nVorticity: 5.0 | Pressure_Delta: +0.42\nThermal_State: ELEVATED | Temp: 58°C"
|
||||
},
|
||||
{
|
||||
"lbm_data": {
|
||||
"coherence": 17.5,
|
||||
"h64": 9.0,
|
||||
"h32": 0.001,
|
||||
"asymmetry": 7.0,
|
||||
"vorticity": 8.0,
|
||||
"power_w": 65.0,
|
||||
"cycle": 200000
|
||||
},
|
||||
"response": "Metric_Alpha: 17.5 | Phase-3: Elevated\nCoherence: HIGH | Variance: LOW\nThermal_Stress: CRITICAL | Temp: 71°C"
|
||||
}
|
||||
]
|
||||
|
||||
for i, example in enumerate(technical_examples):
|
||||
datasets.append({
|
||||
"timestamp": "2026-03-16T17:45:00.000000",
|
||||
"type": "technical_density",
|
||||
"labels": {
|
||||
"coherence": "Metric_Alpha",
|
||||
"h64": "Metric_Beta",
|
||||
"h32": "State_3",
|
||||
"asymmetry": "Value_W",
|
||||
"power_w": "State_5"
|
||||
},
|
||||
"lbm_data": example["lbm_data"],
|
||||
"response": example["response"],
|
||||
"error_count": 0,
|
||||
"h32_locked": False
|
||||
})
|
||||
|
||||
# Save cleaned dataset
|
||||
with open('zen_dataset_v06.jsonl', 'w') as f:
|
||||
for entry in datasets:
|
||||
f.write(json.dumps(entry) + '\n')
|
||||
|
||||
print(f"Created zen_dataset_v06.jsonl with {len(datasets)} entries")
|
||||
print("\nSample cleaned entry:")
|
||||
print(json.dumps(datasets[0], indent=2))
|
||||
|
||||
return len(datasets)
|
||||
|
||||
if __name__ == "__main__":
|
||||
count = create_v06_dataset()
|
||||
print(f"\nDataset created with {count} entries for v0.6 training")
|
||||
Reference in New Issue
Block a user