benchmark2.py

#!/usr/bin/env python3
"""
Test definitivo: phi-2 vs Qwen 3B Flow-trained.
Valutazione semantica: l'output viene transpilato ed eseguito.
Il flow_code è valido se produce risultato CORRETTO (non crash/errore sintassi).
"""
import torch, json, sys, os
sys.path.insert(0, os.path.dirname(__file__))
from flow import flow_run, estimate_tokens
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

def load_model(model_name, lora_path):
    print(f"  Loading {model_name.split('/')[-1]}...")
    m = AutoModelForCausalLM.from_pretrained(
        model_name, device_map="cuda:0",
        torch_dtype=torch.bfloat16,
        trust_remote_code=True,
    )
    m = PeftModel.from_pretrained(m, lora_path)
    m.eval()
    tok = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    tok.pad_token = tok.eos_token
    return m, tok

def generate(model, tokenizer, prompt, max_new=60):
    full = f"Instruct: {prompt}\nOutput:"
    inputs = tokenizer(full, return_tensors="pt")
    inputs = {k: v.to(model.device) for k, v in inputs.items()}
    with torch.no_grad():
        outputs = model.generate(
            **inputs, max_new_tokens=max_new,
            temperature=0.2, do_sample=True,
            pad_token_id=tokenizer.eos_token_id,
        )
    text = tokenizer.decode(outputs[0], skip_special_tokens=True)
    after = text.split("Output:")[-1].strip() if "Output:" in text else text
    # Take only the first line that contains a pipe
    lines = [l.strip() for l in after.split('\n') if l.strip()]
    for line in lines:
        if '|' in line:
            return line
    return after

PHI = "microsoft/phi-2"
PHI_LORA = "/home/diego/workspace/progetti/flow-lang/models/flow-lora"
QWEN = "Qwen/Qwen2.5-Coder-3B-Instruct"
QWEN_LORA = "/home/diego/workspace/progetti/flow-lang/models/qwen3b-flow-lora"

PROMPTS = [
    ("T1: Filter%+map+take", "Da 1..20: filtra pari, triplica, primi 5",
     lambda r: r == "6,12,18,24,30" or r == "[6,12,18,24,30]" or (isinstance(r,list) and r == [6,12,18,24,30])),
    ("T2: Unique+sort desc", "Lista [3,1,4,1,5]: rimuovi duplicati e ordina decrescente",
     lambda r: (isinstance(r,list) and r == [5,4,3,1]) or str(r) == "[5,4,3,1]"),
    ("T3: Flatten+map", "Lista nidificata [[1,2],[3,4],[5]]: appiattisci e raddoppia",
     lambda r: (isinstance(r,list) and r == [2,4,6,8,10])),
    ("T4: Multi-filter", "Da 1..50: filtra multipli di 3 e maggiori di 20, raddoppia",
     lambda r: isinstance(r,list) and len(r) > 5),
    ("T5: Filter+count", "Da 1..30: filtra multipli di 4, conta quanti sono",
     lambda r: r == 7 or r == "7"),
    ("T6: Sort+take", "Da 1..40: filtra dispari, triplica, ordina, primi 8",
     lambda r: isinstance(r,list) and len(r) == 8),
    ("T7: Filter+square+take", "Da 1..25: filtra multipli di 3, eleva al quadrato, primi 4",
     lambda r: isinstance(r,list) and len(r) == 4),
    ("T8: Dict keys", "Dizionario {Alice:25, Bob:30, Carol:22}: estrai chiavi e ordinale",
     lambda r: isinstance(r,list) and len(r) >= 3),
]

print("═"*70)
print("  FLOW BENCHMARK: phi-2 vs Qwen 3B (semantica)")
print("═"*70)

torch.cuda.empty_cache()
import gc

# --- phi-2 ---
print("\n[1] phi-2 2.7B")
phi_m, phi_t = load_model(PHI, PHI_LORA)
print(f"    VRAM: {torch.cuda.memory_allocated()/1024**3:.1f}GB")

phi_results = []
for name, prompt, checker in PROMPTS:
    code = generate(phi_m, phi_t, prompt)
    ok = False
    if '|' in code:
        try:
            r, py = flow_run(code)
            ok = not isinstance(r, str) or not r.startswith('!!')
            if ok and checker:
                ok = checker(r)
        except:
            ok = False
    phi_results.append(ok)
    sym = "✓" if ok else "✗"
    print(f"  {name}: {sym}  {code[:60]}")

phi_ok = sum(phi_results)
print(f"  phi-2: {phi_ok}/{len(PROMPTS)} ({phi_ok/len(PROMPTS)*100:.0f}%)")
del phi_m; torch.cuda.empty_cache(); gc.collect()

# --- Qwen 3B ---
print("\n[2] Qwen 3B Coder")
qwen_m, qwen_t = load_model(QWEN, QWEN_LORA)
print(f"    VRAM: {torch.cuda.memory_allocated()/1024**3:.1f}GB")

qwen_results = []
for name, prompt, checker in PROMPTS:
    code = generate(qwen_m, qwen_t, prompt)
    ok = False
    if '|' in code:
        try:
            r, py = flow_run(code)
            ok = not isinstance(r, str) or not r.startswith('!!')
            if ok and checker:
                ok = checker(r)
        except:
            ok = False
    qwen_results.append(ok)
    sym = "✓" if ok else "✗"
    print(f"  {name}: {sym}  {code[:60]}")

qwen_ok = sum(qwen_results)
print(f"  Qwen 3B: {qwen_ok}/{len(PROMPTS)} ({qwen_ok/len(PROMPTS)*100:.0f}%)")
del qwen_m; torch.cuda.empty_cache()

# Summary
print("\n═"*70)
print("  RISULTATI FINALI")
print("═"*70)
print(f"  phi-2 2.7B:  {phi_ok}/8  {'★★★' if phi_ok >= 6 else '★★' if phi_ok >= 4 else '★'}")
print(f"  Qwen 3B:     {qwen_ok}/8  {'★★★' if qwen_ok >= 6 else '★★' if qwen_ok >= 4 else '★'}")
print(f"  Winner:      {'Qwen 3B' if qwen_ok > phi_ok else 'phi-2' if phi_ok > qwen_ok else 'PARITÀ'}")
print()