diff --git a/benchmark.py b/benchmark.py index 27d7919..ad8ed42 100644 --- a/benchmark.py +++ b/benchmark.py @@ -15,6 +15,7 @@ import json import re import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from pathlib import Path @@ -251,11 +252,15 @@ def print_summary(report: dict, output_path: Path): print(f"{'='*60}\n") -def run_benchmark(config_path: str, resume: bool = False, fresh: bool = False): +def run_benchmark(config_path: str, resume: bool = False, fresh: bool = False, parallel: int | None = None): config = load_config(config_path) model_cfg = config['model'] bench_cfg = config['benchmark'] + parallelism = parallel if parallel is not None else bench_cfg.get('parallel_requests', 1) + if parallelism < 1: + parallelism = 1 + output_dir = Path(config_path).parent / bench_cfg.get('output_dir', 'runs') output_dir.mkdir(exist_ok=True) safe_name = sanitize_model_name(model_cfg['name']) @@ -326,17 +331,13 @@ def run_benchmark(config_path: str, resume: bool = False, fresh: bool = False): global_idx = resume_from - for i, q in enumerate(questions, 1): - qid = q.get('id', f'unknown_{i}') + def process_question(idx: int, q: dict) -> dict: + qid = q.get('id', f'unknown_{idx}') question_text = q.get('text', '?') golden_key = q.get('golden_key', '?') options = q.get('options', {}) - global_idx += 1 - display_q = question_text[:80] + ('...' if len(question_text) > 80 else '') - prompt = build_prompt(q) - response = None timed_out = False @@ -347,55 +348,53 @@ def run_benchmark(config_path: str, resume: bool = False, fresh: bool = False): continue break - is_correct = False - model_choice = '' - if timed_out: - results['timeouts'].append({ - 'id': qid, - 'question': question_text, - 'expected_key': golden_key, - 'options': options, - }) - elif answers_match(golden_key, response or '', options): - is_correct = True - results['correct'].append({ - 'id': qid, - 'question': question_text, - 'expected_key': golden_key, - 'model_response': (response or '')[:200], - }) - else: - model_choice = extract_choice(response or '') - results['incorrect'].append({ - 'id': qid, - 'question': question_text, - 'expected_key': golden_key, - 'model_response': (response or '')[:500], - 'model_choice': model_choice, - 'options': options, - }) + return {'type': 'timeout', 'id': qid, 'question': question_text, 'expected_key': golden_key, 'options': options} - # Running accuracy - answered = len(results['correct']) + len(results['incorrect']) + len(results['timeouts']) - acc = (len(results['correct']) / answered * 100) if answered > 0 else 0.0 + if answers_match(golden_key, response or '', options): + return {'type': 'correct', 'id': qid, 'question': question_text, 'expected_key': golden_key, 'model_response': (response or '')[:200]} - if timed_out: - label = 'TIMEOUT' - elif is_correct: - label = 'CORRECT' - else: - label = f"WRONG (expected: {golden_key}, got: {model_choice or '?'})" + model_choice = extract_choice(response or '') + return {'type': 'incorrect', 'id': qid, 'question': question_text, 'expected_key': golden_key, 'model_response': (response or '')[:500], 'model_choice': model_choice, 'options': options} - status = f" [{global_idx}/{len(questions) + resume_from}] {display_q} {label} [{acc:.1f}%]" - print(f"\r{status}") + total_q = len(questions) + resume_from - # Save checkpoint after each question - completed_ids.add(qid) - save_checkpoint(checkpoint_path, results, list(completed_ids), start_time) + with ThreadPoolExecutor(max_workers=parallelism) as executor: + futures = {executor.submit(process_question, i, q): (i, q) for i, q in enumerate(questions, 1)} - if bench_cfg.get('delay_between_requests', 0) > 0: - time.sleep(bench_cfg['delay_between_requests']) + for future in as_completed(futures): + i, q = futures[future] + qid = q.get('id', f'unknown_{i}') + question_text = q.get('text', '?') + golden_key = q.get('golden_key', '?') + + global_idx += 1 + display_q = question_text[:80] + ('...' if len(question_text) > 80 else '') + + try: + res = future.result() + except Exception as e: + res = {'type': 'error', 'id': qid, 'error': str(e)} + + if res['type'] == 'timeout': + results['timeouts'].append({'id': res['id'], 'question': res['question'], 'expected_key': res['expected_key'], 'options': res.get('options', {})}) + label = 'TIMEOUT' + elif res['type'] == 'correct': + results['correct'].append({'id': res['id'], 'question': res['question'], 'expected_key': res['expected_key'], 'model_response': res['model_response']}) + label = 'CORRECT' + elif res['type'] == 'incorrect': + results['incorrect'].append({'id': res['id'], 'question': res['question'], 'expected_key': res['expected_key'], 'model_response': res['model_response'], 'model_choice': res['model_choice'], 'options': res.get('options', {})}) + label = f"WRONG (expected: {golden_key}, got: {res.get('model_choice', '?')})" + else: + label = f"ERROR: {res.get('error', 'unknown')}" + + answered = len(results['correct']) + len(results['incorrect']) + len(results['timeouts']) + acc = (len(results['correct']) / answered * 100) if answered > 0 else 0.0 + + print(f" [{global_idx}/{total_q}] {display_q} {label} [{acc:.1f}%]") + + completed_ids.add(qid) + save_checkpoint(checkpoint_path, results, list(completed_ids), start_time) # Build and save final report report = build_report(config, results, start_time) @@ -424,6 +423,8 @@ def main(): help='Resume from the latest checkpoint') parser.add_argument('--fresh', '-f', action='store_true', help='Start a new run, ignoring existing checkpoint') + parser.add_argument('--parallel', '-p', type=int, default=None, + help='Number of parallel requests (1 = sequential)') args = parser.parse_args() config_path = Path(args.config) @@ -432,7 +433,7 @@ def main(): print(f" Copy config.example.yaml to config.yaml and fill in your values.", file=sys.stderr) sys.exit(1) - run_benchmark(str(config_path), resume=args.resume, fresh=args.fresh) + run_benchmark(str(config_path), resume=args.resume, fresh=args.fresh, parallel=args.parallel) if __name__ == '__main__': diff --git a/config.example.yaml b/config.example.yaml index c98a7a7..024deed 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -24,6 +24,9 @@ benchmark: # How many times to retry a failed request max_retries: 1 + # Number of parallel requests (1 = sequential) + parallel_requests: 1 + # Delay in seconds between requests (0 = none) delay_between_requests: 0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..08e6d2b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +fastparquet==2026.5.0 +pandas==3.0.5 +pyarrow==25.0.0 +pyyml==0.0.2 +requests==2.34.2