From fd40133142ff3d7c6af469e3477e3a440b9b64e3 Mon Sep 17 00:00:00 2001 From: ARIA Date: Tue, 14 Jul 2026 14:32:24 +0200 Subject: [PATCH] Initial commit: MCQ benchmark with checkpoint/resume support --- .gitignore | 4 + LICENSE | 176 ++++++++++++++++++ README.md | 54 ++++++ benchmark.py | 439 ++++++++++++++++++++++++++++++++++++++++++++ config.example.yaml | 31 ++++ 5 files changed, 704 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 benchmark.py create mode 100644 config.example.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e56fbc8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +runs/ +__pycache__/ +*.pyc +config.yaml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..24dc4ed --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademark. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work, You may choose to offer, and charge a fee for, acceptance + of support, warranty, indemnity, or other liability obligations + and/or rights consistent with this License. However, in accepting + such obligations, You may act only on Your own behalf and on Your + own responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or + additional liability. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 0000000..22b087e --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# MCQ Benchmark + +LLM benchmark for evaluating models on multiple-choice questions from a parquet dataset (financial reasoning). + +## Features + +- **Checkpoint/Resume** — progress is saved after each question. Stop and resume anytime. +- **Live Status** — real-time progress with running accuracy percentage. +- **Configurable** — YAML config for model endpoint, parameters, and benchmark settings. + +## Setup + +1. Copy the example config: + ```bash + cp config.example.yaml config.yaml + ``` + +2. Edit `config.yaml` with your model endpoint and parameters. + +## Usage + +```bash +# Run benchmark (prompts to resume if checkpoint exists) +python3 benchmark.py + +# Auto-resume from latest checkpoint +python3 benchmark.py --resume + +# Start fresh, ignoring existing checkpoint +python3 benchmark.py --fresh + +# Use custom config +python3 benchmark.py --config my_config.yaml +``` + +## Output + +Results are saved to `runs/_YYYYMMDD_HHMMSS.json` with full details: +- Overall accuracy and timing +- Per-question results (correct, incorrect, timeouts) +- Model configuration used + +Checkpoints (`*.checkpoint.json`) are stored in `runs/` and auto-deleted on completion. + +## Live Output Example + +``` + [3048/5074] what is AAPL expecting in return for Q4 2016 based on this sheet:... CORRECT [67.4%] + [3049/5074] calculate the total revenue for the fiscal year 2015... WRONG (expected: B, got: A) [67.4%] +``` + +## License + +Apache 2.0 \ No newline at end of file diff --git a/benchmark.py b/benchmark.py new file mode 100644 index 0000000..27d7919 --- /dev/null +++ b/benchmark.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +""" +MCQ Benchmark +Evaluates local LLMs on multiple-choice questions from a parquet dataset. + +Usage: + python3 benchmark.py [--config config.yaml] [--resume] [--fresh] + +Checkpoints are saved after each question to runs/_.checkpoint.json. +Use --resume to continue from the latest checkpoint. +""" + +import argparse +import json +import re +import sys +import time +from datetime import datetime +from pathlib import Path + +import pandas as pd +import requests +import yaml + + +def load_config(path: str) -> dict: + with open(path, "r") as f: + return yaml.safe_load(f) + + +def sanitize_model_name(name: str) -> str: + sanitized = re.sub(r'[^\w\s-]', '', name) + sanitized = re.sub(r'[\s]+', '_', sanitized) + return sanitized.strip('_') or 'unknown_model' + + +def extract_choice(response: str) -> str: + """Extract the chosen option letter from the model response.""" + if not response: + return "" + + cleaned = response.strip() + + # Check for letter in parentheses: (A), (B), etc. + match = re.search(r'\(([A-Z])\)', cleaned) + if match: + return match.group(1).upper() + + # "Answer: A", "Choice: B", etc. (colon separator) + match = re.search(r'(?:answer|choice|option)\s*:\s*["\']?([A-Z])["\']?', cleaned, re.IGNORECASE) + if match: + return match.group(1).upper() + + # "Answer is A", "Choice is B", etc. + match = re.search(r'(?:answer|choice|option)\s+is\s+["\']?([A-Z])["\']?', cleaned, re.IGNORECASE) + if match: + return match.group(1).upper() + + # "A)", "B)", etc. + match = re.search(r'\b([A-Z])\)', cleaned) + if match: + return match.group(1).upper() + + # Single letter response + match = re.match(r'^\s*([A-Z])\s*$', cleaned) + if match: + return match.group(1).upper() + + return "" + + +def answers_match(expected_key: str, model_response: str, options: dict) -> bool: + """Compare the expected answer key with the model's response.""" + if not expected_key: + return False + + expected = expected_key.strip().upper() + + # Direct match on letter + choice = extract_choice(model_response) + if choice == expected: + return True + + # Check if the model's response contains the correct option text + correct_text = options.get(expected, "") + if correct_text: + model_lower = model_response.lower() + correct_lower = correct_text.lower() + # Remove "million", "billion" suffixes for numeric comparison + clean_correct = re.sub(r'\s*(million|billion|thousand|percent|%)\b', '', correct_lower).strip() + if clean_correct and clean_correct in model_lower: + return True + + return False + + +def build_prompt(row) -> str: + """Build the prompt from a dataset row.""" + query = row.get('query', '') + options = row.get('options', {}) + + # Extract the question text from the query + question_match = re.search(r'Question:\s*(.+?)(?:\nAnswer:|$)', query, re.DOTALL) + question = question_match.group(1).strip() if question_match else "" + + # Extract the context from the query + context_match = re.search(r'Context:\s*(.+?)(?:\nQuestion:|$)', query, re.DOTALL) + context = context_match.group(1).strip() if context_match else "" + + # Format options + options_text = "" + if isinstance(options, dict): + for key, value in sorted(options.items()): + options_text += f" ({key}) {value}\n" + elif isinstance(options, str): + try: + parsed = json.loads(options) + if 'options' in parsed: + for key, value in sorted(parsed['options'].items()): + options_text += f" ({key}) {value}\n" + except (json.JSONDecodeError, TypeError): + pass + + prompt = f"""You are a financial analyst answering multiple-choice questions based on the provided context. + +**Context:** +{context} + +**Question:** {question} + +**Options:** +{options_text} +Answer with ONLY the letter of the correct option (e.g., "A", "B", "C", or "D"). Do not include any explanation.""" + + return prompt + + +def query_model(prompt: str, config: dict) -> str | None: + """Send a request to the LLM endpoint and return the response text.""" + model_cfg = config['model'] + + endpoint = f"{model_cfg['endpoint']}/chat/completions" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {model_cfg['api_key']}", + } + + payload = { + "model": model_cfg['name'], + "messages": [ + {"role": "system", "content": "You are a precise financial analyst. Answer multiple-choice questions based on the provided context. Respond with only the letter of the correct option."}, + {"role": "user", "content": prompt}, + ], + "max_tokens": model_cfg['max_tokens'], + "temperature": model_cfg.get('temperature', 0.0), + } + + try: + resp = requests.post( + endpoint, + headers=headers, + json=payload, + timeout=model_cfg['timeout'], + ) + resp.raise_for_status() + data = resp.json() + return data['choices'][0]['message']['content'] + except requests.exceptions.Timeout: + return None + except Exception as e: + print(f" Warning: Error - {e}", file=sys.stderr) + return "" + + +def find_latest_checkpoint(output_dir: Path, safe_name: str) -> tuple: + """Find the latest checkpoint file for a model and return (path, checkpoint_data).""" + checkpoints = sorted(output_dir.glob(f"{safe_name}_*.checkpoint.json")) + if checkpoints: + with open(checkpoints[-1], 'r') as f: + return checkpoints[-1], json.load(f) + return None, None + + +def save_checkpoint(checkpoint_path: Path, results: dict, completed_ids: list, start_time: float): + """Save checkpoint with current progress.""" + checkpoint = { + 'completed_ids': completed_ids, + 'results': results, + 'start_time': start_time, + 'saved_at': datetime.now().isoformat(), + } + with open(checkpoint_path, 'w') as f: + json.dump(checkpoint, f, indent=2, ensure_ascii=False) + + +def build_report(config: dict, results: dict, start_time: float): + """Build the final report from results.""" + model_cfg = config['model'] + elapsed = time.time() - start_time + total_attempted = len(results['correct']) + len(results['incorrect']) + len(results['timeouts']) + accuracy = (len(results['correct']) / total_attempted * 100) if total_attempted > 0 else 0 + + return { + 'benchmark': 'MCQ Benchmark (parquet)', + 'timestamp': datetime.now().isoformat(), + 'model': { + 'name': model_cfg['name'], + 'endpoint': model_cfg['endpoint'], + 'timeout': model_cfg['timeout'], + 'max_tokens': model_cfg['max_tokens'], + 'temperature': model_cfg.get('temperature', 0.0), + }, + 'summary': { + 'total_questions': total_attempted, + 'correct': len(results['correct']), + 'incorrect': len(results['incorrect']), + 'timeouts': len(results['timeouts']), + 'accuracy_pct': round(accuracy, 1), + 'elapsed_seconds': round(elapsed, 1), + 'avg_seconds_per_question': round(elapsed / total_attempted, 2) if total_attempted > 0 else 0, + }, + 'correct_answers': results['correct'], + 'incorrect_answers': results['incorrect'], + 'timeout_questions': results['timeouts'], + } + + +def print_summary(report: dict, output_path: Path): + """Print the final summary.""" + s = report['summary'] + print(f"\n{'='*60}") + print(f" RESULTS") + print(f"{'='*60}") + print(f" Accuracy: {s['accuracy_pct']:.1f}% ({s['correct']}/{s['total_questions']})") + print(f" Correct: {s['correct']}") + print(f" Wrong: {s['incorrect']}") + print(f" Timeouts: {s['timeouts']}") + if s['total_questions'] > 0: + print(f" Time: {s['elapsed_seconds']:.1f}s ({s['avg_seconds_per_question']:.1f}s/q)") + + if report['incorrect_answers']: + print(f"\n Wrong Answers ({len(report['incorrect_answers'])}):") + for item in report['incorrect_answers'][:10]: + print(f" [{item['id']}] {item['question'][:70]}...") + print(f" Expected: {item['expected_key']}") + print(f" Got: {item['model_response'][:70]}") + if len(report['incorrect_answers']) > 10: + print(f" ... and {len(report['incorrect_answers']) - 10} more") + + print(f"\n Report saved: {output_path}") + print(f"{'='*60}\n") + + +def run_benchmark(config_path: str, resume: bool = False, fresh: bool = False): + config = load_config(config_path) + model_cfg = config['model'] + bench_cfg = config['benchmark'] + + output_dir = Path(config_path).parent / bench_cfg.get('output_dir', 'runs') + output_dir.mkdir(exist_ok=True) + safe_name = sanitize_model_name(model_cfg['name']) + + # Check for existing checkpoint + checkpoint_path, checkpoint_data = find_latest_checkpoint(output_dir, safe_name) + has_checkpoint = checkpoint_data is not None + + if has_checkpoint and not fresh: + if resume or input(f" Found checkpoint: {checkpoint_path.name}\n {len(checkpoint_data['completed_ids'])} questions completed. Resume? [y/N]: ").strip().lower() == 'y': + print(f" Resuming from checkpoint...") + results = checkpoint_data['results'] + completed_ids = set(checkpoint_data['completed_ids']) + start_time = checkpoint_data['start_time'] + resume_from = len(completed_ids) + else: + completed_ids = set() + results = {'correct': [], 'incorrect': [], 'timeouts': [], 'errors': []} + start_time = time.time() + resume_from = 0 + else: + completed_ids = set() + results = {'correct': [], 'incorrect': [], 'timeouts': [], 'errors': []} + start_time = time.time() + resume_from = 0 + + # Load dataset + dataset_path = Path(config_path).parent / bench_cfg['dataset'] + if not dataset_path.exists(): + print(f"Dataset not found: {dataset_path}", file=sys.stderr) + sys.exit(1) + + df = pd.read_parquet(dataset_path) + questions = df.to_dict('records') + + total = len(questions) + max_q = bench_cfg.get('max_questions', 0) or total + if max_q < total: + import random + random.seed(bench_cfg.get('seed', 42)) + random.shuffle(questions) + questions = questions[:max_q] + + # Filter out already completed questions when resuming + if resume_from > 0: + remaining = [q for q in questions if q.get('id', '') not in completed_ids] + skipped = len(questions) - len(remaining) + questions = remaining + print(f" Skipped {skipped} completed questions. {len(questions)} remaining.\n") + + print(f"\n{'='*60}") + print(f" MCQ Benchmark") + print(f"{'='*60}") + print(f" Model: {model_cfg['name']}") + print(f" Endpoint: {model_cfg['endpoint']}") + print(f" Timeout: {model_cfg['timeout']}s") + print(f" Max tokens: {model_cfg['max_tokens']}") + print(f" Temperature: {model_cfg.get('temperature', 0.0)}") + print(f" Questions: {len(questions) + resume_from} / {total}") + if resume_from > 0: + print(f" Resumed: {resume_from} completed") + print(f"{'='*60}\n") + + # Determine checkpoint file path + if not checkpoint_path: + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + checkpoint_path = output_dir / f"{safe_name}_{timestamp}.checkpoint.json" + + global_idx = resume_from + + for i, q in enumerate(questions, 1): + qid = q.get('id', f'unknown_{i}') + question_text = q.get('text', '?') + golden_key = q.get('golden_key', '?') + options = q.get('options', {}) + + global_idx += 1 + display_q = question_text[:80] + ('...' if len(question_text) > 80 else '') + + prompt = build_prompt(q) + + response = None + timed_out = False + + for attempt in range(1 + bench_cfg.get('max_retries', 0)): + response = query_model(prompt, config) + if response is None: + timed_out = True + continue + break + + is_correct = False + model_choice = '' + + if timed_out: + results['timeouts'].append({ + 'id': qid, + 'question': question_text, + 'expected_key': golden_key, + 'options': options, + }) + elif answers_match(golden_key, response or '', options): + is_correct = True + results['correct'].append({ + 'id': qid, + 'question': question_text, + 'expected_key': golden_key, + 'model_response': (response or '')[:200], + }) + else: + model_choice = extract_choice(response or '') + results['incorrect'].append({ + 'id': qid, + 'question': question_text, + 'expected_key': golden_key, + 'model_response': (response or '')[:500], + 'model_choice': model_choice, + 'options': options, + }) + + # Running accuracy + answered = len(results['correct']) + len(results['incorrect']) + len(results['timeouts']) + acc = (len(results['correct']) / answered * 100) if answered > 0 else 0.0 + + if timed_out: + label = 'TIMEOUT' + elif is_correct: + label = 'CORRECT' + else: + label = f"WRONG (expected: {golden_key}, got: {model_choice or '?'})" + + status = f" [{global_idx}/{len(questions) + resume_from}] {display_q} {label} [{acc:.1f}%]" + print(f"\r{status}") + + # Save checkpoint after each question + completed_ids.add(qid) + save_checkpoint(checkpoint_path, results, list(completed_ids), start_time) + + if bench_cfg.get('delay_between_requests', 0) > 0: + time.sleep(bench_cfg['delay_between_requests']) + + # Build and save final report + report = build_report(config, results, start_time) + + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + filename = f"{safe_name}_{timestamp}.json" + output_path = output_dir / filename + + with open(output_path, 'w') as f: + json.dump(report, f, indent=2, ensure_ascii=False) + + # Clean up checkpoint + if checkpoint_path.exists(): + checkpoint_path.unlink() + + print_summary(report, output_path) + + return report + + +def main(): + parser = argparse.ArgumentParser(description='MCQ LLM Benchmark') + parser.add_argument('--config', '-c', default='config.yaml', + help='Path to config.yaml (default: config.yaml)') + parser.add_argument('--resume', '-r', action='store_true', + help='Resume from the latest checkpoint') + parser.add_argument('--fresh', '-f', action='store_true', + help='Start a new run, ignoring existing checkpoint') + args = parser.parse_args() + + config_path = Path(args.config) + if not config_path.exists(): + print(f"Config not found: {config_path}") + print(f" Copy config.example.yaml to config.yaml and fill in your values.", file=sys.stderr) + sys.exit(1) + + run_benchmark(str(config_path), resume=args.resume, fresh=args.fresh) + + +if __name__ == '__main__': + main() diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..c98a7a7 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,31 @@ +# MCQ Benchmark - Configuration +# Copy to config.yaml and fill in your values. + +# Model configuration +model: + name: "your-model-name" + endpoint: "http://localhost:8080/v1" + api_key: "not-needed" + timeout: 300 + max_tokens: 4096 + temperature: 0.0 + +# Benchmark settings +benchmark: + # Path to the parquet dataset file + dataset: "train-00000-of-00001.parquet" + + # Number of questions to run (0 = all) + max_questions: 0 + + # Random seed for reproducibility when max_questions < total + seed: 42 + + # How many times to retry a failed request + max_retries: 1 + + # Delay in seconds between requests (0 = none) + delay_between_requests: 0 + + # Output directory for results + output_dir: "runs"