SWE-bench Pro

lm_eval --model hf --model_args pretrained=<model_path> --tasks swe-bench-pro --batch_size auto

opencompass --datasets swe-bench-pro --models <model_config>

# Standard API Evaluation Loop
from openai import OpenAI

client = OpenAI()
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": prompt}],
    temperature=0.0,
)