BIG-Bench Hard

lm_eval --model hf --model_args pretrained=<model_path> --tasks big-bench-hard --batch_size auto

opencompass --datasets big-bench-hard --models <model_config>

# Standard API Evaluation Loop
from openai import OpenAI

client = OpenAI()
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": prompt}],
    temperature=0.0,
)