{"full_name":"openai/evals","owner":"openai","name":"evals","shard":"1","url":"https://github.com/openai/evals","description":"Evals is a framework for evaluating LLMs and LLM systems, and an open-source registry of benchmarks.","homepage":"","topics":[],"tag_pages":[],"language":"Python","license":"NOASSERTION","avatar":"https://avatars.githubusercontent.com/u/14957082?v=4","created_at":"2023-01-23T20:51:04Z","pushed_at":"2026-04-14T15:29:57Z","metrics":{"stars":19746,"forks":3133,"watchers":271,"issues":307,"prs":1449,"releases":0,"issues_open":180,"issues_closed":127,"prs_open":171,"prs_closed":623,"prs_merged":655,"discussions":80},"growth":{"day":{"stars":3,"forks":0,"base_day":"2026-11-15"},"week":{"stars":52,"forks":5,"base_day":"2026-11-04"},"month":{"stars":133,"forks":17,"base_day":"2026-10-17"}},"history":{"dates":["2026-09-06","2026-09-07","2026-09-08","2026-09-09","2026-09-10","2026-09-11","2026-09-12","2026-09-13","2026-09-14","2026-09-15","2026-09-16","2026-09-17","2026-09-18","2026-09-19","2026-09-20","2026-09-21","2026-09-22","2026-09-23","2026-09-24","2026-09-25","2026-09-26","2026-09-27","2026-09-28","2026-09-29","2026-09-30","2026-10-01","2026-10-02","2026-10-03","2026-10-04","2026-10-05","2026-10-06","2026-10-07","2026-10-08","2026-10-09","2026-10-10","2026-10-11","2026-10-12","2026-10-13","2026-10-14","2026-10-15","2026-10-16","2026-10-17","2026-10-18","2026-10-19","2026-10-20","2026-10-21","2026-10-22","2026-10-23","2026-10-24","2026-10-25","2026-10-26","2026-10-27","2026-10-28","2026-10-29","2026-10-30","2026-10-31","2026-11-01","2026-11-02","2026-11-03","2026-11-04","2026-11-10","2026-11-11","2026-11-12","2026-11-13","2026-11-14","2026-11-15","2026-11-16"],"stars":[19388,19395,19399,19412,19418,19425,19436,19443,19444,19450,19460,19469,19472,19475,19479,19486,19488,19491,19496,19503,19507,19509,19513,19517,19523,19527,19538,19544,19551,19553,19556,19560,19567,19575,19579,19585,19590,19591,19597,19605,19607,19613,19613,19616,19620,19620,19624,19633,19639,19645,19651,19658,19661,19669,19675,19680,19683,19685,19688,19694,19723,19731,19735,19735,19739,19743,19746],"forks":[3074,3077,3077,3080,3083,3082,3083,3083,3084,3085,3089,3091,3089,3091,3091,3092,3092,3092,3093,3093,3093,3093,3093,3094,3096,3097,3099,3102,3102,3104,3103,3106,3108,3111,3112,3113,3113,3112,3113,3114,3115,3116,3120,3119,3119,3119,3119,3119,3119,3120,3121,3123,3121,3124,3128,3128,3131,3131,3129,3128,3127,3127,3128,3128,3134,3133,3133],"watchers":[281,282,282,282,282,282,282,282,283,283,283,283,283,283,283,283,283,282,282,282,282,281,281,281,281,281,280,279,279,279,279,278,278,279,279,279,279,279,279,279,279,279,278,278,278,278,278,278,278,273,272,270,270,270,270,270,270,271,271,271,null,270,270,270,271,271,271],"issues_open":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,180,180,180,180,180,180],"prs_open":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,170,170,170,171,171,171]},"boards":{},"same_owner":[{"full_name":"openai/codex","name":"codex","shard":"1","boards":{"weekly":143,"monthly":101,"total":85}},{"full_name":"openai/whisper","name":"whisper","shard":"1","boards":{"daily_watchers":119,"total":117}},{"full_name":"openai/CLIP","name":"CLIP","shard":"1","boards":{}},{"full_name":"openai/baselines","name":"baselines","shard":"1","boards":{}},{"full_name":"openai/chatgpt-retrieval-plugin","name":"chatgpt-retrieval-plugin","shard":"1","boards":{}},{"full_name":"openai/codex-plugin-cc","name":"codex-plugin-cc","shard":"1","boards":{}},{"full_name":"openai/codex-security","name":"codex-security","shard":"1","boards":{"weekly":67,"weekly_rate":75,"monthly_rate":163}},{"full_name":"openai/gpt-3","name":"gpt-3","shard":"1","boards":{}},{"full_name":"openai/gym","name":"gym","shard":"1","boards":{}},{"full_name":"openai/math","name":"math","shard":"1","boards":{"new":24}},{"full_name":"openai/openai-cookbook","name":"openai-cookbook","shard":"1","boards":{"daily_watchers":127}},{"full_name":"openai/parameter-golf","name":"parameter-golf","shard":"1","boards":{}},{"full_name":"openai/shap-e","name":"shap-e","shard":"1","boards":{}},{"full_name":"openai/skills","name":"skills","shard":"1","boards":{}},{"full_name":"openai/tart","name":"tart","shard":"1","boards":{}},{"full_name":"openai/tiktoken","name":"tiktoken","shard":"1","boards":{}}],"issues":[{"number":873,"title":"Are not merged PRs the result of irrelevancy to the model?","comments":12,"created_at":"2023-04-30T12:43:35Z"},{"number":1826,"title":"False-positive \"distillation\" ban on $200/mo Pro subscriber — automated appeal rejected by bot, requesting human review from Trust & Safety","comments":7,"created_at":"2026-09-05T18:40:22Z"},{"number":1166,"title":"Any website where I can share evaluation results?","comments":7,"created_at":"2023-06-15T04:57:31Z"},{"number":782,"title":"Idea for Evals: Sorting numbers with repeats and negatives","comments":7,"created_at":"2023-04-23T21:01:06Z"},{"number":1393,"title":"Using different models in evaluating mode-graded eval and in generating the completion","comments":6,"created_at":"2023-11-03T11:47:34Z"},{"number":637,"title":"Idea for Evals: Emotion and sentiment analysis Evals","comments":6,"created_at":"2023-04-11T20:58:36Z"},{"number":1839,"title":"Proposal update: Independent black-box evaluation of consumer AI under deployment opacity — historical v0.1 retained","comments":5,"created_at":"2026-09-26T04:17:47Z"},{"number":1384,"title":"Eval-running often hangs on last sample","comments":5,"created_at":"2023-10-17T22:38:29Z"}],"discussions":[{"number":1858,"title":"Can a model distinguish a stop signal from a stale or unrelated alert?","comments":0,"category":"Ideas","created_at":"2026-11-15T01:56:26Z"},{"number":1855,"title":"A zero-latency, local-first runtime firewall for LLMs.","comments":0,"category":"Show and tell","created_at":"2026-10-29T06:16:10Z"},{"number":1852,"title":"A training eval where one score hid the side effect: rewarded to win, the model ended the world more","comments":0,"category":"Show and tell","created_at":"2026-10-17T00:56:51Z"},{"number":1849,"title":"Why no recent developments?","comments":1,"category":"General","created_at":"2026-10-12T12:20:44Z"},{"number":1847,"title":"CML: Machine-Verifiable Continuity and State Transitions for Stateful Agents","comments":0,"category":"General","created_at":"2026-10-07T00:57:40Z"},{"number":1841,"title":"RAG Evidence Lab: offline citation and number checks for exported RAG answers","comments":0,"category":"Show and tell","created_at":"2026-09-30T16:31:57Z"},{"number":1838,"title":"DeciShift: behavioral regression testing for versioned ML decision systems","comments":1,"category":"Show and tell","created_at":"2026-09-25T18:20:58Z"},{"number":1836,"title":"Can a fixed mock evaluation pipeline prove its own pass? (open reproducibility trial)","comments":1,"category":"Show and tell","created_at":"2026-09-21T06:38:14Z"}],"latest_release":null,"readme":true,"readme_zh":false,"news":[],"stale":false,"snapshot_at":"2026-11-16T09:23:14Z","history_start":"2026-09-06"}