@article{mirza_framework_2025,
	title = {A framework for evaluating the chemical knowledge and reasoning abilities of large language models against the expertise of chemists},
	issn = {1755-4330, 1755-4349},
	url = {https://www.nature.com/articles/s41557-025-01815-x},
	doi = {10.1038/s41557-025-01815-x},
	abstract = {Abstract 
            Large language models ({LLMs}) have gained widespread interest owing to their ability to process human language and perform tasks on which they have not been explicitly trained. However, we possess only a limited systematic understanding of the chemical capabilities of {LLMs}, which would be required to improve models and mitigate potential harm. Here we introduce {ChemBench}, an automated framework for evaluating the chemical knowledge and reasoning abilities of state-of-the-art {LLMs} against the expertise of chemists. We curated more than 2,700 question–answer pairs, evaluated leading open- and closed-source {LLMs} and found that the best models, on average, outperformed the best human chemists in our study. However, the models struggle with some basic tasks and provide overconfident predictions. These findings reveal {LLMs}’ impressive chemical capabilities while emphasizing the need for further research to improve their safety and usefulness. They also suggest adapting chemistry education and show the value of benchmarking frameworks for evaluating {LLMs} in specific domains.},
	journaltitle = {Nature Chemistry},
	shortjournal = {Nat. Chem.},
	author = {Mirza, Adrian and Alampara, Nawaf and Kunchapu, Sreekanth and Ríos-García, Martiño and Emoekabu, Benedict and Krishnan, Aswanth and Gupta, Tanya and Schilling-Wilhelmi, Mara and Okereke, Macjonathan and Aneesh, Anagha and Asgari, Mehrdad and Eberhardt, Juliane and Elahi, Amir Mohammad and Elbeheiry, Hani M. and Gil, María Victoria and Glaubitz, Christina and Greiner, Maximilian and Holick, Caroline T. and Hoffmann, Tim and Ibrahim, Abdelrahman and Klepsch, Lea C. and Köster, Yannik and Kreth, Fabian Alexander and Meyer, Jakob and Miret, Santiago and Peschel, Jan Matthias and Ringleb, Michael and Roesner, Nicole C. and Schreiber, Johanna and Schubert, Ulrich S. and Stafast, Leanne M. and Wonanke, A. D. Dinga and Pieler, Michael and Schwaller, Philippe and Jablonka, Kevin Maik},
	urldate = {2025-06-19},
	date = {2025-05-20},
	langid = {english},
}