From 492d25e5ee896c2f4750933573b63484c653b51b Mon Sep 17 00:00:00 2001 From: maraw Date: Thu, 8 Aug 2024 14:32:03 +0200 Subject: [PATCH 01/11] small fixes (typo, abbreviations) + fix incomplete citations --- src/scripts/analyze_performance_per_source.py | 2 +- .../joint_analysis_confidence_performance.py | 7 +- src/tex/appendix.tex | 2 +- src/tex/ms.tex | 4 +- src/tex/references.bib | 125 ++++++++++-------- 5 files changed, 77 insertions(+), 63 deletions(-) diff --git a/src/scripts/analyze_performance_per_source.py b/src/scripts/analyze_performance_per_source.py index c7b089a12..2909e3d6e 100644 --- a/src/scripts/analyze_performance_per_source.py +++ b/src/scripts/analyze_performance_per_source.py @@ -39,7 +39,7 @@ def obtain_score_for_subset(df, subset): "number of isomers", "number of NMR signals", "GFK (chemical safety)", - "DAI", + "DAI (daily allowed intake)", "GHS pictograms", "name to SMILES", "SMILES to name", diff --git a/src/scripts/joint_analysis_confidence_performance.py b/src/scripts/joint_analysis_confidence_performance.py index 208ae502b..b71217a4f 100644 --- a/src/scripts/joint_analysis_confidence_performance.py +++ b/src/scripts/joint_analysis_confidence_performance.py @@ -124,25 +124,26 @@ def make_plot_of_confidence_vs_performance(merged_dicts, suffix: str = ""): average_performance = df.groupby("estimate")["all_correct_"].mean() stdev = df.groupby("estimate")["all_correct_"].apply(sem) ax[i].plot( - average_performance.index - 1, + average_performance.index, average_performance, color=model_color_map[model], marker="o", ) ax[i].errorbar( - average_performance.index - 1, + average_performance.index, average_performance, yerr=stdev, fmt="none", color=model_color_map[model], ) - range_frame(ax[i], np.array([0, 4]), np.array([0, 1])) + range_frame(ax[i], np.array([1, 5]), np.array([0, 1])) # Adjusted x-axis range # set shared y axis label fig.text(0.01, 0.5, "completely correct", va="center", rotation="vertical") ax[-1].set_xlabel("confidence estimate") + ax[-1].set_xticks([1, 2, 3, 4, 5]) # Set x-axis ticks to ordinal scale fig.tight_layout() fig.savefig( diff --git a/src/tex/appendix.tex b/src/tex/appendix.tex index 053f10e23..6eebfd6c7 100644 --- a/src/tex/appendix.tex +++ b/src/tex/appendix.tex @@ -208,7 +208,7 @@ \subsection{Performance as a function of molecular features} \subsection{Influence of model scale} To obtain first insights in how the performance \glspl{llm} depends on scale, we tested the \glspl{llm} of the LLaMA series. -Interestingly, we find that the 7B and 70B models perform comparably, with the 7B showing lower performance (fraction of correct answers for the 7B, 13B, and 70B models are \variable{output/llama/llama_7b.txt}, \variable{output/llama/llama_13b.txt}, \variable{output/llama/llama_70b.txt}). +Interestingly, we find that the 7B and 70B models perform comparably, with the 13B showing lower performance (fraction of correct answers for the 7B, 13B, and 70B models are \variable{output/llama/llama_7b.txt}, \variable{output/llama/llama_13b.txt}, \variable{output/llama/llama_70b.txt}). Note that such analyses are difficult as models are typically not directly comparable in terms of dataset and training protocol.\autocite{biderman2023pythia} diff --git a/src/tex/ms.tex b/src/tex/ms.tex index 011c0f18d..f3cf461e2 100644 --- a/src/tex/ms.tex +++ b/src/tex/ms.tex @@ -106,7 +106,7 @@ \subsection{Benchmark corpus} \centering \includegraphics{figures/question_count_barplot.pdf} \caption{\textbf{Number of questions for different topics.} The topics have been assigned using a combination of a rule-based system (mostly based on the source the question has been sampled from) and a classifier operating on word embeddings of the questions. - The figure shows that not all aspects of chemistry are equally represented in our corpus. The \chembench corpus, by design, currently focuses on safety-related aspects, which is also evident in \Cref{fig:question_diversity}. This figure represents the combined count of \glstext{mcq} and open-ended questions.} + The figure shows that not all aspects of chemistry are equally represented in our corpus. The \chembench corpus, by design, currently focuses on safety-related aspects, which is also evident in \Cref{fig:question_diversity}. This figure represents the combined count of multiple-choice questions (\gls{mcq}) and open-ended questions.} \label{fig:topic_barplot} \script{plot_statistics.py} \end{figure} @@ -323,7 +323,7 @@ \subsection{Model evaluation workflow} We used Galactica (120b)\autocite{taylor2022galactica} with the default settings. -\subparagraph{Instruction-tuned models} In addition, we used Claude 2, Claude3 (Opus),\autocite{anthropicClaudeModelFamily2024} GPT-4,\autocite{openai2024gpt4} GPT-3.5-turbo,\autocite{brown2020language} Gemini Pro,\autocite{gemini} Mixtral-8x7b\autocite{jiang2024mixtral} LLaMA2 (7B, 13B, 70B),\autocite{touvron2023llama} as well as the 7B chat model from Perplexity.AI. +\subparagraph{Instruction-tuned models} In addition, we used Claude 2, Claude3 (Opus),\autocite{anthropicClaudeModelFamily2024} GPT-4,\autocite{openai2024gpt4} GPT-3.5-turbo,\autocite{brown2020language} Gemini Pro,\autocite{gemini} Mixtral-8x7b,\autocite{jiang2024mixtral} LLaMA2 (7B, 13B, 70B),\autocite{touvron2023llama} as well as the 7B chat model from Perplexity.AI. \subparagraph{Tool augmented models} In addition to directly prompting \glspl{llm}, we also investigated the performance of tool-augmented systems. diff --git a/src/tex/references.bib b/src/tex/references.bib index 17155a041..70e56319f 100644 --- a/src/tex/references.bib +++ b/src/tex/references.bib @@ -103,18 +103,13 @@ @article{B_ttcher_2016 } @article{bart, - author = {Lewis, M. and Liu, Y. and Goyal, N. and Ghazvininejad, M. and - Mohamed, A. and Levy, O. and Stoyanov, V. and Zettlemoyer, L.}, - copyright = {arXiv.org perpetual, non-exclusive license}, + author = {Lewis, M. and Liu, Y. and Goyal, N. and Ghazvininejad, M. and Mohamed, A. and Levy, O. and Stoyanov, V. and Zettlemoyer, L.}, date = {2019-10-29}, doi = {10.48550/ARXIV.1910.13461}, + eprint = {1910.13461}, eprintclass = {cs.CL}, eprinttype = {arXiv}, - keywords = {Computation and Language (cs.CL), Machine Learning (cs.LG), - Machine Learning (stat.ML), FOS: Computer and information sciences, - FOS: Computer and information sciences}, - title = {BART: Denoising Sequence-to-Sequence Pre-training for Natural - Language Generation, Translation, and Comprehension}, + title = {BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension}, } @inproceedings{bender2021dangers, @@ -160,17 +155,18 @@ @techreport{blac } @article{bloomberg, - author = {Schmitt, B.}, - date = {2023-12-25}, - doi = {10.1108/qmr-08-2023-0107}, - howpublished = { - www.bloomberg.com/company/press/generative-ai-to-become-a-1-3-trillion-market-by-2032-research-finds - }, - journaltitle = {Qualitative Market Research: An International Journal}, + title = {Transforming qualitative research in phygital settings: the role of generative AI}, + volume = {27}, + ISSN = {1352-2752}, + url = {http://dx.doi.org/10.1108/QMR-08-2023-0107}, + DOI = {10.1108/qmr-08-2023-0107}, + number = {3}, + journal = {Qualitative Market Research: An International Journal}, publisher = {Emerald}, - source = {Crossref}, - title = {Transforming qualitative research in phygital settings: the role of - generative AI}, + author = {Schmitt, Bernd}, + year = {2023}, + month = dec, + pages = {523–526} } @article{Boiko_2023, @@ -185,13 +181,14 @@ @article{Boiko_2023 } @article{bommasani2021opportunities, - author = {Bommasani, R. and Hudson, D. A. a. A., Ehsan and Altman, R. and + author = {Bommasani, R. and Hudson, D. A. and Adeli, E. and Altman, R. and Arora, S. and von Arx, S. and Bernstein, M. S. and Bohg, J. and Bosselut, A. and Brunskill, E. and others}, date = {2021-08-16}, + eprint = {2108.07258}, eprintclass = {cs.LG}, eprinttype = {arXiv}, - title = {On the opportunities and risks of foundation models}, + title = {On the Opportunities and Risks of Foundation Models}, } @article{bran2023chemcrow, @@ -257,12 +254,15 @@ @article{Cai_2024 volume = {222}, } -@article{campbell2023censoring, - author = {Campbell, Q. L. and Herington, J. and White, A. D.}, - date = {2023-04-20}, - eprintclass = {cs.LG}, - eprinttype = {arXiv}, +@misc{campbell2023censoring, + doi = {10.48550/ARXIV.2304.10510}, + url = {https://arxiv.org/abs/2304.10510}, + author = {Campbell, Quintina L. and Herington, Jonathan and White, Andrew D.}, + keywords = {Machine Learning (cs.LG), Cryptography and Security (cs.CR), Computers and Society (cs.CY), Chemical Physics (physics.chem-ph), FOS: Computer and information sciences, FOS: Computer and information sciences, FOS: Physical sciences, FOS: Physical sciences}, title = {Censoring chemical data to mitigate dual use risk}, + publisher = {arXiv}, + year = {2023}, + copyright = {arXiv.org perpetual, non-exclusive license} } @article{castro2023large, @@ -308,15 +308,17 @@ @article{coley2019robotic } @article{Dagdelen_2024, - author = {Dagdelen, J. and Dunn, A. and Lee, S. and Walker, N. and Rosen, A. - S. and Ceder, G. and Persson, K. A. and Jain, A.}, - date = {2024-02}, - doi = {10.1038/s41467-024-45563-x}, - journaltitle = {Nat. Commun.}, - number = {1}, - title = {Structured information extraction from scientific text with large - language models}, + title = {Structured information extraction from scientific text with large language models}, volume = {15}, + ISSN = {2041-1723}, + url = {http://dx.doi.org/10.1038/s41467-024-45563-x}, + DOI = {10.1038/s41467-024-45563-x}, + number = {1}, + journal = {Nature Communications}, + publisher = {Springer Science and Business Media LLC}, + author = {Dagdelen, John and Dunn, Alexander and Lee, Sanghoon and Walker, Nicholas and Rosen, Andrew S. and Ceder, Gerbrand and Persson, Kristin A. and Jain, Anubhav}, + year = {2024}, + month = feb } @article{darvish2024organa, @@ -450,11 +452,15 @@ @inproceedings{gemini Traverses}, } -@article{golovneva2024reverse, - author = {Golovneva, O. and Allen-Zhu, Z. and Weston, J. and Sukhbaatar, S.}, - journaltitle = {arXiv preprint arXiv:2403.13799}, +@misc{golovneva2024reverse, + doi = {10.48550/ARXIV.2403.13799}, + url = {https://arxiv.org/abs/2403.13799}, + author = {Golovneva, Olga and Allen-Zhu, Zeyuan and Weston, Jason and Sukhbaatar, Sainbayar}, + keywords = {Computation and Language (cs.CL), Artificial Intelligence (cs.AI), FOS: Computer and information sciences, FOS: Computer and information sciences}, title = {Reverse Training to Nurse the Reversal Curse}, - date = {2024}, + publisher = {arXiv}, + year = {2024}, + copyright = {arXiv.org perpetual, non-exclusive license} } @article{gopal2023releasing, @@ -734,13 +740,15 @@ @article{miret2024llms date = {2024}, } -@article{moulange2023towards, - author = {Moulange, R. and Langenkamp, M. and Alexanian, T. and Curtis, S. and - Livingston, M.}, - date = {2023-11-27}, - eprintclass = {cs.CY}, - eprinttype = {arXiv}, +@misc{moulange2023towards, + doi = {10.48550/ARXIV.2311.15936}, + url = {https://arxiv.org/abs/2311.15936}, + author = {Moulange, Richard and Langenkamp, Max and Alexanian, Tessa and Curtis, Samuel and Livingston, Morgan}, + keywords = {Computers and Society (cs.CY), Machine Learning (cs.LG), FOS: Computer and information sciences, FOS: Computer and information sciences}, title = {Towards Responsible Governance of Biological Design Tools}, + publisher = {arXiv}, + year = {2023}, + copyright = {arXiv.org perpetual, non-exclusive license} } @misc{mysql, @@ -998,14 +1006,14 @@ @article{spglib } @article{srivastava2022beyond, - author = {Srivastava, A. and Rastogi, A. and Rao, A. and Shoeb, A. A. M. and - Abid, A. and Fisch, A. and Brown, A. R. and Santoro, A. and Gupta, A. - and Garriga-Alonso, A. à. and others}, - date = {2022-06-09}, - eprintclass = {cs.CL}, - eprinttype = {arXiv}, - title = {Beyond the imitation game: Quantifying and extrapolating the - capabilities of language models}, + doi = {10.48550/ARXIV.2206.04615}, + url = {https://arxiv.org/abs/2206.04615}, + author = {Srivastava, Aarohi and Rastogi, Abhinav and Rao, Abhishek and Shoeb, Abu Awal Md and Abid, Abubakar and Fisch, Adam and Brown, Adam R. and Santoro, Adam and Gupta, Aditya and Garriga-Alonso, Adrià and Kluska, Agnieszka and Lewkowycz, Aitor and Agarwal, Akshat and Power, Alethea and Ray, Alex and Warstadt, Alex and Kocurek, Alexander W. and Safaya, Ali and Tazarv, Ali and Xiang, Alice and Parrish, Alicia and Nie, Allen and Hussain, Aman and Askell, Amanda and Dsouza, Amanda and Slone, Ambrose and Rahane, Ameet and Iyer, Anantharaman S. and Andreassen, Anders and Madotto, Andrea and Santilli, Andrea and Stuhlm\"{u}ller, Andreas and Dai, Andrew and La, Andrew and Lampinen, Andrew and Zou, Andy and Jiang, Angela and Chen, Angelica and Vuong, Anh and Gupta, Animesh and Gottardi, Anna and Norelli, Antonio and Venkatesh, Anu and Gholamidavoodi, Arash and Tabassum, Arfa and Menezes, Arul and Kirubarajan, Arun and Mullokandov, Asher and Sabharwal, Ashish and Herrick, Austin and Efrat, Avia and Erdem, Aykut and Karakaş, Ayla and Roberts, B. Ryan and Loe, Bao Sheng and Zoph, Barret and Bojanowski, Bartłomiej and \"{O}zyurt, Batuhan and Hedayatnia, Behnam and Neyshabur, Behnam and Inden, Benjamin and Stein, Benno and Ekmekci, Berk and Lin, Bill Yuchen and Howald, Blake and Orinion, Bryan and Diao, Cameron and Dour, Cameron and Stinson, Catherine and Argueta, Cedrick and Ramírez, César Ferri and Singh, Chandan and Rathkopf, Charles and Meng, Chenlin and Baral, Chitta and Wu, Chiyu and Callison-Burch, Chris and Waites, Chris and Voigt, Christian and Manning, Christopher D. and Potts, Christopher and Ramirez, Cindy and Rivera, Clara E. and Siro, Clemencia and Raffel, Colin and Ashcraft, Courtney and Garbacea, Cristina and Sileo, Damien and Garrette, Dan and Hendrycks, Dan and Kilman, Dan and Roth, Dan and Freeman, Daniel and Khashabi, Daniel and Levy, Daniel and González, Daniel Moseguí and Perszyk, Danielle and Hernandez, Danny and Chen, Danqi and Ippolito, Daphne and Gilboa, Dar and Dohan, David and Drakard, David and Jurgens, David and Datta, Debajyoti and Ganguli, Deep and Emelin, Denis and Kleyko, Denis and Yuret, Deniz and Chen, Derek and Tam, Derek and Hupkes, Dieuwke and Misra, Diganta and Buzan, Dilyar and Mollo, Dimitri Coelho and Yang, Diyi and Lee, Dong-Ho and Schrader, Dylan and Shutova, Ekaterina and Cubuk, Ekin Dogus and Segal, Elad and Hagerman, Eleanor and Barnes, Elizabeth and Donoway, Elizabeth and Pavlick, Ellie and Rodola, Emanuele and Lam, Emma and Chu, Eric and Tang, Eric and Erdem, Erkut and Chang, Ernie and Chi, Ethan A. and Dyer, Ethan and Jerzak, Ethan and Kim, Ethan and Manyasi, Eunice Engefu and Zheltonozhskii, Evgenii and Xia, Fanyue and Siar, Fatemeh and Martínez-Plumed, Fernando and Happé, Francesca and Chollet, Francois and Rong, Frieda and Mishra, Gaurav and Winata, Genta Indra and de Melo, Gerard and Kruszewski, Germán and Parascandolo, Giambattista and Mariani, Giorgio and Wang, Gloria and Jaimovitch-López, Gonzalo and Betz, Gregor and Gur-Ari, Guy and Galijasevic, Hana and Kim, Hannah and Rashkin, Hannah and Hajishirzi, Hannaneh and Mehta, Harsh and Bogar, Hayden and Shevlin, Henry and Sch\"{u}tze, Hinrich and Yakura, Hiromu and Zhang, Hongming and Wong, Hugh Mee and Ng, Ian and Noble, Isaac and Jumelet, Jaap and Geissinger, Jack and Kernion, Jackson and Hilton, Jacob and Lee, Jaehoon and Fisac, Jaime Fernández and Simon, James B. and Koppel, James and Zheng, James and Zou, James and Kocoń, Jan and Thompson, Jana and Wingfield, Janelle and Kaplan, Jared and Radom, Jarema and Sohl-Dickstein, Jascha and Phang, Jason and Wei, Jason and Yosinski, Jason and Novikova, Jekaterina and Bosscher, Jelle and Marsh, Jennifer and Kim, Jeremy and Taal, Jeroen and Engel, Jesse and Alabi, Jesujoba and Xu, Jiacheng and Song, Jiaming and Tang, Jillian and Waweru, Joan and Burden, John and Miller, John and Balis, John U. and Batchelder, Jonathan and Berant, Jonathan and Frohberg, J\"{o}rg and Rozen, Jos and Hernandez-Orallo, Jose and Boudeman, Joseph and Guerr, Joseph and Jones, Joseph and Tenenbaum, Joshua B. and Rule, Joshua S. and Chua, Joyce and Kanclerz, Kamil and Livescu, Karen and Krauth, Karl and Gopalakrishnan, Karthik and Ignatyeva, Katerina and Markert, Katja and Dhole, Kaustubh D. and Gimpel, Kevin and Omondi, Kevin and Mathewson, Kory and Chiafullo, Kristen and Shkaruta, Ksenia and Shridhar, Kumar and McDonell, Kyle and Richardson, Kyle and Reynolds, Laria and Gao, Leo and Zhang, Li and Dugan, Liam and Qin, Lianhui and Contreras-Ochando, Lidia and Morency, Louis-Philippe and Moschella, Luca and Lam, Lucas and Noble, Lucy and Schmidt, Ludwig and He, Luheng and Colón, Luis Oliveros and Metz, Luke and Şenel, L\"{u}tfi Kerem and Bosma, Maarten and Sap, Maarten and ter Hoeve, Maartje and Farooqi, Maheen and Faruqui, Manaal and Mazeika, Mantas and Baturan, Marco and Marelli, Marco and Maru, Marco and Quintana, Maria Jose Ramírez and Tolkiehn, Marie and Giulianelli, Mario and Lewis, Martha and Potthast, Martin and Leavitt, Matthew L. and Hagen, Matthias and Schubert, Mátyás and Baitemirova, Medina Orduna and Arnaud, Melody and McElrath, Melvin and Yee, Michael A. and Cohen, Michael and Gu, Michael and Ivanitskiy, Michael and Starritt, Michael and Strube, Michael and Swędrowski, Michał and Bevilacqua, Michele and Yasunaga, Michihiro and Kale, Mihir and Cain, Mike and Xu, Mimee and Suzgun, Mirac and Walker, Mitch and Tiwari, Mo and Bansal, Mohit and Aminnaseri, Moin and Geva, Mor and Gheini, Mozhdeh and T, Mukund Varma and Peng, Nanyun and Chi, Nathan A. and Lee, Nayeon and Krakover, Neta Gur-Ari and Cameron, Nicholas and Roberts, Nicholas and Doiron, Nick and Martinez, Nicole and Nangia, Nikita and Deckers, Niklas and Muennighoff, Niklas and Keskar, Nitish Shirish and Iyer, Niveditha S. and Constant, Noah and Fiedel, Noah and Wen, Nuan and Zhang, Oliver and Agha, Omar and Elbaghdadi, Omar and Levy, Omer and Evans, Owain and Casares, Pablo Antonio Moreno and Doshi, Parth and Fung, Pascale and Liang, Paul Pu and Vicol, Paul and Alipoormolabashi, Pegah and Liao, Peiyuan and Liang, Percy and Chang, Peter and Eckersley, Peter and Htut, Phu Mon and Hwang, Pinyu and Miłkowski, Piotr and Patil, Piyush and Pezeshkpour, Pouya and Oli, Priti and Mei, Qiaozhu and Lyu, Qing and Chen, Qinlang and Banjade, Rabin and Rudolph, Rachel Etta and Gabriel, Raefer and Habacker, Rahel and Risco, Ramon and Millière, Raphaël and Garg, Rhythm and Barnes, Richard and Saurous, Rif A. and Arakawa, Riku and Raymaekers, Robbe and Frank, Robert and Sikand, Rohan and Novak, Roman and Sitelew, Roman and LeBras, Ronan and Liu, Rosanne and Jacobs, Rowan and Zhang, Rui and Salakhutdinov, Ruslan and Chi, Ryan and Lee, Ryan and Stovall, Ryan and Teehan, Ryan and Yang, Rylan and Singh, Sahib and Mohammad, Saif M. and Anand, Sajant and Dillavou, Sam and Shleifer, Sam and Wiseman, Sam and Gruetter, Samuel and Bowman, Samuel R. and Schoenholz, Samuel S. and Han, Sanghyun and Kwatra, Sanjeev and Rous, Sarah A. and Ghazarian, Sarik and Ghosh, Sayan and Casey, Sean and Bischoff, Sebastian and Gehrmann, Sebastian and Schuster, Sebastian and Sadeghi, Sepideh and Hamdan, Shadi and Zhou, Sharon and Srivastava, Shashank and Shi, Sherry and Singh, Shikhar and Asaadi, Shima and Gu, Shixiang Shane and Pachchigar, Shubh and Toshniwal, Shubham and Upadhyay, Shyam and Shyamolima, and {Debnath} and Shakeri, Siamak and Thormeyer, Simon and Melzi, Simone and Reddy, Siva and Makini, Sneha Priscilla and Lee, Soo-Hwan and Torene, Spencer and Hatwar, Sriharsha and Dehaene, Stanislas and Divic, Stefan and Ermon, Stefano and Biderman, Stella and Lin, Stephanie and Prasad, Stephen and Piantadosi, Steven T. and Shieber, Stuart M. and Misherghi, Summer and Kiritchenko, Svetlana and Mishra, Swaroop and Linzen, Tal and Schuster, Tal and Li, Tao and Yu, Tao and Ali, Tariq and Hashimoto, Tatsu and Wu, Te-Lin and Desbordes, Théo and Rothschild, Theodore and Phan, Thomas and Wang, Tianle and Nkinyili, Tiberius and Schick, Timo and Kornev, Timofei and Tunduny, Titus and Gerstenberg, Tobias and Chang, Trenton and Neeraj, Trishala and Khot, Tushar and Shultz, Tyler and Shaham, Uri and Misra, Vedant and Demberg, Vera and Nyamai, Victoria and Raunak, Vikas and Ramasesh, Vinay and Prabhu, Vinay Uday and Padmakumar, Vishakh and Srikumar, Vivek and Fedus, William and Saunders, William and Zhang, William and Vossen, Wout and Ren, Xiang and Tong, Xiaoyu and Zhao, Xinran and Wu, Xinyi and Shen, Xudong and Yaghoobzadeh, Yadollah and Lakretz, Yair and Song, Yangqiu and Bahri, Yasaman and Choi, Yejin and Yang, Yichi and Hao, Yiding and Chen, Yifu and Belinkov, Yonatan and Hou, Yu and Hou, Yufang and Bai, Yuntao and Seid, Zachary and Zhao, Zhuoye and Wang, Zijian and Wang, Zijie J. and Wang, Zirui and Wu, Ziyi}, + keywords = {Computation and Language (cs.CL), Artificial Intelligence (cs.AI), Computers and Society (cs.CY), Machine Learning (cs.LG), Machine Learning (stat.ML), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models}, + publisher = {arXiv}, + year = {2022}, + copyright = {arXiv.org perpetual, non-exclusive license} } @article{sun2023scieval, @@ -1186,12 +1194,17 @@ @article{zhang2024chemllm } @article{Zheng_2024, - author = {Zheng, Z. and He, Z. and Khattab, O. and Rampal, N. and Zaharia, M. - A. and Borgs, C. and Chayes, J. T. and Yaghi, O. M.}, - doi = {10.1039/d3dd00239j}, - journaltitle = {Digit. Discov.}, title = {Image and data mining in reticular chemistry powered by GPT-4V}, - date = {2024}, + volume = {3}, + ISSN = {2635-098X}, + url = {http://dx.doi.org/10.1039/d3dd00239j}, + DOI = {10.1039/d3dd00239j}, + number = {3}, + journal = {Digital Discovery}, + publisher = {Royal Society of Chemistry (RSC)}, + author = {Zheng, Zhiling and He, Zhiguo and Khattab, Omar and Rampal, Nakul and Zaharia, Matei A. and Borgs, Christian and Chayes, Jennifer T. and Yaghi, Omar M.}, + year = {2024}, + pages = {491–501} } @article{zhong2024benchmarking, From 1f0a14f94c60b84a083914a949abde4684ba9d35 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 10:58:14 +0200 Subject: [PATCH 02/11] Update src/tex/references.bib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Martiño Rios-Garcia <147555961+MrtinoRG@users.noreply.github.com> --- src/tex/references.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tex/references.bib b/src/tex/references.bib index 70e56319f..9a910eb99 100644 --- a/src/tex/references.bib +++ b/src/tex/references.bib @@ -740,7 +740,7 @@ @article{miret2024llms date = {2024}, } -@misc{moulange2023towards, +@article{moulange2023towards, doi = {10.48550/ARXIV.2311.15936}, url = {https://arxiv.org/abs/2311.15936}, author = {Moulange, Richard and Langenkamp, Max and Alexanian, Tessa and Curtis, Samuel and Livingston, Morgan}, From 8be543ac1b766cef1da6799f05ef461f5b25976c Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 10:58:19 +0200 Subject: [PATCH 03/11] Update src/tex/references.bib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Martiño Rios-Garcia <147555961+MrtinoRG@users.noreply.github.com> --- src/tex/references.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tex/references.bib b/src/tex/references.bib index 9a910eb99..61b442ce0 100644 --- a/src/tex/references.bib +++ b/src/tex/references.bib @@ -452,7 +452,7 @@ @inproceedings{gemini Traverses}, } -@misc{golovneva2024reverse, +@article{golovneva2024reverse, doi = {10.48550/ARXIV.2403.13799}, url = {https://arxiv.org/abs/2403.13799}, author = {Golovneva, Olga and Allen-Zhu, Zeyuan and Weston, Jason and Sukhbaatar, Sainbayar}, From db925ad7ced31926fea510c6ac583babeef626a1 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 10:58:24 +0200 Subject: [PATCH 04/11] Update src/tex/references.bib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Martiño Rios-Garcia <147555961+MrtinoRG@users.noreply.github.com> --- src/tex/references.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tex/references.bib b/src/tex/references.bib index 61b442ce0..a12d26a1b 100644 --- a/src/tex/references.bib +++ b/src/tex/references.bib @@ -254,7 +254,7 @@ @article{Cai_2024 volume = {222}, } -@misc{campbell2023censoring, +@article{campbell2023censoring, doi = {10.48550/ARXIV.2304.10510}, url = {https://arxiv.org/abs/2304.10510}, author = {Campbell, Quintina L. and Herington, Jonathan and White, Andrew D.}, From f332a2c374d27255e362b534d3df0fd15d192751 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:03:21 +0200 Subject: [PATCH 05/11] Update src/tex/ms.tex --- src/tex/ms.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tex/ms.tex b/src/tex/ms.tex index f3cf461e2..d342ed9ca 100644 --- a/src/tex/ms.tex +++ b/src/tex/ms.tex @@ -323,7 +323,7 @@ \subsection{Model evaluation workflow} We used Galactica (120b)\autocite{taylor2022galactica} with the default settings. -\subparagraph{Instruction-tuned models} In addition, we used Claude 2, Claude3 (Opus),\autocite{anthropicClaudeModelFamily2024} GPT-4,\autocite{openai2024gpt4} GPT-3.5-turbo,\autocite{brown2020language} Gemini Pro,\autocite{gemini} Mixtral-8x7b,\autocite{jiang2024mixtral} LLaMA2 (7B, 13B, 70B),\autocite{touvron2023llama} as well as the 7B chat model from Perplexity.AI. +\subparagraph{Instruction-tuned models} In addition, we tested Claude 2, Claude3 (Opus),\autocite{anthropicClaudeModelFamily2024} GPT-4,\autocite{openai2024gpt4} GPT-3.5-turbo,\autocite{brown2020language} Gemini Pro,\autocite{gemini} Mixtral-8x7b,\autocite{jiang2024mixtral} LLaMA2 (7B, 13B, 70B),\autocite{touvron2023llama} and LLaMa3 (8B, 70B, 405B). \subparagraph{Tool augmented models} In addition to directly prompting \glspl{llm}, we also investigated the performance of tool-augmented systems. From e1c73babe54d64255ba15647b8a7a78f636dd0d2 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:07:45 +0200 Subject: [PATCH 06/11] Update environment.yml --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d7604f3b3..4773f4e6b 100644 --- a/environment.yml +++ b/environment.yml @@ -7,9 +7,10 @@ dependencies: - transformers==4.37.2 - accelerate==0.26.1 - matplotlib + - pulp=2.7.0 - numpy==1.26.3 - seaborn - textacy==0.13.0 - wordcloud - rdkit - - git+https://github.com/lamalab-org/chem-bench.git \ No newline at end of file + - git+https://github.com/lamalab-org/chem-bench.git From a770216fa714cfb1c41b73c1e855f6f7b1a1e0ec Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:09:13 +0200 Subject: [PATCH 07/11] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 4773f4e6b..125fa4e47 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ dependencies: - transformers==4.37.2 - accelerate==0.26.1 - matplotlib - - pulp=2.7.0 + - pulp==2.7.0 - numpy==1.26.3 - seaborn - textacy==0.13.0 From 4384572d62d2483092b71d76b05620eff25ddfe7 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:11:19 +0200 Subject: [PATCH 08/11] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 125fa4e47..a8ec20a66 100644 --- a/environment.yml +++ b/environment.yml @@ -3,11 +3,11 @@ dependencies: - conda-forge::python=3.10 - pymatgen - pip: + - pulp<2.7.0 - torch==2.2.0 - transformers==4.37.2 - accelerate==0.26.1 - matplotlib - - pulp==2.7.0 - numpy==1.26.3 - seaborn - textacy==0.13.0 From 31e2adda6a3ff176e0db9712f94efecc64b26104 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:11:32 +0200 Subject: [PATCH 09/11] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index a8ec20a66..f5f0563da 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ dependencies: - conda-forge::python=3.10 - pymatgen - pip: - - pulp<2.7.0 + - pulp<2.8.0 - torch==2.2.0 - transformers==4.37.2 - accelerate==0.26.1 From 0618f28b0febf065692f7121e3afac9b55f50a91 Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:13:45 +0200 Subject: [PATCH 10/11] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f5f0563da..9156e74c4 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,6 @@ dependencies: - conda-forge::pip=23.3.1 - - conda-forge::python=3.10 + - conda-forge::python=3.9 - pymatgen - pip: - pulp<2.8.0 From 4ec7e93d75ca6e2a82213d1c7d6109dd1f4c2a1e Mon Sep 17 00:00:00 2001 From: Adrian Mirza Date: Mon, 19 Aug 2024 11:25:08 +0200 Subject: [PATCH 11/11] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 9156e74c4..f5f0563da 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,6 @@ dependencies: - conda-forge::pip=23.3.1 - - conda-forge::python=3.9 + - conda-forge::python=3.10 - pymatgen - pip: - pulp<2.8.0