@article{ashurytahan2025mighty,
author = {Shir Ashury-Tahan and Yifan Mai and Rajmohan C and Ariel Gera and Yotam Perlitz and Asaf Yehudai and Elron Bandel and Leshem Choshen and Eyal Shnarch and Percy Liang and Michal Shmueli-Scheuer},
journal = {arXiv},
title = {The Mighty {ToRR}: A Benchmark for Table Reasoning and Robustness},
year = {2025},
}
@article{kim2025finetuning,
author = {Moo Jin Kim and Chelsea Finn and Percy Liang},
journal = {arXiv},
title = {Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success},
year = {2025},
}
@article{liu2025language,
author = {Ken Ziyu Liu and Christopher A. Choquette-Choo and Matthew Jagielski and Peter Kairouz and Sanmi Koyejo and Percy Liang and Nicolas Papernot},
journal = {arXiv},
title = {Language Models May Verbatim Complete Text They Were Not Explicitly Trained On},
year = {2025},
}
@article{zhu2025independence,
author = {Sally Zhu and Ahmed Ahmed and Rohith Kuditipudi and Percy Liang},
journal = {arXiv},
title = {Independence Tests for Language Models},
year = {2025},
}
s1: simple test-time scaling.
Niklas Muennighoff, Zitong Yang, Weijia Shi, Xiang Lisa Li, Li Fei-Fei, Hannaneh Hajishirzi, Luke Zettlemoyer, Percy Liang, Emmanuel Candès, Tatsunori Hashimoto.
arXiv, 2025.
@article{muennighoff2025s1,
author = {Niklas Muennighoff and Zitong Yang and Weijia Shi and Xiang Lisa Li and Li Fei-Fei and Hannaneh Hajishirzi and Luke Zettlemoyer and Percy Liang and Emmanuel Candès and Tatsunori Hashimoto},
journal = {arXiv},
title = {s1: Simple test-time scaling},
year = {2025},
}
@article{li2025eliciting,
author = {Xiang Lisa Li and Neil Chowdhury and Daniel D. Johnson and Tatsunori Hashimoto and Percy Liang and Sarah Schwettmann and Jacob Steinhardt},
journal = {arXiv},
title = {Eliciting Language Model Behaviors with Investigator Agents},
year = {2025},
}
@article{soni2025videoagent,
author = {Achint Soni and Sreyas Venkataraman and Abhranil Chandra and Sebastian Fischmeister and Percy Liang and Bo Dai and Sherry Yang},
journal = {arXiv},
title = {VideoAgent: Self-Improving Video Generation},
year = {2025},
}
@article{gu2025caching,
author = {Chenchen Gu and Xiang Lisa Li and Rohith Kuditipudi and Percy Liang and Tatsunori Hashimoto},
journal = {arXiv},
title = {Auditing Prompt Caching in Language Model {API}s},
year = {2025},
}
@article{truong2025reliable,
author = {Sang Truong and Yuheng Tu and Percy Liang and Bo Li and Sanmi Koyejo},
journal = {arXiv},
title = {Reliable and Efficient Amortized Model-based Evaluation},
year = {2025},
}
@article{hewitt2025instruction,
author = {John Hewitt and Nelson F. Liu and Percy Liang and Christopher D. Manning},
journal = {arXiv},
title = {Instruction Following without Instruction Tuning},
year = {2025},
}
@article{zhang2025overlap,
author = {Andy K Zhang and Kevin Klyman and Yifan Mai and Yoav Levine and Yian Zhang and Rishi Bommasani and Percy Liang},
journal = {arXiv},
title = {Language model developers should report train-test overlap},
year = {2025},
}
In-House evaluation is not enough: towards robust third-party flaw disclosure for general-purpose AI.
Shayne Longpre, Kevin Klyman, Ruth E. Appel, Sayash Kapoor, Rishi Bommasani, Michelle Sahar, Sean McGregor, Avijit Ghosh, Borhane Blili-Hamelin, Nathan Butters, Alondra Nelson, Amit Elazari, Andrew Sellars, Casey John Ellis, Dane Sherrets, Dawn Song, Harley Geiger, Ilona Cohen, Lauren McIlvenny, Madhulika Srikumar, Mark M. Jaycox, Markus Anderljung, Nadine Farid Johnson, Nicholas Carlini, Nicolas Miailhe, Nik Marda, Peter Henderson, Rebecca S. Portnoff, Rebecca Weiss, Victoria Westerhoff, Yacine Jernite, Rumman Chowdhury, Percy Liang, Arvind Narayanan.
arXiv, 2025.
@article{longpre2025inhouse,
author = {Shayne Longpre and Kevin Klyman and Ruth E. Appel and Sayash Kapoor and Rishi Bommasani and Michelle Sahar and Sean McGregor and Avijit Ghosh and Borhane Blili-Hamelin and Nathan Butters and Alondra Nelson and Amit Elazari and Andrew Sellars and Casey John Ellis and Dane Sherrets and Dawn Song and Harley Geiger and Ilona Cohen and Lauren McIlvenny and Madhulika Srikumar and Mark M. Jaycox and Markus Anderljung and Nadine Farid Johnson and Nicholas Carlini and Nicolas Miailhe and Nik Marda and Peter Henderson and Rebecca S. Portnoff and Rebecca Weiss and Victoria Westerhoff and Yacine Jernite and Rumman Chowdhury and Percy Liang and Arvind Narayanan},
journal = {arXiv},
title = {In-House Evaluation Is Not Enough: Towards Robust Third-Party Flaw Disclosure for General-Purpose {AI}},
year = {2025},
}
@article{shrivastava2025language,
author = {Vaishnavi Shrivastava and Ananya Kumar and Percy Liang},
journal = {arXiv},
title = {Language Models Prefer What They Know: Relative Confidence Estimation via Confidence Preferences},
year = {2025},
}
AILuminate: introducing v1.0 of the AI risk and reliability benchmark from mlcommons.
Shaona Ghosh, Heather Frase, Adina Williams, Sarah Luger, Paul Röttger, Fazl Barez, Sean McGregor, Kenneth Fricklas, Mala Kumar, Quentin Feuillade--Montixi, Kurt Bollacker, Felix Friedrich, Ryan Tsang, Bertie Vidgen, Alicia Parrish, Chris Knotz, Eleonora Presani, Jonathan Bennion, Marisa Ferrara Boston, Mike Kuniavsky, Wiebke Hutiri, James Ezick, Malek Ben Salem, Rajat Sahay, Sujata Goswami, Usman Gohar, Ben Huang, Supheakmungkol Sarin, Elie Alhajjar, Canyu Chen, Roman Eng, Kashyap Ramanandula Manjusha, Virendra Mehta, Eileen Long, Murali Emani, Natan Vidra, Benjamin Rukundo, Abolfazl Shahbazi, Kongtao Chen, Rajat Ghosh, Vithursan Thangarasa, Pierre Peigné, Abhinav Singh, Max Bartolo, Satyapriya Krishna, Mubashara Akhtar, Rafael Gold, Cody Coleman, Luis Oala, Vassil Tashev, Joseph Marvin Imperial, Amy Russ, Sasidhar Kunapuli, Nicolas Miailhe, Julien Delaunay, Bhaktipriya Radharapu, Rajat Shinde, Tuesday, Debojyoti Dutta, Declan Grabb, Ananya Gangavarapu, Saurav Sahay, Agasthya Gangavarapu, Patrick Schramowski, Stephen Singam, Tom David, Xudong Han, Priyanka Mary Mammen, Tarunima Prabhakar, Venelin Kovatchev, Rebecca Weiss, Ahmed Ahmed, Kelvin N. Manyeki, Sandeep Madireddy, Foutse Khomh, Fedor Zhdanov, Joachim Baumann, Nina Vasan, Xianjun Yang, Carlos Mougn, Jibin Rajan Varghese, Hussain Chinoy, Seshakrishna Jitendar, Manil Maskey, Claire V. Hardgrove, Tianhao Li, Aakash Gupta, Emil Joswin, Yifan Mai, Shachi H Kumar, Cigdem Patlak, Kevin Lu, Vincent Alessi, Sree Bhargavi Balija, Chenhe Gu, Robert Sullivan, James Gealy, Matt Lavrisa, James Goel, Peter Mattson, Percy Liang, Joaquin Vanschoren.
arXiv, 2025.
@article{ghosh2025ailuminate,
author = {Shaona Ghosh and Heather Frase and Adina Williams and Sarah Luger and Paul Röttger and Fazl Barez and Sean McGregor and Kenneth Fricklas and Mala Kumar and Quentin Feuillade--Montixi and Kurt Bollacker and Felix Friedrich and Ryan Tsang and Bertie Vidgen and Alicia Parrish and Chris Knotz and Eleonora Presani and Jonathan Bennion and Marisa Ferrara Boston and Mike Kuniavsky and Wiebke Hutiri and James Ezick and Malek Ben Salem and Rajat Sahay and Sujata Goswami and Usman Gohar and Ben Huang and Supheakmungkol Sarin and Elie Alhajjar and Canyu Chen and Roman Eng and Kashyap Ramanandula Manjusha and Virendra Mehta and Eileen Long and Murali Emani and Natan Vidra and Benjamin Rukundo and Abolfazl Shahbazi and Kongtao Chen and Rajat Ghosh and Vithursan Thangarasa and Pierre Peigné and Abhinav Singh and Max Bartolo and Satyapriya Krishna and Mubashara Akhtar and Rafael Gold and Cody Coleman and Luis Oala and Vassil Tashev and Joseph Marvin Imperial and Amy Russ and Sasidhar Kunapuli and Nicolas Miailhe and Julien Delaunay and Bhaktipriya Radharapu and Rajat Shinde and Tuesday and Debojyoti Dutta and Declan Grabb and Ananya Gangavarapu and Saurav Sahay and Agasthya Gangavarapu and Patrick Schramowski and Stephen Singam and Tom David and Xudong Han and Priyanka Mary Mammen and Tarunima Prabhakar and Venelin Kovatchev and Rebecca Weiss and Ahmed Ahmed and Kelvin N. Manyeki and Sandeep Madireddy and Foutse Khomh and Fedor Zhdanov and Joachim Baumann and Nina Vasan and Xianjun Yang and Carlos Mougn and Jibin Rajan Varghese and Hussain Chinoy and Seshakrishna Jitendar and Manil Maskey and Claire V. Hardgrove and Tianhao Li and Aakash Gupta and Emil Joswin and Yifan Mai and Shachi H Kumar and Cigdem Patlak and Kevin Lu and Vincent Alessi and Sree Bhargavi Balija and Chenhe Gu and Robert Sullivan and James Gealy and Matt Lavrisa and James Goel and Peter Mattson and Percy Liang and Joaquin Vanschoren},
journal = {arXiv},
title = {AILuminate: Introducing v1.0 of the {AI} Risk and Reliability Benchmark from MLCommons},
year = {2025},
}
@inproceedings{niklaus2025lawinstruct,
author = {Joel Niklaus and Lucia Zheng and Arya D. McCarthy and Christopher Hahn and Brian M. Rosen and Peter Henderson and Daniel E. Ho and Garrett Honke and Percy Liang and Christopher Manning},
booktitle = {Findings of North America Association for Computational Linguistics (Findings of NAACL)},
title = {LawInstruct: A Resource for Studying Language Model Adaptation to the Legal Domain},
year = {2025},
}
@inproceedings{zeng2025airbench,
author = {Yi Zeng and Yu Yang and Andy Zhou and Jeffrey Ziwei Tan and Yuheng Tu and Yifan Mai and Kevin Klyman and Minzhou Pan and Ruoxi Jia and Dawn Song and Percy Liang and Bo Li},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {{AIR}-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies},
year = {2025},
}
Cybench: a framework for evaluating cybersecurity capabilities and risk of language models.
Andy K. Zhang, Neil Perry, Riya Dulepet, Eliot Jones, Justin W. Lin, Joey Ji, Celeste Menders, Gashon Hussein, Samantha Liu, Donovan Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Mike Yang, Teddy Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Polycarpos Yiorkadjis, Kenny Osele, Gautham Raghupathi, Dan Boneh, Daniel E. Ho, Percy Liang.
International Conference on Learning Representations (ICLR), 2025.
@inproceedings{zhang2025cybench,
author = {Andy K. Zhang and Neil Perry and Riya Dulepet and Eliot Jones and Justin W. Lin and Joey Ji and Celeste Menders and Gashon Hussein and Samantha Liu and Donovan Jasper and Pura Peetathawatchai and Ari Glenn and Vikram Sivashankar and Daniel Zamoshchin and Leo Glikbarg and Derek Askaryar and Mike Yang and Teddy Zhang and Rishi Alluri and Nathan Tran and Rinnara Sangpisit and Polycarpos Yiorkadjis and Kenny Osele and Gautham Raghupathi and Dan Boneh and Daniel E. Ho and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risk of Language Models},
year = {2025},
}
@inproceedings{gao2025equality,
author = {Irena Gao and Percy Liang and Carlos Guestrin},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Model Equality Testing: Which Model Is This {API} Serving?},
year = {2025},
}
@inproceedings{li2025autobencher,
author = {Xiang Lisa Li and Evan Zheran Liu and Percy Liang and Tatsunori Hashimoto},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {AutoBencher: Creating Salient, Novel, Difficult Datasets for Language Models},
year = {2025},
}
@inproceedings{wen2025understanding,
author = {Kaiyue Wen and Zhiyuan Li and Jason Wang and David Hall and Percy Liang and Tengyu Ma},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Understanding Warmup-Stable-Decay Learning Rates: A River Valley Loss Landscape Perspective},
year = {2025},
}
@inproceedings{roohani2025biodiscoveryagent,
author = {Yusuf Roohani and Andrew Lee and Qian Huang and Jian Vora and Zachary Steinhart and Kexin Huang and Alexander Marson and Percy Liang and Jure Leskovec},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {BioDiscoveryAgent: An {AI} Agent for Designing Genetic Perturbation Experiments},
year = {2025},
}
@article{park2024generative,
author = {Joon Sung Park and Carolyn Q. Zou and Aaron Shaw and Benjamin Mako Hill and Carrie Cai and Meredith Ringel Morris and Robb Willer and Percy Liang and Michael S. Bernstein},
journal = {arXiv},
title = {Generative Agent Simulations of 1,000 People},
year = {2024},
}
Machine unlearning doesn't do what you think: lessons for generative AI policy, research, and practice.
A. Feder Cooper, Christopher A. Choquette-Choo, Miranda Bogen, Matthew Jagielski, Katja Filippova, Ken Ziyu Liu, Alexandra Chouldechova, Jamie Hayes, Yangsibo Huang, Niloofar Mireshghallah, Ilia Shumailov, Eleni Triantafillou, Peter Kairouz, Nicole Mitchell, Percy Liang, Daniel E. Ho, Yejin Choi, Sanmi Koyejo, Fernando Delgado, James Grimmelmann, Vitaly Shmatikov, Christopher De Sa, Solon Barocas, Amy Cyphert, Mark Lemley, danah boyd, Jennifer Wortman Vaughan, Miles Brundage, David Bau, Seth Neel, Abigail Z. Jacobs, Andreas Terzis, Hanna Wallach, Nicolas Papernot, Katherine Lee.
arXiv, 2024.
@article{cooper2024machine,
author = {A. Feder Cooper and Christopher A. Choquette-Choo and Miranda Bogen and Matthew Jagielski and Katja Filippova and Ken Ziyu Liu and Alexandra Chouldechova and Jamie Hayes and Yangsibo Huang and Niloofar Mireshghallah and Ilia Shumailov and Eleni Triantafillou and Peter Kairouz and Nicole Mitchell and Percy Liang and Daniel E. Ho and Yejin Choi and Sanmi Koyejo and Fernando Delgado and James Grimmelmann and Vitaly Shmatikov and Christopher De Sa and Solon Barocas and Amy Cyphert and Mark Lemley and danah boyd and Jennifer Wortman Vaughan and Miles Brundage and David Bau and Seth Neel and Abigail Z. Jacobs and Andreas Terzis and Hanna Wallach and Nicolas Papernot and Katherine Lee},
journal = {arXiv},
title = {Machine Unlearning Doesn't Do What You Think: Lessons for Generative {AI} Policy, Research, and Practice},
year = {2024},
}
@article{hewitt2024model,
author = {John Hewitt and Sarah Chen and Lanruo Lora Xie and Edward Adams and Percy Liang and Christopher D. Manning},
journal = {arXiv},
title = {Model Editing with Canonical Examples},
year = {2024},
}
@article{zeng2024ai,
author = {Yi Zeng and Kevin Klyman and Andy Zhou and Yu Yang and Minzhou Pan and Ruoxi Jia and Dawn Song and Percy Liang and Bo Li},
journal = {arXiv},
title = {{AI} Risk Categorization Decoded ({AIR} 2024): From Government Regulations to Corporate Policies},
year = {2024},
}
OpenVLA: an open-source vision-language-action model.
Moo Jin Kim, Karl Pertsch, Siddharth Karamcheti, Ted Xiao, Ashwin Balakrishna, Suraj Nair, Rafael Rafailov, Ethan Foster, Grace Lam, Pannag Sanketi, Quan Vuong, Thomas Kollar, Benjamin Burchfiel, Russ Tedrake, Dorsa Sadigh, Sergey Levine, Percy Liang, Chelsea Finn.
Conference on Robot Learning (CoRL), 2024.
@inproceedings{kim2024openvla,
author = {Moo Jin Kim and Karl Pertsch and Siddharth Karamcheti and Ted Xiao and Ashwin Balakrishna and Suraj Nair and Rafael Rafailov and Ethan Foster and Grace Lam and Pannag Sanketi and Quan Vuong and Thomas Kollar and Benjamin Burchfiel and Russ Tedrake and Dorsa Sadigh and Sergey Levine and Percy Liang and Chelsea Finn},
booktitle = {Conference on Robot Learning (CoRL)},
title = {OpenVLA: An Open-Source Vision-Language-Action Model},
year = {2024},
}
@inproceedings{dubois2024length,
author = {Yann Dubois and Balázs Galambosi and Percy Liang and Tatsunori B. Hashimoto},
booktitle = {Conference on Language Modeling (COLM)},
title = {Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators},
year = {2024},
}
Embodied agent interface: benchmarking llms for embodied decision making.
Manling Li, Shiyu Zhao, Qineng Wang, Kangrui Wang, Yu Zhou, Sanjana Srivastava, Cem Gokmen, Tony Lee, Li Erran Li, Ruohan Zhang, Weiyu Liu, Percy Liang, Li Fei-Fei, Jiayuan Mao, Jiajun Wu.
Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks), 2024.
@inproceedings{li2024embodied,
author = {Manling Li and Shiyu Zhao and Qineng Wang and Kangrui Wang and Yu Zhou and Sanjana Srivastava and Cem Gokmen and Tony Lee and Li Erran Li and Ruohan Zhang and Weiyu Liu and Percy Liang and Li Fei-Fei and Jiayuan Mao and Jiajun Wu},
booktitle = {Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks)},
title = {Embodied Agent Interface: Benchmarking LLMs for Embodied Decision Making},
year = {2024},
}
VHELM: a holistic evaluation of vision language models.
Tony Lee, Haoqin Tu, Chi Heem Wong, Wenhao Zheng, Yiyang Zhou, Yifan Mai, Josselin Somerville Roberts, Michihiro Yasunaga, Huaxiu Yao, Cihang Xie, Percy Liang.
Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks), 2024.
@inproceedings{lee2024vhelm,
author = {Tony Lee and Haoqin Tu and Chi Heem Wong and Wenhao Zheng and Yiyang Zhou and Yifan Mai and Josselin Somerville Roberts and Michihiro Yasunaga and Huaxiu Yao and Cihang Xie and Percy Liang},
booktitle = {Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks)},
title = {VHELM: A Holistic Evaluation of Vision Language Models},
year = {2024},
}
@inproceedings{roberts2024image2struct,
author = {Josselin Somerville Roberts and Tony Lee and Chi Heem Wong and Michihiro Yasunaga and Yifan Mai and Percy Liang},
booktitle = {Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks)},
title = {Image2Struct: Benchmarking Structure Extraction for Vision-Language Models},
year = {2024},
}
The responsible foundation model development cheatsheet: a review of tools & resources.
Shayne Longpre, Stella Biderman, Alon Albalak, Hailey Schoelkopf, Daniel McDuff, Sayash Kapoor, Kevin Klyman, Kyle Lo, Gabriel Ilharco, Nay San, Maribeth Rauh, Aviya Skowron, Bertie Vidgen, Laura Weidinger, Arvind Narayanan, Victor Sanh, David Adelani, Percy Liang, Rishi Bommasani, Peter Henderson, Sasha Luccioni, Yacine Jernite, Luca Soldaini.
Transcations of Machine Learning Research (TMLR), 2024.
@article{longpre2024responsible,
author = {Shayne Longpre and Stella Biderman and Alon Albalak and Hailey Schoelkopf and Daniel McDuff and Sayash Kapoor and Kevin Klyman and Kyle Lo and Gabriel Ilharco and Nay San and Maribeth Rauh and Aviya Skowron and Bertie Vidgen and Laura Weidinger and Arvind Narayanan and Victor Sanh and David Adelani and Percy Liang and Rishi Bommasani and Peter Henderson and Sasha Luccioni and Yacine Jernite and Luca Soldaini},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {The Responsible Foundation Model Development Cheatsheet: A Review of Tools & Resources},
volume = {0},
year = {2024},
}
@article{bommasani2024fmti,
author = {Rishi Bommasani and Kevin Klyman and Sayash Kapoor and Shayne Longpre and Betty Xiong and Nestor Maslej and Percy Liang},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {The 2024 Foundation Model Transparency Index},
volume = {0},
year = {2024},
}
@article{thickstun2024anticipatory,
author = {John Thickstun and D. Hall and Chris Donahue and Percy Liang},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {Anticipatory Music Transformer},
volume = {0},
year = {2024},
}
@article{kuditipudi2024robust,
author = {Rohith Kuditipudi and John Thickstun and Tatsunori Hashimoto and Percy Liang},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {Robust Distortion-free Watermarks for Language Models},
volume = {0},
year = {2024},
}
@inproceedings{huang2024research,
author = {Qian Huang and Jian Vora and Percy Liang and Jure Leskovec},
booktitle = {International Conference on Machine Learning (ICML)},
title = {{MLAgentBench}: Evaluating Language Agents on Machine Learning Experimentation},
year = {2024},
}
@inproceedings{karamcheti2024prismatic,
author = {Siddharth Karamcheti and Suraj Nair and Ashwin Balakrishna and Percy Liang and Thomas Kollar and Dorsa Sadigh},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Prismatic {VLM}s: Investigating the Design Space of Visually-Conditioned Language Models},
year = {2024},
}
On the societal impact of open foundation models.
Sayash Kapoor, Rishi Bommasani, Kevin Klyman, Shayne Longpre, Ashwin Ramaswami, Peter Cihon, Aspen Hopkins, Kevin Bankston, Stella Biderman, Miranda Bogen, Rumman Chowdhury, Alex Engler, Peter Henderson, Yacine Jernite, Seth Lazar, Stefano Maffulli, Alondra Nelson, Joelle Pineau, Aviya Skowron, Dawn Song, Victor Storchan, Daniel Zhang, Daniel E. Ho, Percy Liang, Arvind Narayanan.
International Conference on Machine Learning (Position Paper Track) (ICML), 2024.
@inproceedings{kapoor2024open,
author = {Sayash Kapoor and Rishi Bommasani and Kevin Klyman and Shayne Longpre and Ashwin Ramaswami and Peter Cihon and Aspen Hopkins and Kevin Bankston and Stella Biderman and Miranda Bogen and Rumman Chowdhury and Alex Engler and Peter Henderson and Yacine Jernite and Seth Lazar and Stefano Maffulli and Alondra Nelson and Joelle Pineau and Aviya Skowron and Dawn Song and Victor Storchan and Daniel Zhang and Daniel E. Ho and Percy Liang and Arvind Narayanan},
booktitle = {International Conference on Machine Learning (Position Paper Track) (ICML)},
title = {On the Societal Impact of Open Foundation Models},
year = {2024},
}
A safe harbor for AI evaluation and red teaming.
Shayne Longpre, Sayash Kapoor, Kevin Klyman, Ashwin Ramaswami, Rishi Bommasani, Borhane Blili-Hamelin, Yangsibo Huang, Aviya Skowron, Zheng-Xin Yong, Suhas Kotha, Yi Zeng, Weiyan Shi, Xianjun Yang, Reid Southen, Alexander Robey, Patrick Chao, Diyi Yang, Ruoxi Jia, Daniel Kang, Sandy Pentland, Arvind Narayanan, Percy Liang, Peter Henderson.
International Conference on Machine Learning (Position Paper Track) (ICML), 2024.
@inproceedings{longpre2024safe,
author = {Shayne Longpre and Sayash Kapoor and Kevin Klyman and Ashwin Ramaswami and Rishi Bommasani and Borhane Blili-Hamelin and Yangsibo Huang and Aviya Skowron and Zheng-Xin Yong and Suhas Kotha and Yi Zeng and Weiyan Shi and Xianjun Yang and Reid Southen and Alexander Robey and Patrick Chao and Diyi Yang and Ruoxi Jia and Daniel Kang and Sandy Pentland and Arvind Narayanan and Percy Liang and Peter Henderson},
booktitle = {International Conference on Machine Learning (Position Paper Track) (ICML)},
title = {A Safe Harbor for {AI} Evaluation and Red Teaming},
year = {2024},
}
@inproceedings{liu2024sophia,
author = {Hong Liu and Zhiyuan Li and David Hall and Percy Liang and Tengyu Ma},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training},
year = {2024},
}
@inproceedings{li2024consistency,
author = {Xiang Lisa Li and Vaishnavi Shrivastava and Siyan Li and Tatsunori Hashimoto and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Benchmarking and Improving Generator-Validator Consistency of Language Models},
year = {2024},
}
@inproceedings{yasunaga2024analogical,
author = {Michihiro Yasunaga and Xinyun Chen and Yujia Li and Panupong Pasupat and J. Leskovec and Percy Liang and Ed H. Chi and Denny Zhou},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Large Language Models as Analogical Reasoners},
year = {2024},
}
@inproceedings{gu2024watermarks,
author = {Chenchen Gu and Xiang Lisa Li and Percy Liang and Tatsunori Hashimoto},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {On the Learnability of Watermarks for Language Models},
year = {2024},
}
MedAlign: a clinician-generated dataset for instruction following with electronic medical records.
S. Fleming, A. Lozano, W. Haberkorn, Jenelle A. Jindal, E. Reis, Rahul Thapa, L. Blankemeier, Julian Z. Genkins, E. Steinberg, A. Nayak, Birju S. Patel, Chia-Chun Chiang, A. Callahan, Zepeng Huo, S. Gatidis, S. Adams, Oluseyi Fayanju, Shreya J. Shah, Thomas Savage, Ethan Goh, A. Chaudhari, N. Aghaeepour, Christopher D. Sharp, M. Pfeffer, Percy Liang, Jonathan H. Chen, K. Morse, E. Brunskill, Jason Alan Fries, N. Shah.
Association for the Advancement of Artificial Intelligence (AAAI), 2024.
@inproceedings{fleming2024medalign,
author = {S. Fleming and A. Lozano and W. Haberkorn and Jenelle A. Jindal and E. Reis and Rahul Thapa and L. Blankemeier and Julian Z. Genkins and E. Steinberg and A. Nayak and Birju S. Patel and Chia-Chun Chiang and A. Callahan and Zepeng Huo and S. Gatidis and S. Adams and Oluseyi Fayanju and Shreya J. Shah and Thomas Savage and Ethan Goh and A. Chaudhari and N. Aghaeepour and Christopher D. Sharp and M. Pfeffer and Percy Liang and Jonathan H. Chen and K. Morse and E. Brunskill and Jason Alan Fries and N. Shah},
booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
title = {MedAlign: A Clinician-Generated Dataset for Instruction Following with Electronic Medical Records},
year = {2024},
}
The foundation model transparency index.
Rishi Bommasani, Kevin Klyman, Shayne Longpre, Sayash Kapoor, Nestor Maslej, Betty Xiong, Daniel Zhang, Percy Liang.
Transcations of Machine Learning Research (TMLR), 2023.
@article{bommasani2023fmti,
author = {Rishi Bommasani and Kevin Klyman and Shayne Longpre and Sayash Kapoor and Nestor Maslej and Betty Xiong and Daniel Zhang and Percy Liang},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {The Foundation Model Transparency Index},
volume = {0},
year = {2023},
}
@inproceedings{xie2023data,
author = {Sang Michael Xie and Shibani Santurkar and Tengyu Ma and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Data Selection for Language Models via Importance Resampling},
year = {2023},
}
@inproceedings{xie2023doremi,
author = {Sang Michael Xie and Hieu Pham and Xuanyi Dong and Nan Du and Hanxiao Liu and Yifeng Lu and Percy Liang and Quoc V. Le and Tengyu Ma and A. Yu},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {DoReMi: Optimizing Data Mixtures Speeds Up Language Model Pretraining},
year = {2023},
}
@inproceedings{huang2023prodigy,
author = {Qian Huang and Hongyu Ren and Peng Chen and Gregor Krvzmanc and D. Zeng and Percy Liang and J. Leskovec},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {{PRODIGY}: Enabling In-context Learning Over Graphs},
year = {2023},
}
Lexinvariant language models.
Qian Huang, E. Zelikman, Sarah Chen, Yuhuai Wu, G. Valiant, Percy Liang.
Advances in Neural Information Processing Systems (NeurIPS), 2023.
@inproceedings{huang2023lexinvariant,
author = {Qian Huang and E. Zelikman and Sarah Chen and Yuhuai Wu and G. Valiant and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Lexinvariant Language Models},
year = {2023},
}
@inproceedings{toups2023ecosystem,
author = {Connor Toups and Rishi Bommasani and Kathleen A. Creel and Sarah H. Bana and Dan Jurafsky and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Ecosystem-level Analysis of Deployed Machine Learning Reveals Homogeneous Outcomes},
year = {2023},
}
@inproceedings{dubois2023alpacafarm,
author = {Yann Dubois and Xuechen Li and Rohan Taori and Tianyi Zhang and Ishaan Gulrajani and Jimmy Ba and Carlos Guestrin and Percy Liang and Tatsunori B. Hashimoto},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {{AlpacaFarm}: A Simulation Framework for Methods that Learn from Human Feedback},
year = {2023},
}
@inproceedings{narayanan2023cheaply,
author = {Deepak Narayanan and Keshav Santhanam and Peter Henderson and Rishi Bommasani and Tony Lee and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Cheaply Evaluating Inference Efficiency Metrics for Autoregressive Transformer APIs},
year = {2023},
}
Holistic evaluation of text-to-image models.
Tony Lee, Michihiro Yasunaga, Chenlin Meng, Yifan Mai, Joon Sung Park, Agrim Gupta, Yunzhi Zhang, Deepak Narayanan, Hannah Teufel, Marco Bellagente, Minguk Kang, Taesung Park, J. Leskovec, Jun-Yan Zhu, Fei-Fei Li, Jiajun Wu, Stefano Ermon, Percy Liang.
Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks), 2023.
@inproceedings{lee2023holistic,
author = {Tony Lee and Michihiro Yasunaga and Chenlin Meng and Yifan Mai and Joon Sung Park and Agrim Gupta and Yunzhi Zhang and Deepak Narayanan and Hannah Teufel and Marco Bellagente and Minguk Kang and Taesung Park and J. Leskovec and Jun-Yan Zhu and Fei-Fei Li and Jiajun Wu and Stefano Ermon and Percy Liang},
booktitle = {Neural Information Processing Systems Track on Datasets and Benchmarks (NeurIPS Datasets and Benchmarks)},
title = {Holistic Evaluation of Text-To-Image Models},
year = {2023},
}
@article{liu2023lost,
author = {Nelson F. Liu and Kevin Lin and John Hewitt and Ashwin Paranjape and Michele Bevilacqua and Fabio Petroni and Percy Liang},
journal = {Transactions of the Association for Computational Linguistics (TACL)},
title = {Lost in the Middle: How Language Models Use Long Contexts},
volume = {0},
year = {2023},
}
@inproceedings{liu2023evaluating,
author = {Nelson F. Liu and Tianyi Zhang and Percy Liang},
booktitle = {Findings of Empirical Methods in Natural Language Processing (Findings of EMNLP)},
title = {Evaluating Verifiability in Generative Search Engines},
year = {2023},
}
Evaluating human-language model interaction.
Mina Lee, Megha Srivastava, Amelia Hardy, John Thickstun, Esin Durmus, Ashwin Paranjape, Ines Gerard-Ursin, Xiang Lisa Li, Faisal Ladhak, Frieda Rong, Rose E. Wang, Minae Kwon, Joon Sung Park, Hancheng Cao, Tony Lee, Rishi Bommasani, Michael Bernstein, Percy Liang.
Transcations of Machine Learning Research (TMLR), 2023.
@article{lee2023evaluating,
author = {Mina Lee and Megha Srivastava and Amelia Hardy and John Thickstun and Esin Durmus and Ashwin Paranjape and Ines Gerard-Ursin and Xiang Lisa Li and Faisal Ladhak and Frieda Rong and Rose E. Wang and Minae Kwon and Joon Sung Park and Hancheng Cao and Tony Lee and Rishi Bommasani and Michael Bernstein and Percy Liang},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {Evaluating Human-Language Model Interaction},
volume = {0},
year = {2023},
}
@article{zhang2023benchmarking,
author = {Tianyi Zhang and Faisal Ladhak and Esin Durmus and Percy Liang and K. McKeown and Tatsunori Hashimoto},
journal = {arXiv},
title = {Benchmarking Large Language Models for News Summarization},
year = {2023},
}
@article{khattab2023dsp,
author = {Omar Khattab and Keshav Santhanam and Xiang Lisa Li and David Hall and Percy Liang and Christopher Potts and Matei Zaharia},
journal = {arXiv},
title = {{Demonstrate-Search-Predict}: Composing retrieval and language models for knowledge-intensive {NLP}},
year = {2023},
}
@inproceedings{park2023generative,
author = {Joon Sung Park and Joseph C. O'Brien and Carrie J. Cai and Meredith Ringel Morris and Percy Liang and Michael S. Bernstein},
booktitle = {User Interface Software and Technology (UIST)},
title = {Generative Agents: Interactive Simulacra of Human Behavior},
year = {2023},
}
Foundation models and fair use.
Peter Henderson, Xuechen Li, Dan Jurafsky, Tatsunori Hashimoto, Mark A. Lemley, Percy Liang.
Journal of Machine Learning Research (JMLR), 2023.
@article{henderson2023fairuse,
author = {Peter Henderson and Xuechen Li and Dan Jurafsky and Tatsunori Hashimoto and Mark A. Lemley and Percy Liang},
journal = {Journal of Machine Learning Research (JMLR)},
title = {Foundation Models and Fair Use},
volume = {0},
year = {2023},
}
@article{bommasani2023ecosystem,
author = {Rishi Bommasani and Dilara Soylu and Thomas Liao and Kathleen A. Creel and Percy Liang},
journal = {arXiv},
title = {Ecosystem Graphs: The Social Footprint of Foundation Models},
year = {2023},
}
@inproceedings{karamcheti2023voltron,
author = {Siddharth Karamcheti and Suraj Nair and Annie S. Chen and T. Kollar and Chelsea Finn and Dorsa Sadigh and Percy Liang},
booktitle = {Robotics: Science and Systems (RSS)},
title = {Language-driven Representation Learning for Robotics},
year = {2023},
}
@inproceedings{sheng2023flexgen,
author = {Ying Sheng and Lianmin Zheng and Binhang Yuan and Zhuohan Li and Max Ryabinin and Daniel Y. Fu and Zhiqiang Xie and Beidi Chen and Clark W. Barrett and Joseph Gonzalez and Percy Liang and Christopher Ré and I. Stoica and Ce Zhang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {High-throughput Generative Inference of Large Language Models with a Single {GPU}},
year = {2023},
}
Retrieval-Augmented multimodal language modeling.
Michihiro Yasunaga, Armen Aghajanyan, Weijia Shi, Rich James, Jure Leskovec, Percy Liang, Mike Lewis, Luke Zettlemoyer, Wen-tau Yih.
International Conference on Machine Learning (ICML), 2023.
@inproceedings{yasunaga2023retrieval,
author = {Michihiro Yasunaga and Armen Aghajanyan and Weijia Shi and Rich James and Jure Leskovec and Percy Liang and Mike Lewis and Luke Zettlemoyer and Wen-tau Yih},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Retrieval-Augmented Multimodal Language Modeling},
year = {2023},
}
@inproceedings{santurkar2023opinions,
author = {Shibani Santurkar and Esin Durmus and Faisal Ladhak and Cinoo Lee and Percy Liang and Tatsunori Hashimoto},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Whose Opinions Do Language Models Reflect?},
year = {2023},
}
@inproceedings{gao2023targeted,
author = {Irena Gao and Shiori Sagawa and Pang Wei Koh and Tatsunori Hashimoto and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Out-of-Domain Robustness via Targeted Augmentations},
year = {2023},
}
@inproceedings{liu2023sample,
author = {Nelson F. Liu and Ananya Kumar and Percy Liang and Robin Jia},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Are Sample-Efficient {NLP} Models More Robust?},
year = {2023},
}
@inproceedings{liu2023concurrence,
author = {Nelson F. Liu and Tony Lee and Robin Jia and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Do Question Answering Modeling Improvements Hold Across Benchmarks?},
year = {2023},
}
@inproceedings{zhang2023beyond,
author = {Yuhui Zhang and Michihiro Yasunaga and Zhengping Zhou and Jeff Z. HaoChen and James Zou and Percy Liang and Serena Yeung},
booktitle = {Findings of the Association for Computational Linguistics (Findings of ACL)},
title = {Beyond Positive Scaling: How Negation Impacts Scaling Trends of Language Models},
year = {2023},
}
@inproceedings{li2023contrastive,
author = {Xiang Lisa Li and Ari Holtzman and Daniel Fried and Percy Liang and Jason Eisner and Tatsunori Hashimoto and Luke Zettlemoyer and M. Lewis},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Contrastive Decoding: Open-ended Text Generation as Optimization},
year = {2023},
}
Backpack language models.
John Hewitt, John Thickstun, Christopher D. Manning, Percy Liang.
Association for Computational Linguistics (ACL), 2023. Outstanding paper award.
@inproceedings{hewitt2023backpack,
author = {John Hewitt and John Thickstun and Christopher D. Manning and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Backpack Language Models},
year = {2023},
}
@inproceedings{cui2023corrections,
author = {Yuchen Cui and Siddharth Karamcheti and Raj Palleti and Nidhya Shivakumar and Percy Liang and Dorsa Sadigh},
booktitle = {ACM/IEEE International Conference on Human Robot Interaction (HRI)},
title = {"No, to the Right"-- Online Language Corrections for Robotic Manipulation via Shared Autonomy},
year = {2023},
}
@inproceedings{santurkar2023captions,
author = {Shibani Santurkar and Yann Dubois and Rohan Taori and Percy Liang and Tatsunori Hashimoto},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Is a Caption Worth a Thousand Images? A Controlled Study for Representation Learning},
year = {2023},
}
@inproceedings{lee2023surgical,
author = {Yoonho Lee and Annie S. Chen and Fahim Tajwar and Ananya Kumar and Huaxiu Yao and Percy Liang and Chelsea Finn},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Surgical Fine-Tuning Improves Adaptation to Distribution Shifts},
year = {2023},
}
Holistic evaluation of language models.
Percy Liang, Rishi Bommasani, Tony Lee, Dimitris Tsipras, Dilara Soylu, Michihiro Yasunaga, Yian Zhang, D. Narayanan, Yuhuai Wu, Ananya Kumar, Benjamin Newman, Binhang Yuan, Bobby Yan, Ce Zhang, Christian Cosgrove, Christopher D. Manning, Christopher Ré, Diana Acosta-Navas, Drew A. Hudson, E. Zelikman, Esin Durmus, Faisal Ladhak, Frieda Rong, Hongyu Ren, Huaxiu Yao, Jue Wang, Keshav Santhanam, Laurel J. Orr, Lucia Zheng, Mert Yuksekgonul, Mirac Suzgun, Nathan S. Kim, Neel Guha, Niladri S. Chatterji, O. Khattab, Peter Henderson, Qian Huang, Ryan Chi, Sang Michael Xie, Shibani Santurkar, S. Ganguli, Tatsunori Hashimoto, Thomas F. Icard, Tianyi Zhang, Vishrav Chaudhary, William Wang, Xuechen Li, Yifan Mai, Yuhui Zhang, Yuta Koreeda.
Transcations of Machine Learning Research (TMLR), 2023.
@article{liang2023helm,
author = {Percy Liang and Rishi Bommasani and Tony Lee and Dimitris Tsipras and Dilara Soylu and Michihiro Yasunaga and Yian Zhang and D. Narayanan and Yuhuai Wu and Ananya Kumar and Benjamin Newman and Binhang Yuan and Bobby Yan and Ce Zhang and Christian Cosgrove and Christopher D. Manning and Christopher Ré and Diana Acosta-Navas and Drew A. Hudson and E. Zelikman and Esin Durmus and Faisal Ladhak and Frieda Rong and Hongyu Ren and Huaxiu Yao and Jue Wang and Keshav Santhanam and Laurel J. Orr and Lucia Zheng and Mert Yuksekgonul and Mirac Suzgun and Nathan S. Kim and Neel Guha and Niladri S. Chatterji and O. Khattab and Peter Henderson and Qian Huang and Ryan Chi and Sang Michael Xie and Shibani Santurkar and S. Ganguli and Tatsunori Hashimoto and Thomas F. Icard and Tianyi Zhang and Vishrav Chaudhary and William Wang and Xuechen Li and Yifan Mai and Yuhui Zhang and Yuta Koreeda},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {Holistic Evaluation of Language Models},
volume = {0},
year = {2023},
}
@inproceedings{hewitt2022truncation,
author = {John Hewitt and Christopher D. Manning and Percy Liang},
booktitle = {Findings of Empirical Methods in Natural Language Processing (Findings of EMNLP)},
title = {Truncation Sampling as Language Model Desmoothing},
year = {2022},
}
@inproceedings{wu2022synthetic,
author = {Yuhuai Wu and Felix Li and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Insights into Pre-training via Simpler Synthetic Tasks},
year = {2022},
}
@inproceedings{dubois2022issl,
author = {Yann Dubois and Tatsunori Hashimoto and S. Ermon and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Improving Self-Supervised Learning by Characterizing Idealized Representations},
year = {2022},
}
@inproceedings{yuan2022decentralized,
author = {Binhang Yuan and Yongjun He and Jared Quincy Davis and Tianyi Zhang and Tri Dao and Beidi Chen and Percy Liang and Christopher Re and Ce Zhang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Decentralized Training of Foundation Models in Heterogeneous Environments},
year = {2022},
}
@inproceedings{li2022diffusion,
author = {Xiang Lisa Li and John Thickstun and Ishaan Gulrajani and Percy Liang and Tatsunori Hashimoto},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Diffusion-{LM} Improves Controllable Text Generation},
year = {2022},
}
@inproceedings{garg2022incontext,
author = {Shivam Garg and Dimitris Tsipras and Percy Liang and G. Valiant},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {What Can Transformers Learn In-Context? A Case Study of Simple Function Classes},
year = {2022},
}
Deep bidirectional language-knowledge graph pretraining.
Michihiro Yasunaga, Antoine Bosselut, Hongyu Ren, Xikun Zhang, Christopher D. Manning, Percy Liang*, Jure Leskovec*.
Advances in Neural Information Processing Systems (NeurIPS), 2022.
@inproceedings{yasunaga2022dragon,
author = {Michihiro Yasunaga and Antoine Bosselut and Hongyu Ren and Xikun Zhang and Christopher D. Manning and Percy Liang and Jure Leskovec},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Deep Bidirectional Language-Knowledge Graph Pretraining},
year = {2022},
}
@inproceedings{bommasani2022homogenization,
author = {Rishi Bommasani and Kathleen A. Creel and Ananya Kumar and Dan Jurafsky and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Picking on the Same Person: Does Algorithmic Monoculture lead to Outcome Homogenization?},
year = {2022},
}
@inproceedings{donahue2022melody,
author = {Chris Donahue and John Thickstun and Percy Liang},
booktitle = {International Society for Music Information Retrieval (ISMIR)},
title = {Melody transcription via generative pre-training},
year = {2022},
}
@inproceedings{park2022social,
author = {J. Park and Lindsay Popowski and Carrie J. Cai and M. Morris and Percy Liang and Michael S. Bernstein},
booktitle = {User Interface Software and Technology (UIST)},
title = {Social Simulacra: Creating Populated Prototypes for Social Computing Systems},
year = {2022},
}
Emergent abilities of large language models.
Jason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, Ed Chi, Tatsunori Hashimoto, Oriol Vinyals, Percy Liang, Jeff Dean, William Fedus.
Transcations of Machine Learning Research (TMLR), 2022.
@article{wei2022emergent,
author = {Jason Wei and Yi Tay and Rishi Bommasani and Colin Raffel and Barret Zoph and Sebastian Borgeaud and Dani Yogatama and Maarten Bosma and Denny Zhou and Donald Metzler and Ed Chi and Tatsunori Hashimoto and Oriol Vinyals and Percy Liang and Jeff Dean and William Fedus},
journal = {Transcations of Machine Learning Research (TMLR)},
title = {Emergent Abilities of Large Language Models},
volume = {0},
year = {2022},
}
@inproceedings{kumar2022calibrated,
author = {Ananya Kumar and Tengyu Ma and Percy Liang and Aditi Raghunathan},
booktitle = {Uncertainty in Artificial Intelligence (UAI)},
title = {Calibrated ensembles can mitigate accuracy tradeoffs under distribution shift},
year = {2022},
}
@inproceedings{karamcheti2022lilac,
author = {Siddharth Karamcheti and Raj Palleti and Yuchen Cui and Percy Liang and Dorsa Sadigh},
booktitle = {ACL Workshop for Learning with Natural Language Supervision (NL Supervision)},
title = {Shared Autonomy for Robotic Manipulation with Language Corrections},
year = {2022},
}
@inproceedings{shen2022connect,
author = {Kendrick Shen and Robbie Jones and Ananya Kumar and Sang Michael Xie and Jeff Z. HaoChen and Tengyu Ma and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Connect, Not Collapse: Explaining Contrastive Learning for Unsupervised Domain Adaptation},
year = {2022},
}
@inproceedings{yasunaga2022linkbert,
author = {Michihiro Yasunaga and Jure Leskovec and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {{LinkBERT}: Pretraining Language Models with Document Links},
year = {2022},
}
Extending the WILDS benchmark for unsupervised adaptation.
Shiori Sagawa, Pang Wei Koh, Tony Lee, Irena Gao, Sang Michael Xie, Kendrick Shen, Ananya Kumar, Weihua Hu, Michihiro Yasunaga, H. Marklund, Sara Beery, E. David, I. Stavness, Wei Guo, J. Leskovec, Kate Saenko, Tatsunori B. Hashimoto, S. Levine, Chelsea Finn, Percy Liang.
International Conference on Learning Representations (ICLR), 2022.
@inproceedings{sagawa2022uwilds,
author = {Shiori Sagawa and Pang Wei Koh and Tony Lee and Irena Gao and Sang Michael Xie and Kendrick Shen and Ananya Kumar and Weihua Hu and Michihiro Yasunaga and H. Marklund and Sara Beery and E. David and I. Stavness and Wei Guo and J. Leskovec and Kate Saenko and Tatsunori B. Hashimoto and S. Levine and Chelsea Finn and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Extending the {WILDS} Benchmark for Unsupervised Adaptation},
year = {2022},
}
@inproceedings{kumar2022finetuning,
author = {Ananya Kumar and Aditi Raghunathan and Robbie Jones and Tengyu Ma and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Fine-tuning can Distort Pretrained Features and Underperform Out-of-Distribution},
year = {2022},
}
@inproceedings{xie2022incontext,
author = {Sang Michael Xie and Aditi Raghunathan and Percy Liang and Tengyu Ma},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {An Explanation of In-context Learning as Implicit {B}ayesian Inference},
year = {2022},
}
@inproceedings{li2022privacy,
author = {Xuechen Li and Florian Tramèr and Percy Liang and Tatsunori B. Hashimoto},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Large Language Models Can Be Strong Differentially Private Learners},
year = {2022},
}
@inproceedings{zhang2022greaselm,
author = {Xikun Zhang and Antoine Bosselut and Michihiro Yasunaga and Hongyu Ren and Percy Liang and Christopher D. Manning and J. Leskovec},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {GreaseLM: Graph REASoning Enhanced Language Models for Question Answering},
year = {2022},
}
@inproceedings{lee2022coauthor,
author = {Mina Lee and Percy Liang and Qian Yang},
booktitle = {Conference on Human Factors in Computing Systems (CHI)},
title = {Co{A}uthor: Designing a Human-{AI} Collaborative Writing Dataset for Exploring Language Model Capabilities},
year = {2022},
}
Beyond the imitation game: quantifying and extrapolating the capabilities of language models.
Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R. Brown, Adam Santoro, Aditya Gupta, Adrià Garriga-Alonso, Agnieszka Kluska, Aitor Lewkowycz, Akshat Agarwal, Alethea Power, Alex Ray, Alex Warstadt, Alexander W. Kocurek, Ali Safaya, Ali Tazarv, Alice Xiang, Alicia Parrish, Allen Nie, Aman Hussain, Amanda Askell, Amanda Dsouza, Ambrose Slone, Ameet Rahane, Anantharaman S. Iyer, Anders Andreassen, Andrea Madotto, Andrea Santilli, Andreas Stuhlmüller, Andrew Dai, Andrew La, Andrew Lampinen, Andy Zou, Angela Jiang, Angelica Chen, Anh Vuong, Animesh Gupta, Anna Gottardi, Antonio Norelli, Anu Venkatesh, Arash Gholamidavoodi, Arfa Tabassum, Arul Menezes, Arun Kirubarajan, Asher Mullokandov, Ashish Sabharwal, Austin Herrick, Avia Efrat, Aykut Erdem, Ayla Karakaş, B. Ryan Roberts, Bao Sheng Loe, Barret Zoph, Bartłomiej Bojanowski, Batuhan Özyurt, Behnam Hedayatnia, Behnam Neyshabur, Benjamin Inden, Benno Stein, Berk Ekmekci, Bill Yuchen Lin, Blake Howald, Bryan Orinion, Cameron Diao, Cameron Dour, Catherine Stinson, Cedrick Argueta, César Ferri Ramírez, Chandan Singh, Charles Rathkopf, Chenlin Meng, Chitta Baral, Chiyu Wu, Chris Callison-Burch, Chris Waites, Christian Voigt, Christopher D. Manning, Christopher Potts, Cindy Ramirez, Clara E. Rivera, Clemencia Siro, Colin Raffel, Courtney Ashcraft, Cristina Garbacea, Damien Sileo, Dan Garrette, Dan Hendrycks, Dan Kilman, Dan Roth, Daniel Freeman, Daniel Khashabi, Daniel Levy, Daniel Moseguí González, Danielle Perszyk, Danny Hernandez, Danqi Chen, Daphne Ippolito, Dar Gilboa, David Dohan, David Drakard, David Jurgens, Debajyoti Datta, Deep Ganguli, Denis Emelin, Denis Kleyko, Deniz Yuret, Derek Chen, Derek Tam, Dieuwke Hupkes, Diganta Misra, Dilyar Buzan, Dimitri Coelho Mollo, Diyi Yang, Dong-Ho Lee, Dylan Schrader, Ekaterina Shutova, Ekin Dogus Cubuk, Elad Segal, Eleanor Hagerman, Elizabeth Barnes, Elizabeth Donoway, Ellie Pavlick, Emanuele Rodola, Emma Lam, Eric Chu, Eric Tang, Erkut Erdem, Ernie Chang, Ethan A. Chi, Ethan Dyer, Ethan Jerzak, Ethan Kim, Eunice Engefu Manyasi, Evgenii Zheltonozhskii, Fanyue Xia, Fatemeh Siar, Fernando Martínez-Plumed, Francesca Happé, Francois Chollet, Frieda Rong, Gaurav Mishra, Genta Indra Winata, Gerard de Melo, Germán Kruszewski, Giambattista Parascandolo, Giorgio Mariani, Gloria Wang, Gonzalo Jaimovitch-López, Gregor Betz, Guy Gur-Ari, Hana Galijasevic, Hannah Kim, Hannah Rashkin, Hannaneh Hajishirzi, Harsh Mehta, Hayden Bogar, Henry Shevlin, Hinrich Schütze, Hiromu Yakura, Hongming Zhang, Hugh Mee Wong, Ian Ng, Isaac Noble, Jaap Jumelet, Jack Geissinger, Jackson Kernion, Jacob Hilton, Jaehoon Lee, Jaime Fernández Fisac, James B. Simon, James Koppel, James Zheng, James Zou, Jan Kocoń, Jana Thompson, Janelle Wingfield, Jared Kaplan, Jarema Radom, Jascha Sohl-Dickstein, Jason Phang, Jason Wei, Jason Yosinski, Jekaterina Novikova, Jelle Bosscher, Jennifer Marsh, Jeremy Kim, Jeroen Taal, Jesse Engel, Jesujoba Alabi, Jiacheng Xu, Jiaming Song, Jillian Tang, Joan Waweru, John Burden, John Miller, John U. Balis, Jonathan Batchelder, Jonathan Berant, Jörg Frohberg, Jos Rozen, Jose Hernandez-Orallo, Joseph Boudeman, Joseph Guerr, Joseph Jones, Joshua B. Tenenbaum, Joshua S. Rule, Joyce Chua, Kamil Kanclerz, Karen Livescu, Karl Krauth, Karthik Gopalakrishnan, Katerina Ignatyeva, Katja Markert, Kaustubh D. Dhole, Kevin Gimpel, Kevin Omondi, Kory Mathewson, Kristen Chiafullo, Ksenia Shkaruta, Kumar Shridhar, Kyle McDonell, Kyle Richardson, Laria Reynolds, Leo Gao, Li Zhang, Liam Dugan, Lianhui Qin, Lidia Contreras-Ochando, Louis-Philippe Morency, Luca Moschella, Lucas Lam, Lucy Noble, Ludwig Schmidt, Luheng He, Luis Oliveros Colón, Luke Metz, Lütfi Kerem Şenel, Maarten Bosma, Maarten Sap, Maartje ter Hoeve, Maheen Farooqi, Manaal Faruqui, Mantas Mazeika, Marco Baturan, Marco Marelli, Marco Maru, Maria Jose Ramírez Quintana, Marie Tolkiehn, Mario Giulianelli, Martha Lewis, Martin Potthast, Matthew L. Leavitt, Matthias Hagen, Mátyás Schubert, Medina Orduna Baitemirova, Melody Arnaud, Melvin McElrath, Michael A. Yee, Michael Cohen, Michael Gu, Michael Ivanitskiy, Michael Starritt, Michael Strube, Michał Swędrowski, Michele Bevilacqua, Michihiro Yasunaga, Mihir Kale, Mike Cain, Mimee Xu, Mirac Suzgun, Mitch Walker, Mo Tiwari, Mohit Bansal, Moin Aminnaseri, Mor Geva, Mozhdeh Gheini, Mukund Varma T, Nanyun Peng, Nathan A. Chi, Nayeon Lee, Neta Gur-Ari Krakover, Nicholas Cameron, Nicholas Roberts, Nick Doiron, Nicole Martinez, Nikita Nangia, Niklas Deckers, Niklas Muennighoff, Nitish Shirish Keskar, Niveditha S. Iyer, Noah Constant, Noah Fiedel, Nuan Wen, Oliver Zhang, Omar Agha, Omar Elbaghdadi, Omer Levy, Owain Evans, Pablo Antonio Moreno Casares, Parth Doshi, Pascale Fung, Paul Pu Liang, Paul Vicol, Pegah Alipoormolabashi, Peiyuan Liao, Percy Liang, Peter Chang, Peter Eckersley, Phu Mon Htut, Pinyu Hwang, Piotr Miłkowski, Piyush Patil, Pouya Pezeshkpour, Priti Oli, Qiaozhu Mei, Qing Lyu, Qinlang Chen, Rabin Banjade, Rachel Etta Rudolph, Raefer Gabriel, Rahel Habacker, Ramon Risco, Raphaël Millière, Rhythm Garg, Richard Barnes, Rif A. Saurous, Riku Arakawa, Robbe Raymaekers, Robert Frank, Rohan Sikand, Roman Novak, Roman Sitelew, Ronan LeBras, Rosanne Liu, Rowan Jacobs, Rui Zhang, Ruslan Salakhutdinov, Ryan Chi, Ryan Lee, Ryan Stovall, Ryan Teehan, Rylan Yang, Sahib Singh, Saif M. Mohammad, Sajant Anand, Sam Dillavou, Sam Shleifer, Sam Wiseman, Samuel Gruetter, Samuel R. Bowman, Samuel S. Schoenholz, Sanghyun Han, Sanjeev Kwatra, Sarah A. Rous, Sarik Ghazarian, Sayan Ghosh, Sean Casey, Sebastian Bischoff, Sebastian Gehrmann, Sebastian Schuster, Sepideh Sadeghi, Shadi Hamdan, Sharon Zhou, Shashank Srivastava, Sherry Shi, Shikhar Singh, Shima Asaadi, Shixiang Shane Gu, Shubh Pachchigar, Shubham Toshniwal, Shyam Upadhyay, Shyamolima, Debnath, Siamak Shakeri, Simon Thormeyer, Simone Melzi, Siva Reddy, Sneha Priscilla Makini, Soo-Hwan Lee, Spencer Torene, Sriharsha Hatwar, Stanislas Dehaene, Stefan Divic, Stefano Ermon, Stella Biderman, Stephanie Lin, Stephen Prasad, Steven T. Piantadosi, Stuart M. Shieber, Summer Misherghi, Svetlana Kiritchenko, Swaroop Mishra, Tal Linzen, Tal Schuster, Tao Li, Tao Yu, Tariq Ali, Tatsu Hashimoto, Te-Lin Wu, Théo Desbordes, Theodore Rothschild, Thomas Phan, Tianle Wang, Tiberius Nkinyili, Timo Schick, Timofei Kornev, Titus Tunduny, Tobias Gerstenberg, Trenton Chang, Trishala Neeraj, Tushar Khot, Tyler Shultz, Uri Shaham, Vedant Misra, Vera Demberg, Victoria Nyamai, Vikas Raunak, Vinay Ramasesh, Vinay Uday Prabhu, Vishakh Padmakumar, Vivek Srikumar, William Fedus, William Saunders, William Zhang, Wout Vossen, Xiang Ren, Xiaoyu Tong, Xinran Zhao, Xinyi Wu, Xudong Shen, Yadollah Yaghoobzadeh, Yair Lakretz, Yangqiu Song, Yasaman Bahri, Yejin Choi, Yichi Yang, Yiding Hao, Yifu Chen, Yonatan Belinkov, Yu Hou, Yufang Hou, Yuntao Bai, Zachary Seid, Zhuoye Zhao, Zijian Wang, Zijie J. Wang, Zirui Wang, Ziyi Wu.
arXiv, 2022.
@article{srivastava2022beyond,
author = {Aarohi Srivastava and Abhinav Rastogi and Abhishek Rao and Abu Awal Md Shoeb and Abubakar Abid and Adam Fisch and Adam R. Brown and Adam Santoro and Aditya Gupta and Adrià Garriga-Alonso and Agnieszka Kluska and Aitor Lewkowycz and Akshat Agarwal and Alethea Power and Alex Ray and Alex Warstadt and Alexander W. Kocurek and Ali Safaya and Ali Tazarv and Alice Xiang and Alicia Parrish and Allen Nie and Aman Hussain and Amanda Askell and Amanda Dsouza and Ambrose Slone and Ameet Rahane and Anantharaman S. Iyer and Anders Andreassen and Andrea Madotto and Andrea Santilli and Andreas Stuhlmüller and Andrew Dai and Andrew La and Andrew Lampinen and Andy Zou and Angela Jiang and Angelica Chen and Anh Vuong and Animesh Gupta and Anna Gottardi and Antonio Norelli and Anu Venkatesh and Arash Gholamidavoodi and Arfa Tabassum and Arul Menezes and Arun Kirubarajan and Asher Mullokandov and Ashish Sabharwal and Austin Herrick and Avia Efrat and Aykut Erdem and Ayla Karakaş and B. Ryan Roberts and Bao Sheng Loe and Barret Zoph and Bartłomiej Bojanowski and Batuhan Özyurt and Behnam Hedayatnia and Behnam Neyshabur and Benjamin Inden and Benno Stein and Berk Ekmekci and Bill Yuchen Lin and Blake Howald and Bryan Orinion and Cameron Diao and Cameron Dour and Catherine Stinson and Cedrick Argueta and César Ferri Ramírez and Chandan Singh and Charles Rathkopf and Chenlin Meng and Chitta Baral and Chiyu Wu and Chris Callison-Burch and Chris Waites and Christian Voigt and Christopher D. Manning and Christopher Potts and Cindy Ramirez and Clara E. Rivera and Clemencia Siro and Colin Raffel and Courtney Ashcraft and Cristina Garbacea and Damien Sileo and Dan Garrette and Dan Hendrycks and Dan Kilman and Dan Roth and Daniel Freeman and Daniel Khashabi and Daniel Levy and Daniel Moseguí González and Danielle Perszyk and Danny Hernandez and Danqi Chen and Daphne Ippolito and Dar Gilboa and David Dohan and David Drakard and David Jurgens and Debajyoti Datta and Deep Ganguli and Denis Emelin and Denis Kleyko and Deniz Yuret and Derek Chen and Derek Tam and Dieuwke Hupkes and Diganta Misra and Dilyar Buzan and Dimitri Coelho Mollo and Diyi Yang and Dong-Ho Lee and Dylan Schrader and Ekaterina Shutova and Ekin Dogus Cubuk and Elad Segal and Eleanor Hagerman and Elizabeth Barnes and Elizabeth Donoway and Ellie Pavlick and Emanuele Rodola and Emma Lam and Eric Chu and Eric Tang and Erkut Erdem and Ernie Chang and Ethan A. Chi and Ethan Dyer and Ethan Jerzak and Ethan Kim and Eunice Engefu Manyasi and Evgenii Zheltonozhskii and Fanyue Xia and Fatemeh Siar and Fernando Martínez-Plumed and Francesca Happé and Francois Chollet and Frieda Rong and Gaurav Mishra and Genta Indra Winata and Gerard de Melo and Germán Kruszewski and Giambattista Parascandolo and Giorgio Mariani and Gloria Wang and Gonzalo Jaimovitch-López and Gregor Betz and Guy Gur-Ari and Hana Galijasevic and Hannah Kim and Hannah Rashkin and Hannaneh Hajishirzi and Harsh Mehta and Hayden Bogar and Henry Shevlin and Hinrich Schütze and Hiromu Yakura and Hongming Zhang and Hugh Mee Wong and Ian Ng and Isaac Noble and Jaap Jumelet and Jack Geissinger and Jackson Kernion and Jacob Hilton and Jaehoon Lee and Jaime Fernández Fisac and James B. Simon and James Koppel and James Zheng and James Zou and Jan Kocoń and Jana Thompson and Janelle Wingfield and Jared Kaplan and Jarema Radom and Jascha Sohl-Dickstein and Jason Phang and Jason Wei and Jason Yosinski and Jekaterina Novikova and Jelle Bosscher and Jennifer Marsh and Jeremy Kim and Jeroen Taal and Jesse Engel and Jesujoba Alabi and Jiacheng Xu and Jiaming Song and Jillian Tang and Joan Waweru and John Burden and John Miller and John U. Balis and Jonathan Batchelder and Jonathan Berant and Jörg Frohberg and Jos Rozen and Jose Hernandez-Orallo and Joseph Boudeman and Joseph Guerr and Joseph Jones and Joshua B. Tenenbaum and Joshua S. Rule and Joyce Chua and Kamil Kanclerz and Karen Livescu and Karl Krauth and Karthik Gopalakrishnan and Katerina Ignatyeva and Katja Markert and Kaustubh D. Dhole and Kevin Gimpel and Kevin Omondi and Kory Mathewson and Kristen Chiafullo and Ksenia Shkaruta and Kumar Shridhar and Kyle McDonell and Kyle Richardson and Laria Reynolds and Leo Gao and Li Zhang and Liam Dugan and Lianhui Qin and Lidia Contreras-Ochando and Louis-Philippe Morency and Luca Moschella and Lucas Lam and Lucy Noble and Ludwig Schmidt and Luheng He and Luis Oliveros Colón and Luke Metz and Lütfi Kerem Şenel and Maarten Bosma and Maarten Sap and Maartje ter Hoeve and Maheen Farooqi and Manaal Faruqui and Mantas Mazeika and Marco Baturan and Marco Marelli and Marco Maru and Maria Jose Ramírez Quintana and Marie Tolkiehn and Mario Giulianelli and Martha Lewis and Martin Potthast and Matthew L. Leavitt and Matthias Hagen and Mátyás Schubert and Medina Orduna Baitemirova and Melody Arnaud and Melvin McElrath and Michael A. Yee and Michael Cohen and Michael Gu and Michael Ivanitskiy and Michael Starritt and Michael Strube and Michał Swędrowski and Michele Bevilacqua and Michihiro Yasunaga and Mihir Kale and Mike Cain and Mimee Xu and Mirac Suzgun and Mitch Walker and Mo Tiwari and Mohit Bansal and Moin Aminnaseri and Mor Geva and Mozhdeh Gheini and Mukund Varma T and Nanyun Peng and Nathan A. Chi and Nayeon Lee and Neta Gur-Ari Krakover and Nicholas Cameron and Nicholas Roberts and Nick Doiron and Nicole Martinez and Nikita Nangia and Niklas Deckers and Niklas Muennighoff and Nitish Shirish Keskar and Niveditha S. Iyer and Noah Constant and Noah Fiedel and Nuan Wen and Oliver Zhang and Omar Agha and Omar Elbaghdadi and Omer Levy and Owain Evans and Pablo Antonio Moreno Casares and Parth Doshi and Pascale Fung and Paul Pu Liang and Paul Vicol and Pegah Alipoormolabashi and Peiyuan Liao and Percy Liang and Peter Chang and Peter Eckersley and Phu Mon Htut and Pinyu Hwang and Piotr Miłkowski and Piyush Patil and Pouya Pezeshkpour and Priti Oli and Qiaozhu Mei and Qing Lyu and Qinlang Chen and Rabin Banjade and Rachel Etta Rudolph and Raefer Gabriel and Rahel Habacker and Ramon Risco and Raphaël Millière and Rhythm Garg and Richard Barnes and Rif A. Saurous and Riku Arakawa and Robbe Raymaekers and Robert Frank and Rohan Sikand and Roman Novak and Roman Sitelew and Ronan LeBras and Rosanne Liu and Rowan Jacobs and Rui Zhang and Ruslan Salakhutdinov and Ryan Chi and Ryan Lee and Ryan Stovall and Ryan Teehan and Rylan Yang and Sahib Singh and Saif M. Mohammad and Sajant Anand and Sam Dillavou and Sam Shleifer and Sam Wiseman and Samuel Gruetter and Samuel R. Bowman and Samuel S. Schoenholz and Sanghyun Han and Sanjeev Kwatra and Sarah A. Rous and Sarik Ghazarian and Sayan Ghosh and Sean Casey and Sebastian Bischoff and Sebastian Gehrmann and Sebastian Schuster and Sepideh Sadeghi and Shadi Hamdan and Sharon Zhou and Shashank Srivastava and Sherry Shi and Shikhar Singh and Shima Asaadi and Shixiang Shane Gu and Shubh Pachchigar and Shubham Toshniwal and Shyam Upadhyay and Shyamolima and Debnath and Siamak Shakeri and Simon Thormeyer and Simone Melzi and Siva Reddy and Sneha Priscilla Makini and Soo-Hwan Lee and Spencer Torene and Sriharsha Hatwar and Stanislas Dehaene and Stefan Divic and Stefano Ermon and Stella Biderman and Stephanie Lin and Stephen Prasad and Steven T. Piantadosi and Stuart M. Shieber and Summer Misherghi and Svetlana Kiritchenko and Swaroop Mishra and Tal Linzen and Tal Schuster and Tao Li and Tao Yu and Tariq Ali and Tatsu Hashimoto and Te-Lin Wu and Théo Desbordes and Theodore Rothschild and Thomas Phan and Tianle Wang and Tiberius Nkinyili and Timo Schick and Timofei Kornev and Titus Tunduny and Tobias Gerstenberg and Trenton Chang and Trishala Neeraj and Tushar Khot and Tyler Shultz and Uri Shaham and Vedant Misra and Vera Demberg and Victoria Nyamai and Vikas Raunak and Vinay Ramasesh and Vinay Uday Prabhu and Vishakh Padmakumar and Vivek Srikumar and William Fedus and William Saunders and William Zhang and Wout Vossen and Xiang Ren and Xiaoyu Tong and Xinran Zhao and Xinyi Wu and Xudong Shen and Yadollah Yaghoobzadeh and Yair Lakretz and Yangqiu Song and Yasaman Bahri and Yejin Choi and Yichi Yang and Yiding Hao and Yifu Chen and Yonatan Belinkov and Yu Hou and Yufang Hou and Yuntao Bai and Zachary Seid and Zhuoye Zhao and Zijian Wang and Zijie J. Wang and Zirui Wang and Ziyi Wu},
journal = {arXiv},
title = {Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models},
year = {2022},
}
How does contrastive pre-training connect disparate domains?
Kendrick Shen, Robbie Matthew Jones, Ananya Kumar, Sang Michael Xie, Percy Liang.
NeurIPS Workshop on Distribution Shifts, 2021.
@inproceedings{shen2021connect,
author = {Kendrick Shen and Robbie Matthew Jones and Ananya Kumar and Sang Michael Xie and Percy Liang},
booktitle = {NeurIPS Workshop on Distribution Shifts},
title = {How Does Contrastive Pre-training Connect Disparate Domains?},
year = {2021},
}
On the opportunities and risks of foundation models.
Rishi Bommasani, Drew A. Hudson, Ehsan Adeli, Russ Altman, Simran Arora, Sydney von Arx, Michael S. Bernstein, Jeannette Bohg, Antoine Bosselut, Emma Brunskill, Erik Brynjolfsson, Shyamal Buch, Dallas Card, Rodrigo Castellon, Niladri Chatterji, Annie Chen, Kathleen Creel, Jared Quincy Davis, Dorottya Demszky, Chris Donahue, Moussa Doumbouya, Esin Durmus, Stefano Ermon, John Etchemendy, Kawin Ethayarajh, Li Fei-Fei, Chelsea Finn, Trevor Gale, Lauren Gillespie, Karan Goel, Noah Goodman, Shelby Grossman, Neel Guha, Tatsunori Hashimoto, Peter Henderson, John Hewitt, Daniel E. Ho, Jenny Hong, Kyle Hsu, Jing Huang, Thomas Icard, Saahil Jain, Dan Jurafsky, Pratyusha Kalluri, Siddharth Karamcheti, Geoff Keeling, Fereshte Khani, Omar Khattab, Pang Wei Koh, Mark Krass, Ranjay Krishna, Rohith Kuditipudi, Ananya Kumar, Faisal Ladhak, Mina Lee, Tony Lee, Jure Leskovec, Isabelle Levent, Xiang Lisa Li, Xuechen Li, Tengyu Ma, Ali Malik, Christopher D. Manning, Suvir Mirchandani, Eric Mitchell, Zanele Munyikwa, Suraj Nair, Avanika Narayan, Deepak Narayanan, Ben Newman, Allen Nie, Juan Carlos Niebles, Hamed Nilforoshan, Julian Nyarko, Giray Ogut, Laurel Orr, Isabel Papadimitriou, Joon Sung Park, Chris Piech, Eva Portelance, Christopher Potts, Aditi Raghunathan, Rob Reich, Hongyu Ren, Frieda Rong, Yusuf Roohani, Camilo Ruiz, Jack Ryan, Christopher Ré, Dorsa Sadigh, Shiori Sagawa, Keshav Santhanam, Andy Shih, Krishnan Srinivasan, Alex Tamkin, Rohan Taori, Armin W. Thomas, Florian Tramèr, Rose E. Wang, William Wang, Bohan Wu, Jiajun Wu, Yuhuai Wu, Sang Michael Xie, Michihiro Yasunaga, Jiaxuan You, Matei Zaharia, Michael Zhang, Tianyi Zhang, Xikun Zhang, Yuhui Zhang, Lucia Zheng, Kaitlyn Zhou, Percy Liang.
arXiv preprint arXiv:2108.07258, 2021.
@article{bommasani2021opportunities,
author = {Rishi Bommasani and Drew A. Hudson and Ehsan Adeli and Russ Altman and Simran Arora and Sydney von Arx and Michael S. Bernstein and Jeannette Bohg and Antoine Bosselut and Emma Brunskill and Erik Brynjolfsson and Shyamal Buch and Dallas Card and Rodrigo Castellon and Niladri Chatterji and Annie Chen and Kathleen Creel and Jared Quincy Davis and Dorottya Demszky and Chris Donahue and Moussa Doumbouya and Esin Durmus and Stefano Ermon and John Etchemendy and Kawin Ethayarajh and Li Fei-Fei and Chelsea Finn and Trevor Gale and Lauren Gillespie and Karan Goel and Noah Goodman and Shelby Grossman and Neel Guha and Tatsunori Hashimoto and Peter Henderson and John Hewitt and Daniel E. Ho and Jenny Hong and Kyle Hsu and Jing Huang and Thomas Icard and Saahil Jain and Dan Jurafsky and Pratyusha Kalluri and Siddharth Karamcheti and Geoff Keeling and Fereshte Khani and Omar Khattab and Pang Wei Koh and Mark Krass and Ranjay Krishna and Rohith Kuditipudi and Ananya Kumar and Faisal Ladhak and Mina Lee and Tony Lee and Jure Leskovec and Isabelle Levent and Xiang Lisa Li and Xuechen Li and Tengyu Ma and Ali Malik and Christopher D. Manning and Suvir Mirchandani and Eric Mitchell and Zanele Munyikwa and Suraj Nair and Avanika Narayan and Deepak Narayanan and Ben Newman and Allen Nie and Juan Carlos Niebles and Hamed Nilforoshan and Julian Nyarko and Giray Ogut and Laurel Orr and Isabel Papadimitriou and Joon Sung Park and Chris Piech and Eva Portelance and Christopher Potts and Aditi Raghunathan and Rob Reich and Hongyu Ren and Frieda Rong and Yusuf Roohani and Camilo Ruiz and Jack Ryan and Christopher Ré and Dorsa Sadigh and Shiori Sagawa and Keshav Santhanam and Andy Shih and Krishnan Srinivasan and Alex Tamkin and Rohan Taori and Armin W. Thomas and Florian Tramèr and Rose E. Wang and William Wang and Bohan Wu and Jiajun Wu and Yuhuai Wu and Sang Michael Xie and Michihiro Yasunaga and Jiaxuan You and Matei Zaharia and Michael Zhang and Tianyi Zhang and Xikun Zhang and Yuhui Zhang and Lucia Zheng and Kaitlyn Zhou and Percy Liang},
journal = {arXiv preprint arXiv:2108.07258},
title = {On the Opportunities and Risks of Foundation Models},
year = {2021},
}
@inproceedings{yasunaga2021language,
author = {Michihiro Yasunaga and Jure Leskovec and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {{LM-Critic}: Language Models for Unsupervised Grammatical Error Correction},
year = {2021},
}
@inproceedings{hewitt2021conditional,
author = {John Hewitt and Kawin Ethayarajh and Percy Liang and Christopher D. Manning},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Conditional probing: measuring usable information beyond a baseline},
year = {2021},
}
@inproceedings{castellon2021calm,
author = {Rodrigo Castellon and Chris Donahue and Percy Liang},
booktitle = {International Society for Music Information Retrieval (ISMIR)},
title = {Codified audio language modeling learns useful representations for music information retrieval},
year = {2021},
}
@inproceedings{yasunaga2021break,
author = {Michihiro Yasunaga and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {{Break-It-Fix-It}: Unsupervised Learning for Program Repair},
year = {2021},
}
@inproceedings{davis2021catformer,
author = {Jared Quincy Davis and Albert Gu and Krzysztof Choromanski and Tri Dao and Christopher Re and Chelsea Finn and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Catformer: Designing Stable Transformers via Sensitivity Analysis},
year = {2021},
}
@inproceedings{liu2021jtt,
author = {Evan Zheran Liu and Behzad Haghgoo and Annie S. Chen and Aditi Raghunathan and Pang Wei Koh and Shiori Sagawa and Percy Liang and Chelsea Finn},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Just Train Twice: Improving Group Robustness without Training Group Information},
year = {2021},
}
@inproceedings{liu2021dream,
author = {Evan Zheran Liu and Aditi Raghunathan and Percy Liang and Chelsea Finn},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Decoupling Exploration and Exploitation for Meta-Reinforcement Learning without Sacrifices},
year = {2021},
}
@inproceedings{xie2021composed,
author = {Sang Michael Xie and Tengyu Ma and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Composed Fine-Tuning: Freezing Pre-Trained Denoising Autoencoders for Improved Generalization},
year = {2021},
}
WILDS: a benchmark of in-the-wild distribution shifts.
Pang Wei Koh*, Shiori Sagawa*, Henrik Marklund, Sang Michael Xie, Marvin Zhang, Akshay Balsubramani, Weihua Hu, Michihiro Yasunaga, Richard Lanas Phillips, Irena Gao, Tony Lee, Etienne David, Ian Stavness, Wei Guo, Berton A. Earnshaw, Imran S. Haque, Sara Beery, Jure Leskovec, Anshul Kundaje, Emma Pierson, Sergey Levine, Chelsea Finn, Percy Liang.
International Conference on Machine Learning (ICML), 2021.
@inproceedings{koh2021wilds,
author = {Pang Wei Koh and Shiori Sagawa and Henrik Marklund and Sang Michael Xie and Marvin Zhang and Akshay Balsubramani and Weihua Hu and Michihiro Yasunaga and Richard Lanas Phillips and Irena Gao and Tony Lee and Etienne David and Ian Stavness and Wei Guo and Berton A. Earnshaw and Imran S. Haque and Sara Beery and Jure Leskovec and Anshul Kundaje and Emma Pierson and Sergey Levine and Chelsea Finn and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {{WILDS}: A Benchmark of in-the-Wild Distribution Shifts},
year = {2021},
}
@inproceedings{miller2021line,
author = {John Miller and Rohan Taori and Aditi Raghunathan and Shiori Sagawa and Pang Wei Koh and Vaishaal Shankar and Percy Liang and Yair Carmon and Ludwig Schmidt},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Accuracy on the Line: on the Strong Correlation Between Out-of-Distribution and In-Distribution Generalization},
year = {2021},
}
@inproceedings{lee2021swords,
author = {Mina Lee and C. Donahue and Robin Jia and Alexander Iyabor and Percy Liang},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {Swords: A Benchmark for Lexical Substitution with Improved Data Coverage and Quality},
year = {2021},
}
@inproceedings{yasunaga2021qagnn,
author = {Michihiro Yasunaga and Hongyu Ren and Antoine Bosselut and Percy Liang and Jure Leskovec},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {{QA-GNN}: Reasoning with Language Models and Knowledge Graphs for Question Answering},
year = {2021},
}
@inproceedings{li2021prefix,
author = {Xiang Lisa Li and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Prefix-Tuning: Optimizing Continuous Prompts for Generation},
year = {2021},
}
@inproceedings{jones2021selective,
author = {Erik Jones and Shiori Sagawa and Pang Wei Koh and Ananya Kumar and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Selective Classification Can Magnify Disparities Across Groups},
year = {2021},
}
@inproceedings{xie2021innout,
author = {Sang Michael Xie and Ananya Kumar and Robbie Jones and Fereshte Khani and Tengyu Ma and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {In-{N}-Out: Pre-Training and Self-Training using Auxiliary Information for Out-of-Distribution Robustness},
year = {2021},
}
@inproceedings{khani2021removing,
author = {Fereshte Khani and Percy Liang},
booktitle = {ACM Conference on Fairness, Accountability, and Transparency (FAccT)},
title = {Removing Spurious Features can Hurt Accuracy and Affect Groups Disproportionately},
year = {2021},
}
@inproceedings{gu2021beyond,
author = {Yu Gu and Sue Kase and Michelle T. Vanni and Brian M. Sadler and Percy Liang and Xifeng Yan and Yu Su},
booktitle = {World Wide Web (WWW)},
title = {Beyond I.I.D.: Three Levels of Generalization for Question Answering on Knowledge Bases},
year = {2021},
}
@article{koh2021stronger,
author = {Pang Wei Koh and Jacob Steinhardt and Percy Liang},
journal = {Machine Learning},
title = {Stronger Data Poisoning Attacks Break Data Sanitization Defenses},
volume = {1},
year = {2021},
}
@inproceedings{karamcheti2020decomposition,
author = {Siddharth Karamcheti and Dorsa Sadigh and Percy Liang},
booktitle = {EMNLP Workshop for Interactive and Executable Semantic Parsing (IntEx-SemPar)},
title = {Learning Adaptive Language Interfaces through Decomposition},
year = {2020},
}
@article{liu2020explore,
author = {Evan Zheran Liu and Aditi Raghunathan and Percy Liang and Chelsea Finn},
journal = {arXiv preprint arXiv:2008.02790},
title = {Explore then Execute: Adapting without Rewards via Factorized Meta-Reinforcement Learning},
year = {2020},
}
@article{liu2020learning,
author = {Evan Zheran Liu and Ramtin Keramati and Sudarshan Seshadri and Kelvin Guu and Panupong Pasupat and Emma Brunskill and Percy Liang},
journal = {arXiv preprint arXiv:2007.05896},
title = {Learning Abstract Models for Strategic Exploration and Fast Reward Transfer},
year = {2020},
}
@inproceedings{mussmann2020pairwise,
author = {Stephen Mussmann and Robin Jia and Percy Liang},
booktitle = {Findings of Empirical Methods in Natural Language Processing (Findings of EMNLP)},
title = {On the Importance of Adaptive Data Collection for Extremely Imbalanced Pairwise Tasks},
year = {2020},
}
@inproceedings{hewitt2020rnn,
author = {John Hewitt and Michael Hahn and Surya Ganguli and Percy Liang and Christopher D. Manning},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {{RNN}s can generate bounded hierarchical languages with optimal memory},
year = {2020},
}
The EOS decision and length extrapolation.
Benjamin Newman, John Hewitt, Percy Liang, Christopher D. Manning.
Proceedings of the Third BlackboxNLP Workshop on Analyzing and Interpreting Neural Networks for NLP, 2020. Outstanding paper award.
@inproceedings{newman2020eos,
author = {Benjamin Newman and John Hewitt and Percy Liang and Christopher D. Manning},
booktitle = {Proceedings of the Third BlackboxNLP Workshop on Analyzing and Interpreting Neural Networks for NLP},
title = {The {EOS} Decision and Length Extrapolation},
year = {2020},
}
@inproceedings{dathathri2020sdp,
author = {Sumanth Dathathri and Krishnamurthy Dvijotham and Alexey Kurakin and Aditi Raghunathan and Jonathan Uesato and Rudy Bunel and Shreya Shankar and Jacob Steinhardt and Ian Goodfellow and Percy Liang and Pushmeet Kohli},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Enabling certification of verification-agnostic networks via memory-efficient semidefinite programming},
year = {2020},
}
Task-Oriented dialogue as dataflow synthesis.
Semantic Machines, Jacob Andreas, John Bufe, David Burkett, Charles Chen, Josh Clausman, Jean Crawford, Kate Crim, Jordan DeLoach, Leah Dorner, Jason Eisner, Hao Fang, Alan Guo, David Hall, Kristin Hayes, Kellie Hill, Diana Ho, Wendy Iwaszuk, Smriti Jha, Dan Klein, Jayant Krishnamurthy, Theo Lanman, Percy Liang, Christopher H. Lin, Ilya Lintsbakh, Andy McGovern, Aleksandr Nisnevich, Adam Pauls, Dmitrij Petters, Brent Read, Dan Roth, Subhro Roy, Jesse Rusak, Beth Short, Div Slomin, Ben Snyder, Stephon Striplin, Yu Su, Zachary Tellman, Sam Thomson, Andrei Vorobev, Izabela Witoszko, Jason Wolfe, Abby Wray, Yuchen Zhang, Alexander Zotov.
Transactions of the Association for Computational Linguistics (TACL), 2020.
@article{semanticmachines2020dataflow,
author = {Semantic Machines and Jacob Andreas and John Bufe and David Burkett and Charles Chen and Josh Clausman and Jean Crawford and Kate Crim and Jordan DeLoach and Leah Dorner and Jason Eisner and Hao Fang and Alan Guo and David Hall and Kristin Hayes and Kellie Hill and Diana Ho and Wendy Iwaszuk and Smriti Jha and Dan Klein and Jayant Krishnamurthy and Theo Lanman and Percy Liang and Christopher H. Lin and Ilya Lintsbakh and Andy McGovern and Aleksandr Nisnevich and Adam Pauls and Dmitrij Petters and Brent Read and Dan Roth and Subhro Roy and Jesse Rusak and Beth Short and Div Slomin and Ben Snyder and Stephon Striplin and Yu Su and Zachary Tellman and Sam Thomson and Andrei Vorobev and Izabela Witoszko and Jason Wolfe and Abby Wray and Yuchen Zhang and Alexander Zotov},
journal = {Transactions of the Association for Computational Linguistics (TACL)},
title = {Task-Oriented Dialogue as Dataflow Synthesis},
volume = {8},
year = {2020},
}
@inproceedings{sagawa2020overparameterization,
author = {Shiori Sagawa and Aditi Raghunathan and Pang Wei Koh and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {An investigation of why overparameterization exacerbates spurious correlations},
year = {2020},
}
Concept bottleneck models.
Pang Wei Koh*, Thao Nguyen*, Yew Siang Tang*, Stephen Mussmann, Emma Pierson, Been Kim, Percy Liang.
International Conference on Machine Learning (ICML), 2020.
@inproceedings{koh2020bottleneck,
author = {Pang Wei Koh and Thao Nguyen and Yew Siang Tang and Stephen Mussmann and Emma Pierson and Been Kim and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Concept Bottleneck Models},
year = {2020},
}
@inproceedings{khani2020noise,
author = {Fereshte Khani and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Feature Noise Induces Loss Discrepancy Across Groups},
year = {2020},
}
@inproceedings{yasunaga2020repair,
author = {Michihiro Yasunaga and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Graph-based, Self-Supervised Program Repair from Diagnostic Feedback},
year = {2020},
}
@inproceedings{raghunathan2020understanding,
author = {Aditi Raghunathan and Sang Michael Xie and Fanny Yang and John C. Duchi and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Understanding and Mitigating the Tradeoff Between Robustness and Accuracy},
year = {2020},
}
@inproceedings{kumar2020gradual,
author = {Ananya Kumar and Tengyu Ma and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Understanding Self-Training for Gradual Domain Adaptation},
year = {2020},
}
@inproceedings{srivasta2020human,
author = {Megha Srivastava and Tatsunori Hashimoto and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Robustness to Spurious Correlations via Human Annotations},
year = {2020},
}
@inproceedings{jones2020roben,
author = {Erik Jones and Robin Jia and Aditi Raghunathan and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Robust Encodings: A Framework for Combating Adversarial Typos},
year = {2020},
}
@inproceedings{kamath2020squads,
author = {Amita Kamath and Robin Jia and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Selective Question Answering under Domain Shift},
year = {2020},
}
@inproceedings{mu2020shaping,
author = {Jesse Mu and Percy Liang and Noah Goodman},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Shaping Visual Representations with Language for Few-shot Classification},
year = {2020},
}
@inproceedings{murty2020expbert,
author = {Shikhar Murty and Pang Wei Koh and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {{ExpBERT}: Representation Engineering with Natural Language Explanations},
year = {2020},
}
@inproceedings{donahue2020infilling,
author = {Chris Donahue and Mina Lee and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Enabling Language Models to Fill in the Blanks},
year = {2020},
}
@inproceedings{sagawa2020group,
author = {Shiori Sagawa and Pang Wei Koh and Tatsunori B. Hashimoto and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Distributionally Robust Neural Networks for Group Shifts: On the Importance of Regularization for Worst-Case Generalization},
year = {2020},
}
@inproceedings{hu2020pretraining,
author = {Weihua Hu and Bowen Liu and Joseph Gomes and Marinka Zitnik and Percy Liang and Vijay Pande and Jure Leskovec},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Strategies for Pre-training Graph Neural Networks},
year = {2020},
}
@inproceedings{coleman2020selection,
author = {Cody Coleman and Christopher Yeh and Stephen Mussmann and Baharan Mirzasoleiman and Peter Bailis and Percy Liang and Jure Leskovec and Matei Zaharia},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Selection via Proxy: Efficient Data Selection for Deep Learning},
year = {2020},
}
@inproceedings{li2020greedy,
author = {Ray Li and Percy Liang and Stephen Mussmann},
booktitle = {Symposium on Discrete Algorithms (SODA)},
title = {A Tight Analysis of Greedy Yields Subexponential Time Approximation for Uniform Decision Tree},
year = {2020},
}
@inproceedings{jia2019certified,
author = {Robin Jia and Aditi Raghunathan and Kerem Göksel and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Certified Robustness to Adversarial Word Substitutions},
year = {2019},
}
@inproceedings{oren2019drolm,
author = {Yonatan Oren and Shiori Sagawa and Tatsunori Hashimoto and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Distributionally Robust Language Modeling},
year = {2019},
}
@inproceedings{hewitt2019control,
author = {John Hewitt and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Designing and Interpreting Probes with Control Tasks},
year = {2019},
}
@inproceedings{kulal2019spoc,
author = {Sumith Kulal and Panupong Pasupat and Kartik Chandra and Mina Lee and Oded Padon and Alex Aiken and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {SPoC: Search-based Pseudocode to Code},
year = {2019},
}
@inproceedings{carmon2019unlabeled,
author = {Yair Carmon and Aditi Raghunathan and Ludwig Schmidt and Percy Liang and John C. Duchi},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Unlabeled Data Improves Adversarial Robustness},
year = {2019},
}
@inproceedings{kumar2019calibration,
author = {Ananya Kumar and Percy Liang and Tengyu Ma},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Verified Uncertainty Calibration},
year = {2019},
}
@inproceedings{koh2019influence,
author = {Pang Wei Koh and Kai-Siang Ang and Hubert H. K. Teo and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {On the Accuracy of Influence Functions for Measuring Group Effects},
year = {2019},
}
@inproceedings{lee2019autocomplete,
author = {Mina Lee and Tatsunori Hashimoto and Percy Liang},
booktitle = {Emergent Communication Workshop at Neural Information Processing Systems (NeurIPS)},
title = {Learning Autocomplete Systems as a Communication Game},
year = {2019},
}
@article{raghunathan2019hurt,
author = {Aditi Raghunathan and Sang Michael Xie and Fanny Yang and John C. Duchi and Percy Liang},
journal = {arXiv preprint arXiv:1906.06032},
title = {Adversarial Training Can Hurt Generalization},
year = {2019},
}
@article{monajemi2019painless,
author = {Hatef Monajemi and Riccardo Murri and Eric Jonas and Percy Liang and Victoria Stodden and David L. Donoho},
journal = {Harvard Data Science Review},
title = {Ambitious Data Science Can Be Painless},
volume = {1},
year = {2019},
}
@inproceedings{hashimoto2019huse,
author = {Tatsunori Hashimoto and Hugh Zhang and Percy Liang},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {Unifying Human and Statistical Evaluation for Natural Language Generation},
year = {2019},
}
@inproceedings{peng2019pun,
author = {Nanyun Peng and He He and Percy Liang},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {Pun Generation with Surprise},
year = {2019},
}
@inproceedings{selsam2019sat,
author = {Daniel Selsam and Matthew Lamm and Benedikt Bünz and Percy Liang and Leonardo de Moura and David L. Dill},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Learning a {SAT} Solver from Single-Bit Supervision},
year = {2019},
}
@inproceedings{zhang2019discretization,
author = {Yuchen Zhang and Percy Liang},
booktitle = {Artificial Intelligence and Statistics (AISTATS)},
title = {Defending against Whitebox Adversarial Attacks via Randomized Discretization},
year = {2019},
}
@inproceedings{pierson2019aging,
author = {Emma Pierson and Pang Wei Koh and Tatsunori Hashimoto and Daphne Koller and Jure Leskovec and Nick Eriksson and Percy Liang},
booktitle = {Artificial Intelligence and Statistics (AISTATS)},
title = {Inferring Multidimensional Rates of Aging from Cross-Sectional Data},
year = {2019},
}
@inproceedings{shi2019frangel,
author = {Kensen Shi and Jacob Steinhardt and Percy Liang},
booktitle = {Principles of Programming Languages (POPL)},
title = {{F}r{A}ngel: Component-Based Synthesis with Control Structures},
year = {2019},
}
@inproceedings{raghunathan2018sdp,
author = {Aditi Raghunathan and Jacob Steinhardt and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Semidefinite relaxations for certifying robustness to adversarial examples},
year = {2018},
}
@inproceedings{mussmann2018sgd,
author = {Stephen Mussmann and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Uncertainty Sampling is Preconditioned Stochastic Gradient Descent on Zero-One Loss},
year = {2018},
}
@inproceedings{hashimoto2018edit,
author = {Tatsunori Hashimoto and Kelvin Guu and Yonatan Oren and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {A Retrieve-and-Edit Framework for Predicting Structured Outputs},
year = {2018},
}
QuAC: question answering in context.
Eunsol Choi, He He, Mohit Iyyer, Mark Yatskar, Wen-tau Yih, Yejin Choi, Percy Liang, Luke Zettlemoyer.
Empirical Methods in Natural Language Processing (EMNLP), 2018.
@inproceedings{choi2018quac,
author = {Eunsol Choi and He He and Mohit Iyyer and Mark Yatskar and Wen-tau Yih and Yejin Choi and Percy Liang and Luke Zettlemoyer},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {{QuAC}: Question Answering in Context},
year = {2018},
}
@inproceedings{he2018negotiation,
author = {He He and Derek Chen and Anusha Balakrishnan and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Decoupling Strategy and Generation in Negotiation Dialogues},
year = {2018},
}
@inproceedings{pasupat2018elements,
author = {Panupong Pasupat and Tian-Shun Jiang and Evan Zheran Liu and Kelvin Guu and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Mapping Natural Language Commands to Web Elements},
year = {2018},
}
@inproceedings{lamm2018tap,
author = {Matthew Lamm and Arun Chaganty and Christopher D. Manning and Dan Jurafsky and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Textual Analogy Parsing: What's Shared and What's Compared among Analogous Facts},
year = {2018},
}
@inproceedings{mussmann2018accuracy,
author = {Stephen Mussmann and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {On the Relationship between Data Efficiency and Error in Active Learning},
year = {2018},
}
@inproceedings{hashimoto2018repeated,
author = {Tatsunori B. Hashimoto and Megha Srivastava and Hongseok Namkoong and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Fairness Without Demographics in Repeated Loss Minimization},
year = {2018},
}
@inproceedings{hancock2018babble,
author = {Braden Hancock and Paroma Varma and Stephanie Wang and Martin Bringmann and Percy Liang and Christopher Ré},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Training Classifiers with Natural Language Explanations},
year = {2018},
}
@inproceedings{chaganty2018evaluation,
author = {Arun Chaganty and Stephen Mussmann and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {The price of debiasing automatic metrics in natural language evaluation},
year = {2018},
}
@inproceedings{rajpurkar2018squadrun,
author = {Pranav Rajpurkar and Robin Jia and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Know What You Don't Know: Unanswerable Questions for {SQuAD}},
year = {2018},
}
@article{khani2018pip,
author = {Fereshte Khani and Noah D. Goodman and Percy Liang},
journal = {Transactions of the Association for Computational Linguistics (TACL)},
title = {Planning, Inference and Pragmatics in Sequential Language Games},
volume = {6},
year = {2018},
}
@article{guu2018edit,
author = {Kelvin Guu and Tatsunori B. Hashimoto and Yonatan Oren and Percy Liang},
journal = {Transactions of the Association for Computational Linguistics (TACL)},
title = {Generating Sentences by Editing Prototypes},
volume = {0},
year = {2018},
}
@inproceedings{li2018style,
author = {Juncen Li and Robin Jia and He He and Percy Liang},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {Delete, Retrieve, Generate: A Simple Approach to Sentiment and Style Transfer},
year = {2018},
}
@inproceedings{liu2018workflow,
author = {Evan Zheran Liu and Kelvin Guu and Panupong Pasupat and Tianlin Shi and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Reinforcement Learning on Web Interfaces using Workflow-Guided Exploration},
year = {2018},
}
@inproceedings{raghunathan2018certified,
author = {Aditi Raghunathan and Jacob Steinhardt and Percy Liang},
booktitle = {International Conference on Learning Representations (ICLR)},
title = {Certified defenses against adversarial examples},
year = {2018},
}
@inproceedings{bastani2018active,
author = {Osbert Bastani and Rahul Sharma and Alex Aiken and Percy Liang},
booktitle = {Programming Language Design and Implementation (PLDI)},
title = {Active Learning of Points-To Specifications},
year = {2018},
}
@inproceedings{sharan2018prediction,
author = {Vatsal Sharan and Sham Kakade and Percy Liang and Gregory Valiant},
booktitle = {Symposium on Theory of Computing (STOC)},
title = {Prediction with a Short Memory},
year = {2018},
}
@inproceedings{steinhardt2017certified,
author = {Jacob Steinhardt and Pang Wei Koh and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Certified Defenses for Data Poisoning Attacks},
year = {2017},
}
@inproceedings{hashimoto2017transformation,
author = {Tatsunori B. Hashimoto and John Duchi and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Unsupervised Transformation Learning via Convex Relaxations},
year = {2017},
}
Learning overcomplete HMMs.
Vatsal Sharan, Sham Kakade, Percy Liang, Gregory Valiant.
Advances in Neural Information Processing Systems (NeurIPS), 2017.
@inproceedings{sharan2017overcomplete,
author = {Vatsal Sharan and Sham Kakade and Percy Liang and Gregory Valiant},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Learning Overcomplete {HMM}s},
year = {2017},
}
@inproceedings{jia2017adversarial,
author = {Robin Jia and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Adversarial Examples for Evaluating Reading Comprehension Systems},
year = {2017},
}
@inproceedings{zhang2017macro,
author = {Yuchen Zhang and Panupong Pasupat and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Macro Grammars and Holistic Triggering for Efficient Semantic Parsing},
year = {2017},
}
@inproceedings{chaganty2017unbiased,
author = {Arun Chaganty and Ashwin Paranjape and Percy Liang and Chris Manning},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Importance sampling for unbiased on-demand evaluation of knowledge base population},
year = {2017},
}
@inproceedings{zhang2017convexified,
author = {Yuchen Zhang and Percy Liang and Martin J. Wainwright},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Convexified Convolutional Neural Networks},
year = {2017},
}
@inproceedings{selsam2017bugfree,
author = {Daniel Selsam and Percy Liang and David Dill},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Developing Bug-Free Machine Learning Systems With Formal Mathematics},
year = {2017},
}
@inproceedings{shi2017wob,
author = {Tianlin Shi and Andrej Karpathy and Linxi Fan and Jonathan Hernandez and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {World of Bits: An Open-Domain Platform for Web-Based Agents},
year = {2017},
}
@inproceedings{zhang2017hitting,
author = {Yuchen Zhang and Percy Liang and Moses Charikar},
booktitle = {Conference on Learning Theory (COLT)},
title = {A Hitting Time Analysis of Stochastic Gradient {L}angevin Dynamics},
year = {2017},
}
@inproceedings{bastani2017synthesizing,
author = {Osbert Bastani and Rahul Sharma and Alex Aiken and Percy Liang},
booktitle = {Programming Language Design and Implementation (PLDI)},
title = {Synthesizing Program Input Grammars},
year = {2017},
}
@inproceedings{wang2017naturalizing,
author = {Sida I. Wang and Sam Ginn and Percy Liang and Christopher D. Manning},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Naturalizing a Programming Language via Interactive Learning},
year = {2017},
}
@inproceedings{he2017symmetric,
author = {He He and Anusha Balakrishnan and Mihail Eric and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
pages = {1766--1776},
title = {Learning Symmetric Collaborative Dialogue Agents with Dynamic Knowledge Graph Embeddings},
year = {2017},
}
@inproceedings{guu2017bridging,
author = {Kelvin Guu and Panupong Pasupat and Evan Zheran Liu and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {From Language to Programs: Bridging Reinforcement Learning and Maximum Marginal Likelihood},
year = {2017},
}
@inproceedings{steinhardt2016risk,
author = {Jacob Steinhardt and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Unsupervised Risk Estimation Using Only Conditional Independence Structure},
year = {2016},
}
@inproceedings{rajpurkar2016squad,
author = {Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {{SQuAD}: 100,000+ Questions for Machine Comprehension of Text},
year = {2016},
}
@inproceedings{wang2016games,
author = {Sida I. Wang and Percy Liang and Chris Manning},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Learning Language Games through Interaction},
year = {2016},
}
@inproceedings{long2016projections,
author = {Reginald Long and Panupong Pasupat and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Simpler Context-Dependent Logical Forms via Model Projections},
year = {2016},
}
@inproceedings{pasupat2016inferring,
author = {Panupong Pasupat and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Inferring Logical Forms From Denotations},
year = {2016},
}
@inproceedings{khani2016unanimity,
author = {Fereshte Khani and Martin Rinard and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Unanimous Prediction for 100% Precision with Application to Learning Semantic Mappings},
year = {2016},
}
@inproceedings{chaganty2016perspectives,
author = {Arun Tejasvi Chaganty and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {How Much is 131 Million Dollars? {P}utting Numbers in Perspective with Compositional Descriptions},
year = {2016},
}
@inproceedings{raghunathan2016linear,
author = {Aditi Raghunathan and Roy Frostig and John Duchi and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Estimation from Indirect Supervision with Linear Moments},
year = {2016},
}
@incollection{wager2016levy,
author = {Stefan Wager and Will Fithian and Percy Liang},
booktitle = {Perturbations, Optimization and Statistics},
title = {Data Augmentation via {L}évy Processes},
year = {2016},
}
@article{liang2016executable,
author = {Percy Liang},
journal = {Communications of the ACM},
title = {Learning Executable Semantic Parsers for Natural Language Understanding},
volume = {59},
year = {2016},
}
@inproceedings{wang2015overnight,
author = {Yushi Wang and Jonathan Berant and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Building a Semantic Parser Overnight},
year = {2015},
}
@article{berant2015agenda,
author = {Jonathan Berant and Percy Liang},
journal = {Transactions of the Association for Computational Linguistics (TACL)},
pages = {545--558},
title = {Imitation Learning of Agenda-Based Semantic Parsers},
volume = {3},
year = {2015},
}
@inproceedings{steinhardt2015relaxed,
author = {Jacob Steinhardt and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Learning with Relaxed Supervision},
year = {2015},
}
@inproceedings{wang2015polynomial,
author = {Sida I. Wang and Arun Chaganty and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Estimating Mixture Models via Mixture of Polynomials},
year = {2015},
}
@inproceedings{werling2015onthejob,
author = {Keenon Werling and Arun Chaganty and Percy Liang and Chris Manning},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {On-the-Job Learning with {B}ayesian Decision Theory},
year = {2015},
}
@inproceedings{kuleshov2015calibrated,
author = {Volodymyr Kuleshov and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Calibrated Structured Prediction},
year = {2015},
}
@inproceedings{guu2015traversing,
author = {Kelvin Guu and John Miller and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Traversing Knowledge Graphs in Vector Space},
year = {2015},
}
@inproceedings{misra2015environment,
author = {Dipendra K. Misra and Kejia Tao and Percy Liang and Ashutosh Saxena},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Environment-Driven Lexicon Induction for High-Level Instructions},
year = {2015},
}
@inproceedings{shi2015sample,
author = {Tianlin Shi and Jacob Steinhardt and Percy Liang},
booktitle = {Artificial Intelligence and Statistics (AISTATS)},
pages = {875--884},
title = {Learning Where To Sample in Structured Prediction},
year = {2015},
}
@inproceedings{kuleshov2015tensor,
author = {Volodymyr Kuleshov and Arun Chaganty and Percy Liang},
booktitle = {Artificial Intelligence and Statistics (AISTATS)},
title = {Tensor factorization via matrix factorization},
year = {2015},
}
@article{steinhardt2014sparse,
author = {Jacob Steinhardt and Stefan Wager and Percy Liang},
journal = {arXiv preprint arXiv:1412.4182},
title = {The Statistics of Streaming Sparse Regression},
year = {2014},
}
@inproceedings{ramanathan2014linking,
author = {Vignesh Ramanathan and Armand Joulin and Percy Liang and Li Fei-Fei},
booktitle = {European Conference on Computer Vision (ECCV)},
title = {Linking people with "their" names using coreference resolution},
year = {2014},
}
@article{liang2014talking,
author = {Percy Liang},
journal = {XRDS: Crossroads, The ACM Magazine for Students},
number = {1},
pages = {18--21},
title = {Talking to computers in natural language},
volume = {21},
year = {2014},
}
@inproceedings{berant2014paraphrasing,
author = {Jonathan Berant and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Semantic Parsing via Paraphrasing},
year = {2014},
}
@inproceedings{pasupat2014extraction,
author = {Panupong Pasupat and Percy Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Zero-shot Entity Extraction from Web Pages},
year = {2014},
}
@inproceedings{chaganty2014graphical,
author = {Arun Chaganty and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Estimating Latent-Variable Graphical Models using Moments and Likelihoods},
year = {2014},
}
@inproceedings{steinhardt2014eg,
author = {Jacob Steinhardt and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Adaptivity and Optimism: An Improved Exponentiated Gradient Algorithm},
year = {2014},
}
@inproceedings{wager2014altitude,
author = {Stefan Wager and Will Fithian and Sida I. Wang and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Altitude Training: Strong Bounds for Single-Layer Dropout},
year = {2014},
}
@inproceedings{frostig2014lowrank,
author = {Roy Frostig and Sida I. Wang and Percy Liang and Chris Manning},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Simple {MAP} inference via low-rank relaxations},
year = {2014},
}
@inproceedings{wang2014iqp,
author = {Sida I. Wang and Roy Frostig and Percy Liang and Chris Manning},
booktitle = {International Conference on Learning Representations Workshop (ICLR)},
title = {Relaxations for inference in restricted {B}oltzmann machines},
year = {2014},
}
@inproceedings{berant2013freebase,
author = {Jonathan Berant and Andrew Chou and Roy Frostig and Percy Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Semantic Parsing on {F}reebase from Question-Answer Pairs},
year = {2013},
}
@inproceedings{wang2013noising,
author = {Sida I. Wang and Mengqiu Wang and Stefan Wager and Percy Liang and Chris Manning},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Feature Noising for Log-linear Structured Prediction},
year = {2013},
}
@inproceedings{wager2013dropout,
author = {Stefan Wager and Sida I. Wang and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Dropout Training as Adaptive Regularization},
year = {2013},
}
@inproceedings{chaganty13regression,
author = {Arun Chaganty and Percy Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Spectral Experts for Estimating Mixtures of Linear Regressions},
year = {2013},
}
@inproceedings{ramanathan2013event,
author = {Vignesh Ramanathan and Percy Liang and Li Fei-Fei},
booktitle = {International Conference on Computer Vision (ICCV)},
title = {Video Event Understanding using Natural Language Descriptions},
year = {2013},
}
@inproceedings{sharma13algebraic,
author = {Rahul Sharma and Saurabh Gupta and Bharath Hariharan and Alex Aiken and Percy Liang and Aditya V. Nori},
booktitle = {European Symposium on Programming (ESOP)},
title = {A Data Driven Approach for Algebraic Loop Invariants},
year = {2013},
}
This paper explores unsupervised learning of parsing models along two directions. First, which models are identifiable from infinite data? We use a general technique for numerically checking identifiability based on the rank of a Jacobian matrix, and apply it to several standard constituency and dependency parsing models. Second, for identifiable models, how do we estimate the parameters efficiently? EM suffers from local optima, while recent work using spectral methods cannot be directly applied since the topology of the parse tree varies across sentences. We develop a strategy, unmixing, which deals with this additional complexity for restricted classes of parsing models.
@inproceedings{hsu12identifiability,
author = {Daniel Hsu and Sham M. Kakade and Percy Liang},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Identifiability and Unmixing of Latent Parse Trees},
year = {2012},
}
Compositional question answering begins by mapping questions to logical forms, but training a semantic parser to perform this mapping typically requires the costly annotation of the target logical forms. In this paper, we learn to map questions to answers via latent logical forms, which are induced automatically from question-answer pairs. In tackling this challenging learning problem, we introduce a new semantic representation which highlights a parallel between dependency syntax and efficient evaluation of logical forms. On two standard semantic parsing benchmarks (GEO and JOBS), our system obtains the highest published accuracies, despite using less supervision than existing systems.
Task: learn to map questions to answers via latent logical forms.
Contribution: new tree-based semantic representation.
Result: surpass state-of-the-art on semantic parsing with less supervision.
@inproceedings{liang11dcs,
author = {Percy Liang and Michael I. Jordan and Dan Klein},
booktitle = {Association for Computational Linguistics (ACL)},
pages = {590--599},
title = {Learning Dependency-Based Compositional Semantics},
year = {2011},
}
Many static analyses do not scale as they are made more precise. For example, increasing the amount of context sensitivity in a k-limited pointer analysis causes the number of contexts to grow exponentially with k. Iterative refinement techniques can mitigate this growth by starting with a coarse abstraction and only refining parts of the abstraction that are deemed relevant with respect to a given client.
In this paper, we introduce a new technique called pruning that uses client feedback in a different way. The basic idea is to use coarse abstractions to prune away parts of the program analysis deemed irrelevant for proving a client query, and then using finer abstractions on the sliced program analysis. For a k-limited pointer analysis, this approach amounts to adaptively refining and pruning a set of prefix patterns representing the contexts relevant for the client. By pruning, we are able to scale up to much more expensive abstractions than before. We also prove that the pruned analysis is both sound and complete, that is, it yields the same results as an analysis that uses a more expensive abstraction directly without pruning.
Idea: run cheap analysis, use client feedback to prune away irrelvant parts of program analysis (think program slicing); then run expensive analysis.
Theoretical result: pruning is sound and complete.
Empirical result: we can use much richer $k$-object-sensitivity abstractions.
@inproceedings{liang11pruning,
author = {Percy Liang and Mayur Naik},
booktitle = {Programming Language Design and Implementation (PLDI)},
title = {Scaling up Abstraction Refinement via Pruning},
year = {2011},
}
Static analyses are generally parametrized by an abstraction which is chosen from a family of abstractions. We are interested in flexible families of abstractions with many parameters, as these families can allow one to increase precision in ways tailored to the client without sacrificing scalability. For example, we consider k-limited points-to analyses where each call site and allocation site in a program can have a different k value. We then ask a natural question in this paper: What is the minimal (coarsest) abstraction in a given family which is able to prove a set of queries? In addressing this question, we make the following two contributions: (i) We introduce two machine learning algorithms for efficiently finding a minimal abstraction; and (ii) for a static race detector backed by a k-limited points-to analysis, we show empirically that minimal abstractions are actually quite coarse: It suffices to provide context/object sensitivity to a very small fraction (0.4--2.3%) of the sites to yield equally precise results as providing context/object sensitivity uniformly to all sites.
Question: how small is the smallest abstraction needed to prove a query?
Empirical answer: very small (less than 2.5% sites need to be treated context-sensitively for k-limited analyses for race detection).
Found this answer using a new machine learning algorithm that exploits this sparsity.
@inproceedings{liang11minimal,
author = {Percy Liang and Omer Tripp and Mayur Naik},
booktitle = {Principles of Programming Languages (POPL)},
title = {Learning Minimal Abstractions},
year = {2011},
}
@inproceedings{golland2010pragmatics,
author = {Dave Golland and Percy Liang and Dan Klein},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
pages = {410--419},
title = {A Game-theoretic Approach to Generating Spatial Descriptions},
year = {2010},
}
We present a simple, robust generation system which performs content selection and surface realization in a unified, domain-independent framework. In our approach, we break up the end-to-end generation process into a sequence of local decisions, arranged hierarchically and each trained discriminatively. We deployed our system in three different domains---Robocup sportscasting, technical weather forecasts, and common weather forecasts, obtaining results comparable to state-of-the-art domain-specific systems both in terms of BLEU scores and human evaluation.
Model natural language generation as a sequence of local decisions, each backed by a log-linear model.
Advantage: can use arbitrary expressive features, works across multiple domains.
@inproceedings{angeli10generation,
author = {Gabor Angeli and Percy Liang and Dan Klein},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {A Simple Domain-Independent Probabilistic Approach to Generation},
year = {2010},
}
The quality of a static analysis of heap-manipulating programs is largely determined by its heap abstraction. Object allocation sites are a commonly-used abstraction, but are too coarse for some clients. The goal of this paper is to investigate how various refinements of allocation sites can improve precision. In particular, we consider abstractions that use call stack, object recency, and heap connectivity information. We measure the precision of these abstractions dynamically for four different clients motivated by concurrency and on nine Java programs chosen from the DaCapo benchmark suite. Our dynamic results shed new light on aspects of heap abstractions that matter for precision, which allows us to more effectively navigate the large space of possible heap abstractions.
Question: what aspects of a heap abstraction matter?
Methodology: run program (9 DaCapo benchmarks) dynamically, compute static heap abstractions (3 dimensions of refinement: context sensitivity, object recency, and shape analysis), answer client queries (4 clients based on concurrency).
@inproceedings{liang10abstraction,
author = {Percy Liang and Omer Tripp and Mayur Naik and Mooly Sagiv},
booktitle = {Object-Oriented Programming, Systems, Languages, and Applications (OOPSLA)},
title = {A Dynamic Evaluation of Static Heap Abstractions},
year = {2010},
}
We are interested in learning programs for multiple related tasks given only a few training examples per task. Since the program for a single task is underdetermined by its data, we introduce a nonparametric hierarchical Bayesian prior over programs which shares statistical strength across multiple tasks. The key challenge is to parametrize this multi-task sharing. For this, we introduce a new representation of programs based on combinatory logic and provide an MCMC algorithm that can perform safe program transformations on this representation to reveal shared inter-program substructures.
Programs are trees, subprograms are subtrees, which can be shared across tasks. Combinators refactor programs to expose the appropriate subprograms.
@inproceedings{liang10programs,
author = {Percy Liang and Michael I. Jordan and Dan Klein},
booktitle = {International Conference on Machine Learning (ICML)},
pages = {639--646},
title = {Learning Programs: A Hierarchical {B}ayesian Approach},
year = {2010},
}
A learning problem might have several measures of complexity (e.g., norm and dimensionality) that affect the generalization error. What is the interaction between these complexities? Dimension-free learning theory bounds and parametric asymptotic analyses each provide a partial picture of the full learning curve. In this paper, we use high-dimensional asymptotics on two classical problems---mean estimation and linear regression---to explore the learning curve more completely. We show that these curves exhibit multiple regimes, where in each regime, the excess risk is controlled by a subset of the problem complexities.
Goal: understand excess risk as a function of sample size and problem complexity. On simple examples, show that asymptotic risk has multiple regimes, each controlled by different complexities.
@inproceedings{liang10regimes,
author = {Percy Liang and Nati Srebro},
booktitle = {International Conference on Machine Learning (ICML)},
title = {On the Interaction between Norm and Dimensionality: Multiple Regimes in Learning},
year = {2010},
}
Most existing algorithms for learning latent-variable models---such as EM and existing Gibbs samplers---are token-based, meaning that they update the variables associated with one sentence at a time. The incremental nature of these methods makes them susceptible to local optima/slow mixing. In this paper, we introduce a type-based sampler, which updates a block of variables, identified by a type, which spans multiple sentences. We show improvements on part-of-speech induction, word segmentation, and learning tree-substitution grammars.
NLP perspective: goal is to avoid local optima by processing all tokens associated with a type at once instead of one token or sentence at a time.
Sampling perspective: new type of block sampling that exploits exchangeability.
@inproceedings{liang10type,
author = {Percy Liang and Michael I. Jordan and Dan Klein},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {Type-Based {MCMC}},
year = {2010},
}
Many types of regularization schemes have been employed in statistical learning, each one motivated by some assumption about the problem domain. In this paper, we present a unified asymptotic analysis of smooth regularizers, which allows us to see how the validity of these assumptions impacts the success of a particular regularizer. In addition, our analysis motivates an algorithm for optimizing regularization parameters, which in turn can be analyzed within our framework. We apply our analysis to several examples, including hybrid generative-discriminative learning and multi-task learning.
Setting: estimator defined by minimizing loss plus regularization.
Question: what is the best regularizer to use?
This is hard to optimize, so use a Taylor expansion instead, yielding a interpretable closed form solution.
@inproceedings{liang09regularization,
author = {Percy Liang and Francis Bach and Guillaume Bouchard and Michael I. Jordan},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Asymptotically Optimal Regularization in Smooth Parametric Models},
year = {2009},
}
Probabilistic context-free grammars (PCFGs) have played an important role in the modeling of syntax in natural language processing and other applications, but choosing the proper model complexity is often difficult. We present a nonparametric Bayesian generalization of the PCFG based on the hierarchical Dirichlet process (HDP). In our HDP-PCFG model, the effective complexity of the grammar can grow with increasing data. We describe an efficient variational inference algorithm for our model and present experiments on both a synthetic grammar induction task and a large-scale natural language parsing task.
Details of the EMNLP 2007 paper + general background, empirical intuitions, and derivations for structured mean-field + a small grammar induction experiment.
@incollection{liang09hdppcfg,
author = {Percy Liang and Michael I. Jordan and Dan Klein},
booktitle = {The Oxford Handbook of Applied Bayesian Analysis},
title = {Probabilistic grammars and hierarchical {D}irichlet processes},
year = {2009},
}
A central problem in grounded language acquisition is learning the correspondences between a rich world state and a stream of text which references that world state. To deal with the high degree of ambiguity present in this setting, we present a generative model that simultaneously segments the text into utterances and maps each utterance to a meaning representation grounded in the world state. We show that our model generalizes across three domains of increasing difficulty---Robocup sportscasting, weather forecasts (a new domain), and NFL recaps.
Stuff happens in the world. A text talks about it. Our goal: learn the correspondence between the two.
Approach: probabilistic model capturing identification of entities/events in the world, segmentation of the text, and alignment between the two.
@inproceedings{liang09semantics,
author = {Percy Liang and Michael I. Jordan and Dan Klein},
booktitle = {Association for Computational Linguistics and International Joint Conference on Natural Language Processing (ACL-IJCNLP)},
pages = {91--99},
title = {Learning Semantic Correspondences with Less Supervision},
year = {2009},
}
Given a model family and a set of unlabeled examples, one could either label specific examples or state general constraints---both provide information about the desired model. In general, what is the most cost-effective way to learn? To address this question, we introduce measurements, a general class of mechanisms for providing information about a target model. We present a Bayesian decision-theoretic framework, which allows us to both integrate diverse measurements and choose new measurements to make. We use a variational inference algorithm, which exploits exponential family duality. The merits of our approach are demonstrated on two sequence labeling tasks.
Goal: learning with minimum human effort.
Things human can do: label data, provide constraints---in general, make measurements.
Use Bayesian decision theory to choose optimal measurements.
@inproceedings{liang09measurements,
author = {Percy Liang and Michael I. Jordan and Dan Klein},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Learning from Measurements in Exponential Families},
year = {2009},
}
The (batch) EM algorithm plays an important role in unsupervised induction, but it sometimes suffers from slow convergence. In this paper, we show that online variants (1) provide significant speedups and (2) can even find better solutions than those found by batch EM. We support these findings on four unsupervised tasks: part-of-speech tagging, document classification, word segmentation, and word alignment.
What you'd expect: online is faster than batch.
What you might not expect: online gets better accuarcy than batch.
@inproceedings{liang09online,
author = {Percy Liang and Dan Klein},
booktitle = {North American Association for Computational Linguistics (NAACL)},
pages = {611--619},
title = {Online {EM} for Unsupervised Models},
year = {2009},
}
Statistical and computational concerns have motivated parameter estimators based on various forms of likelihood, e.g., joint, conditional, and pseudolikelihood. In this paper, we present a unified framework for studying these estimators, which allows us to compare their relative (statistical) efficiencies. Our asymptotic analysis suggests that modeling more of the data tends to reduce variance, but at the cost of being more sensitive to model misspecification. We present experiments validating our analysis.
Derive general expression for the asymptotic risk of composite likelihood estimators in exponential families.
This allows us to compare the various estimators.
@inproceedings{liang08asymptotics,
author = {Percy Liang and Michael I. Jordan},
booktitle = {International Conference on Machine Learning (ICML)},
pages = {584--591},
title = {An Asymptotic Analysis of Generative, Discriminative, and Pseudolikelihood Estimators},
year = {2008},
}
Structured models often achieve excellent performance but can be slow at test time. We investigate structure compilation, where we replace structure with features, which are often computationally simpler but unfortunately statistically more complex. We analyze this tradeoff theoretically and empirically on three natural language processing tasks. We also introduce a simple method to transfer predictive power from structure to features via unlabeled data, while incurring a minimal statistical penalty.
How much do we lose by throwing out edge features in CRFs and adding node features?
Studies the approximation, estimation, computational aspects of the tradeoff.
@inproceedings{liang08structure,
author = {Percy Liang and Hal {Daum{é} III} and Dan Klein},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Structure Compilation: Trading Structure for Features},
year = {2008},
}
We identify four types of errors that unsupervised induction systems make and study each one in turn. Our contributions include (1) using a meta-model to analyze the incorrect biases of a model in a systematic way, (2) providing an efficient and robust method of measuring distance between two parameter settings of a model, and (3) showing that local optima issues which typically plague EM can be somewhat alleviated by increasing the number of training examples. We conduct our analyses on three models: the HMM, the PCFG, and a simple dependency model.
Error decomposition: approximation, identifiability, estimation, optimization errors.
Used meta-model to analyze approximation error.
Empirically observed that more data reduces optimization error.
@inproceedings{liang08errors,
author = {Percy Liang and Dan Klein},
booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
title = {Analyzing the Errors of Unsupervised Learning},
year = {2008},
}
We present a method for learning bilingual translation lexicons from monolingual corpora. Word types in each language are characterized by purely monolingual features, such as context counts and orthographic substrings. Translations are induced using a generative model based on canonical correlation analysis, which explains the monolingual lexicons in terms of latent matchings. We show that high-precision lexicons can be learned in a variety of language pairs and from a range of corpus types.
By using CCA, can do word alignment without the usual sentence-aligned corpora.
@inproceedings{haghighi08lexicon,
author = {Aria Haghighi and Percy Liang and Taylor Berg-Kirkpatrick and Dan Klein},
booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
title = {Learning Bilingual Lexicons from Monolingual Corpora},
year = {2008},
}
The learning of probabilistic models with many hidden variables
and non-decomposable dependencies is an important and challenging problem.
In contrast to traditional approaches based on approximate inference in a single
intractable model, our approach is to train a set of tractable
submodels by encouraging them to agree on the hidden variables. This allows
us to capture non-decomposable aspects of the data while still maintaining
tractability. We propose an objective function for our approach,
derive EM-style algorithms for parameter estimation, and demonstrate their
effectiveness on three challenging real-world learning tasks.
Setting: unsupervised learning.
Alternative to approximate inference: make two tractable models and train them to agree.
Advantage: maintain existing tractable inference procedures as black-boxes.
@inproceedings{liang08agreement,
author = {Percy Liang and Dan Klein and Michael I. Jordan},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {Agreement-Based Learning},
year = {2008},
}
We present a probabilistic approach to language change in which word forms are
represented by phoneme sequences that undergo stochastic edits along the
branches of a phylogenetic tree. This framework combines the
advantages of the classical comparative method with the robustness
of corpus-based probabilistic models. We use this framework to
explore the consequences of two different schemes for defining
probabilistic models of phonological change, evaluating these
schemes by reconstructing ancient word forms of Romance languages.
The result is an efficient inference procedure for automatically
inferring ancient word forms from modern languages, which can be
generalized to support inferences about linguistic phylogenies.
Feature-based generative model of phonemes of words in a phylogeny of languages.
@inproceedings{bouchard08language,
author = {Alexandre Bouchard-Côté and Percy Liang and Tom Griffiths and Dan Klein},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
title = {A Probabilistic Approach to Language Change},
year = {2008},
}
@inproceedings{liang07tutorial,
author = {Percy Liang and Dan Klein},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Structured {B}ayesian Nonparametric Models with Variational Inference (tutorial)},
year = {2007},
}
We introduce a new inference algorithm for Dirichlet process mixture
models. While Gibbs sampling and variational methods focus on local
moves, the new algorithm makes more global moves. This is done by
introducing a permutation of the data points as an auxiliary variable.
The algorithm is a blocked sampler which alternates between sampling the
clustering and sampling the permutation. The key to the efficiency of
this approach is that it is possible to use dynamic programming to
consider all exponentially many clusterings consistent with a given
permutation. We also show that random projections can be used to
effectively sample the permutation. The result is a stochastic
hill-climbing algorithm that yields burn-in times significantly
smaller than those of collapsed Gibbs sampling.
Task: clustering.
Idea: conditioned on a permutation of the data points, one can consider all possible clusterings of those data points which are consistent with the permutation using dynamic programming.
Idea: if data are ordered, can cluster using dynamic programming.
Let this ordering be a random auxiliary variable and we get a sampler.
@inproceedings{liang07permdp,
author = {Percy Liang and Michael I. Jordan and Ben Taskar},
booktitle = {International Conference on Machine Learning (ICML)},
title = {A permutation-augmented sampler for {D}irichlet process mixture models},
year = {2007},
}
We present a nonparametric Bayesian model of tree structures based on the
hierarchical Dirichlet process (HDP). Our HDP-PCFG model allows the complexity
of the grammar to grow as more training data is available. In addition to
presenting a fully Bayesian model for the PCFG, we also develop an efficient
variational inference procedure. On synthetic data, we recover the correct
grammar without having to specify its complexity in advance. We also show that
our techniques can be applied to full-scale parsing applications by
demonstrating its effectiveness in learning state-split grammars.
A PCFG with an infinite number of states.
Learning: variational inference.
@inproceedings{liang07infpcfg,
author = {Percy Liang and Slav Petrov and Michael I. Jordan and Dan Klein},
booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
title = {The Infinite {PCFG} using Hierarchical {D}irichlet Processes},
year = {2007},
}
We present a probabilistic model of diachronic phonology in which individual
word forms undergo stochastic edits along the branches of a phylogenetic tree.
Our approach allows us to achieve three goals with a single unified
model: (1) reconstruction of both ancient and modern word forms, (2) discovery
of general phonological changes, and (3) selection among different
phylogenies. We learn our model using a Monte Carlo EM algorithm and present
quantitative results validating the model.
Generative model of phonemes of words in a phylogeny of languages
@inproceedings{bouchard07diachronic,
author = {Alexandre Bouchard-Côté and Percy Liang and Tom Griffiths and Dan Klein},
booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
title = {A Probabilistic Approach to Diachronic Phonology},
year = {2007},
}
We present a perceptron-style discriminative approach to machine
translation in which large feature sets can be exploited. Unlike
discriminative reranking approaches, our system can take advantage of learned
features in all stages of decoding. We first discuss several challenges to
error-driven discriminative approaches. In particular, we explore different
ways of updating parameters given a training example. We find that making
frequent but smaller updates is preferable to making fewer but larger updates.
Then, we discuss an array of features and show both how they quantitatively
increase BLEU score and how they qualitatively interact on specific examples.
One particular feature we investigate is a novel way to introduce learning into
the initial phrase extraction process, which has previously been entirely
heuristic.
Task: machine translation.
Idea: treat machine translation as a structured classification task (learn a map from input sentence to output sentence). Use a Perceptron-like algorithm: decode and update towards maximum BLEU scoring translation on the n-best list.
@inproceedings{liang06discrimative,
author = {Percy Liang and Alexandre Bouchard-Côté and Dan Klein and Ben Taskar},
booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
title = {An End-to-End Discriminative Approach to Machine Translation},
year = {2006},
}
We present an unsupervised approach to symmetric
word alignment in which two simple asymmetric models are
trained jointly to maximize a
combination of data likelihood and agreement between the models.
Compared to the standard practice of intersecting predictions of
independently-trained models, joint training provides a 32% reduction
in AER. Moreover, a simple and efficient pair of HMM aligners
provides a 29% reduction in AER over symmetrized IBM model 4
predictions.
Task: unsupervised word alignment.
Idea: Jointly train two HMM models (one in each direction) to encourage agreement. Uses a simple EM-like algorithm for training.
Result: performance competitive with supervised methods (4.9 AER on Hansards).
@inproceedings{liang06alignment,
author = {Percy Liang and Ben Taskar and Dan Klein},
booktitle = {North American Association for Computational Linguistics (NAACL)},
pages = {104--111},
title = {Alignment by Agreement},
year = {2006},
}
Task: named-entity recognition and Chinese word segmentation
Idea: create features based on unlabeled data to use in Perceptron learning in Markov or semi-Markov models
@mastersthesis{liang05meng,
author = {Percy Liang},
school = {Massachusetts Institute of Technology},
title = {Semi-Supervised Learning for Natural Language},
year = {2005},
}
We introduce the first definition of hyperacyclicity for hypergraphs, a generalization of acyclicity in graphs.
We provide a dynamic data structure for maintaining hyperacyclicity, a generalization of Tarjan's Union-Find algorithm.
@techreport{liang05hypercycle,
author = {Percy Liang and Nathan Srebro},
institution = {Massachusetts Institute of Technology},
title = {A Data Structure for Maintaining Acyclicity in Hypergraphs},
year = {2005},
}
@inproceedings{liang05mcmaster,
author = {Percy Liang and Nathan Srebro},
booktitle = {Mathematical Programing for Data Mining and Machine Learning Workshop at McMaster University},
title = {Linear Programming in Bounded Tree-width {M}arkov Networks},
year = {2005},
}
In parsing sequences using dynamic programming, the subproblems are continguous subsequences (quadratic in number of terminals). In parsing documents or images, the subproblems would be subsets of the terminals (exponential in number of terminals). We introduce (and unify) several ways to constrain these subsets using the geometric structure of the terminals.
@inproceedings{liang05geometric,
author = {Percy Liang and Mukund Narasimhan and Michael Shilman and Paul Viola},
booktitle = {International Conference on Document Analysis and Recognition (ICDAR)},
title = {Efficient Geometric Algorithms for Parsing in Two Dimensions},
year = {2005},
}
Use a greedy procedure to find the maximum likelihood (or MDL) bounded tree-width Markov network (for tree-width 1, equivalent to Chow-Liu maximum spanning trees).
@techreport{liang04markov,
author = {Percy Liang and Nathan Srebro},
institution = {Massachusetts Institute of Technology},
title = {Methods and Experiments With Bounded Tree-width {M}arkov Networks},
year = {2004},
}
Use linear programming to find worst case inputs to a dynamic program in order to explore the tightness of a bound for approximating maximum weight hypertrees with windmill farms.
@techreport{liang03maxwmfarm,
author = {Percy Liang and Nathan Srebro},
institution = {Massachusetts Institute of Technology},
title = {How Much Of A Hypertree Can Be Captured By Windmills?},
year = {2003},
}