@article{steinhardt2017certified,
author = {J. Steinhardt and P. W. Koh and P. Liang},
journal = {arXiv preprint arXiv:1706.03691},
title = {Certified Defenses for Data Poisoning Attacks},
year = {2017},
}
@inproceedings{koh2017understanding,
author = {P. W. Koh and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Understanding Black-box Predictions via Influence Functions},
year = {2017},
}
@inproceedings{zhang2017convexified,
author = {Y. Zhang and P. Liang and M. J. Wainwright},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Convexified Convolutional Neural Networks},
year = {2017},
}
Developing bug-free machine learning systems with formal mathematics.
Daniel Selsam, Percy Liang, David Dill.
International Conference on Machine Learning (ICML), 2017.
@inproceedings{selsam2017bugfree,
author = {D. Selsam and P. Liang and D. Dill},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Developing Bug-Free Machine Learning Systems With Formal Mathematics},
year = {2017},
}
World of bits: an open-domain platform for web-based agents.
Tianlin Shi, Andrej Karpathy, Linxi Fan, Jonathan Hernandez, Percy Liang.
International Conference on Machine Learning (ICML), 2017.
@inproceedings{shi2017wob,
author = {T. Shi and A. Karpathy and L. Fan and J. Hernandez and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {World of Bits: An Open-Domain Platform for Web-Based Agents},
year = {2017},
}
@inproceedings{zhang2017hitting,
author = {Y. Zhang and P. Liang and M. Charikar},
booktitle = {Conference on Learning Theory (COLT)},
title = {A Hitting Time Analysis of Stochastic Gradient {L}angevin Dynamics},
year = {2017},
}
@article{kakade2017prediction,
author = {S. Kakade and P. Liang and V. Sharan and G. Valiant},
journal = {arXiv preprint arXiv:1612.02526},
title = {Prediction with a Short Memory},
year = {2017},
}
@inproceedings{bastani2017synthesizing,
author = {O. Bastani and R. Sharma and A. Aiken and P. Liang},
booktitle = {Programming Language Design and Implementation (PLDI)},
title = {Synthesizing Program Input Grammars},
year = {2017},
}
@inproceedings{wang2017naturalizing,
author = {S. I. Wang and S. Ginn and P. Liang and C. D. Manning},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Naturalizing a Programming Language via Interactive Learning},
year = {2017},
}
@inproceedings{he2017symmetric,
author = {H. He and A. Balakrishnan and M. Eric and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Learning Symmetric Collaborative Dialogue Agents with Dynamic Knowledge Graph Embeddings},
year = {2017},
}
@inproceedings{guu2017bridging,
author = {K. Guu and P. Pasupat and E. Z. Liu and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {From Language to Programs: Bridging Reinforcement Learning and Maximum Marginal Likelihood},
year = {2017},
}
@inproceedings{steinhardt2016risk,
author = {J. Steinhardt and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Unsupervised Risk Estimation Using Only Conditional Independence Structure},
year = {2016},
}
@inproceedings{rajpurkar2016squad,
author = {P. Rajpurkar and J. Zhang and K. Lopyrev and P. Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {SQuAD: 100,000+ Questions for Machine Comprehension of Text},
year = {2016},
}
@inproceedings{wang2016games,
author = {S. I. Wang and P. Liang and C. Manning},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Learning Language Games through Interaction},
year = {2016},
}
@inproceedings{jia2016recombination,
author = {R. Jia and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Data Recombination for Neural Semantic Parsing},
year = {2016},
}
@inproceedings{long2016projections,
author = {R. Long and P. Pasupat and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Simpler Context-Dependent Logical Forms via Model Projections},
year = {2016},
}
@inproceedings{pasupat2016inferring,
author = {P. Pasupat and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Inferring Logical Forms From Denotations},
year = {2016},
}
@inproceedings{khani2016unanimity,
author = {F. Khani and M. Rinard and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Unanimous Prediction for 100% Precision with Application to Learning Semantic Mappings},
year = {2016},
}
@inproceedings{chaganty2016perspectives,
author = {A. T. Chaganty and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {How Much is 131 Million Dollars? {P}utting Numbers in Perspective with Compositional Descriptions},
year = {2016},
}
@inproceedings{raghunathan2016linear,
author = {A. Raghunathan and R. Frostig and J. Duchi and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Estimation from Indirect Supervision with Linear Moments},
year = {2016},
}
@incollection{wager2016levy,
author = {S. Wager and W. Fithian and P. Liang},
booktitle = {Perturbations, Optimization and Statistics},
title = {Data Augmentation via {L}évy Processes},
year = {2016},
}
@article{liang2016executable,
author = {P. Liang},
journal = {Communications of the ACM},
title = {Learning Executable Semantic Parsers for Natural Language Understanding},
volume = {59},
year = {2016},
}
@article{berant2015agenda,
author = {J. Berant and P. Liang},
journal = {Transactions of the Association for Computational Linguistics (TACL)},
pages = {545--558},
title = {Imitation Learning of Agenda-Based Semantic Parsers},
volume = {3},
year = {2015},
}
@inproceedings{steinhardt2015relaxed,
author = {J. Steinhardt and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Learning with Relaxed Supervision},
year = {2015},
}
@inproceedings{wang2015polynomial,
author = {S. I. Wang and A. Chaganty and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Estimating Mixture Models via Mixture of Polynomials},
year = {2015},
}
@inproceedings{werling2015onthejob,
author = {K. Werling and A. Chaganty and P. Liang and C. Manning},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {On-the-Job Learning with {B}ayesian Decision Theory},
year = {2015},
}
@inproceedings{kuleshov2015calibrated,
author = {V. Kuleshov and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Calibrated Structured Prediction},
year = {2015},
}
@inproceedings{guu2015traversing,
author = {K. Guu and J. Miller and P. Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Traversing Knowledge Graphs in Vector Space},
year = {2015},
}
@inproceedings{pasupat2015compositional,
author = {P. Pasupat and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Compositional Semantic Parsing on Semi-Structured Tables},
year = {2015},
}
@inproceedings{misra2015environment,
author = {D. K. Misra and K. Tao and P. Liang and A. Saxena},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Environment-Driven Lexicon Induction for High-Level Instructions},
year = {2015},
}
@inproceedings{steinhardt2015rcm,
author = {J. Steinhardt and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Reified Context Models},
year = {2015},
}
@inproceedings{shi2015sample,
author = {T. Shi and J. Steinhardt and P. Liang},
booktitle = {Artificial Intelligence and Statistics (AISTATS)},
pages = {875--884},
title = {Learning Where To Sample in Structured Prediction},
year = {2015},
}
@inproceedings{kuleshov2015tensor,
author = {V. Kuleshov and A. Chaganty and P. Liang},
booktitle = {Artificial Intelligence and Statistics (AISTATS)},
title = {Tensor factorization via matrix factorization},
year = {2015},
}
@inproceedings{wang2015overnight,
author = {Y. Wang and J. Berant and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Building a Semantic Parser Overnight},
year = {2015},
}
@article{liang2014talking,
author = {P. Liang},
journal = {XRDS: Crossroads, The ACM Magazine for Students},
number = {1},
pages = {18--21},
title = {Talking to computers in natural language},
volume = {21},
year = {2014},
}
@inproceedings{berant2014paraphrasing,
author = {J. Berant and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Semantic Parsing via Paraphrasing},
year = {2014},
}
@inproceedings{pasupat2014extraction,
author = {P. Pasupat and P. Liang},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Zero-shot Entity Extraction from Web Pages},
year = {2014},
}
@inproceedings{chaganty2014graphical,
author = {A. Chaganty and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Estimating Latent-Variable Graphical Models using Moments and Likelihoods},
year = {2014},
}
@inproceedings{steinhardt2014eg,
author = {J. Steinhardt and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Adaptivity and Optimism: An Improved Exponentiated Gradient Algorithm},
year = {2014},
}
@inproceedings{steinhardt2014filtering,
author = {J. Steinhardt and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
pages = {727--735},
title = {Filtering with Abstract Particles},
year = {2014},
}
@inproceedings{wager2014altitude,
author = {S. Wager and W. Fithian and S. I. Wang and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Altitude Training: Strong Bounds for Single-Layer Dropout},
year = {2014},
}
@inproceedings{frostig2014lowrank,
author = {R. Frostig and S. I. Wang and P. Liang and C. Manning},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Simple {MAP} inference via low-rank relaxations},
year = {2014},
}
@inproceedings{wang2014iqp,
author = {S. I. Wang and R. Frostig and P. Liang and C. Manning},
booktitle = {International Conference on Learning Representations Workshop (ICLR)},
title = {Relaxations for inference in restricted {B}oltzmann machines},
year = {2014},
}
@inproceedings{ramanathan2014linking,
author = {V. Ramanathan and A. Joulin and P. Liang and L. Fei-Fei},
booktitle = {European Conference on Computer Vision (ECCV)},
title = {Linking people with "their" names using coreference resolution},
year = {2014},
}
@article{steinhardt2014sparse,
author = {J. Steinhardt and S. Wager and P. Liang},
journal = {arXiv preprint arXiv:1412.4182},
title = {The Statistics of Streaming Sparse Regression},
year = {2014},
}
@inproceedings{berant2013freebase,
author = {J. Berant and A. Chou and R. Frostig and P. Liang},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Semantic Parsing on {F}reebase from Question-Answer Pairs},
year = {2013},
}
@inproceedings{wang2013noising,
author = {S. I. Wang and M. Wang and S. Wager and P. Liang and C. Manning},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {Feature Noising for Log-linear Structured Prediction},
year = {2013},
}
@inproceedings{wager2013dropout,
author = {S. Wager and S. I. Wang and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Dropout Training as Adaptive Regularization},
year = {2013},
}
@inproceedings{chaganty13regression,
author = {A. Chaganty and P. Liang},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Spectral Experts for Estimating Mixtures of Linear Regressions},
year = {2013},
}
@inproceedings{ramanathan2013event,
author = {V. Ramanathan and P. Liang and L. Fei-Fei},
booktitle = {International Conference on Computer Vision (ICCV)},
title = {Video Event Understanding using Natural Language Descriptions},
year = {2013},
}
@inproceedings{sharma13algebraic,
author = {R. Sharma and S. Gupta and B. Hariharan and A. Aiken and P. Liang and and A. V. Nori},
booktitle = {European Symposium on Programming (ESOP)},
title = {A Data Driven Approach for Algebraic Loop Invariants},
year = {2013},
}
This paper explores unsupervised learning of parsing models along two directions. First, which models are identifiable from infinite data? We use a general technique for numerically checking identifiability based on the rank of a Jacobian matrix, and apply it to several standard constituency and dependency parsing models. Second, for identifiable models, how do we estimate the parameters efficiently? EM suffers from local optima, while recent work using spectral methods cannot be directly applied since the topology of the parse tree varies across sentences. We develop a strategy, unmixing, which deals with this additional complexity for restricted classes of parsing models.
@inproceedings{hsu12identifiability,
author = {D. Hsu and S. M. Kakade and P. Liang},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Identifiability and Unmixing of Latent Parse Trees},
year = {2012},
}
Compositional question answering begins by mapping questions to logical forms, but training a semantic parser to perform this mapping typically requires the costly annotation of the target logical forms. In this paper, we learn to map questions to answers via latent logical forms, which are induced automatically from question-answer pairs. In tackling this challenging learning problem, we introduce a new semantic representation which highlights a parallel between dependency syntax and efficient evaluation of logical forms. On two standard semantic parsing benchmarks (GEO and JOBS), our system obtains the highest published accuracies, despite using less supervision than existing systems.
Task: learn to map questions to answers via latent logical forms.
Contribution: new tree-based semantic representation.
Result: surpass state-of-the-art on semantic parsing with less supervision.
@inproceedings{liang11dcs,
author = {P. Liang and M. I. Jordan and D. Klein},
booktitle = {Association for Computational Linguistics (ACL)},
pages = {590--599},
title = {Learning Dependency-Based Compositional Semantics},
year = {2011},
}
Many static analyses do not scale as they are made more precise. For example, increasing the amount of context sensitivity in a k-limited pointer analysis causes the number of contexts to grow exponentially with k. Iterative refinement techniques can mitigate this growth by starting with a coarse abstraction and only refining parts of the abstraction that are deemed relevant with respect to a given client.
In this paper, we introduce a new technique called pruning that uses client feedback in a different way. The basic idea is to use coarse abstractions to prune away parts of the program analysis deemed irrelevant for proving a client query, and then using finer abstractions on the sliced program analysis. For a k-limited pointer analysis, this approach amounts to adaptively refining and pruning a set of prefix patterns representing the contexts relevant for the client. By pruning, we are able to scale up to much more expensive abstractions than before. We also prove that the pruned analysis is both sound and complete, that is, it yields the same results as an analysis that uses a more expensive abstraction directly without pruning.
Idea: run cheap analysis, use client feedback to prune away irrelvant parts of program analysis (think program slicing); then run expensive analysis.
Theoretical result: pruning is sound and complete.
Empirical result: we can use much richer $k$-object-sensitivity abstractions.
@inproceedings{liang11pruning,
author = {P. Liang and M. Naik},
booktitle = {Programming Language Design and Implementation (PLDI)},
title = {Scaling up Abstraction Refinement via Pruning},
year = {2011},
}
Static analyses are generally parametrized by an abstraction which is chosen from a family of abstractions. We are interested in flexible families of abstractions with many parameters, as these families can allow one to increase precision in ways tailored to the client without sacrificing scalability. For example, we consider k-limited points-to analyses where each call site and allocation site in a program can have a different k value. We then ask a natural question in this paper: What is the minimal (coarsest) abstraction in a given family which is able to prove a set of queries? In addressing this question, we make the following two contributions: (i) We introduce two machine learning algorithms for efficiently finding a minimal abstraction; and (ii) for a static race detector backed by a k-limited points-to analysis, we show empirically that minimal abstractions are actually quite coarse: It suffices to provide context/object sensitivity to a very small fraction (0.4--2.3%) of the sites to yield equally precise results as providing context/object sensitivity uniformly to all sites.
Question: how small is the smallest abstraction needed to prove a query?
Empirical answer: very small (less than 2.5% sites need to be treated context-sensitively for k-limited analyses for race detection).
Found this answer using a new machine learning algorithm that exploits this sparsity.
@inproceedings{liang11minimal,
author = {P. Liang and O. Tripp and M. Naik},
booktitle = {Principles of Programming Languages (POPL)},
title = {Learning Minimal Abstractions},
year = {2011},
}
Language is sensitive to both semantic and pragmatic effects. To capture both effects, we model language use as a cooperative game between two players: a speaker, who generates an utterance, and a listener, who responds with an action. Specifically, we consider the task of generating spatial references to objects, wherein the listener must accurately identify an object described by the speaker. We show that a speaker model that acts optimally with respect to an explicit, embedded listener model substantially outperforms one that is trained to directly generate spatial descriptions.
Observation: people say things to be understood (pragmatics), not simply what's technically correct (semantics).
Goal: model language as a game between speaker and listener. Speaker maximizes utility (communictative success) with respect to its model of the listener.
@inproceedings{golland10pragmatics,
author = {D. Golland and P. Liang and D. Klein},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {A Game-Theoretic Approach to Generating Spatial Descriptions},
year = {2010},
}
We present a simple, robust generation system which performs content selection and surface realization in a unified, domain-independent framework. In our approach, we break up the end-to-end generation process into a sequence of local decisions, arranged hierarchically and each trained discriminatively. We deployed our system in three different domains---Robocup sportscasting, technical weather forecasts, and common weather forecasts, obtaining results comparable to state-of-the-art domain-specific systems both in terms of BLEU scores and human evaluation.
Model natural language generation as a sequence of local decisions, each backed by a log-linear model.
Advantage: can use arbitrary expressive features, works across multiple domains.
@inproceedings{angeli10generation,
author = {G. Angeli and P. Liang and D. Klein},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
title = {A Simple Domain-Independent Probabilistic Approach to Generation},
year = {2010},
}
The quality of a static analysis of heap-manipulating programs is largely determined by its heap abstraction. Object allocation sites are a commonly-used abstraction, but are too coarse for some clients. The goal of this paper is to investigate how various refinements of allocation sites can improve precision. In particular, we consider abstractions that use call stack, object recency, and heap connectivity information. We measure the precision of these abstractions dynamically for four different clients motivated by concurrency and on nine Java programs chosen from the DaCapo benchmark suite. Our dynamic results shed new light on aspects of heap abstractions that matter for precision, which allows us to more effectively navigate the large space of possible heap abstractions.
Question: what aspects of a heap abstraction matter?
Methodology: run program (9 DaCapo benchmarks) dynamically, compute static heap abstractions (3 dimensions of refinement: context sensitivity, object recency, and shape analysis), answer client queries (4 clients based on concurrency).
@inproceedings{liang10abstraction,
author = {P. Liang and O. Tripp and M. Naik and M. Sagiv},
booktitle = {Object-Oriented Programming, Systems, Languages, and Applications (OOPSLA)},
title = {A Dynamic Evaluation of Static Heap Abstractions},
year = {2010},
}
We are interested in learning programs for multiple related tasks given only a few training examples per task. Since the program for a single task is underdetermined by its data, we introduce a nonparametric hierarchical Bayesian prior over programs which shares statistical strength across multiple tasks. The key challenge is to parametrize this multi-task sharing. For this, we introduce a new representation of programs based on combinatory logic and provide an MCMC algorithm that can perform safe program transformations on this representation to reveal shared inter-program substructures.
Programs are trees, subprograms are subtrees, which can be shared across tasks. Combinators refactor programs to expose the appropriate subprograms.
@inproceedings{liang10programs,
author = {P. Liang and M. I. Jordan and D. Klein},
booktitle = {International Conference on Machine Learning (ICML)},
pages = {639--646},
title = {Learning Programs: A Hierarchical {B}ayesian Approach},
year = {2010},
}
A learning problem might have several measures of complexity (e.g., norm and dimensionality) that affect the generalization error. What is the interaction between these complexities? Dimension-free learning theory bounds and parametric asymptotic analyses each provide a partial picture of the full learning curve. In this paper, we use high-dimensional asymptotics on two classical problems---mean estimation and linear regression---to explore the learning curve more completely. We show that these curves exhibit multiple regimes, where in each regime, the excess risk is controlled by a subset of the problem complexities.
Goal: understand excess risk as a function of sample size and problem complexity. On simple examples, show that asymptotic risk has multiple regimes, each controlled by different complexities.
@inproceedings{liang10regimes,
author = {P. Liang and N. Srebro},
booktitle = {International Conference on Machine Learning (ICML)},
title = {On the Interaction between Norm and Dimensionality: Multiple Regimes in Learning},
year = {2010},
}
Most existing algorithms for learning latent-variable models---such as EM and existing Gibbs samplers---are token-based, meaning that they update the variables associated with one sentence at a time. The incremental nature of these methods makes them susceptible to local optima/slow mixing. In this paper, we introduce a type-based sampler, which updates a block of variables, identified by a type, which spans multiple sentences. We show improvements on part-of-speech induction, word segmentation, and learning tree-substitution grammars.
NLP perspective: goal is to avoid local optima by processing all tokens associated with a type at once instead of one token or sentence at a time.
Sampling perspective: new type of block sampling that exploits exchangeability.
@inproceedings{liang10type,
author = {P. Liang and M. I. Jordan and D. Klein},
booktitle = {North American Association for Computational Linguistics (NAACL)},
title = {Type-Based {MCMC}},
year = {2010},
}
Many types of regularization schemes have been employed in statistical learning, each one motivated by some assumption about the problem domain. In this paper, we present a unified asymptotic analysis of smooth regularizers, which allows us to see how the validity of these assumptions impacts the success of a particular regularizer. In addition, our analysis motivates an algorithm for optimizing regularization parameters, which in turn can be analyzed within our framework. We apply our analysis to several examples, including hybrid generative-discriminative learning and multi-task learning.
Setting: estimator defined by minimizing loss plus regularization.
Question: what is the best regularizer to use?
This is hard to optimize, so use a Taylor expansion instead, yielding a interpretable closed form solution.
@inproceedings{liang09regularization,
author = {P. Liang and F. Bach and G. Bouchard and M. I. Jordan},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Asymptotically Optimal Regularization in Smooth Parametric Models},
year = {2009},
}
Probabilistic context-free grammars (PCFGs) have played an important role in the modeling of syntax in natural language processing and other applications, but choosing the proper model complexity is often difficult. We present a nonparametric Bayesian generalization of the PCFG based on the hierarchical Dirichlet process (HDP). In our HDP-PCFG model, the effective complexity of the grammar can grow with increasing data. We describe an efficient variational inference algorithm for our model and present experiments on both a synthetic grammar induction task and a large-scale natural language parsing task.
Details of the EMNLP 2007 paper + general background, empirical intuitions, and derivations for structured mean-field + a small grammar induction experiment.
@incollection{liang09hdppcfg,
author = {P. Liang and M. I. Jordan and D. Klein},
booktitle = {The Oxford Handbook of Applied Bayesian Analysis},
title = {Probabilistic grammars and hierarchical {D}irichlet processes},
year = {2009},
}
A central problem in grounded language acquisition is learning the correspondences between a rich world state and a stream of text which references that world state. To deal with the high degree of ambiguity present in this setting, we present a generative model that simultaneously segments the text into utterances and maps each utterance to a meaning representation grounded in the world state. We show that our model generalizes across three domains of increasing difficulty---Robocup sportscasting, weather forecasts (a new domain), and NFL recaps.
Stuff happens in the world. A text talks about it. Our goal: learn the correspondence between the two.
Approach: probabilistic model capturing identification of entities/events in the world, segmentation of the text, and alignment between the two.
@inproceedings{liang09semantics,
author = {P. Liang and M. I. Jordan and D. Klein},
booktitle = {Association for Computational Linguistics and International Joint Conference on Natural Language Processing (ACL-IJCNLP)},
pages = {91--99},
title = {Learning Semantic Correspondences with Less Supervision},
year = {2009},
}
Given a model family and a set of unlabeled examples, one could either label specific examples or state general constraints---both provide information about the desired model. In general, what is the most cost-effective way to learn? To address this question, we introduce measurements, a general class of mechanisms for providing information about a target model. We present a Bayesian decision-theoretic framework, which allows us to both integrate diverse measurements and choose new measurements to make. We use a variational inference algorithm, which exploits exponential family duality. The merits of our approach are demonstrated on two sequence labeling tasks.
Goal: learning with minimum human effort.
Things human can do: label data, provide constraints---in general, make measurements.
Use Bayesian decision theory to choose optimal measurements.
@inproceedings{liang09measurements,
author = {P. Liang and M. I. Jordan and D. Klein},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Learning from Measurements in Exponential Families},
year = {2009},
}
The (batch) EM algorithm plays an important role in unsupervised induction, but it sometimes suffers from slow convergence. In this paper, we show that online variants (1) provide significant speedups and (2) can even find better solutions than those found by batch EM. We support these findings on four unsupervised tasks: part-of-speech tagging, document classification, word segmentation, and word alignment.
What you'd expect: online is faster than batch.
What you might not expect: online gets better accuarcy than batch.
@inproceedings{liang09online,
author = {P. Liang and D. Klein},
booktitle = {North American Association for Computational Linguistics (NAACL)},
pages = {611--619},
title = {Online {EM} for Unsupervised Models},
year = {2009},
}
Statistical and computational concerns have motivated parameter estimators based on various forms of likelihood, e.g., joint, conditional, and pseudolikelihood. In this paper, we present a unified framework for studying these estimators, which allows us to compare their relative (statistical) efficiencies. Our asymptotic analysis suggests that modeling more of the data tends to reduce variance, but at the cost of being more sensitive to model misspecification. We present experiments validating our analysis.
Derive general expression for the asymptotic risk of composite likelihood estimators in exponential families.
This allows us to compare the various estimators.
@inproceedings{liang08asymptotics,
author = {P. Liang and M. I. Jordan},
booktitle = {International Conference on Machine Learning (ICML)},
pages = {584--591},
title = {An Asymptotic Analysis of Generative, Discriminative, and Pseudolikelihood Estimators},
year = {2008},
}
Structured models often achieve excellent performance but can be slow at test time. We investigate structure compilation, where we replace structure with features, which are often computationally simpler but unfortunately statistically more complex. We analyze this tradeoff theoretically and empirically on three natural language processing tasks. We also introduce a simple method to transfer predictive power from structure to features via unlabeled data, while incurring a minimal statistical penalty.
How much do we lose by throwing out edge features in CRFs and adding node features?
Studies the approximation, estimation, computational aspects of the tradeoff.
@inproceedings{liang08structure,
author = {P. Liang and H. Daume and D. Klein},
booktitle = {International Conference on Machine Learning (ICML)},
title = {Structure Compilation: Trading Structure for Features},
year = {2008},
}
We identify four types of errors that unsupervised induction systems make and study each one in turn. Our contributions include (1) using a meta-model to analyze the incorrect biases of a model in a systematic way, (2) providing an efficient and robust method of measuring distance between two parameter settings of a model, and (3) showing that local optima issues which typically plague EM can be somewhat alleviated by increasing the number of training examples. We conduct our analyses on three models: the HMM, the PCFG, and a simple dependency model.
Error decomposition: approximation, identifiability, estimation, optimization errors.
Used meta-model to analyze approximation error.
Empirically observed that more data reduces optimization error.
@inproceedings{liang08errors,
author = {P. Liang and D. Klein},
booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
title = {Analyzing the Errors of Unsupervised Learning},
year = {2008},
}
We present a method for learning bilingual translation lexicons from monolingual corpora. Word types in each language are characterized by purely monolingual features, such as context counts and orthographic substrings. Translations are induced using a generative model based on canonical correlation analysis, which explains the monolingual lexicons in terms of latent matchings. We show that high-precision lexicons can be learned in a variety of language pairs and from a range of corpus types.
By using CCA, can do word alignment without the usual sentence-aligned corpora.
@inproceedings{haghighi08lexicon,
author = {A. Haghighi and P. Liang and T. Berg-Kirkpatrick and D. Klein},
booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
title = {Learning Bilingual Lexicons from Monolingual Corpora},
year = {2008},
}
The learning of probabilistic models with many hidden variables
and non-decomposable dependencies is an important and challenging problem.
In contrast to traditional approaches based on approximate inference in a single
intractable model, our approach is to train a set of tractable
submodels by encouraging them to agree on the hidden variables. This allows
us to capture non-decomposable aspects of the data while still maintaining
tractability. We propose an objective function for our approach,
derive EM-style algorithms for parameter estimation, and demonstrate their
effectiveness on three challenging real-world learning tasks.
Setting: unsupervised learning.
Alternative to approximate inference: make two tractable models and train them to agree.
Advantage: maintain existing tractable inference procedures as black-boxes.
@inproceedings{liang08agreement,
author = {P. Liang and D. Klein and M. I. Jordan},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {Agreement-Based Learning},
year = {2008},
}
We present a probabilistic approach to language change in which word forms are
represented by phoneme sequences that undergo stochastic edits along the
branches of a phylogenetic tree. This framework combines the
advantages of the classical comparative method with the robustness
of corpus-based probabilistic models. We use this framework to
explore the consequences of two different schemes for defining
probabilistic models of phonological change, evaluating these
schemes by reconstructing ancient word forms of Romance languages.
The result is an efficient inference procedure for automatically
inferring ancient word forms from modern languages, which can be
generalized to support inferences about linguistic phylogenies.
Feature-based generative model of phonemes of words in a phylogeny of languages.
@inproceedings{bouchard08language,
author = {A. Bouchard-Côté and P. Liang and T. Griffiths and D. Klein},
booktitle = {Advances in Neural Information Processing Systems (NIPS)},
title = {A Probabilistic Approach to Language Change},
year = {2008},
}
@inproceedings{liang07tutorial,
author = {P. Liang and D. Klein},
booktitle = {Association for Computational Linguistics (ACL)},
title = {Structured {B}ayesian Nonparametric Models with Variational Inference (tutorial)},
year = {2007},
}
We introduce a new inference algorithm for Dirichlet process mixture
models. While Gibbs sampling and variational methods focus on local
moves, the new algorithm makes more global moves. This is done by
introducing a permutation of the data points as an auxiliary variable.
The algorithm is a blocked sampler which alternates between sampling the
clustering and sampling the permutation. The key to the efficiency of
this approach is that it is possible to use dynamic programming to
consider all exponentially many clusterings consistent with a given
permutation. We also show that random projections can be used to
effectively sample the permutation. The result is a stochastic
hill-climbing algorithm that yields burn-in times significantly
smaller than those of collapsed Gibbs sampling.
Task: clustering.
Idea: conditioned on a permutation of the data points, one can consider all possible clusterings of those data points which are consistent with the permutation using dynamic programming.
Idea: if data are ordered, can cluster using dynamic programming.
Let this ordering be a random auxiliary variable and we get a sampler.
@inproceedings{liang07permdp,
author = {P. Liang and M. I. Jordan and B. Taskar},
booktitle = {International Conference on Machine Learning (ICML)},
title = {A permutation-augmented sampler for {D}irichlet process mixture models},
year = {2007},
}
We present a nonparametric Bayesian model of tree structures based on the
hierarchical Dirichlet process (HDP). Our HDP-PCFG model allows the complexity
of the grammar to grow as more training data is available. In addition to
presenting a fully Bayesian model for the PCFG, we also develop an efficient
variational inference procedure. On synthetic data, we recover the correct
grammar without having to specify its complexity in advance. We also show that
our techniques can be applied to full-scale parsing applications by
demonstrating its effectiveness in learning state-split grammars.
A PCFG with an infinite number of states.
Learning: variational inference.
@inproceedings{liang07infpcfg,
author = {P. Liang and S. Petrov and M. I. Jordan and D. Klein},
booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
title = {The Infinite {PCFG} using Hierarchical {D}irichlet Processes},
year = {2007},
}
We present a probabilistic model of diachronic phonology in which individual
word forms undergo stochastic edits along the branches of a phylogenetic tree.
Our approach allows us to achieve three goals with a single unified
model: (1) reconstruction of both ancient and modern word forms, (2) discovery
of general phonological changes, and (3) selection among different
phylogenies. We learn our model using a Monte Carlo EM algorithm and present
quantitative results validating the model.
Generative model of phonemes of words in a phylogeny of languages
@inproceedings{bouchard07diachronic,
author = {A. Bouchard-Côté and P. Liang and T. Griffiths and D. Klein},
booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
title = {A Probabilistic Approach to Diachronic Phonology},
year = {2007},
}
We present a perceptron-style discriminative approach to machine
translation in which large feature sets can be exploited. Unlike
discriminative reranking approaches, our system can take advantage of learned
features in all stages of decoding. We first discuss several challenges to
error-driven discriminative approaches. In particular, we explore different
ways of updating parameters given a training example. We find that making
frequent but smaller updates is preferable to making fewer but larger updates.
Then, we discuss an array of features and show both how they quantitatively
increase BLEU score and how they qualitatively interact on specific examples.
One particular feature we investigate is a novel way to introduce learning into
the initial phrase extraction process, which has previously been entirely
heuristic.
Task: machine translation.
Idea: treat machine translation as a structured classification task (learn a map from input sentence to output sentence). Use a Perceptron-like algorithm: decode and update towards maximum BLEU scoring translation on the n-best list.
@inproceedings{liang06discrimative,
author = {P. Liang and A. Bouchard-Côté and D. Klein and B. Taskar},
booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
title = {An End-to-End Discriminative Approach to Machine Translation},
year = {2006},
}
We present an unsupervised approach to symmetric
word alignment in which two simple asymmetric models are
trained jointly to maximize a
combination of data likelihood and agreement between the models.
Compared to the standard practice of intersecting predictions of
independently-trained models, joint training provides a 32% reduction
in AER. Moreover, a simple and efficient pair of HMM aligners
provides a 29% reduction in AER over symmetrized IBM model 4
predictions.
Task: unsupervised word alignment.
Idea: Jointly train two HMM models (one in each direction) to encourage agreement. Uses a simple EM-like algorithm for training.
Result: performance competitive with supervised methods (4.9 AER on Hansards).
@inproceedings{liang06alignment,
author = {P. Liang and B. Taskar and D. Klein},
booktitle = {North American Association for Computational Linguistics (NAACL)},
pages = {104--111},
title = {Alignment by Agreement},
year = {2006},
}
Task: named-entity recognition and Chinese word segmentation
Idea: create features based on unlabeled data to use in Perceptron learning in Markov or semi-Markov models
@mastersthesis{liang05meng,
author = {P. Liang},
school = {Massachusetts Institute of Technology},
title = {Semi-Supervised Learning for Natural Language},
year = {2005},
}
We introduce the first definition of hyperacyclicity for hypergraphs, a generalization of acyclicity in graphs.
We provide a dynamic data structure for maintaining hyperacyclicity, a generalization of Tarjan's Union-Find algorithm.
@techreport{liang05hypercycle,
author = {P. Liang and N. Srebro},
institution = {Massachusetts Institute of Technology},
title = {A Data Structure for Maintaining Acyclicity in Hypergraphs},
year = {2005},
}
@inproceedings{liang05mcmaster,
author = {P. Liang and N. Srebro},
booktitle = {Mathematical Programing for Data Mining and Machine Learning Workshop at McMaster University},
title = {Linear Programming in Bounded Tree-width {M}arkov Networks},
year = {2005},
}
In parsing sequences using dynamic programming, the subproblems are continguous subsequences (quadratic in number of terminals). In parsing documents or images, the subproblems would be subsets of the terminals (exponential in number of terminals). We introduce (and unify) several ways to constrain these subsets using the geometric structure of the terminals.
@inproceedings{liang05geometric,
author = {P. Liang and M. Narasimhan and M. Shilman and P. Viola},
booktitle = {International Conference on Document Analysis and Recognition (ICDAR)},
title = {Efficient Geometric Algorithms for Parsing in Two Dimensions},
year = {2005},
}
Use a greedy procedure to find the maximum likelihood (or MDL) bounded tree-width Markov network (for tree-width 1, equivalent to Chow-Liu maximum spanning trees).
@techreport{liang04markov,
author = {P. Liang and N. Srebro},
institution = {Massachusetts Institute of Technology},
title = {Methods and Experiments With Bounded Tree-width {M}arkov Networks},
year = {2004},
}
Use linear programming to find worst case inputs to a dynamic program in order to explore the tightness of a bound for approximating maximum weight hypertrees with windmill farms.
@techreport{liang03maxwmfarm,
author = {P. Liang and N. Srebro},
institution = {Massachusetts Institute of Technology},
title = {How Much Of A Hypertree Can Be Captured By Windmills?},
year = {2003},
}