@@ -25,6 +25,11 @@ A statistics package for comparing multiple parameters (e.g., multiple treatment
:target: https://github.com/psf/black
|
Motivation
==========
Multiple inference techniques outperform standard methods like OLS and IV estimation for comparing multiple parameters. For example, `this post <https://gitlab.com/dsbowen/conditional-inference/-/blob/master/examples/bayes_primer.ipynb>_` shows how to apply Bayesian estimators to a randomized control trial testing many interventions to increase vaccination rates.
title={The Megastudy Paradigm: A New Direction for Behavioral Research in Cognitive Science},
author={Cortese, Michael J},
booktitle={New Methods in Cognitive Psychology},
pages={67--85},
year={2019},
publisher={Routledge}
}
@article{milkman2021megastudy,
title={A megastudy of text-based nudges encouraging patients to get vaccinated at an upcoming doctor’s appointment},
author={Milkman, Katherine L and Patel, Mitesh S and Gandhi, Linnea and Graci, Heather N and Gromet, Dena M and Ho, Hung and Kay, Joseph S and Lee, Timothy W and Akinola, Modupe and Beshears, John and others},
journal={Proceedings of the National Academy of Sciences},
volume={118},
number={20},
year={2021},
publisher={National Acad Sciences}
}
@article{milkman2022680,
title={A 680,000-person megastudy of nudges to encourage vaccination in pharmacies},
author={Milkman, Katherine L and Gandhi, Linnea and Patel, Mitesh S and Graci, Heather N and Gromet, Dena M and Ho, Hung and Kay, Joseph S and Lee, Timothy W and Rothschild, Jake and Bogard, Jonathan E and others},
journal={Proceedings of the National Academy of Sciences},
volume={119},
number={6},
year={2022},
publisher={National Acad Sciences}
}
@article{milkman2021megastudies,
title={Megastudies improve the impact of applied behavioural science},
author={Milkman, Katherine L and Gromet, Dena and Ho, Hung and Kay, Joseph S and Lee, Timothy W and Pandiloski, Pepi and Park, Yeji and Rai, Aneesh and Bazerman, Max and Beshears, John and others},
journal={Nature},
volume={600},
number={7889},
pages={478--483},
year={2021},
publisher={Nature Publishing Group}
}
@article{dellavigna2018motivates,
title={What motivates effort? Evidence and expert forecasts},
author={DellaVigna, Stefano and Pope, Devin},
journal={The Review of Economic Studies},
volume={85},
number={2},
pages={1029--1069},
year={2018},
publisher={Oxford University Press}
}
@article{lai2014reducing,
title={Reducing implicit racial preferences: I. A comparative investigation of 17 interventions.},
author={Lai, Calvin K and Marini, Maddalena and Lehr, Steven A and Cerruti, Carlo and Shin, Jiyun-Elizabeth L and Joy-Gaba, Jennifer A and Ho, Arnold K and Teachman, Bethany A and Wojcik, Sean P and Koleva, Spassena P and others},
journal={Journal of Experimental Psychology: General},
volume={143},
number={4},
pages={1765},
year={2014},
publisher={American Psychological Association}
}
@article{karlan2007does,
title={Does price matter in charitable giving? Evidence from a large-scale natural field experiment},
author={Karlan, Dean and List, John A},
journal={American Economic Review},
volume={97},
number={5},
pages={1774--1793},
year={2007}
}
@techreport{banerjee2021selecting,
title={Selecting the most effective nudge: Evidence from a large-scale experiment on immunization},
author={Banerjee, Abhijit and Chandrasekhar, Arun G and Dalpath, Suresh and Duflo, Esther and Floretta, John and Jackson, Matthew O and Kannan, Harini and Loza, Francine N and Sankar, Anirudh and Schrimpf, Anna and others},
year={2021},
institution={National Bureau of Economic Research}
}
@article{caria2020adaptive,
title={An adaptive targeted field experiment: Job search assistance for refugees in Jordan},
author={Caria, Stefano and Kasy, Maximilian and Quinn, Simon and Shami, Soha and Teytelboym, Alex and others},
year={2020},
publisher={CESifo Working Paper}
}
@techreport{chetty2018opportunity,
title={The opportunity atlas: Mapping the childhood roots of social mobility},
author={Chetty, Raj and Friedman, John N and Hendren, Nathaniel and Jones, Maggie R and Porter, Sonya R},
year={2018},
institution={National Bureau of Economic Research}
}
@article{chetty2018impacts,
title={The impacts of neighborhoods on intergenerational mobility II: County-level estimates},
author={Chetty, Raj and Hendren, Nathaniel},
journal={The Quarterly Journal of Economics},
volume={133},
number={3},
pages={1163--1228},
year={2018},
publisher={Oxford University Press}
}
@article{chetty2014land,
title={Where is the land of opportunity? The geography of intergenerational mobility in the United States},
author={Chetty, Raj and Hendren, Nathaniel and Kline, Patrick and Saez, Emmanuel},
journal={The Quarterly Journal of Economics},
volume={129},
number={4},
pages={1553--1623},
year={2014},
publisher={Oxford University Press}
}
@techreport{andrews2019inference,
title={Inference on winners},
author={Andrews, Isaiah and Kitagawa, Toru and McCloskey, Adam},
year={2019},
institution={National Bureau of Economic Research}
}
@article{andrewsinference,
title={Inference for Losers},
author={Andrews, Isaiah and Bowen, Dillon and Kitagawa, Toru and McCloskey, Adam},
year={2022}
}
@article{romano2005stepwise,
title={Stepwise multiple testing as formalized data snooping},
author={Romano, Joseph P and Wolf, Michael},
journal={Econometrica},
volume={73},
number={4},
pages={1237--1282},
year={2005},
publisher={Wiley Online Library}
}
@techreport{mogstad2020inference,
title={Inference for ranks with applications to mobility across neighborhoods and academic achievement across countries},
author={Mogstad, Magne and Romano, Joseph P and Shaikh, Azeem and Wilhelm, Daniel},
year={2020},
institution={National Bureau of Economic Research}
}
@inproceedings{stein1956inadmissibility,
title={Inadmissibility of the usual estimator for the mean of a multivariate normal distribution},
author={Stein, Charles and others},
booktitle={Proceedings of the Third Berkeley symposium on mathematical statistics and probability},
volume={1},
number={1},
pages={197--206},
year={1956}
}
@incollection{james1992estimation,
title={Estimation with quadratic loss},
author={James, William and Stein, Charles},
booktitle={Breakthroughs in statistics},
pages={443--460},
year={1992},
publisher={Springer}
}
@inproceedings{dimmery2019shrinkage,
title={Shrinkage estimators in online experiments},
author={Dimmery, Drew and Bakshy, Eytan and Sekhon, Jasjeet},
booktitle={Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining},
pages={2914--2922},
year={2019}
}
@article{cai2021nonparametric,
title={Nonparametric empirical bayes estimation and testing for sparse and heteroscedastic signals},
author={Cai, Junhui and Han, Xu and Ritov, Ya'acov and Zhao, Linda},
journal={arXiv preprint arXiv:2106.08881},
year={2021}
}
@article{brown2009nonparametric,
title={Nonparametric empirical Bayes and compound decision approaches to estimation of a high-dimensional vector of normal means},
author={Brown, Lawrence D and Greenshtein, Eitan},
journal={The Annals of Statistics},
pages={1685--1704},
year={2009},
publisher={JSTOR}
}
@article{hernandez2017applying,
title={Applying Behavioral Insights to Improve Tax Collection},
author={Hernandez, Marco and Jamison, Julian and Korczyc, Ewa and Mazar, Nina and Sormani, Roberto},
year={2017},
publisher={World Bank, Washington, DC}
}
@inproceedings{seabold2010statsmodels,
title={statsmodels: Econometric and statistical modeling with python},
author={Seabold, Skipper and Perktold, Josef},
booktitle={9th Python in Science Conference},
year={2010},
}
@article{salvatier2016probabilistic,
title={Probabilistic programming in Python using PyMC3},
author={Salvatier, John and Wiecki, Thomas V and Fonnesbeck, Christopher},
journal={PeerJ Computer Science},
volume={2},
pages={e55},
year={2016},
publisher={PeerJ Inc.}
}
@ARTICLE{2020SciPy-NMeth,
author={Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and
Haberland, Matt and Reddy, Tyler and Cournapeau, David and
Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and
Bright, Jonathan and {van der Walt}, St{\'e}fan J. and
Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and
Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and
Kern, Robert and Larson, Eric and Carey, C J and
Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and
{VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and
Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and
Harris, Charles R. and Archibald, Anne M. and
Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and
{van Mulbregt}, Paul and {SciPy 1.0 Contributors}},
title={{{SciPy} 1.0: Fundamental Algorithms for Scientific
Computing in Python}},
journal={Nature Methods},
year={2020},
volume={17},
pages={261--272},
adsurl={https://rdcu.be/b08Wh},
doi={10.1038/s41592-019-0686-2},
}
@article{botev2017normal,
title={The normal law under linear restrictions: simulation and estimation via minimax tilting},
author={Botev, Zdravko I},
journal={Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
volume={79},
number={1},
pages={125--148},
year={2017},
publisher={Wiley Online Library}
}
@inproceedings{botev2015efficient,
title={Efficient probability estimation and simulation of the truncated multivariate student-t distribution},
-name:Wharton School of Business, University of Pennsylvania
index:1
date:14 May 2022
bibliography:paper.bib
---
# Summary
Scientists often want to compare many parameters. For example, scientists often run randomized control trials to study the effects of many treatments, use observational data to compare many geographic regions, and study how public policy will impact many subgroups of people. Multiple Inference implements many of the latest econometric and statistical tools for making such comparisons, including inference after ranking, simultaneous confidence sets, and Bayesian estimators. It uses a `statsmodels`-like API and provides template notebooks for ease of use. In just a few clicks, researchers can upload a `.csv` file of conventional estimates (e.g., OLS or IV estimates) to a Jupyter binder and click "run" to apply a suite of multiple inference tools to their data.
# Statement of need
Researchers often want to compare multiple parameters. For example, there is a recent trend in social science to run large-scale studies and randomized control trials designed to test the effectiveness of many behavioral interventions [@cortese2019megastudy]. Researchers have used large-scale field studies to test the effectiveness of many text messages reminding patients to get vaccinated [@milkman2021megastudy; @milkman2022680; @banerjee2021selecting], behavioral nudges encouraging 24 Hour Fitness customers to exercise more often [@milkman2021megastudies], monetary and social incentives to exert effort [@dellavigna2018motivates], behavioral interventions to decrease implicit racial bias [@lai2014reducing], donation matching schemes to increase charitable giving [@karlan2007does], and job training programs to increase employment among refugees in Jordan [@caria2020adaptive]. Researchers also perform multiple comparisons using observational data. For example, economists often use observation data to compare many neighborhoods in terms of intergenerational mobility [@chetty2018opportunity; @chetty2018impacts; @chetty2014land].
Researchers tend to ask the same set of questions when comparing many parameters.
1. Which parameters are significantly different from zero?
2. Which parameters are significantly better than the average (i.e., the average value across all parameters)?
3. Which parameters are significantly different from which other parameters?
4. What is the ranking of each parameter?
5. Which parameters might be the largest (i.e., the highest-ranked)?
6. What are the values of the parameters given their rank? e.g., What is the value of the parameter with the largest estimated value?
7. How are the parameters distributed?
Researchers often use conventional estimators like ordinary least squares (OLS) and instrumental variables (IV) to answer such questions [@milkman2021megastudy; @milkman2022680; @milkman2021megastudies; @lai2014reducing]. Unfortunately, conventional estimators overestimate the value of the top-performing parameter (i.e., the parameter with the largest estimated value) and exaggerate the variability of the parameters [@andrews2019inference; @andrewsinference]. These problems lead researchers to overstate the effectiveness of the top-performing treatments and the differences between treatment effects.
Statisticians and econometricians have advanced multiple inference tools in recent years. Recent publications describe new statistical techniques for inference after ranking [@andrews2019inference; @andrewsinference], multiple hypothesis testing [@romano2005stepwise], rank estimation [@mogstad2020inference], and Bayesian estimation [@stein1956inadmissibility; @james1992estimation; @dimmery2019shrinkage; @cai2021nonparametric; @brown2009nonparametric]. However, these techniques are mathematically complex and often inaccessible to all but professional statisticians.
Multiple Inference solves this problem by implementing many of the latest multiple inference tools in an easy-to-use `statsmodels`-like API. Additionally, Multiple Inference provides Jupyter binders with boilerplate code and narrative explanations to help researchers interpret the output. These binders allow researchers to upload a `.csv` file of their conventional estimates and click "run" to apply multiple inference tools to their data without downloading any software or writing a single line of code.
Multiple Inference initially implemented the inference after ranking techniques in @andrews2019inference and extended in @andrewsinference. The latter paper uses Multiple Inference to compare many United States commuting zones regarding intergenerational mobility. The World Bank Group is currently using Multiple Inference to reanalyze the results of a multi-treatment study designed to improve tax collection in Poland (see @hernandez2017applying for an earlier version of the paper).
# State of the field
Multiple Inference's defining features are inference after ranking, rank estimation, and hypothesis testing tools. Most importantly, Multiple Inference contains the only implementation of the inference after ranking techniques developed in @andrews2019inference and @andrewsinference in any language. These techniques correct for the winner's curse when performing inference on top-performing parameters (e.g., the parameters that rank in the top five according to conventional estimates). Specifically, Multiple Inference implements computationally efficient algorithms for computing quantile-unbiased point estimates and confidence intervals with correct coverage for parameters of specific ranks.
@mogstad2020inference has an associated R package for estimating rankings. For example, it may estimate that a particular parameter has a 95% chance of being one of the three largest parameters. It also computes sets of parameters that contain all of the truly largest $K$ parameters with 95% confidence. Multiple Inference is the only Python implementation of these techniques.
`statsmodels` implements multiple testing corrections based on p-values, such as the Holm-Bonferroni correction. Multiple Inference implements multiple hypothesis tests using a more powerful stepdown method based on simultaneous confidence sets for jointly Gaussian distributed estimates [@romano2005stepwise].
Bayesian estimators are essential tools for multiple inference, and robust packages for Bayesian analysis already exist in Python. For example, `statsmodels` implements two Bayesian models (binomial and Poisson) with independent Gaussian priors [@seabold2010statsmodels]. `pymc3` is a comprehensive package for Bayesian inference [@salvatier2016probabilistic]. Additionally, @dimmery2019shrinkage implements a Gaussian prior Bayesian model fit using Stein-type estimation. It distinguishes itself by incorporating uncertainty about the estimates of the prior parameters into the posterior distribution.
Multiple Inference aims to be a one-stop-shop for multiple inference and therefore includes parametric and nonparametric Bayesian estimators. Its Gaussian prior Bayesian estimator is most similar to the Stein-type estimator from @dimmery2019shrinkage. However, @dimmery2019shrinkage does not account for correlated errors. For example, if we underestimate the prior mean and shrink all posterior estimates towards the estimated prior mean, we will underestimate many parameters. Multiple Inference accounts for this correlated uncertainty in its James-Stein fit method of the Gaussian prior Bayesian model. Additionally, Multiple Inference provides a maximum likelihood fit method for the Gaussian prior model, also accounting for correlated uncertainty about the prior parameters.
Multiple Inference also implements several "intermediate products" that researchers can use in other applications. The most notable is a truncated normal distribution with two advantages over `scipy`'s truncated normal distribution [@2020SciPy-NMeth]. First, `scipy`'s truncated normal required a convex truncation set, whereas Multiple Inference's truncated normal accepts both convex and concave truncation sets. Second, Multiple Inference uses an exponential tilting method to improve accuracy when the truncation set is far from the mean of the underlying normal [@botev2017normal; @botev2015efficient]. Multiple Inference uses the same exponential tilting method as the R package `TruncatedNormal` [@botev2021truncatednormal]. We can see the advantage of Multiple Inference's implementation for the cumulative distribution function of a standard normal truncated to the interval $[8, 9]$ evaluated at 8.7.
```python
>>>fromscipy.statsimporttruncnorm
>>>truncnorm(8,9).cdf(8.7)
1.0709836154559238
>>>fromconditional_inference.statsimporttruncnorm
>>>truncnorm([(8,9)]).cdf(8.7)
0.9978948153314305
```
# Acknowledgements
I would like to thank Sarah Reed and Christian Kaps for feedback on this paper. I would also like to thank Isaiah Andrews, Toru Kitagawa, Adam McCloskey, and Jeff Rowley for feedback on my early drafts of the software.
