{ "meta": { "title": "Replication Lab", "lede": "Classic studies that forecast Supreme Court decisions, rebuilt from their published methods and run on the cases the Court has decided since OT2022. Each study page profiles the paper, checks our rebuild against the authors' own reported results, then applies the same method to recent terms, scored against the simplest baseline: the petitioner always wins.", "updated": "2026-10-05" }, "studies": [ { "slug": "ruger-2004", "status": "published", "short": "Ruger, Kim, Martin & Quinn (2004)", "title": "The Supreme Court Forecasting Project", "authors": "Theodore W. Ruger, Pauline T. Kim, Andrew D. Martin & Kevin M. Quinn", "citations": [ { "text": "Theodore W. Ruger, Pauline T. Kim, Andrew D. Martin & Kevin M. Quinn, The Supreme Court Forecasting Project: Legal and Political Science Approaches to Predicting Supreme Court Decisionmaking, 104 Columbia Law Review 1150 (2004).", "url": "https://scholar.google.com/scholar?q=%22The+Supreme+Court+Forecasting+Project%3A+Legal+and+Political+Science+Approaches%22" }, { "text": "Andrew D. Martin, Kevin M. Quinn, Theodore W. Ruger & Pauline T. Kim, Competing Approaches to Predicting Supreme Court Decision Making, 2 Perspectives on Politics 761 (2004).", "url": "https://doi.org/10.1017/S1537592704040502" } ], "oneLine": "Six case characteristics and a set of classification trees beat 83 legal experts at forecasting the 2002 Term.", "question": "Can a simple statistical model, which knows nothing about the law or facts of a case, forecast the Court's decisions as well as legal experts can?", "design": "A prospective contest. For every case argued in the 2002 Term, the authors posted the model's forecast before oral argument, and 83 legal experts (71 academics and 12 appellate lawyers, including 38 former Supreme Court clerks) each forecast one to three cases in their field without seeing the model or each other. Both forecast the outcome (affirm or reverse) and every Justice's vote.", "data": [ "Training data: the 628 cases the second Rehnquist Court decided before the 2002 Term. The same nine Justices had sat together since OT1994, so every Justice had eight terms of votes.", "Six variables per case: circuit of origin (state courts and three-judge district courts count as the circuit they sit in), issue area, type of petitioner, type of respondent, the ideological direction of the lower-court ruling, and whether the petitioner argued that a law or practice is unconstitutional.", "Variables came from the Supreme Court Database for training; for the 2002 cases the authors coded them from the parties' briefs before argument.", "Test data: 68 of the 76 cases argued in OT2002. Dismissed cases, evenly divided decisions and three cases that could not be scored as affirm or reverse were left out." ], "method": [ "Classification trees, one question at a time: each tree splits cases on the six variables until it reaches a group that mostly went one way.", "Two trees run first and ask whether the Court will rule unanimously liberal or unanimously conservative. If exactly one says yes, that is the forecast.", "Otherwise nine trees forecast each Justice's vote, in a fixed order (Scalia, Thomas, Rehnquist, Stevens, O'Connor, Ginsburg, Breyer, Souter, Kennedy). A later Justice's tree may use the votes predicted for the Justices before them. The majority of the nine votes is the forecast.", "The authors chose this design from 24 candidates by fitting each on part of the training data and scoring it on the rest, then refit the winner on all 628 cases." ], "findings": [ { "label": "Case outcomes", "paper": "75% (51 of 68)", "note": "68% as first posted. At the end of the Term the authors found a software error that had fed some cases into the trees with wrong inputs (a Ninth Circuit case entered as Fifth Circuit, for example); 75% is the corrected model. The experts got 59.1%, and their per-case consensus 65.6%." }, { "label": "Justices' votes", "paper": "66.7%", "note": "The experts did slightly better on votes: 67.9%. They did best with the Justices at the ends of the spectrum; the model did better with O'Connor and Kennedy in the middle." }, { "label": "By issue", "paper": "Economic activity 87.5%; judicial power 50%", "note": "The experts got 51.3% of economic activity cases and 73.7% of judicial power cases." } ], "takeaway": "Knowing a case's broad shape (where it came from, who is suing whom, which way the lower court leaned) predicted this stable Court's decisions better than case-specific expertise did.", "script": "src/research/replication/ruger.py", "data_file": "/data/replication/ruger.json" }, { "slug": "katz-2017", "status": "published", "short": "Katz, Bommarito & Blackman (2017)", "title": "A general approach for predicting the behavior of the Supreme Court of the United States", "oneLine": "One random forest, retrained every term, forecasts two centuries of Supreme Court votes and cases.", "authors": "Daniel Martin Katz, Michael J. Bommarito II & Josh Blackman", "citations": [ { "text": "Daniel Martin Katz, Michael J. Bommarito II & Josh Blackman, A general approach for predicting the behavior of the Supreme Court of the United States, PLOS ONE 12(4): e0174698 (2017).", "url": "https://doi.org/10.1371/journal.pone.0174698" }, { "text": "Replication code and per-case predictions: github.com/mjbommar/scotus-predict-v2.", "url": "https://github.com/mjbommar/scotus-predict-v2" }, { "text": "An earlier preprint (arXiv:1407.6333, 2014) used extremely randomized trees on 1953–2013; the PLOS ONE paper supersedes it.", "url": "https://arxiv.org/abs/1407.6333" } ], "question": "Can a single model, using only information available before each decision, predict the Court's decisions and every Justice's vote across its whole history, term after term, as Justices come and go?", "design": "A simulated forecast of every term from 1816 to 2015. Before each term the model is retrained on all earlier terms only, then predicts every vote and case in that term. It is scored against a strong baseline: the most frequent outcome of the previous ten terms, a window the authors picked because it was close to the best possible in hindsight.", "data": [ "Every Justice vote in the Supreme Court Database's Legacy release, 1791–2015: about 244,000 votes in 28,000 cases, of every decision type.", "Each vote is coded affirm, reverse or other from the vote and the Court's disposition (a dissent from a reversal counts as affirm; jurisdictional dissents and unclear dispositions are other). A case counts as reversed when the Court reverses or vacates.", "About 1,500 inputs per vote: the Justice, natural court, argument month, the parties (raw and grouped), jurisdiction, agency action, source and origin court mapped to circuit, lower-court disagreement and disposition, cert reason, issue, issue area, law type and law, plus the decision month and the weeks from argument to decision.", "\"Behavior\" inputs computed from earlier terms only: each Justice's and the whole Court's reversal rate, ideological direction and agreement with the majority, in the previous term and over all prior terms, the differences between Justice and Court, and how far the lower court's ruling sits from the Justice's usual direction." ], "method": [ "A random forest: hundreds of decision trees, each grown on a random sample of votes and inputs, voting together.", "Retrained at the start of every term on all earlier terms, with 125 trees plus 5 for each term elapsed. A 3-fold grid search picks the minimum leaf size (1 or 2).", "Each vote is predicted as affirm, reverse or other. The case is predicted reversed when at least half the predicted votes are to reverse.", "Baseline (M = 10): predict the most frequent vote outcome of the previous ten terms." ], "findings": [ { "label": "Case outcomes, 1816–2015", "paper": "70.2%", "note": "The ten-term baseline gets 67.5%." }, { "label": "Justice votes, 1816–2015", "paper": "71.9%", "note": "The baseline gets 66.2%." }, { "label": "The last century", "paper": "about 5 points better than the baseline", "note": "The authors' published predictions: 67.5% vs 62.7% for the last 100 terms. In the 19th century the model did no better than the baseline." }, { "label": "The Roberts Court", "paper": "\"less predictable than its immediate predecessors\"", "note": "In the authors' own predictions for OT2005–OT2014 the model trails the baseline (68.0% vs 69.0%)." } ], "takeaway": "A general-purpose model can follow the Court across changes of personnel and centuries of history, and beat a well-tuned baseline over the long run; the edge is a few points, and it was shrinking in the most recent terms the paper covers.", "script": "src/research/replication/katz.py", "data_file": "/data/replication/katz.json" }, { "slug": "kaufman-2019", "status": "published", "short": "Kaufman, Kraft & Sen (2019)", "title": "Improving Supreme Court Forecasting Using Boosted Decision Trees", "oneLine": "Boosted decision trees that add what the Justices did at oral argument: whom they questioned, how much, and whom they cut off.", "authors": "Aaron Russell Kaufman, Peter Kraft & Maya Sen", "citations": [ { "text": "Aaron Russell Kaufman, Peter Kraft & Maya Sen, Improving Supreme Court Forecasting Using Boosted Decision Trees, Political Analysis 27(3): 381–387 (2019).", "url": "https://doi.org/10.1017/pan.2018.59" }, { "text": "Replication archive (code, feature table and per-case predictions): Harvard Dataverse, doi:10.7910/DVN/JJCXTH.", "url": "https://doi.org/10.7910/DVN/JJCXTH" }, { "text": "Oral-argument counts build on the CourtCast project, github.com/nasrallah/CourtCast.", "url": "https://github.com/nasrallah/CourtCast" } ], "question": "Does what happens at oral argument (which side the Justices question, how much they talk to each side, whom they interrupt) improve forecasts built from a case's characteristics, and is boosting a good tool for it?", "design": "A cross-validated test on cases argued OT2005–OT2014: the model is fit on nine-tenths of the cases and scored on the remaining tenth, ten times over, and compared with \"the petitioner always wins\" and with earlier models refit on the same cases.", "data": [ "704 argued cases with a clear winner, OT2005–OT2014 (697 scored).", "Eight case variables from the Supreme Court Database: origin court (by circuit), whether a federal or a state agency acted, the lower court's ideological direction and disposition, issue area, cert reason and jurisdiction.", "47 oral-argument variables from the Oyez transcripts, for the five Justices who sat and spoke throughout (Breyer, Ginsburg, Kennedy, Roberts, Scalia): for each, the share of turns spent on each side, how often those turns are questions, which side drew more of their questions, turns and words, and how often they cut a lawyer off mid-sentence; plus which side had more lawyers arguing (usually because the Solicitor General joined it) and the net number of interruptions." ], "method": [ "AdaBoost: decision trees are fit one after another, each giving more weight to the cases the earlier ones got wrong, and their votes are combined.", "Each step is a forest of shallow (three-level) extremely randomized trees.", "Scored by tenfold cross-validation, with the cases kept in docket order." ], "findings": [ { "label": "Case variables + oral argument", "paper": "74.04%", "note": "Against 67.98% for the petitioner always winning (the authors' figure for all cases since 2000): about seven more correct calls a term." }, { "label": "Oral argument only", "paper": "72.5%", "note": "Most of the gain comes from the argument." }, { "label": "Case variables only", "paper": "60.6%", "note": "Below the base rate: no model the authors tried beat it with case variables alone." }, { "label": "By kind of case", "paper": "6–3 decisions +13 points; federalism +16; government a party +9", "note": "Weaker gains in criminal procedure, civil rights and First Amendment cases." } ], "takeaway": "How the Justices behave at argument says more about the outcome than the case's paperwork does; on its own, the paperwork did worse than simply picking the petitioner.", "script": "src/research/replication/kaufman.py", "data_file": "/data/replication/kaufman.json" }, { "slug": "george-epstein-1992", "status": "published", "short": "George & Epstein (1992)", "title": "On the Nature of Supreme Court Decision Making", "oneLine": "Law or politics? A legal model and a political model of the Court's death-penalty decisions, tested head to head.", "authors": "Tracey E. George & Lee Epstein", "citations": [ { "text": "Tracey E. George & Lee Epstein, On the Nature of Supreme Court Decision Making, American Political Science Review 86(2): 323–337 (1992).", "url": "https://doi.org/10.2307/1964223" }, { "text": "Full text on Lee Epstein's site: epstein.wustl.edu/onnature.", "url": "https://epstein.wustl.edu/onnature" } ], "question": "Does the law (the doctrine the Court has announced) or politics (who appointed the Justices, which party governs, who the litigants are) better explain how the Court decides death-penalty cases?", "design": "Two probit models of every death-penalty decision from the 1971 to the 1988 terms, each judged by how many outcomes it classifies correctly, then combined; with a look at the 1989 term, which neither model had seen.", "data": [ "64 decisions involving the imposition of the death penalty, 1971–1988 terms, from the U.S. Reports. The outcome: the Court upheld the death sentence (45%) or not.", "Legal variables, coded from the parties' briefs and the lower-court opinions: whether the crime was an intentional murder, and whether the defense raised four doctrinal claims (a jury death-qualified by excluding opponents of capital punishment; a failure to consider the defendant's particular, mitigating circumstances; vague or overbroad aggravating factors; a state psychiatric examination).", "Extralegal variables: Court change (one point per appointment from Justice Stevens on), the political environment (one point each for a Republican President and Senate), expert defense counsel (the NAACP Legal Defense Fund, the ACLU, or appointed by the Court), a repeat-player state (Texas, Georgia, Florida), a Solicitor General amicus brief, and whether the state appealed." ], "method": [ "Probit regression of the outcome on each set of variables, then on both together.", "Each model judged by the share of cases it classifies correctly, against 55% for guessing that every case comes out for the defendant.", "Errors examined by era and by ideological direction." ], "findings": [ { "label": "Legal model", "paper": "75% correct", "note": "Every claim variable in the expected direction; the strongest, whether the crime was an intentional murder." }, { "label": "Extralegal model", "paper": "81% correct", "note": "Strongest effects: the political environment, who appealed and the Solicitor General's support for the state." }, { "label": "Integrated model", "paper": "88% correct", "note": "Extralegal factors carry about twice the weight of legal ones overall; legal factors dominated only at the start (1971–74)." }, { "label": "The 1989 term (out of sample)", "paper": "Legal model: 10 of 11 for the defendant; extralegal: all 11 for the state", "note": "The state won 10. Each model drifts in its own direction: the legal one toward the defendant, the political one toward the state." } ], "takeaway": "Neither law nor politics alone explains the death-penalty cases; the two models make opposite mistakes, and together they classify nearly nine cases in ten.", "script": "src/research/replication/george_epstein.py", "data_file": "/data/replication/george-epstein.json" } ] }