diff --git a/benchmarks/python/benchmark_callaway.py b/benchmarks/python/benchmark_callaway.py index 5fb959d82..98072261f 100644 --- a/benchmarks/python/benchmark_callaway.py +++ b/benchmarks/python/benchmark_callaway.py @@ -105,7 +105,6 @@ def main(): time="time", unit="unit", first_treat="first_treat", - aggregate="all", ) cs = CallawaySantAnna( @@ -122,8 +121,12 @@ def main(): time="time", unit="unit", first_treat="first_treat", - aggregate="all", # Get event study and group aggregations ) + # Post-fit aggregations stay inside the timed region: the R `aggte` + # yardstick is fit + simple/dynamic/group aggregation (the 3.x + # fit(aggregate="all") route, deprecated since 3.9 and removed at the 4.0 cut). + es = results.aggregate("event_study") + grp = results.aggregate("group") estimation_time = estimation_timer.elapsed total_time = estimation_time @@ -145,25 +148,25 @@ def main(): } ) - # Format event study effects (if available) + # Format event study effects from the post-fit container: one record per + # row INCLUDING the reference row (the legacy dict carried it too); + # run_refresh.py joins these records against R on `event_time`. es_effects = [] - if results.event_study_effects: - for rel_t, effect_data in sorted(results.event_study_effects.items()): - es_effects.append({ - "event_time": int(rel_t), - "att": float(effect_data["effect"]), - "se": float(effect_data["se"]), - }) - - # Format group effects (if available) + for row in es.to_dataframe().sort_values("event_time").itertuples(index=False): + es_effects.append({ + "event_time": int(row.event_time), + "att": float(row.att), + "se": float(row.se), + }) + + # Format group effects from the post-fit container (joined on `group`). grp_effects = [] - if results.group_effects: - for g, effect_data in sorted(results.group_effects.items()): - grp_effects.append({ - "group": int(g), - "att": float(effect_data["effect"]), - "se": float(effect_data["se"]), - }) + for row in grp.to_dataframe().sort_values("label").itertuples(index=False): + grp_effects.append({ + "group": int(row.label), + "att": float(row.att), + "se": float(row.se), + }) # Build output output = { diff --git a/benchmarks/python/benchmark_imputation.py b/benchmarks/python/benchmark_imputation.py index 27f556fed..6161e4d61 100644 --- a/benchmarks/python/benchmark_imputation.py +++ b/benchmarks/python/benchmark_imputation.py @@ -79,8 +79,10 @@ def main(): time="time", unit="unit", first_treat="first_treat", - aggregate="event_study", ) + # Post-fit event study stays inside the timed region (the 3.x + # fit(aggregate="event_study") route, deprecated since 3.9 and removed at the 4.0 cut). + es = results.aggregate("event_study") estimation_time = estimation_timer.elapsed total_time = estimation_time @@ -90,18 +92,17 @@ def main(): n_periods = len(df["time"].unique()) n_obs = len(df) - # Format event study effects (if available) + # Format event study effects from the post-fit container, skipping the + # reference-period marker row exactly as the legacy reader did. es_effects = [] - if results.event_study_effects: - for rel_t, effect_data in sorted(results.event_study_effects.items()): - # Skip reference period marker (n_obs == 0) - if effect_data.get("n_obs", 1) == 0: - continue - es_effects.append({ - "event_time": int(rel_t), - "att": float(effect_data["effect"]), - "se": float(effect_data["se"]), - }) + for row in es.to_dataframe().sort_values("event_time").itertuples(index=False): + if bool(row.is_reference): + continue + es_effects.append({ + "event_time": int(row.event_time), + "att": float(row.att), + "se": float(row.se), + }) # Build output output = { diff --git a/benchmarks/speed_review/bench_brfss_panel.py b/benchmarks/speed_review/bench_brfss_panel.py index 412401c86..69b0c1687 100644 --- a/benchmarks/speed_review/bench_brfss_panel.py +++ b/benchmarks/speed_review/bench_brfss_panel.py @@ -93,20 +93,28 @@ def cs_fit(): results["cs"] = cs.fit( results["panel"], outcome="y_mean", unit="state", time="year", first_treat="first_treat", - survey_design=results["stage2"], aggregate="all", + survey_design=results["stage2"], ) + # Materialize BOTH post-fit containers ONCE inside the timed fit + # phase (the 3.x fit(aggregate="all") workload = fit + event-study + # + group aggregation): the fit is bootstrapped, so every + # aggregate() call replays the bootstrap. + results["cs_es"] = results["cs"].aggregate("event_study") + results["cs_grp"] = results["cs"].aggregate("group") def inspect_pretrends(): - es = results["cs"].event_study_effects or {} + es = results["cs_es"].to_dataframe() + pre = es[es["event_time"] < 0] results["pretrends"] = { - rel_t: eff for rel_t, eff in es.items() if rel_t < 0 + int(r.event_time): {"effect": float(r.att), "se": float(r.se)} + for r in pre.itertuples(index=False) } def honest_grid(): out = {} for M in (0.5, 1.0, 1.5): out[M] = compute_honest_did( - results["cs"], method="relative_magnitude", M=M, + results["cs_es"], method="relative_magnitude", M=M, ) results["honest"] = out diff --git a/benchmarks/speed_review/bench_callaway.py b/benchmarks/speed_review/bench_callaway.py index 033daacb7..9fe6b0806 100644 --- a/benchmarks/speed_review/bench_callaway.py +++ b/benchmarks/speed_review/bench_callaway.py @@ -57,15 +57,18 @@ def bench_fit(n_units, n_bootstrap=0, covariates=None, n_cohorts=5, n_runs=3, times = [] for _ in range(n_runs): start = time.perf_counter() - cs.fit( + res = cs.fit( df, outcome="outcome", unit="unit", time="time", first_treat="first_treat", covariates=covariates, - aggregate="all", ) + # Keep "fit + aggregations" as the measured work (the 3.x + # fit(aggregate="all") route, deprecated since 3.9 and removed at the 4.0 cut). + res.aggregate("event_study") + res.aggregate("group") elapsed = time.perf_counter() - start times.append(elapsed) diff --git a/benchmarks/speed_review/bench_campaign_staggered.py b/benchmarks/speed_review/bench_campaign_staggered.py index 1e213858b..a74a831b2 100644 --- a/benchmarks/speed_review/bench_campaign_staggered.py +++ b/benchmarks/speed_review/bench_campaign_staggered.py @@ -1,7 +1,7 @@ """ Scenario 1: Staggered marketing campaign. -CallawaySantAnna with covariates + bootstrap + aggregate='all', wrapped in +CallawaySantAnna with covariates + bootstrap + post-fit aggregate(), wrapped in the 8-step Baker workflow: Bacon -> CS fit -> event-study pre-trend inspection -> HonestDiD M-grid -> SunAbraham + ImputationDiD robustness -> with/without-covariates refit -> practitioner_next_steps. @@ -64,21 +64,27 @@ def cs_fit(): control_group="never_treated", estimation_method="dr", cluster="unit", n_bootstrap=999, seed=123, ) - results["cs"] = cs.fit( - **fit_kwargs, covariates=covars, aggregate="all", - ) + results["cs"] = cs.fit(**fit_kwargs, covariates=covars) + # Materialize BOTH post-fit containers ONCE inside the timed fit + # phase (the 3.x fit(aggregate="all") workload = fit + event-study + # + group aggregation) and reuse them below: the fit has + # n_bootstrap=999, so every aggregate() call replays the bootstrap. + results["cs_es"] = results["cs"].aggregate("event_study") + results["cs_grp"] = results["cs"].aggregate("group") def inspect_pretrends(): - es = results["cs"].event_study_effects or {} + es = results["cs_es"].to_dataframe() + pre = es[es["event_time"] < 0] results["pretrends"] = { - rel_t: eff for rel_t, eff in es.items() if rel_t < 0 + int(r.event_time): {"effect": float(r.att), "se": float(r.se)} + for r in pre.itertuples(index=False) } def honest_did_grid(): out = {} for M in (0.5, 1.0, 1.5, 2.0): out[M] = compute_honest_did( - results["cs"], method="relative_magnitude", M=M, + results["cs_es"], method="relative_magnitude", M=M, ) results["honest"] = out @@ -88,7 +94,8 @@ def sun_abraham(): def imputation(): bjs = ImputationDiD(cluster="unit") - results["bjs"] = bjs.fit(**fit_kwargs, aggregate="event_study") + results["bjs"] = bjs.fit(**fit_kwargs) + results["bjs_es"] = results["bjs"].aggregate("event_study") def cs_no_covariates(): # Match phase 2's estimator config exactly; the only axis that @@ -99,7 +106,10 @@ def cs_no_covariates(): control_group="never_treated", estimation_method="dr", cluster="unit", n_bootstrap=999, seed=123, ) - results["cs_nocov"] = cs.fit(**fit_kwargs, aggregate="all") + results["cs_nocov"] = cs.fit(**fit_kwargs) + # Same fit + event-study + group workload as the covariate arm. + results["cs_nocov_es"] = results["cs_nocov"].aggregate("event_study") + results["cs_nocov_grp"] = results["cs_nocov"].aggregate("group") def next_steps(): results["guidance"] = practitioner_next_steps(results["cs"]) @@ -137,7 +147,7 @@ def run_scale(scale, config): "n_obs": int(len(data)), "covariates": covars, "n_bootstrap": 999, - "aggregate": "all", + "aggregate": "fit + post-fit aggregate", "estimation_method": "dr", }, ) diff --git a/benchmarks/speed_review/bench_memory_scaling.py b/benchmarks/speed_review/bench_memory_scaling.py index 6e0e4ced8..150e67885 100644 --- a/benchmarks/speed_review/bench_memory_scaling.py +++ b/benchmarks/speed_review/bench_memory_scaling.py @@ -128,7 +128,6 @@ def _run_one(cfg): "dose", "period", "unit", - aggregate="event_study", survey_design=SurveyDesign(weights="w"), ) elif est == "twfe": diff --git a/benchmarks/speed_review/bench_solve_ols_fastpath.py b/benchmarks/speed_review/bench_solve_ols_fastpath.py index 3c668b475..94437eb71 100644 --- a/benchmarks/speed_review/bench_solve_ols_fastpath.py +++ b/benchmarks/speed_review/bench_solve_ols_fastpath.py @@ -124,8 +124,11 @@ def fit(d): time="time", first_treat="first_treat", covariates=covs, - aggregate="all", ) + # fit + aggregations is the measured work (the 3.x + # fit(aggregate="all") route, deprecated since 3.9 and removed at the 4.0 cut). + res.aggregate("event_study") + res.aggregate("group") return float(res.overall_att), float(res.overall_se) return df, fit diff --git a/benchmarks/speed_review/validate_results.py b/benchmarks/speed_review/validate_results.py index 9578a907e..684ec19e3 100644 --- a/benchmarks/speed_review/validate_results.py +++ b/benchmarks/speed_review/validate_results.py @@ -64,8 +64,11 @@ def run_estimator(df, estimation_method="reg", covariates=None, control_group="n time="time", first_treat="first_treat", covariates=covariates, - aggregate="all", ) + # Post-fit containers (the 3.x fit(aggregate="all") route is deprecated and goes at the 4.0 cut); + # the fit is bootstrapped, so each call replays the fit-time bootstrap. + es_container = results.aggregate("event_study") + grp_container = results.aggregate("group") out = { "overall_att": float(results.overall_att), @@ -87,31 +90,30 @@ def run_estimator(df, estimation_method="reg", covariates=None, control_group="n } out["group_time_effects"] = gt_effects - # Event study - if results.event_study_effects: - es = {} - for e, data in sorted(results.event_study_effects.items()): - es[str(e)] = { - "effect": float(data["effect"]), - "se": float(data["se"]), - "t_stat": float(data["t_stat"]), - "p_value": float(data["p_value"]), - "conf_int": [float(data["conf_int"][0]), float(data["conf_int"][1])], - } - out["event_study"] = es + # Event study (same keyed shape as the legacy dict, one record per + # container row including the reference row) + es = {} + for row in es_container.to_dataframe().sort_values("event_time").itertuples(index=False): + es[str(int(row.event_time))] = { + "effect": float(row.att), + "se": float(row.se), + "t_stat": float(row.t_stat), + "p_value": float(row.p_value), + "conf_int": [float(row.conf_int_lower), float(row.conf_int_upper)], + } + out["event_study"] = es # Group effects - if results.group_effects: - ge = {} - for g_key, data in sorted(results.group_effects.items()): - ge[str(g_key)] = { - "effect": float(data["effect"]), - "se": float(data["se"]), - "t_stat": float(data["t_stat"]), - "p_value": float(data["p_value"]), - "conf_int": [float(data["conf_int"][0]), float(data["conf_int"][1])], - } - out["group_effects"] = ge + ge = {} + for row in grp_container.to_dataframe().sort_values("label").itertuples(index=False): + ge[str(row.label)] = { + "effect": float(row.att), + "se": float(row.se), + "t_stat": float(row.t_stat), + "p_value": float(row.p_value), + "conf_int": [float(row.conf_int_lower), float(row.conf_int_upper)], + } + out["group_effects"] = ge return out diff --git a/changelog.d/20261020-v4-pr3a-aggregate-test-migration.md b/changelog.d/20261020-v4-pr3a-aggregate-test-migration.md new file mode 100644 index 000000000..5c44f562e --- /dev/null +++ b/changelog.d/20261020-v4-pr3a-aggregate-test-migration.md @@ -0,0 +1,25 @@ +### Internal +- **4.0 cut, PR 3a - tests-first migration to the post-fit aggregation containers.** The + test and benchmark suites now read event-study / group surfaces from + `results.aggregate("event_study")` / `results.aggregate("group")` (the container API that + has existed since 3.9) instead of the deprecated fit-time `fit(aggregate=...)` keyword and + the legacy `event_study_effects` / `group_effects` fields, so the keyword removals in PRs 3b + and 3c land against already-migrated suites. No library source changes and no ledger row + flips in this PR; the fit-time keywords still work and still warn. Bootstrapped + ImputationDiD / TwoStageDiD / ContinuousDiD sites stay on the fit-time route until PR 3c + adds the post-fit relay for them. +- **Committed per-backend aggregation oracles.** `tests/test_aggregate_contract.py` no longer + uses the fit-time route as its numerical reference: every analytic surface is pinned at + `rtol=1e-12` (confidence-interval bounds at `1e-9`: scipy's `t.ppf` moved by up to 3.4e-11 + relative between 1.13 and 1.14+) and every bootstrapped CS / EfficientDiD replay at the + replay-parity tolerances against `tests/fixtures/v4_aggregate_oracles.json`, captured on the unmodified + tree at `8724fecd` by `tests/_capture_v4_aggregate_oracles.py` under both weight backends + (see `tests/fixtures/README.md`). New `tests/helpers/legacy_surface.py` (`es_dict`, + `group_dict`, `legacy_standin`) gives consumer tests a faithful stand-in for the 3.x + fit-time surface where their subject is the native (legacy-pickle) route. +- **Benchmarks** (`benchmarks/python/benchmark_callaway.py`, `benchmark_imputation.py`, the + `speed_review` scripts) call `aggregate()` post-fit inside the timed region and emit the + same JSON row shape the R refresh joins on. +- `docs/v4-design.md` section 9 records the three-way split of cut-sequence item 3 (3a + tests-first, 3b mechanical removals, 3c the Imputation/TwoStage/ContinuousDiD removals + + relay). diff --git a/docs/v4-design.md b/docs/v4-design.md index fe79acd26..815b6f084 100644 --- a/docs/v4-design.md +++ b/docs/v4-design.md @@ -1002,7 +1002,8 @@ re-enumerate the cells' M-id lists: cut's PR order; order and rationale only - the M-id rosters live in the phase-5 cell above and the lifecycle facts in the ledger. Every PR lands on main under the section 2 cut window with `__version__` unchanged; each flips -its rows terminal and names them in its changelog fragment. +its rows terminal and names them in its changelog fragment (3a excepted: a +tests-first PR that flips no rows). 0. Spec/ledger refresh (the PR that wrote this subsection): the cut-window rule, the roster/M-080/M-008 amendments, the [M-150] locator, the id @@ -1017,7 +1018,16 @@ its rows terminal and names them in its changelog fragment. [M-045]/[M-115]; `_deprecation.py` stays (section 8 rule 11 corollary). 3. Fit-time `aggregate=`/`balance_e=` removals, plus the runtime messages that still steer users to the deprecated route (readers recorded in the rows' - `code_refs`) and the benchmark/guide sweep. + `code_refs`) and the benchmark/guide sweep. Split three ways (2026-10-20) + so each piece fits the AI reviewer's input cap: + - 3a - tests-first migration of the test and benchmark suites to the + post-fit containers; flips no rows, by design (it carries the test bulk + so 3b/3c fit the reviewer) (SHIPPED, PR 3a). + - 3b - the mechanical removals on the estimators whose post-fit route + already exists (CallawaySantAnna, EfficientDiD, StackedDiD, dCDH, HAD, + the HAD pretest workflow). + - 3c - the ImputationDiD/TwoStageDiD/ContinuousDiD removals plus the + bootstrapped-fit relay (section 6 amendment). 4. The three merges: the MultiPeriodDiD class, `MultiPeriodDiDResults`, `PeriodEffect` and `EventStudy`; the QDiD class and `QDiDResults`; the `StaggeredTripleDifference` CLASS and `SDDD` - but NOT diff --git a/tests/_capture_v4_aggregate_oracles.py b/tests/_capture_v4_aggregate_oracles.py new file mode 100644 index 000000000..8550d8b33 --- /dev/null +++ b/tests/_capture_v4_aggregate_oracles.py @@ -0,0 +1,573 @@ +"""Capture the pre-removal aggregation oracles for tests/test_aggregate_contract.py. + +MUST run on a tree where the fit-time ``aggregate=`` / ``balance_e=`` keywords still +exist (they are removed at the 4.0 cut, PRs 3b/3c), ONCE PER BACKEND:: + + DIFF_DIFF_BACKEND=python python tests/_capture_v4_aggregate_oracles.py > numpy.json + DIFF_DIFF_BACKEND=rust python tests/_capture_v4_aggregate_oracles.py > rust.json + +and the two payloads are merged into ``tests/fixtures/v4_aggregate_oracles.json`` +(``{"analytic": ..., "boot": {"numpy": ..., "rust": ...}, "provenance": ...}``). + +Why. The 4.0 cut (PRs 3b/3c) deletes the deprecated fit-time aggregation route that +``test_aggregate_contract.py`` used as its NUMERICAL REFERENCE (the ``*_fit_time`` +fixtures and ``_boot_fit_time`` helpers): post-fit ``aggregate()`` was asserted +equal to ``fit(aggregate=...)``. Once that route is removed there is no independent +second implementation left to disagree with, so the reference becomes these +committed numbers, captured while the route still existed: + +* ANALYTIC oracles - the ``fit(aggregate="all")`` (ContinuousDiD: + ``"eventstudy"``) and ``balance_e=1`` surfaces on every design arm the contract + file exercises. After the removal the kit recompute must reproduce them. +* BOOTSTRAPPED oracles, per weight backend - ``generate_bootstrap_weights_batch`` + draws a DIFFERENT seeded stream under the Rust backend than under NumPy, so + the percentile surfaces are backend-specific. After the removal the CS/EDiD + replays and the Imputation/TwoStage/ContinuousDiD relays must reproduce them, + and a bootstrapped PLAIN fit's overall quintet must be unchanged (the + always-on surface materialization draws no extra weights). + +The panel builders are imported from the contract test module itself so the +oracle panels are, by construction, the test panels. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import warnings +from typing import Any, Dict, Optional + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "helpers")) + +import test_aggregate_contract as T # noqa: E402 (the contract module's panel builders) + +from diff_diff import CallawaySantAnna, SurveyDesign # noqa: E402 +from diff_diff.bootstrap_chunking import effective_weight_backend # noqa: E402 + + +# --------------------------------------------------------------------------- # +# serialization helpers +# --------------------------------------------------------------------------- # +def _f(x: Any) -> Optional[float]: + if x is None: + return None + try: + v = float(x) + except (TypeError, ValueError): + return None + return None if np.isnan(v) else v + + +def _key(k: Any) -> Any: + # JSON object keys must be strings; keep a lossless repr of the label. + if isinstance(k, (np.integer, int)): + return str(int(k)) + if isinstance(k, (np.floating, float)): + return repr(float(k)) + return str(k) + + +def _row(d: Dict[str, Any]) -> Dict[str, Any]: + ci = d.get("conf_int", (np.nan, np.nan)) + out = { + "effect": _f(d.get("effect")), + "se": _f(d.get("se")), + "t_stat": _f(d.get("t_stat")), + "p_value": _f(d.get("p_value")), + "ci_lo": _f(ci[0]), + "ci_hi": _f(ci[1]), + } + for k in ("n_obs", "n_groups", "n_units", "df_used"): + if k in d: + out[k] = _f(d[k]) + cb = d.get("cband_conf_int") + if cb is not None: + out["cband_lo"], out["cband_hi"] = _f(cb[0]), _f(cb[1]) + return out + + +def _dict_surface(surface: Optional[Dict[Any, Dict[str, Any]]]) -> Optional[Dict[str, Any]]: + if surface is None: + return None + return {_key(k): _row(v) for k, v in sorted(surface.items(), key=lambda kv: float(kv[0]))} + + +def _overall(r: Any) -> Dict[str, Optional[float]]: + ci = getattr(r, "overall_conf_int", None) or (np.nan, np.nan) + return { + "att": _f(r.overall_att), + "se": _f(r.overall_se), + "t_stat": _f(getattr(r, "overall_t_stat", np.nan)), + "p_value": _f(r.overall_p_value), + "ci_lo": _f(ci[0]), + "ci_hi": _f(ci[1]), + } + + +def _matrix(m: Any) -> Optional[list]: + if m is None: + return None + arr = np.asarray(m, dtype=float) + return [[_f(x) for x in row] for row in arr] + + +def _ref_labels(r: Any) -> Optional[list]: + """The event-time labels the post-fit container marks ``is_reference``. + + Stored so the oracle check can assert reference MEMBERSHIP independently of + the container under test (a surface that wrongly marks an estimated horizon + as reference must fail, not skip). Where the post-fit route is closed on + the capture tree (bootstrapped Imputation/TwoStage/ContinuousDiD) the + legacy zero-count sentinel rows (``n_obs``/``n_groups`` == 0) are the + reference set; a surface with neither yields ``None`` (no membership pin). + """ + try: + es = _quiet(r.aggregate, "event_study") + except NotImplementedError: + legacy = r.event_study_effects or {} + refs = [ + _key(k) + for k, v in legacy.items() + if ("n_obs" in v or "n_groups" in v) and float(v.get("n_obs", v.get("n_groups"))) == 0.0 + ] + return sorted(refs, key=float) + labels = list(es.event_time) + flags = np.asarray(es.is_reference, dtype=bool) + return sorted((_key(lab) for lab, f in zip(labels, flags) if f), key=float) + + +def _es_bundle(r: Any) -> Dict[str, Any]: + """Event-study surface + its provenance side fields where the class has them.""" + out: Dict[str, Any] = {"es": _dict_surface(r.event_study_effects)} + out["ref_labels"] = None if r.event_study_effects is None else _ref_labels(r) + for attr in ("event_study_vcov_index", "reference_event_times"): + v = getattr(r, attr, None) + out[attr] = None if v is None else [_f(x) for x in v] + out["event_study_vcov"] = _matrix(getattr(r, "event_study_vcov", None)) + out["event_study_df"] = _f(getattr(r, "event_study_df", None)) + out["cband_crit_value"] = _f(getattr(r, "cband_crit_value", None)) + return out + + +def _all_bundle(r: Any) -> Dict[str, Any]: + out = _es_bundle(r) + out["group"] = _dict_surface(getattr(r, "group_effects", None)) + out["overall"] = _overall(r) + return out + + +def _quiet(fn, *a, **kw): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(*a, **kw) + + +# --------------------------------------------------------------------------- # +# analytic oracles (backend-independent in principle; the two runs are diffed) +# --------------------------------------------------------------------------- # +def capture_analytic() -> Dict[str, Any]: + out: Dict[str, Any] = {} + panel = T._panel() + + # ---- CallawaySantAnna -------------------------------------------------- + cs: Dict[str, Any] = {} + cs["default_all"] = _all_bundle( + _quiet(CallawaySantAnna().fit, panel, aggregate="all", **T.FIT_KW) + ) + cs["default_balance_e1"] = _es_bundle( + _quiet(CallawaySantAnna().fit, panel, aggregate="event_study", balance_e=1, **T.FIT_KW) + ) + cs["rcs_all"] = _all_bundle( + _quiet(CallawaySantAnna(panel=False).fit, T._rcs(), aggregate="all", **T.FIT_KW) + ) + cs["anticipation1_all"] = _all_bundle( + _quiet(CallawaySantAnna(anticipation=1).fit, panel, aggregate="all", **T.FIT_KW) + ) + cs["universal_all"] = _all_bundle( + _quiet(CallawaySantAnna(base_period="universal").fit, panel, aggregate="all", **T.FIT_KW) + ) + cs["nyt_anticipation1_all"] = _all_bundle( + _quiet( + CallawaySantAnna(control_group="not_yet_treated", anticipation=1).fit, + panel, + aggregate="all", + **T.FIT_KW, + ) + ) + thinned = panel.drop(panel.index[::13]).reset_index(drop=True) + cs["unbalanced_all"] = _all_bundle( + _quiet( + CallawaySantAnna(allow_unbalanced_panel=True).fit, thinned, aggregate="all", **T.FIT_KW + ) + ) + cs["survey_all"] = _all_bundle( + _quiet( + CallawaySantAnna().fit, + T._survey_panel(), + survey_design=T._survey_design(), + aggregate="all", + **T.FIT_KW, + ) + ) + out["CallawaySantAnna"] = cs + + # ---- EfficientDiD ------------------------------------------------------ + ed: Dict[str, Any] = {} + ep = T._efficient_panel() + ed["default_all"] = _all_bundle(T._fit_efficient(ep, aggregate="all")) + ed["default_balance_e1"] = _es_bundle( + T._fit_efficient(ep, aggregate="event_study", balance_e=1) + ) + ed["fractional_balance_e1"] = _es_bundle( + T._fit_efficient(T._efficient_fractional_panel(), aggregate="event_study", balance_e=1) + ) + ed["clustered_all"] = _all_bundle( + T._fit_efficient(T._efficient_clustered_panel(), est_kw={"cluster": "cl"}, aggregate="all") + ) + d_sw, _ = T._efficient_survey_panel() + ed["survey_weights_all"] = _all_bundle( + T._fit_efficient(d_sw, survey_design=T._efficient_survey_design(), aggregate="all") + ) + for tag, degenerate in (("healthy", None), ("dropped", "dropped"), ("undefined", "undefined")): + d_rep, rep_cols = T._efficient_survey_panel(replicate=True, degenerate=degenerate) + ed[f"replicate_{tag}_all"] = _all_bundle( + T._fit_efficient( + d_rep, survey_design=T._efficient_survey_design(rep_cols), aggregate="all" + ) + ) + out["EfficientDiD"] = ed + + # ---- ImputationDiD ----------------------------------------------------- + im: Dict[str, Any] = {} + ip = T._imputation_panel() + im["default_all"] = _all_bundle(T._fit_imputation(ip, aggregate="all")) + im["default_balance_e1"] = _es_bundle( + T._fit_imputation(ip, aggregate="event_study", balance_e=1) + ) + im["pretrends_all"] = _all_bundle( + T._fit_imputation(ip, est_kw={"pretrends": True}, aggregate="all") + ) + im["clustered_all"] = _all_bundle( + T._fit_imputation( + T._imputation_clustered_panel(), est_kw={"cluster": "cl"}, aggregate="all" + ) + ) + d_sw, _ = T._imputation_survey_panel() + im["survey_tsl_all"] = _all_bundle( + T._fit_imputation(d_sw, survey_design=T._imputation_survey_design(), aggregate="all") + ) + for degenerate in (None, "dropped", "undefined", "cohort_zero"): + d_rep, rep_cols = T._imputation_survey_panel(replicate=True, degenerate=degenerate) + sd = T._imputation_survey_design(rep_cols) + tag = degenerate or "none" + # LEVEL-MATCHED references: the joint replicate stack couples rows, so + # fit(aggregate="event_study") and fit(aggregate="group") are separate. + im[f"replicate_{tag}_event_study"] = _es_bundle( + T._fit_imputation(d_rep, survey_design=sd, aggregate="event_study") + ) + im[f"replicate_{tag}_group"] = _all_bundle( + T._fit_imputation(d_rep, survey_design=sd, aggregate="group") + ) + d_cov = T._imputation_panel(seed=23).copy() + d_cov["x1"] = np.random.default_rng(23).normal(size=len(d_cov)) + 0.1 * d_cov["period"] + im["covariates_all"] = _all_bundle(T._fit_imputation(d_cov, covariates=["x1"], aggregate="all")) + out["ImputationDiD"] = im + + # ---- TwoStageDiD ------------------------------------------------------- + ts: Dict[str, Any] = {} + tp = T._twostage_panel() + ts["default_all"] = _all_bundle(T._fit_twostage(tp, aggregate="all")) + ts["default_balance_e1"] = _es_bundle(T._fit_twostage(tp, aggregate="event_study", balance_e=1)) + ts["pretrends_all"] = _all_bundle( + T._fit_twostage(tp, est_kw={"pretrends": True}, aggregate="all") + ) + ts["prop5_all"] = _all_bundle(T._fit_twostage(T._twostage_prop5_panel(), aggregate="all")) + ts["clustered_all"] = _all_bundle( + T._fit_twostage(T._twostage_clustered_panel(), est_kw={"cluster": "cl"}, aggregate="all") + ) + d_sw, _ = T._twostage_survey_panel() + ts["survey_tsl_all"] = _all_bundle( + T._fit_twostage(d_sw, survey_design=T._twostage_survey_design(), aggregate="all") + ) + ts["survey_psu_all"] = _all_bundle( + T._fit_twostage(d_sw, survey_design=T._twostage_survey_design(psu=True), aggregate="all") + ) + ts["always_treated_survey_all"] = _all_bundle( + T._fit_twostage( + T._twostage_always_treated_panel(), + survey_design=T._twostage_survey_design(), + aggregate="all", + ) + ) + for degenerate in (None, "dropped", "undefined", "cohort_zero"): + d_rep, rep_cols = T._twostage_survey_panel(replicate=True, degenerate=degenerate) + sd = T._twostage_survey_design(rep_cols) + tag = degenerate or "none" + ts[f"replicate_{tag}_event_study"] = _es_bundle( + T._fit_twostage(d_rep, survey_design=sd, aggregate="event_study") + ) + ts[f"replicate_{tag}_group"] = _all_bundle( + T._fit_twostage(d_rep, survey_design=sd, aggregate="group") + ) + ts["always_treated_survey_psu_all"] = _all_bundle( + T._fit_twostage( + T._twostage_always_treated_panel(), + survey_design=T._twostage_survey_design(psu=True), + aggregate="all", + ) + ) + d_cov = T._twostage_panel(seed=23).copy() + d_cov["x1"] = np.random.default_rng(23).normal(size=len(d_cov)) + 0.1 * d_cov["period"] + ts["covariates_all"] = _all_bundle(T._fit_twostage(d_cov, covariates=["x1"], aggregate="all")) + out["TwoStageDiD"] = ts + + # ---- ContinuousDiD (no group level; "eventstudy" is the only ES spelling) - + co: Dict[str, Any] = {} + cp = T._cont_panel() + co["default_eventstudy"] = _all_bundle(T._fit_cont(cp, aggregate="eventstudy")) + co["covariate_eventstudy"] = _all_bundle( + T._fit_cont(T._cont_covariate_panel(), covariates=["x1"], aggregate="eventstudy") + ) + co["anticipation1_eventstudy"] = _all_bundle( + T._fit_cont(T._cont_panel(seed=47), est_kw={"anticipation": 1}, aggregate="eventstudy") + ) + d_sw, _ = T._cont_survey_panel() + co["survey_tsl_eventstudy"] = _all_bundle( + T._fit_cont(d_sw, survey_design=T._cont_survey_design(), aggregate="eventstudy") + ) + co["empty_post_eventstudy"] = _all_bundle( + T._fit_cont(T._cont_empty_post_panel(), aggregate="eventstudy") + ) + # The remaining TestContinuousAggregate.test_event_study_across_designs + # arms (previously pinned only structurally); data built exactly as there. + co["multi_cohort_eventstudy"] = _all_bundle( + T._fit_cont(T._cont_panel(seed=43, cohort_periods=[3, 4, 5]), aggregate="eventstudy") + ) + d_zd, _ = T._cont_survey_panel(zero_dose_treated=True) + co["survey_zero_dose_drop_eventstudy"] = _all_bundle( + T._fit_cont(d_zd, survey_design=T._cont_survey_design(), aggregate="eventstudy") + ) + for tag, degenerate in (("healthy", None), ("dropped", "dropped"), ("undefined", "undefined")): + d_rep, rep_cols = T._cont_survey_panel(replicate=True, degenerate=degenerate) + co[f"replicate_{tag}_eventstudy"] = _all_bundle( + T._fit_cont( + d_rep, + survey_design=T._cont_survey_design(rep_cols=rep_cols), + aggregate="eventstudy", + ) + ) + co["discrete_eventstudy"] = _all_bundle( + T._fit_cont( + T._cont_discrete_panel(), est_kw={"treatment_type": "discrete"}, aggregate="eventstudy" + ) + ) + co["lowest_dose_eventstudy"] = _all_bundle( + T._fit_cont( + T._cont_lowest_dose_panel(), + est_kw={"control_group": "lowest_dose"}, + aggregate="eventstudy", + ) + ) + co["not_yet_treated_eventstudy"] = _all_bundle( + T._fit_cont( + T._cont_panel(seed=53, cohort_periods=[3, 5]), + est_kw={"control_group": "not_yet_treated"}, + aggregate="eventstudy", + ) + ) + co["nondefault_config_eventstudy"] = _all_bundle( + T._fit_cont( + T._cont_panel(seed=59), + est_kw={"alpha": 0.10, "base_period": "universal"}, + aggregate="eventstudy", + ) + ) + out["ContinuousDiD"] = co + return out + + +# --------------------------------------------------------------------------- # +# bootstrapped oracles (backend-specific) +# --------------------------------------------------------------------------- # +def capture_boot() -> Dict[str, Any]: + out: Dict[str, Any] = {} + panel = T._panel() + + # ---- CallawaySantAnna: fit-time reference surfaces the replay must match - + cs: Dict[str, Any] = {} + cs["default_all"] = _all_bundle(T._boot_fit_time(panel)) + cs["default_plain_overall"] = _overall(T._boot_fit(panel)) + d = panel.copy() + d["psu"] = d["unit"] % 7 + cs["psu_cluster_all"] = _all_bundle(T._boot_fit_time(d, cluster="psu")) + d2 = panel.copy() + rng = np.random.default_rng(5) + wmap = {u: rng.uniform(0.5, 2.0) for u in d2["unit"].unique()} + d2["w"] = d2["unit"].map(wmap) + d2["psu"] = d2["unit"] % 7 + d2["stratum"] = d2["unit"] % 2 + sd = SurveyDesign(weights="w", strata="stratum", psu="psu", nest=True) + cs["stratified_survey_all"] = _all_bundle( + T._boot_fit_time(d2, fit_kwargs={"survey_design": sd}) + ) + cs["cband_all"] = _all_bundle(T._boot_fit_time(panel, cband=True)) + # The three design arms TestBootstrapReplayDesigns previously pinned only + # structurally: non-census FPC survey, repeated cross-sections, unbalanced. + d3 = panel.copy() + rng3 = np.random.default_rng(6) + wmap3 = {u: rng3.uniform(0.5, 2.0) for u in d3["unit"].unique()} + d3["w"] = d3["unit"].map(wmap3) + d3["psu"] = d3["unit"] % 7 + d3["fpc"] = 100 + sd3 = SurveyDesign(weights="w", psu="psu", fpc="fpc") + cs["fpc_survey_all"] = _all_bundle(T._boot_fit_time(d3, fit_kwargs={"survey_design": sd3})) + cs["rcs_all"] = _all_bundle(T._boot_fit_time(T._rcs(), panel=False)) + thinned = panel.drop(panel.index[::13]).reset_index(drop=True) + cs["unbalanced_all"] = _all_bundle(T._boot_fit_time(thinned, allow_unbalanced_panel=True)) + out["CallawaySantAnna"] = cs + + # ---- EfficientDiD ------------------------------------------------------ + ed: Dict[str, Any] = {} + ep = T._efficient_panel() + ed["default_all"] = _all_bundle(T._efficient_boot_fit_time(ep)) + ed["default_plain_overall"] = _overall(T._efficient_boot_fit(ep)) + ed["clustered_all"] = _all_bundle( + T._efficient_boot_fit_time(T._efficient_clustered_panel(), est_kw={"cluster": "cl"}) + ) + d_sw, _ = T._efficient_survey_panel() + ed["survey_weights_all"] = _all_bundle( + T._efficient_boot_fit_time(d_sw, survey_design=T._efficient_survey_design()) + ) + d_ps, sd_ps = T.TestEfficientBootstrapReplayDesigns._psu_survey_frame(strata=True) + ed["stratified_psu_survey_all"] = _all_bundle( + T._efficient_boot_fit_time(d_ps, survey_design=sd_ps) + ) + d_fpc, sd_fpc = T.TestEfficientBootstrapReplayDesigns._psu_survey_frame(fpc=200.0) + ed["fpc_survey_all"] = _all_bundle(T._efficient_boot_fit_time(d_fpc, survey_design=sd_fpc)) + ed["fractional_balance_e1_es"] = _es_bundle( + T._efficient_boot_fit_time( + T._efficient_fractional_panel(), aggregate="event_study", balance_e=1 + ) + ) + # Arms TestEfficientBootstrapReplay(Designs) previously pinned only + # structurally: covariates, anticipation and the alpha/Mammen wiring. + d_cov = ep.copy() + rng_cov = np.random.default_rng(3) + xmap = {u: rng_cov.normal() for u in d_cov["unit"].unique()} + d_cov["x1"] = d_cov["unit"].map(xmap) + ed["covariates_all"] = _all_bundle(T._efficient_boot_fit_time(d_cov, covariates=["x1"])) + ed["anticipation1_all"] = _all_bundle( + T._efficient_boot_fit_time(ep, est_kw={"anticipation": 1}) + ) + ed["alpha10_mammen_all"] = _all_bundle( + T._efficient_boot_fit_time(ep, est_kw={"alpha": 0.10, "bootstrap_weights": "mammen"}) + ) + out["EfficientDiD"] = ed + + # ---- ImputationDiD: the fit-time route the relay must reproduce --------- + im: Dict[str, Any] = {} + ip = T._imputation_panel() + boot_kw = {"n_bootstrap": 19, "seed": 1} + im["default_all"] = _all_bundle(T._fit_imputation(ip, est_kw=boot_kw, aggregate="all")) + im["default_plain_overall"] = _overall(T._fit_imputation(ip, est_kw=boot_kw)) + im["pretrends_all"] = _all_bundle( + T._fit_imputation(ip, est_kw={**boot_kw, "pretrends": True}, aggregate="all") + ) + im["pretrends_plain_overall"] = _overall( + T._fit_imputation(ip, est_kw={**boot_kw, "pretrends": True}) + ) + d_sw, _ = T._imputation_survey_panel() + im["survey_tsl_all"] = _all_bundle( + T._fit_imputation( + d_sw, est_kw=boot_kw, survey_design=T._imputation_survey_design(), aggregate="all" + ) + ) + im["survey_tsl_plain_overall"] = _overall( + T._fit_imputation(d_sw, est_kw=boot_kw, survey_design=T._imputation_survey_design()) + ) + out["ImputationDiD"] = im + + # ---- TwoStageDiD ------------------------------------------------------- + ts: Dict[str, Any] = {} + tp = T._twostage_panel() + ts["default_all"] = _all_bundle(T._fit_twostage(tp, est_kw=boot_kw, aggregate="all")) + ts["default_plain_overall"] = _overall(T._fit_twostage(tp, est_kw=boot_kw)) + ts["pretrends_all"] = _all_bundle( + T._fit_twostage(tp, est_kw={**boot_kw, "pretrends": True}, aggregate="all") + ) + ts["prop5_all"] = _all_bundle( + T._fit_twostage(T._twostage_prop5_panel(), est_kw=boot_kw, aggregate="all") + ) + ts["prop5_plain_overall"] = _overall(T._fit_twostage(T._twostage_prop5_panel(), est_kw=boot_kw)) + out["TwoStageDiD"] = ts + + # ---- ContinuousDiD ----------------------------------------------------- + co: Dict[str, Any] = {} + cp = T._cont_panel() + cont_kw = {"n_bootstrap": 30, "seed": 3} + + def _cont_bundle(r: Any) -> Dict[str, Any]: + b = _es_bundle(r) + b["overall"] = _overall(r) + b["overall_acrt"] = _f(getattr(r, "overall_acrt", np.nan)) + b["overall_acrt_se"] = _f(getattr(r, "overall_acrt_se", np.nan)) + return b + + co["default_eventstudy"] = _cont_bundle(T._fit_cont(cp, est_kw=cont_kw, aggregate="eventstudy")) + co["default_plain"] = _cont_bundle(T._fit_cont(cp, est_kw=cont_kw)) + out["ContinuousDiD"] = co + return out + + +def dump_fixture(obj: Any) -> str: + """The fixture's canonical text: sorted keys, one indent level per nesting + level, and every LEAF container (a dict or list of scalars - an oracle + row, a label list, a vcov matrix row) on a single line. Diffable per row + without the ~3x size of a fully indented dump.""" + + def _fmt(o: Any, ind: int = 0) -> str: + pad = " " * ind + if isinstance(o, dict): + if all(not isinstance(v, (dict, list)) for v in o.values()): + return json.dumps(o, sort_keys=True) + items = [f"{pad} {json.dumps(k)}: {_fmt(v, ind + 1)}" for k, v in sorted(o.items())] + return "{\n" + ",\n".join(items) + f"\n{pad}}}" + if isinstance(o, list): + if all(not isinstance(v, (dict, list)) for v in o): + return json.dumps(o) + items = [f"{pad} {_fmt(v, ind + 1)}" for v in o] + return "[\n" + ",\n".join(items) + f"\n{pad}]" + return json.dumps(o) + + return _fmt(obj) + "\n" + + +def _git_head() -> str: + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=HERE, text=True, stderr=subprocess.DEVNULL + ).strip() + except Exception: # noqa: BLE001 + return "unknown" + + +if __name__ == "__main__": + import diff_diff + + payload = { + "provenance": { + "captured_at_head": _git_head(), + "diff_diff_version": diff_diff.__version__, + "weight_backend": effective_weight_backend(), + "numpy": np.__version__, + }, + "analytic": capture_analytic(), + "boot": capture_boot(), + } + sys.stdout.write(dump_fixture(payload)) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index fed6ee02c..fc90e4186 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -32,3 +32,36 @@ capturing `2> ` from a real run and applying the same rules. There is deliberately NO 0.153.4 transcript: the format was first observed on that CLI, but the live pin is 0.162.1 and the older CLI is no longer installed. + +## `v4_aggregate_oracles.json` - pre-removal aggregation oracles (4.0 PRs 3a-3c) + +Numeric reference for `tests/test_aggregate_contract.py`, captured by +`tests/_capture_v4_aggregate_oracles.py` on the UNMODIFIED tree at the commit +named in `provenance.captured_at_head` - i.e. while the deprecated fit-time +`fit(aggregate=...)` / `fit(balance_e=...)` route still existed and could serve +as an independent second implementation of every aggregation surface. Layout: + +- `analytic` - per estimator, the `fit(aggregate="all")` (ContinuousDiD: + `"eventstudy"`) and `balance_e=1` surfaces on every design arm the contract + file exercises. The NumPy and Rust captures agreed to rtol 1e-12 / + atol 1e-13, so one copy is stored; the post-fit kit recompute must reproduce + it after the removal. +- `boot` - keyed by weight backend (`"numpy"` / `"rust"`, the + `effective_weight_backend()` vocabulary), because the seeded multiplier + stream differs per backend: the fit-time bootstrapped surfaces the CS/EDiD + replays and the Imputation/TwoStage/ContinuousDiD relays must reproduce, plus + the overall quintets of bootstrapped PLAIN fits, which the always-on surface + materialization must leave unchanged. A pin whose backend key is absent skips. + +Rows are stored as `{label: {effect, se, t_stat, p_value, ci_lo, ci_hi, +n_obs|n_groups, df_used, cband_lo/hi}}` with NaN serialized as `null` (the file +is written by the script's `dump_fixture()`: sorted keys, one indent level per +nesting level, every leaf row / label list / matrix row on one line); labels +are stringified numbers (`repr(float)` for fractional horizons). Every +event-study bundle also carries `ref_labels`, the labels the post-fit container +marks `is_reference` (the legacy zero-count sentinel rows where the post-fit +route was closed at capture): the contract test pins reference MEMBERSHIP +against it, so a surface that mis-marks an estimated horizon as reference +fails instead of having that row skipped. Re-capturing +on a post-removal tree is NOT a substitute: the fit-time route is gone, so +nothing independent remains to disagree with. diff --git a/tests/fixtures/v4_aggregate_oracles.json b/tests/fixtures/v4_aggregate_oracles.json new file mode 100644 index 000000000..e5423871b --- /dev/null +++ b/tests/fixtures/v4_aggregate_oracles.json @@ -0,0 +1,2608 @@ +{ + "analytic": { + "CallawaySantAnna": { + "anticipation1_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.16215645467523254, "ci_lo": -0.03617378917805543, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.21313113221560442, "se": 0.05059537966454783, "t_stat": 1.2450016813042433}, + "-2": {"ci_hi": 0.13040474791524795, "ci_lo": -0.12039901995021106, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 0.9376752677805849, "se": 0.06398172870618214, "t_stat": 0.07819207270082792}, + "-3": {"ci_hi": 0.15112521874149912, "ci_lo": -0.15077979064092137, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 0.9982107330753857, "se": 0.07701799924993742, "t_stat": 0.002242515411603801}, + "-4": {"ci_hi": 0.17259215344718637, "ci_lo": -0.21742570722209376, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.8217436524938793, "se": 0.09949618047721573, "t_stat": -0.2253028888137777}, + "0": {"ci_hi": 1.6238338587609582, "ci_lo": 1.4099865007439334, "effect": 1.5169101797524458, "n_groups": 4.0, "p_value": 3.699718967459427e-170, "se": 0.05455389989403515, "t_stat": 27.8057147646433}, + "1": {"ci_hi": 1.9803139761216155, "ci_lo": 1.7353326515827125, "effect": 1.857823313852164, "n_groups": 4.0, "p_value": 3.450291518119317e-194, "se": 0.06249638423748716, "t_stat": 29.72689278778767}, + "2": {"ci_hi": 2.2916019077590333, "ci_lo": 1.9779459480986163, "effect": 2.134773927928825, "n_groups": 3.0, "p_value": 8.157759365200267e-157, "se": 0.08001574573168058, "t_stat": 26.679423011158732}, + "3": {"ci_hi": 2.6214371422372893, "ci_lo": 2.2290675525850934, "effect": 2.4252523474111913, "n_groups": 2.0, "p_value": 1.0947709569017035e-129, "se": 0.1000961223642774, "t_stat": 24.2292337617738}, + "4": {"ci_hi": 2.9414501991423885, "ci_lo": 2.512945391902159, "effect": 2.7271977955222737, "n_groups": 1.0, "p_value": 2.2342327100197705e-137, "se": 0.10931445950543507, "t_stat": 24.948188994034027} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.009899489929554682, 0.0014181451479144305, 1.7962010870062695e-05, 0.0008272097767922567, 0.0009571149337842271, 0.0006504845372393448, 0.0015578018056016977, 0.001658920513869697, 0.003938387070789138], + [0.0014181451479144305, 0.005931772208463364, -0.0016041354657931298, 0.001043139715992548, 0.001302221046313346, 0.0007839814765109008, 0.0003878155641285595, 0.0013963852711583369, 0.0021060996028580614], + [1.7962010870062695e-05, -0.0016041354657931298, 0.004093661608231491, -0.0008755179531925678, -0.0009275974387503431, -0.0010884005146517633, -0.0007291420134011654, 0.00015510766539264986, 0.0008487902045114534], + [0.0008272097767922567, 0.001043139715992548, -0.0008755179531925678, 0.0025598924433997403, 0.001566884417001228, 0.0016182850516806785, 0.0023131849334562336, 0.0021132518896300337, 0.002116463519493331], + [0.0009571149337842271, 0.001302221046313346, -0.0009275974387503431, 0.001566884417001228, 0.0029761279936484073, 0.0023174098071227353, 0.0031446906417378377, 0.003351739456096941, 0.0027336343834122685], + [0.0006504845372393448, 0.0007839814765109008, -0.0010884005146517633, 0.0016182850516806785, 0.0023174098071227353, 0.003905798042759632, 0.00368157999942266, 0.004213778264835235, 0.003634661181110358], + [0.0015578018056016977, 0.0003878155641285595, -0.0007291420134011654, 0.0023131849334562336, 0.0031446906417378377, 0.00368157999942266, 0.006402519564996956, 0.0055122791922999174, 0.004845535909388172], + [0.001658920513869697, 0.0013963852711583369, 0.00015510766539264986, 0.0021132518896300337, 0.003351739456096941, 0.004213778264835235, 0.0055122791922999174, 0.010019233712364389, 0.006076503520543358], + [0.003938387070789138, 0.0021060996028580614, 0.0008487902045114534, 0.002116463519493331, 0.0027336343834122685, 0.003634661181110358, 0.004845535909388172, 0.006076503520543358, 0.011949651056965403] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.001158549206537, "ci_lo": 1.6456968521569708, "df_used": null, "effect": 1.8234277006817539, "p_value": 6.251621614501271e-90, "se": 0.09068067062798163, "t_stat": 20.108229108300097}, + "4": {"ci_hi": 1.7705350957130925, "ci_lo": 1.3968571482584176, "df_used": null, "effect": 1.583696121985755, "p_value": 5.595782544754999e-62, "se": 0.09532775867367943, "t_stat": 16.613168546288534}, + "5": {"ci_hi": 1.555025020923708, "ci_lo": 1.1795502831315854, "df_used": null, "effect": 1.3672876520276467, "p_value": 3.1608615059507244e-46, "se": 0.09578613197839843, "t_stat": 14.274380056770596}, + "6": {"ci_hi": 1.343333825178485, "ci_lo": 0.9264836130975396, "df_used": null, "effect": 1.1349087191380123, "p_value": 1.371527279081729e-26, "se": 0.10634129386279717, "t_stat": 10.672323778590519} + }, + "overall": {"att": 1.5407179766413293, "ci_hi": 1.6648589094422162, "ci_lo": 1.4165770438404424, "p_value": 1.0614623238290957e-130, "se": 0.06333837447019167, "t_stat": 24.325189737327765}, + "ref_labels": [], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.16215645467523254, "ci_lo": -0.03617378917805543, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.21313113221560442, "se": 0.05059537966454783, "t_stat": 1.2450016813042433}, + "-2": {"ci_hi": 0.13040474791524795, "ci_lo": -0.12039901995021106, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 0.9376752677805849, "se": 0.06398172870618214, "t_stat": 0.07819207270082792}, + "-3": {"ci_hi": 0.15112521874149912, "ci_lo": -0.15077979064092137, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 0.9982107330753857, "se": 0.07701799924993742, "t_stat": 0.002242515411603801}, + "-4": {"ci_hi": 0.17259215344718637, "ci_lo": -0.21742570722209376, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.8217436524938793, "se": 0.09949618047721573, "t_stat": -0.2253028888137777}, + "0": {"ci_hi": 1.5499821105859704, "ci_lo": 1.3578555834217447, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 2.2325981431362827e-193, "se": 0.04901276979569397, "t_stat": 29.66408250471068}, + "1": {"ci_hi": 1.9062076201705753, "ci_lo": 1.6834563420365758, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 5.922208367217076e-219, "se": 0.056825349825566536, "t_stat": 31.585058193448287}, + "2": {"ci_hi": 2.25597724608233, "ci_lo": 1.966785600885477, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 3.865135818581669e-180, "se": 0.07377473450480726, "t_stat": 28.619302226649467}, + "3": {"ci_hi": 2.5420267744233493, "ci_lo": 2.1754679645254207, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 2.1790525859688993e-140, "se": 0.09351161878210465, "t_stat": 25.22411011802289}, + "4": {"ci_hi": 2.891765222268375, "ci_lo": 2.445148849615699, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 2.6173950505761986e-121, "se": 0.11393484170513579, "t_stat": 23.420904404712505} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.009899489929554682, 0.0014181451479144305, 1.7962010870062695e-05, 0.0008272097767922567, 0.00012990515699197038, -0.0001767252395529118, 0.00136922250527329, 0.00015679289707945984, -0.0011655025362148886], + [0.0014181451479144305, 0.005931772208463364, -0.0016041354657931298, 0.001043139715992548, 0.00025908133032079815, -0.00025915823948164665, -0.00016227218117475977, -0.00047757140817898024, 0.0006039719860678242], + [1.7962010870062695e-05, -0.0016041354657931298, 0.004093661608231491, -0.0008755179531925678, -5.2079485557775425e-05, -0.00021288256145919604, -0.00019555049733657937, 0.0002670041009863298, 0.0006602109041830465], + [0.0008272097767922567, 0.001043139715992548, -0.0008755179531925678, 0.0025598924433997403, -0.0009930080263985117, -0.0009416073917190606, 0.00012801894702241804, -0.00046309611763112037, -0.0008627001959194525], + [0.00012990515699197038, 0.00025908133032079815, -5.2079485557775425e-05, -0.0009930080263985117, 0.00240225160304569, 0.001692132781840569, 0.0017656899409002383, 0.002239277942519361, 0.001670576325608205], + [-0.0001767252395529118, -0.00025915823948164665, -0.00021288256145919604, -0.0009416073917190606, 0.001692132781840569, 0.003229120382798015, 0.002336229148550142, 0.0030193445758969484, 0.0024956522813723604], + [0.00136922250527329, -0.00016227218117475977, -0.00019555049733657937, 0.00012801894702241804, 0.0017656899409002383, 0.002336229148550142, 0.005442711451254799, 0.004512076081287001, 0.0030230559230661896], + [0.00015679289707945984, -0.00047757140817898024, 0.0002670041009863298, -0.00046309611763112037, 0.002239277942519361, 0.0030193445758969484, 0.004512076081287001, 0.008744422847249671, 0.004837919396035595], + [-0.0011655025362148886, 0.0006039719860678242, 0.0006602109041830465, -0.0008627001959194525, 0.001670576325608205, 0.0024956522813723604, 0.0030230559230661896, 0.004837919396035595, 0.012981148154374351] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.291199373047453, "ci_lo": 1.944049285596187, "df_used": null, "effect": 2.11762432932182, "p_value": 2.316557917855421e-126, "se": 0.08856032309510324, "t_stat": 23.911659932041392}, + "4": {"ci_hi": 2.0617043593451987, "ci_lo": 1.711862954885753, "df_used": null, "effect": 1.886783657115476, "p_value": 3.3273822810518135e-99, "se": 0.089246896172315, "t_stat": 21.141168354723906}, + "5": {"ci_hi": 2.119567287249691, "ci_lo": 1.6940862982609604, "df_used": null, "effect": 1.9068267927553257, "p_value": 4.372566413685329e-69, "se": 0.10854306312383041, "t_stat": 17.56746804335104}, + "6": {"ci_hi": 1.6377784847646388, "ci_lo": 1.2215842196706277, "df_used": null, "effect": 1.4296813522176333, "p_value": 2.4975231449802003e-41, "se": 0.10617395737291561, "t_stat": 13.465461659266898} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 2.021352186881718, "ci_lo": 1.7950905785143174, "p_value": 1.136922422856294e-239, "se": 0.057720858687231855, "t_stat": 33.05947669694189}, + "ref_labels": [], + "reference_event_times": null + }, + "default_balance_e1": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.16215645467523254, "ci_lo": -0.03617378917805543, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.21313113221560442, "se": 0.05059537966454783, "t_stat": 1.2450016813042433}, + "-2": {"ci_hi": 0.13040474791524795, "ci_lo": -0.12039901995021106, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 0.9376752677805849, "se": 0.06398172870618214, "t_stat": 0.07819207270082792}, + "-3": {"ci_hi": 0.15112521874149912, "ci_lo": -0.15077979064092137, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 0.9982107330753857, "se": 0.07701799924993742, "t_stat": 0.002242515411603801}, + "-4": {"ci_hi": 0.17259215344718637, "ci_lo": -0.21742570722209376, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.8217436524938793, "se": 0.09949618047721573, "t_stat": -0.2253028888137777}, + "0": {"ci_hi": 1.5499821105859704, "ci_lo": 1.3578555834217447, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 2.2325981431362827e-193, "se": 0.04901276979569397, "t_stat": 29.66408250471068}, + "1": {"ci_hi": 1.9062076201705753, "ci_lo": 1.6834563420365758, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 5.922208367217076e-219, "se": 0.056825349825566536, "t_stat": 31.585058193448287}, + "2": {"ci_hi": 2.25597724608233, "ci_lo": 1.966785600885477, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 3.865135818581669e-180, "se": 0.07377473450480726, "t_stat": 28.619302226649467}, + "3": {"ci_hi": 2.5420267744233493, "ci_lo": 2.1754679645254207, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 2.1790525859688993e-140, "se": 0.09351161878210465, "t_stat": 25.22411011802289}, + "4": {"ci_hi": 2.891765222268375, "ci_lo": 2.445148849615699, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 2.6173950505761986e-121, "se": 0.11393484170513579, "t_stat": 23.420904404712505} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.009899489929554682, 0.0014181451479144305, 1.7962010870062695e-05, 0.0008272097767922567, 0.00012990515699197038, -0.0001767252395529118, 0.00136922250527329, 0.00015679289707945984, -0.0011655025362148886], + [0.0014181451479144305, 0.005931772208463364, -0.0016041354657931298, 0.001043139715992548, 0.00025908133032079815, -0.00025915823948164665, -0.00016227218117475977, -0.00047757140817898024, 0.0006039719860678242], + [1.7962010870062695e-05, -0.0016041354657931298, 0.004093661608231491, -0.0008755179531925678, -5.2079485557775425e-05, -0.00021288256145919604, -0.00019555049733657937, 0.0002670041009863298, 0.0006602109041830465], + [0.0008272097767922567, 0.001043139715992548, -0.0008755179531925678, 0.0025598924433997403, -0.0009930080263985117, -0.0009416073917190606, 0.00012801894702241804, -0.00046309611763112037, -0.0008627001959194525], + [0.00012990515699197038, 0.00025908133032079815, -5.2079485557775425e-05, -0.0009930080263985117, 0.00240225160304569, 0.001692132781840569, 0.0017656899409002383, 0.002239277942519361, 0.001670576325608205], + [-0.0001767252395529118, -0.00025915823948164665, -0.00021288256145919604, -0.0009416073917190606, 0.001692132781840569, 0.003229120382798015, 0.002336229148550142, 0.0030193445758969484, 0.0024956522813723604], + [0.00136922250527329, -0.00016227218117475977, -0.00019555049733657937, 0.00012801894702241804, 0.0017656899409002383, 0.002336229148550142, 0.005442711451254799, 0.004512076081287001, 0.0030230559230661896], + [0.00015679289707945984, -0.00047757140817898024, 0.0002670041009863298, -0.00046309611763112037, 0.002239277942519361, 0.0030193445758969484, 0.004512076081287001, 0.008744422847249671, 0.004837919396035595], + [-0.0011655025362148886, 0.0006039719860678242, 0.0006602109041830465, -0.0008627001959194525, 0.001670576325608205, 0.0024956522813723604, 0.0030230559230661896, 0.004837919396035595, 0.012981148154374351] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "ref_labels": [], + "reference_event_times": null + }, + "nyt_anticipation1_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.16898652052958094, "ci_lo": -0.048480857757560364, "effect": 0.060252831386010286, "n_groups": 4.0, "p_value": 0.27744402968527837, "se": 0.055477391422111895, "t_stat": 1.0860790286184043}, + "-2": {"ci_hi": 0.1531272415947705, "ci_lo": -0.14116681702474623, "effect": 0.005980212285012129, "n_groups": 3.0, "p_value": 0.9365116284230134, "se": 0.07507639450032519, "t_stat": 0.07965502771961466}, + "-3": {"ci_hi": 0.15612812234521145, "ci_lo": -0.1611207229084996, "effect": -0.002496300281644076, "n_groups": 2.0, "p_value": 0.9753937140992579, "se": 0.08093231502112526, "t_stat": -0.03084429601442245}, + "-4": {"ci_hi": 0.17655605408274752, "ci_lo": -0.15099909485656046, "effect": 0.01277847961309353, "n_groups": 1.0, "p_value": 0.8784589892018173, "se": 0.08356152243689711, "t_stat": 0.15292301097964572}, + "0": {"ci_hi": 1.6190218747793874, "ci_lo": 1.3947871861554513, "effect": 1.5069045304674193, "n_groups": 4.0, "p_value": 6.215168391036675e-153, "se": 0.05720377782262088, "t_stat": 26.342744969398215}, + "1": {"ci_hi": 1.9848516008294237, "ci_lo": 1.7257995457680584, "effect": 1.855325573298741, "n_groups": 4.0, "p_value": 2.009892663515184e-173, "se": 0.06608592226814752, "t_stat": 28.07444474741002}, + "2": {"ci_hi": 2.2916019077590333, "ci_lo": 1.9779459480986163, "effect": 2.134773927928825, "n_groups": 3.0, "p_value": 8.157759365200267e-157, "se": 0.08001574573168058, "t_stat": 26.679423011158732}, + "3": {"ci_hi": 2.6214371422372893, "ci_lo": 2.2290675525850934, "effect": 2.4252523474111913, "n_groups": 2.0, "p_value": 1.0947709569017035e-129, "se": 0.1000961223642774, "t_stat": 24.2292337617738}, + "4": {"ci_hi": 2.9414501991423885, "ci_lo": 2.512945391902159, "effect": 2.7271977955222737, "n_groups": 1.0, "p_value": 2.2342327100197705e-137, "se": 0.10931445950543507, "t_stat": 24.948188994034027} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.006982528031972061, -0.002062635923061642, -0.0008521579095748296, 0.00031240630315058606, 0.00032769885398484857, -0.0003782597916087535, 0.001150538836179655, 0.000870761236611041, 0.0013127956902630456], + [-0.002062635923061642, 0.006550039614678656, -0.004568553311449108, 0.00127478987494083, 0.0006331677330158608, 0.0011507376755470764, 0.0006032674293146269, 0.0007183207911100319, 0.0007248508788632691], + [-0.0008521579095748296, -0.004568553311449108, 0.005636465011168458, -0.0020405600439165616, -0.0015467054943632346, -0.0016363909584042869, -0.0009574869853712986, -0.0004151160830631299, -7.20422781517411e-05], + [0.00031240630315058606, 0.00127478987494083, -0.0020405600439165616, 0.0030777409590022147, 0.0019075611287429478, 0.0019971237841853806, 0.0023832983249850986, 0.001898141749956585, 0.0015406091619219232], + [0.00032769885398484857, 0.0006331677330158608, -0.0015467054943632346, 0.0019075611287429478, 0.003272272197179773, 0.0026678538938474283, 0.0031907550177352586, 0.002930252306232027, 0.0022056032396164287], + [-0.0003782597916087535, 0.0011507376755470764, -0.0016363909584042869, 0.0019971237841853806, 0.0026678538938474283, 0.004367349122031634, 0.003556177967363973, 0.004042714330630157, 0.003316364854418563], + [0.001150538836179655, 0.0006032674293146269, -0.0009574869853712986, 0.0023832983249850986, 0.0031907550177352586, 0.003556177967363973, 0.006402519564996956, 0.0055122791922999174, 0.004845535909388172], + [0.000870761236611041, 0.0007183207911100319, -0.0004151160830631299, 0.001898141749956585, 0.002930252306232027, 0.004042714330630157, 0.0055122791922999174, 0.010019233712364389, 0.006076503520543358], + [0.0013127956902630456, 0.0007248508788632691, -7.20422781517411e-05, 0.0015406091619219232, 0.0022056032396164287, 0.003316364854418563, 0.004845535909388172, 0.006076503520543358, 0.011949651056965403] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 1.981973943645983, "ci_lo": 1.6659436003511565, "df_used": null, "effect": 1.8239587719985697, "p_value": 2.531136739367264e-113, "se": 0.0806214669727693, "t_stat": 22.623735842150204}, + "4": {"ci_hi": 1.7445034465870992, "ci_lo": 1.4003968549396029, "df_used": null, "effect": 1.572450150763351, "p_value": 9.380506582158234e-72, "se": 0.08778390683751462, "t_stat": 17.91273830719233}, + "5": {"ci_hi": 1.5494440248799441, "ci_lo": 1.1811692108788567, "df_used": null, "effect": 1.3653066178794004, "p_value": 7.556702913570926e-48, "se": 0.09394938297488932, "t_stat": 14.532363860700586}, + "6": {"ci_hi": 1.343333825178485, "ci_lo": 0.9264836130975396, "df_used": null, "effect": 1.1349087191380123, "p_value": 1.371527279081729e-26, "se": 0.10634129386279717, "t_stat": 10.672323778590519} + }, + "overall": {"att": 1.5373308897077675, "ci_hi": 1.6631713120329592, "ci_lo": 1.4114904673825759, "p_value": 1.0690753370413593e-126, "se": 0.06420547689539442, "t_stat": 23.943921360672007}, + "ref_labels": [], + "reference_event_times": null + }, + "rcs_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2403450441027863, "ci_lo": -0.15489149333418528, "effect": 0.0427267753843005, "n_groups": 3.0, "p_value": 0.6717400281394849, "se": 0.10082750003432384, "t_stat": 0.4237611303439577}, + "-2": {"ci_hi": 0.17115957452791575, "ci_lo": -0.3048366237302024, "effect": -0.06683852460114334, "n_groups": 2.0, "p_value": 0.5820250319711249, "se": 0.12142983289813371, "t_stat": -0.550429190306253}, + "-3": {"ci_hi": 0.35558895871387397, "ci_lo": -0.5032241289119656, "effect": -0.07381758509904579, "n_groups": 1.0, "p_value": 0.7361699127881354, "se": 0.21908899714485766, "t_stat": -0.33692967725914114}, + "0": {"ci_hi": 1.5865265533564499, "ci_lo": 1.2041504592952552, "effect": 1.3953385063258525, "n_groups": 3.0, "p_value": 2.0566559862275245e-46, "se": 0.09754671439815442, "t_stat": 14.304310657051227}, + "1": {"ci_hi": 1.689486444970567, "ci_lo": 1.2566722806147976, "effect": 1.4730793627926824, "n_groups": 3.0, "p_value": 1.3286466967659662e-40, "se": 0.11041380550095627, "t_stat": 13.341441825224694}, + "2": {"ci_hi": 1.6686972798572384, "ci_lo": 1.1200096679184746, "effect": 1.3943534738878565, "n_groups": 2.0, "p_value": 2.2459160911686854e-23, "se": 0.13997390162950485, "t_stat": 9.961524667495182}, + "3": {"ci_hi": 1.747716353586616, "ci_lo": 0.895616157001141, "effect": 1.3216662552938785, "n_groups": 1.0, "p_value": 1.2012334137479834e-09, "se": 0.21737649347302626, "t_stat": 6.080079010281215} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.04799998866993945, 0.0028663044393755083, 0.0034510262203502843, -0.008883550229415534, -0.008883550229415534, -0.0133253253441233, -0.0266506506882466], + [0.0028663044393755083, 0.014745204317668672, -0.0016881921246718495, -1.6328703418135744e-05, -0.0010759168507964553, -0.00171569726828955, 0.0], + [0.0034510262203502843, -0.0016881921246718495, 0.010166184763171573, -0.002316583035464825, -0.0024658163560372474, -0.00252054780176246, -0.003093275322057309], + [-0.008883550229415534, -1.6328703418135744e-05, -0.002316583035464825, 0.00951536148987511, 0.006483631048457987, 0.008909532790016333, 0.011976825551472843], + [-0.008883550229415534, -0.0010759168507964553, -0.0024658163560372474, 0.006483631048457987, 0.012191208445203, 0.010851395237176778, 0.01380418913898376], + [-0.0133253253441233, -0.00171569726828955, -0.00252054780176246, 0.008909532790016333, 0.010851395237176778, 0.0195926931373863, 0.02070628370847564], + [-0.0266506506882466, 0.0, -0.003093275322057309, 0.011976825551472843, 0.01380418913898376, 0.02070628370847564, 0.04725253991462863] + ], + "event_study_vcov_index": [-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], + "group": { + "3": {"ci_hi": 1.7023879448065795, "ci_lo": 0.9093105841362874, "df_used": null, "effect": 1.3058492644714335, "p_value": 1.0865162277084667e-10, "se": 0.20231937089813515, "t_stat": 6.454395635348775}, + "4": {"ci_hi": 1.7143549504307611, "ci_lo": 1.166561687346286, "df_used": null, "effect": 1.4404583188885236, "p_value": 6.503396978963198e-25, "se": 0.1397457472191832, "t_stat": 10.307707730305719}, + "5": {"ci_hi": 2.0063369583151927, "ci_lo": 1.1645178375587, "df_used": null, "effect": 1.5854273979369462, "p_value": 1.553008079232467e-13, "se": 0.21475372185322136, "t_stat": 7.382537467828125} + }, + "overall": {"att": 1.4128474233805774, "ci_hi": 1.6211217491808865, "ci_lo": 1.2045730975802684, "p_value": 2.4552846403822455e-40, "se": 0.10626436375522731, "t_stat": 13.295590106152376}, + "ref_labels": [], + "reference_event_times": null + }, + "survey_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.06305242208539347, "ci_lo": -0.5490449829528414, "effect": -0.24299628043372395, "n_groups": 2.0, "p_value": 0.11175118200872357, "se": 0.14436904451889626, "t_stat": -1.6831605504039926}, + "-2": {"ci_hi": 0.3350718459671376, "ci_lo": -0.2633067648049015, "effect": 0.03588254058111806, "n_groups": 2.0, "p_value": 0.8025443593928684, "se": 0.1411333352937636, "t_stat": 0.2542456784318881}, + "-3": {"ci_hi": 0.19793310050233898, "ci_lo": -0.5516009118064895, "effect": -0.1768339056520753, "n_groups": 1.0, "p_value": 0.33206463602490016, "se": 0.1767847867034837, "t_stat": -1.0002778460155284}, + "-4": {"ci_hi": 0.6478420295428329, "ci_lo": -0.18534467096860502, "effect": 0.23124867928711396, "n_groups": 1.0, "p_value": 0.2565066902270988, "se": 0.19651507565397106, "t_stat": 1.1767477813982208}, + "0": {"ci_hi": 2.539200804244224, "ci_lo": 2.055758501083632, "effect": 2.297479652663928, "n_groups": 2.0, "p_value": 8.534005357227127e-13, "se": 0.1140245046177732, "t_stat": 20.148999202981983}, + "1": {"ci_hi": 2.3684274400757257, "ci_lo": 1.8848236369228923, "effect": 2.126625538499309, "n_groups": 2.0, "p_value": 2.8158800286713316e-12, "se": 0.11406259594013106, "t_stat": 18.644372600598427}, + "2": {"ci_hi": 2.4296734240114835, "ci_lo": 1.991857868303597, "effect": 2.2107656461575402, "n_groups": 2.0, "p_value": 3.3410990731014977e-13, "se": 0.1032629985567557, "t_stat": 21.40907853787001}, + "3": {"ci_hi": 2.4272117780839584, "ci_lo": 1.5981841492089865, "effect": 2.0126979636464726, "n_groups": 1.0, "p_value": 1.834373613234197e-08, "se": 0.1955341187126412, "t_stat": 10.293333853435332}, + "4": {"ci_hi": 2.2581507682745747, "ci_lo": 1.6255363921826604, "effect": 1.9418435802286176, "n_groups": 1.0, "p_value": 6.295655126959685e-10, "se": 0.14920816895082636, "t_stat": 13.014324844831915} + }, + "event_study_df": 16.0, + "event_study_vcov": [ + [0.03861817495928597, -0.021365623038974126, 0.0042117990032355355, -0.012967146006298651, 0.00827494055165137, 0.0009206953628820844, 0.010303204607573075, -0.0011550709391477945, 0.005691841846969219], + [-0.021365623038974126, 0.03125286080979622, -0.015122597193305243, 0.017472268577626102, -0.011232408987329663, -0.0010544032984044866, -0.013558945269131846, -0.00906892563503207, -0.007455894336591355], + [0.0042117990032355355, -0.015122597193305243, 0.019918618331141894, -0.012670318924642418, 0.006624329308282802, 0.002215918716922371, 0.005448531417766656, 0.015499209743999747, 0.005239687227570692], + [-0.012967146006298651, 0.017472268577626102, -0.012670318924642418, 0.020842421015299047, -0.011459445052354291, -0.00389443927122821, -0.008075958598388753, -0.00832417806279835, 0.00043026738464958], + [0.00827494055165137, -0.011232408987329663, 0.006624329308282802, -0.011459445052354291, 0.013001587653328582, 0.006791784084885521, 0.006793856776640319, 0.00968029338087329, 0.0012905597546445317], + [0.0009206953628820844, -0.0010544032984044866, 0.002215918716922371, -0.00389443927122821, 0.006791784084885521, 0.013010275792601602, 0.0006308325080012554, 0.010624647518131438, 0.0012869544023455094], + [0.010303204607573075, -0.013558945269131846, 0.005448531417766656, -0.008075958598388753, 0.006793856776640319, 0.0006308325080012554, 0.010663246870932532, 0.006359533491099557, 0.005028141534539605], + [-0.0011550709391477945, -0.00906892563503207, 0.015499209743999747, -0.00832417806279835, 0.00968029338087329, 0.010624647518131438, 0.006359533491099557, 0.03823359158072926, 0.019474129949428095], + [0.005691841846969219, -0.007455894336591355, 0.005239687227570692, 0.00043026738464958, 0.0012905597546445317, 0.0012869544023455094, 0.005028141534539605, 0.019474129949428095, 0.022263077681658344] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "4": {"ci_hi": 2.3932158987173153, "ci_lo": 1.7839734164980532, "df_used": 16.0, "effect": 2.0885946576076844, "p_value": 1.2243052838757856e-10, "se": 0.14369568358621185, "t_stat": 14.534846179666966}, + "6": {"ci_hi": 2.574596615542614, "ci_lo": 1.9503444240305765, "df_used": 16.0, "effect": 2.262470519786595, "p_value": 5.318287531848915e-11, "se": 0.14723586750345774, "t_stat": 15.366300060910515} + }, + "overall": {"att": 2.1521431960690123, "ci_hi": 2.340781870520201, "ci_lo": 1.9635045216178235, "p_value": 5.0187150689601094e-14, "se": 0.08898448176929655, "t_stat": 24.185601278756828}, + "ref_labels": [], + "reference_event_times": null + }, + "unbalanced_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.16012719694144062, "ci_lo": -0.09078378126266735, "effect": 0.034671707839386626, "n_groups": 4.0, "p_value": 0.5838069214176409, "se": 0.06302869996810885, "t_stat": 0.5500939707931427}, + "-2": {"ci_hi": 0.14035515384162178, "ci_lo": -0.16776653508017997, "effect": -0.013705690619279079, "n_groups": 3.0, "p_value": 0.8599018175846732, "se": 0.07739999909020026, "t_stat": -0.17707610827368056}, + "-3": {"ci_hi": 0.25409114937877186, "ci_lo": -0.11442174474108975, "effect": 0.06983470231884105, "n_groups": 2.0, "p_value": 0.4528538399580236, "se": 0.09257023667958403, "t_stat": 0.7543969295504976}, + "-4": {"ci_hi": 0.27954685715066224, "ci_lo": -0.22588498625503867, "effect": 0.026830935447811788, "n_groups": 1.0, "p_value": 0.8331764965116059, "se": 0.1269642015680625, "t_stat": 0.21132677649635245}, + "0": {"ci_hi": 1.583633352529269, "ci_lo": 1.3551117136955353, "effect": 1.4693725331124021, "n_groups": 4.0, "p_value": 5.401371013270853e-40, "se": 0.057404510210610285, "t_stat": 25.59681334657242}, + "1": {"ci_hi": 1.95535841775937, "ci_lo": 1.6577079699959099, "effect": 1.8065331938776399, "n_groups": 4.0, "p_value": 3.097701291949369e-38, "se": 0.07476962906021313, "t_stat": 24.16132347564291}, + "2": {"ci_hi": 2.313352493326447, "ci_lo": 1.9360514513136506, "effect": 2.124701972320049, "n_groups": 3.0, "p_value": 5.4649059574744875e-36, "se": 0.09477781460536361, "t_stat": 22.417714326574153}, + "3": {"ci_hi": 2.6087141999545014, "ci_lo": 2.1012301789584176, "effect": 2.3549721894564595, "n_groups": 2.0, "p_value": 2.1033523976880707e-30, "se": 0.12747970745206683, "t_stat": 18.473310274436766}, + "4": {"ci_hi": 3.106089460328497, "ci_lo": 2.46405333636247, "effect": 2.7850713983454836, "n_groups": 1.0, "p_value": 1.5233162640510215e-28, "se": 0.16127912184545357, "t_stat": 17.268641882948064} + }, + "event_study_df": 79.0, + "event_study_vcov": [ + [0.016119908479815606, 0.0035942753027381385, -0.0012718892569674746, 0.0005483518851426506, 0.0012760551811625178, 0.0012916447901105146, 0.0030506250361085735, 0.0017616109124787799, -0.00042836426353781525], + [0.0035942753027381385, 0.008569248718914206, -0.002874296420655473, 0.0014917550301567185, -1.9541673634603666e-05, -0.00034010869666223755, -0.0003508399474333856, -0.0009770107176791537, 0.0010140565269676507], + [-0.0012718892569674746, -0.002874296420655473, 0.005990759859163002, -0.0010372234180487227, 0.00031170187300190704, -0.00033854881182831793, 0.00019062040642808453, 0.0007906542136677535, 0.0006324857211530199], + [0.0005483518851426506, 0.0014917550301567185, -0.0010372234180487227, 0.003972617019669884, -0.001566557370020595, -0.0014157252516670714, -0.0002414496184740009, -0.0017430671876029575, -0.003303599620206488], + [0.0012760551811625178, -1.9541673634603666e-05, 0.00031170187300190704, -0.001566557370020595, 0.00329527779252006, 0.0028345215091216294, 0.003041351380631268, 0.004132688308340615, 0.0038745930942409184], + [0.0012916447901105146, -0.00034010869666223755, -0.00033854881182831793, -0.0014157252516670714, 0.0028345215091216294, 0.005590497429801868, 0.004398402134370539, 0.005961173311620288, 0.0048140845687418745], + [0.0030506250361085735, -0.0003508399474333856, 0.00019062040642808453, -0.0002414496184740009, 0.003041351380631268, 0.004398402134370539, 0.008982834141368672, 0.008950138054963279, 0.0065743580437580525], + [0.0017616109124787799, -0.0009770107176791537, 0.0007906542136677535, -0.0017430671876029575, 0.004132688308340615, 0.005961173311620288, 0.008950138054963279, 0.016251075812064544, 0.01085446502511428], + [-0.00042836426353781525, 0.0010140565269676507, 0.0006324857211530199, -0.003303599620206488, 0.0038745930942409184, 0.0048140845687418745, 0.0065743580437580525, 0.01085446502511428, 0.026010955143240665] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.4509080543538264, "ci_lo": 1.9701921490451717, "df_used": 79.0, "effect": 2.210550101699499, "p_value": 3.773054485148066e-30, "se": 0.1207555714877882, "t_stat": 18.305988489508728}, + "4": {"ci_hi": 2.0227664116733526, "ci_lo": 1.5877739785566771, "df_used": 79.0, "effect": 1.805270195115015, "p_value": 2.375021142908374e-27, "se": 0.10926986037655856, "t_stat": 16.521208949053403}, + "5": {"ci_hi": 2.2002394561757734, "ci_lo": 1.686890232065834, "df_used": 79.0, "effect": 1.9435648441208038, "p_value": 5.969424344121814e-25, "se": 0.12895304325411575, "t_stat": 15.071880392080407}, + "6": {"ci_hi": 1.7179628664250794, "ci_lo": 1.1502559144336753, "df_used": 79.0, "effect": 1.4341093904293774, "p_value": 8.583408496949165e-16, "se": 0.1426076746541095, "t_stat": 10.056326869558495} + }, + "overall": {"att": 1.926624614441337, "ci_hi": 2.0792031887291786, "ci_lo": 1.7740460401534954, "p_value": 1.9560163450676445e-39, "se": 0.0766553081828663, "t_stat": 25.133609923597813}, + "ref_labels": [], + "reference_event_times": null + }, + "universal_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.0, "n_groups": 0.0, "p_value": null, "se": null, "t_stat": null}, + "-2": {"ci_hi": 0.03617378917805543, "ci_lo": -0.16215645467523254, "effect": -0.06299133274858856, "n_groups": 4.0, "p_value": 0.21313113221560442, "se": 0.05059537966454783, "t_stat": -1.2450016813042433}, + "-3": {"ci_hi": 0.06838126948770742, "ci_lo": -0.20720337839548894, "effect": -0.06941105445389076, "n_groups": 3.0, "p_value": 0.32349272599668843, "se": 0.07030349793592457, "t_stat": -0.9873058452532875}, + "-4": {"ci_hi": 0.07343770410619782, "ci_lo": -0.2871278687484276, "effect": -0.10684508232111489, "n_groups": 2.0, "p_value": 0.24540693448700368, "se": 0.0919827036870883, "t_stat": -1.161577971056235}, + "-5": {"ci_hi": 0.07775474760100509, "ci_lo": -0.4812942317749539, "effect": -0.2017697420869744, "n_groups": 1.0, "p_value": 0.15713744806197613, "se": 0.1426171561788038, "t_stat": -1.41476486765736}, + "0": {"ci_hi": 1.5499821105859704, "ci_lo": 1.3578555834217447, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 2.2325981431362827e-193, "se": 0.04901276979569397, "t_stat": 29.66408250471068}, + "1": {"ci_hi": 1.9062076201705753, "ci_lo": 1.6834563420365758, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 5.922208367217076e-219, "se": 0.056825349825566536, "t_stat": 31.585058193448287}, + "2": {"ci_hi": 2.25597724608233, "ci_lo": 1.966785600885477, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 3.865135818581669e-180, "se": 0.07377473450480726, "t_stat": 28.619302226649467}, + "3": {"ci_hi": 2.5420267744233493, "ci_lo": 2.1754679645254207, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 2.1790525859688993e-140, "se": 0.09351161878210465, "t_stat": 25.22411011802289}, + "4": {"ci_hi": 2.891765222268375, "ci_lo": 2.445148849615699, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 2.6173950505761986e-121, "se": 0.11393484170513579, "t_stat": 23.420904404712505} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.020339653236529313, 0.008299600016515161, 0.004644459180849902, 0.0035261611838809707, 0.0006806452582775707, 0.0013185834869704068, -0.0030392506904203674, -0.0016058600966099958, -0.00222980765181627], + [0.008299600016515161, 0.008460817777586687, 0.004608171451967142, 0.0022419298162267513, 0.0010036353834108813, 0.0013947123761282787, 4.5797789597992955e-05, -0.000533115979999261, -0.0026879714502901484], + [0.004644459180849902, 0.004608171451967142, 0.00494258182202655, 0.0015444919316732438, 0.0010237466370474947, 0.00114150703204825, 0.00013947087589322887, -0.00025656594934418523, -0.001791980966860099], + [0.0035261611838809707, 0.0022419298162267513, 0.0015444919316732438, 0.0025598924433997403, 0.0009930080263985117, 0.0009416073917190606, -0.00012801894702241804, 0.00046309611763112037, 0.0008627001959194525], + [0.0006806452582775707, 0.0010036353834108813, 0.0010237466370474947, 0.0009930080263985117, 0.00240225160304569, 0.001692132781840569, 0.0017656899409002383, 0.002239277942519361, 0.001670576325608205], + [0.0013185834869704068, 0.0013947123761282787, 0.00114150703204825, 0.0009416073917190606, 0.001692132781840569, 0.003229120382798015, 0.002336229148550142, 0.0030193445758969484, 0.0024956522813723604], + [-0.0030392506904203674, 4.5797789597992955e-05, 0.00013947087589322887, -0.00012801894702241804, 0.0017656899409002383, 0.002336229148550142, 0.005442711451254799, 0.004512076081287001, 0.0030230559230661896], + [-0.0016058600966099958, -0.000533115979999261, -0.00025656594934418523, 0.00046309611763112037, 0.002239277942519361, 0.0030193445758969484, 0.004512076081287001, 0.008744422847249671, 0.004837919396035595], + [-0.00222980765181627, -0.0026879714502901484, -0.001791980966860099, 0.0008627001959194525, 0.001670576325608205, 0.0024956522813723604, 0.0030230559230661896, 0.004837919396035595, 0.012981148154374351] + ], + "event_study_vcov_index": [-5.0, -4.0, -3.0, -2.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.291199373047453, "ci_lo": 1.944049285596187, "df_used": null, "effect": 2.11762432932182, "p_value": 2.316557917855421e-126, "se": 0.08856032309510324, "t_stat": 23.911659932041392}, + "4": {"ci_hi": 2.0617043593451987, "ci_lo": 1.711862954885753, "df_used": null, "effect": 1.886783657115476, "p_value": 3.3273822810518135e-99, "se": 0.089246896172315, "t_stat": 21.141168354723906}, + "5": {"ci_hi": 2.119567287249691, "ci_lo": 1.6940862982609604, "df_used": null, "effect": 1.9068267927553257, "p_value": 4.372566413685329e-69, "se": 0.10854306312383041, "t_stat": 17.56746804335104}, + "6": {"ci_hi": 1.6377784847646388, "ci_lo": 1.2215842196706277, "df_used": null, "effect": 1.4296813522176333, "p_value": 2.4975231449802003e-41, "se": 0.10617395737291561, "t_stat": 13.465461659266898} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 2.021352186881718, "ci_lo": 1.7950905785143174, "p_value": 1.136922422856294e-239, "se": 0.057720858687231855, "t_stat": 33.05947669694189}, + "ref_labels": ["-1"], + "reference_event_times": [-1.0] + } + }, + "ContinuousDiD": { + "anticipation1_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.5632543047168012, "ci_lo": -0.18564639552062256, "effect": 0.1888039545980893, "p_value": 0.32303232796586, "se": 0.191049607580715, "t_stat": 0.98824570743137}, + "0": {"ci_hi": 5.30547494834385, "ci_lo": 4.169640550858567, "effect": 4.7375577496012085, "p_value": 4.350257064173576e-60, "se": 0.2897589972174488, "t_stat": 16.349993598459072}, + "1": {"ci_hi": 5.317072613738213, "ci_lo": 4.1600244549419285, "effect": 4.738548534340071, "p_value": 5.395693979499819e-58, "se": 0.2951707704638792, "t_stat": 16.053583242314772}, + "2": {"ci_hi": 5.050399273403116, "ci_lo": 4.0167442129497495, "effect": 4.533571743176433, "p_value": 3.0140883116910266e-66, "se": 0.26369236083078684, "t_stat": 17.192654837982417}, + "3": {"ci_hi": 5.41701771690124, "ci_lo": 3.926300352709818, "effect": 4.671659034805529, "p_value": 1.0985838480664398e-34, "se": 0.38029202984085686, "t_stat": 12.284399009783368} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 3.670258050724984, "ci_hi": 4.093891186267547, "ci_lo": 3.2466249151824207, "p_value": 1.1417921640251344e-64, "se": 0.21614332655300164, "t_stat": 16.980667917244165}, + "ref_labels": [], + "reference_event_times": null + }, + "covariate_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.8466320434003214, "ci_lo": 0.03107198281704182, "effect": 0.4388520131086816, "p_value": 0.03491790738504462, "se": 0.20805485891993764, "t_stat": 2.1093091283081153}, + "-2": {"ci_hi": 0.8949050740331412, "ci_lo": -0.29675595081445433, "effect": 0.2990745616093435, "p_value": 0.32521607850518663, "se": 0.30400074548493383, "t_stat": 0.9837954875152288}, + "0": {"ci_hi": 4.75055556934943, "ci_lo": 3.5946059553831886, "effect": 4.17258076236631, "p_value": 1.8784249668176888e-45, "se": 0.29489052428621776, "t_stat": 14.149592539353502}, + "1": {"ci_hi": 4.8345246260246055, "ci_lo": 3.622062884176901, "effect": 4.228293755100753, "p_value": 1.5295257670198974e-42, "se": 0.3093071483485022, "t_stat": 13.670210267292804}, + "2": {"ci_hi": 4.8129176265252065, "ci_lo": 3.486334315160728, "effect": 4.149625970842967, "p_value": 1.4532437701217397e-34, "se": 0.3384203285949126, "t_stat": 12.26175149723363}, + "3": {"ci_hi": 5.672056393915652, "ci_lo": 3.8489347206733435, "effect": 4.760495557294497, "p_value": 1.3729802053226833e-24, "se": 0.46509060564960863, "t_stat": 10.235630433010668} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.240788455640767, "ci_hi": 4.8099854259752535, "ci_lo": 3.6715914853062808, "p_value": 2.7007827424712023e-48, "se": 0.29041195390539787, "t_stat": 14.602664933765814}, + "ref_labels": [], + "reference_event_times": null + }, + "default_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.18345278379253552, "ci_lo": -0.6610708496665441, "effect": -0.23880903293700426, "p_value": 0.26766650634754985, "se": 0.21544366124086317, "t_stat": -1.1084523515872622}, + "-2": {"ci_hi": 0.17899313361928743, "ci_lo": -1.0125456115269085, "effect": -0.4167762389538105, "p_value": 0.17034010648367948, "se": 0.3039695511103524, "t_stat": -1.3711118019268482}, + "0": {"ci_hi": 5.03076924232823, "ci_lo": 4.044519050989685, "effect": 4.537644146658957, "p_value": 1.0308619836747304e-72, "se": 0.2515990597577204, "t_stat": 18.035219014842596}, + "1": {"ci_hi": 5.037696997495454, "ci_lo": 3.89498577951186, "effect": 4.466341388503657, "p_value": 5.516726744027995e-53, "se": 0.2915133203969955, "t_stat": 15.321225741661475}, + "2": {"ci_hi": 5.402617560241142, "ci_lo": 4.357209148111314, "effect": 4.879913354176228, "p_value": 8.580400877050968e-75, "se": 0.2666907199254364, "t_stat": 18.298024601458177}, + "3": {"ci_hi": 4.825180975813696, "ci_lo": 3.195317878669007, "effect": 4.010249427241352, "p_value": 5.165463128438495e-22, "se": 0.4157890425540575, "t_stat": 9.644913686536055} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.641957051061234, "ci_hi": 5.123728191937191, "ci_lo": 4.160185910185277, "p_value": 1.5261011214223842e-79, "se": 0.24580611923285658, "t_stat": 18.884627712070195}, + "ref_labels": [], + "reference_event_times": null + }, + "discrete_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.5619082136567884, "ci_lo": -0.7121108279945735, "effect": -0.07510130716889252, "p_value": 0.8172578998689504, "se": 0.32501082971438805, "t_stat": -0.23107324526659556}, + "0": {"ci_hi": 5.530019822624903, "ci_lo": 4.1105996839060825, "effect": 4.820309753265493, "p_value": 1.972332614192793e-40, "se": 0.3621036279021004, "t_stat": 13.311962051285299}, + "1": {"ci_hi": 5.564388141892863, "ci_lo": 4.074140349923281, "effect": 4.819264245908072, "p_value": 7.978797081512949e-37, "se": 0.38017223880756623, "t_stat": 12.676528567746011}, + "2": {"ci_hi": 5.51890591141352, "ci_lo": 4.157402508188198, "effect": 4.838154209800859, "p_value": 4.186154204297231e-44, "se": 0.34732867898713643, "t_stat": 13.92961336768866}, + "3": {"ci_hi": 5.214873874272675, "ci_lo": 3.5436120576994483, "effect": 4.379242965986061, "p_value": 9.474895045250283e-25, "se": 0.42635013443000136, "t_stat": 10.271470822545385} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.714242793740121, "ci_hi": 5.382696768302874, "ci_lo": 4.0457888191773685, "p_value": 1.86320276717105e-43, "se": 0.3410542131566869, "t_stat": 13.822561375526263}, + "ref_labels": [], + "reference_event_times": null + }, + "empty_post_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": -0.07599983738405297, "p_value": null, "se": null, "t_stat": null}, + "-2": {"ci_hi": null, "ci_lo": null, "effect": -0.03497868484900135, "p_value": null, "se": null, "t_stat": null}, + "-3": {"ci_hi": null, "ci_lo": null, "effect": 0.032986095023899153, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": null, "ci_hi": null, "ci_lo": null, "p_value": null, "se": null, "t_stat": null}, + "ref_labels": [], + "reference_event_times": null + }, + "lowest_dose_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.549749845376697, "ci_lo": -0.05887573023666892, "effect": 0.24543705757001405, "p_value": 0.11393118672318944, "se": 0.1552644794532264, "t_stat": 1.580767593684892}, + "0": {"ci_hi": 1.236890217299402, "ci_lo": 0.747916732983003, "effect": 0.9924034751412025, "p_value": 1.7805154171470266e-15, "se": 0.1247404258887815, "t_stat": 7.955748652213429}, + "1": {"ci_hi": 1.2310196044566133, "ci_lo": 0.5381082171933003, "effect": 0.8845639108249568, "p_value": 5.611145260183066e-07, "se": 0.17676635711903627, "t_stat": 5.004141767934282}, + "2": {"ci_hi": 1.479593369697431, "ci_lo": 0.6790685620345311, "effect": 1.079330965865981, "p_value": 1.2559669395513567e-07, "se": 0.20421926473581598, "t_stat": 5.285157437336949} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 0.9854327839440468, "ci_hi": 1.2684089095433002, "ci_lo": 0.7024566583447933, "p_value": 8.770760720267569e-12, "se": 0.1443782272691402, "t_stat": 6.82535589044925}, + "ref_labels": [], + "reference_event_times": null + }, + "multi_cohort_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.833379480786332, "ci_lo": 0.094690106401973, "effect": 0.46403479359415245, "p_value": 0.013799280971322733, "se": 0.18844462964907682, "t_stat": 2.4624463666504157}, + "-2": {"ci_hi": 0.5545098180168937, "ci_lo": -0.30684515049962907, "effect": 0.12383233375863228, "p_value": 0.5730626073280713, "se": 0.21973744806302076, "t_stat": 0.56354679118289}, + "-3": {"ci_hi": 0.8242552113775771, "ci_lo": -0.6558204716877593, "effect": 0.08421736984490891, "p_value": 0.8234991465124396, "se": 0.3775772653834419, "t_stat": 0.2230467180257356}, + "0": {"ci_hi": 5.020308039874349, "ci_lo": 4.128133508478971, "effect": 4.57422077417666, "p_value": 7.736385604384005e-90, "se": 0.2275997259216845, "t_stat": 20.09765501980709}, + "1": {"ci_hi": 4.989666568835305, "ci_lo": 3.965353248851536, "effect": 4.4775099088434205, "p_value": 8.147921917214769e-66, "se": 0.2613092199814439, "t_stat": 17.134909779155045}, + "2": {"ci_hi": 5.127246292175807, "ci_lo": 3.863652584279217, "effect": 4.495449438227512, "p_value": 3.336377493759456e-44, "se": 0.3223512569270801, "t_stat": 13.945810173293163}, + "3": {"ci_hi": 5.4005927952318435, "ci_lo": 3.494843937267625, "effect": 4.447718366249735, "p_value": 5.773172163078618e-20, "se": 0.4861693564260676, "t_stat": 9.148495904690169} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.558197399894351, "ci_hi": 5.025871039604077, "ci_lo": 4.090523760184626, "p_value": 2.390411591934585e-81, "se": 0.2386133844288341, "t_stat": 19.102857162875637}, + "ref_labels": [], + "reference_event_times": null + }, + "nondefault_config_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.2920715827120267, "ci_lo": -0.3558979846789758, "effect": -0.031913200983474554, "p_value": 0.8712888166640185, "se": 0.19696876268313676, "t_stat": -0.16202163504886943}, + "-3": {"ci_hi": 0.5806420101806341, "ci_lo": -0.32786538499656526, "effect": 0.12638831259203442, "p_value": 0.647202223308722, "se": 0.27616663887017195, "t_stat": 0.4576523547851503}, + "0": {"ci_hi": 4.815922783406208, "ci_lo": 3.9182175918875615, "effect": 4.367070187646885, "p_value": 1.20881417099687e-57, "se": 0.2728830021132122, "t_stat": 16.00345259260633}, + "1": {"ci_hi": 4.9133685741575075, "ci_lo": 3.94434934106944, "effect": 4.428858957613474, "p_value": 4.3005379311491336e-51, "se": 0.29456093150489676, "t_stat": 15.03545950573506}, + "2": {"ci_hi": 5.221285872197571, "ci_lo": 4.24133157128994, "effect": 4.731308721743756, "p_value": 8.30979122862741e-57, "se": 0.2978849560990577, "t_stat": 15.883006593224607}, + "3": {"ci_hi": 5.7015009438566455, "ci_lo": 4.0336552739219105, "effect": 4.867578108889278, "p_value": 7.920903005336168e-22, "se": 0.5069890848056418, "t_stat": 9.60095247564413} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.529321090618209, "ci_hi": 4.974201516254072, "ci_lo": 4.084440664982346, "p_value": 6.032979674501608e-63, "se": 0.2704680941491386, "t_stat": 16.74623065935718}, + "ref_labels": [], + "reference_event_times": null + }, + "not_yet_treated_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.5012703771792749, "ci_lo": -0.3550670045504556, "effect": 0.07310168631440965, "p_value": 0.7379067077472231, "se": 0.2184574279130664, "t_stat": 0.3346266914004863}, + "-2": {"ci_hi": 0.27372284113543316, "ci_lo": -0.8731695817291951, "effect": -0.29972337029688095, "p_value": 0.30563920908057307, "se": 0.292579973895227, "t_stat": -1.024415192559324}, + "-3": {"ci_hi": 1.0572424398211584, "ci_lo": -0.1323401232887942, "effect": 0.4624511582661821, "p_value": 0.12753991837639397, "se": 0.30347051591080965, "t_stat": 1.5238750851239105}, + "0": {"ci_hi": 5.647486803298519, "ci_lo": 4.287943978120929, "effect": 4.967715390709724, "p_value": 1.5659947817194822e-46, "se": 0.3468285223354844, "t_stat": 14.323260835810077}, + "1": {"ci_hi": 5.623309223201617, "ci_lo": 4.164048457666904, "effect": 4.89367884043426, "p_value": 1.8035236613438092e-39, "se": 0.37226723986899146, "t_stat": 13.145607016498275}, + "2": {"ci_hi": 5.74776186698851, "ci_lo": 3.642676532028821, "effect": 4.695219199508665, "p_value": 2.268479846591243e-18, "se": 0.537021432935588, "t_stat": 8.743075995761652}, + "3": {"ci_hi": 6.258049889426443, "ci_lo": 3.9152645940472848, "effect": 5.086657241736864, "p_value": 1.7251308053084342e-17, "se": 0.5976602921938237, "t_stat": 8.510950632281993} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.936165168753279, "ci_hi": 5.614634294602472, "ci_lo": 4.257696042904086, "p_value": 3.906246501899678e-46, "se": 0.34616407811616506, "t_stat": 14.259611209851792}, + "ref_labels": [], + "reference_event_times": null + }, + "replicate_dropped_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.6128612359837633, "ci_lo": -1.0197838160714088, "effect": -0.20346129004382268, "p_value": 0.5499267560566133, "se": 0.3175633293252132, "t_stat": -0.6406951661457743}, + "-2": {"ci_hi": 1.2956159776556055, "ci_lo": -1.7474527942772249, "effect": -0.22591840831080967, "p_value": 0.7183799069545708, "se": 0.5919027221281904, "t_stat": -0.3816816511647699}, + "0": {"ci_hi": 5.773876911076424, "ci_lo": 4.2127215350404486, "effect": 4.993299223058436, "p_value": 1.5178650099141507e-05, "se": 0.3036579801493718, "t_stat": 16.44382677050737}, + "1": {"ci_hi": 6.097357394054634, "ci_lo": 4.105133708587274, "effect": 5.101245551320954, "p_value": 4.516727384558041e-05, "se": 0.38750442756750636, "t_stat": 13.164354232913317}, + "2": {"ci_hi": 6.18527229131092, "ci_lo": 3.6992893614446523, "effect": 4.942280826377786, "p_value": 0.00015392508323412295, "se": 0.48354479429574254, "t_stat": 10.220936890812684}, + "3": {"ci_hi": 7.068022220123719, "ci_lo": 2.1018050904899255, "effect": 4.584913655306822, "p_value": 0.005120741206266744, "se": 0.9659714117610402, "t_stat": 4.7464278957781705} + }, + "event_study_df": 5.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.970179123231916, "ci_hi": 5.767623342780594, "ci_lo": 4.172734903683239, "p_value": 1.7251593185108033e-05, "se": 0.3102193474230632, "t_stat": 16.02149951161431}, + "ref_labels": [], + "reference_event_times": null + }, + "replicate_healthy_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.5633860537380702, "ci_lo": -0.9703086338257154, "effect": -0.20346129004382268, "p_value": 0.5503145417109129, "se": 0.32429987270296884, "t_stat": -0.6273862778545582}, + "-2": {"ci_hi": 1.2025427192693312, "ci_lo": -1.6543795358909505, "effect": -0.22591840831080967, "p_value": 0.7194914022663185, "se": 0.6040964549094622, "t_stat": -0.37397737807395137}, + "0": {"ci_hi": 6.228967629247149, "ci_lo": 3.7576308168697232, "effect": 4.993299223058436, "p_value": 2.8850821894992677e-05, "se": 0.5225643800939539, "t_stat": 9.5553761665896}, + "1": {"ci_hi": 6.221426028067218, "ci_lo": 3.9810650745746896, "effect": 5.101245551320954, "p_value": 1.3106260122918867e-05, "se": 0.473724515001368, "t_stat": 10.768379912334119}, + "2": {"ci_hi": 6.298324304449607, "ci_lo": 3.586237348305965, "effect": 4.942280826377786, "p_value": 5.65015060321704e-05, "se": 0.5734710185596742, "t_stat": 8.618187609185176}, + "3": {"ci_hi": 7.398347327084652, "ci_lo": 1.7714799835289923, "effect": 4.584913655306822, "p_value": 0.0062638912988894815, "se": 1.1898015804763624, "t_stat": 3.8535111488683302} + }, + "event_study_df": 7.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.970179123231916, "ci_hi": 5.975540183921501, "ci_lo": 3.964818062542332, "p_value": 7.578088964806102e-06, "se": 0.4251673643338405, "t_stat": 11.68993563515694}, + "ref_labels": [], + "reference_event_times": null + }, + "replicate_undefined_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": -0.20346129004382268, "p_value": null, "se": null, "t_stat": null}, + "-2": {"ci_hi": null, "ci_lo": null, "effect": -0.22591840831080967, "p_value": null, "se": null, "t_stat": null}, + "0": {"ci_hi": null, "ci_lo": null, "effect": 4.993299223058436, "p_value": null, "se": null, "t_stat": null}, + "1": {"ci_hi": null, "ci_lo": null, "effect": 5.101245551320954, "p_value": null, "se": null, "t_stat": null}, + "2": {"ci_hi": null, "ci_lo": null, "effect": 4.942280826377786, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": 4.584913655306822, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.970179123231916, "ci_hi": null, "ci_lo": null, "p_value": null, "se": null, "t_stat": null}, + "ref_labels": [], + "reference_event_times": null + }, + "survey_tsl_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.25733287333638755, "ci_lo": -0.6642554534240335, "effect": -0.20346129004382296, "p_value": 0.38333257869972726, "se": 0.23241944910681148, "t_stat": -0.8754056118183104}, + "-2": {"ci_hi": 0.6371493818107479, "ci_lo": -1.0889861984323679, "effect": -0.22591840831081, "p_value": 0.6048643496860584, "se": 0.4353217906459717, "t_stat": -0.5189687563665739}, + "0": {"ci_hi": 5.973028938394066, "ci_lo": 4.013569507722805, "effect": 4.993299223058435, "p_value": 3.246693284087112e-17, "se": 0.4941647677164561, "t_stat": 10.10452292285538}, + "1": {"ci_hi": 6.1397756445086955, "ci_lo": 4.062715458133212, "effect": 5.101245551320954, "p_value": 2.1735245550567547e-16, "se": 0.5238230240784919, "t_stat": 9.73849051460663}, + "2": {"ci_hi": 5.962609096234344, "ci_lo": 3.921952556521221, "effect": 4.942280826377782, "p_value": 4.3837006332936423e-16, "se": 0.5146422269079298, "t_stat": 9.60333328275832}, + "3": {"ci_hi": 6.319934893921338, "ci_lo": 2.8498924166922945, "effect": 4.584913655306816, "p_value": 8.257561797152738e-07, "se": 0.8751254085105956, "t_stat": 5.239150424291793} + }, + "event_study_df": 106.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 4.970179123231914, "ci_hi": 5.932401595552254, "ci_lo": 4.0079566509115745, "p_value": 1.599954979442042e-17, "se": 0.48533430912916814, "t_stat": 10.24073309828409}, + "ref_labels": [], + "reference_event_times": null + }, + "survey_zero_dose_drop_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2579783114757419, "ci_lo": -0.6823769045463565, "effect": -0.2121992965353073, "p_value": 0.3727954345275856, "se": 0.2370449871161837, "t_stat": -0.8951857582683296}, + "-2": {"ci_hi": 0.6358967771933868, "ci_lo": -1.0877335938150068, "effect": -0.22591840831081, "p_value": 0.6042191857972364, "se": 0.4344931916442371, "t_stat": -0.519958454253046}, + "0": {"ci_hi": 6.1531107837135925, "ci_lo": 4.146590978166178, "effect": 5.149850880939885, "p_value": 3.228072174210494e-17, "se": 0.5058040337845873, "t_stat": 10.181514058729537}, + "1": {"ci_hi": 6.27477006557415, "ci_lo": 4.156726314864836, "effect": 5.215748190219493, "p_value": 2.64607508402481e-16, "se": 0.5339170188498257, "t_stat": 9.768836740689327}, + "2": {"ci_hi": 6.076229804261528, "ci_lo": 3.9662668210216037, "effect": 5.021248312641566, "p_value": 1.4095417244422216e-15, "se": 0.5318800168871277, "t_stat": 9.44056582916734}, + "3": {"ci_hi": 6.637106054564792, "ci_lo": 2.8360174061800913, "effect": 4.736561730372442, "p_value": 3.0247519933455853e-06, "se": 0.9581794138340265, "t_stat": 4.94329314738638} + }, + "event_study_df": 102.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": null, + "overall": {"att": 5.078141579779009, "ci_hi": 6.07043452495823, "ci_lo": 4.085848634599788, "p_value": 3.776987096564807e-17, "se": 0.500274926746322, "t_stat": 10.150701760742086}, + "ref_labels": [], + "reference_event_times": null + } + }, + "EfficientDiD": { + "clustered_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.07345692105801568, "ci_lo": -0.3420838450325637, "effect": -0.134313461987274, "n_groups": 2.0, "p_value": 0.20514759243179237, "se": 0.10600724538009676, "t_stat": -1.267021527685992}, + "-2": {"ci_hi": 0.2615490713261905, "ci_lo": -0.041502571869040994, "effect": 0.11002324972857475, "n_groups": 2.0, "p_value": 0.15469720219064959, "se": 0.07731051325066791, "t_stat": 1.4231343849941938}, + "-3": {"ci_hi": 0.2867769840849672, "ci_lo": 0.00042351549121141807, "effect": 0.1436002497880893, "n_groups": 2.0, "p_value": 0.04932616233519001, "se": 0.07305069655679274, "t_stat": 1.9657615403632507}, + "-4": {"ci_hi": 0.04605843215891903, "ci_lo": -0.27927254536649015, "effect": -0.11660705660378556, "n_groups": 1.0, "p_value": 0.16002014565445544, "se": 0.08299412134395796, "t_stat": -1.4050038088905517}, + "-5": {"ci_hi": -0.028013067573306683, "ci_lo": -0.45006607365998713, "effect": -0.2390395706166469, "n_groups": 1.0, "p_value": 0.026409082760295216, "se": 0.10766856162046362, "t_stat": -2.220142695499842}, + "0": {"ci_hi": 2.1895302268223618, "ci_lo": 1.8649513450386688, "effect": 2.0272407859305153, "n_groups": 2.0, "p_value": 2.246085494735641e-132, "se": 0.08280225665979828, "t_stat": 24.48291710526255}, + "1": {"ci_hi": 2.376261607445581, "ci_lo": 2.116113815800811, "effect": 2.246187711623196, "n_groups": 2.0, "p_value": 4.1928789869885235e-251, "se": 0.06636545204319641, "t_stat": 33.84573814341205}, + "2": {"ci_hi": 2.4288248251575846, "ci_lo": 1.8712668915342754, "effect": 2.15004585834593, "n_groups": 1.0, "p_value": 1.2710002118261154e-51, "se": 0.14223678037485762, "t_stat": 15.115962641164936}, + "3": {"ci_hi": 2.8445382047477983, "ci_lo": 2.397886784454424, "effect": 2.621212494601111, "n_groups": 1.0, "p_value": 4.2082659776578635e-117, "se": 0.11394378259409464, "t_stat": 23.004436353835427} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3316219868783983, "ci_lo": 2.1037326180078195, "df_used": null, "effect": 2.217677302443109, "p_value": 0.0, "se": 0.058136111343917704, "t_stat": 38.14629584225065}, + "6": {"ci_hi": 2.3650715410094088, "ci_lo": 2.05207761627639, "df_used": null, "effect": 2.2085745786428994, "p_value": 2.1080412209649096e-168, "se": 0.07984685616722419, "t_stat": 27.660131965840414} + }, + "overall": {"att": 2.214245127895489, "ci_hi": 2.327117735710094, "ci_lo": 2.1013725200808837, "p_value": 0.0, "se": 0.057589123425190354, "t_stat": 38.449015998165805}, + "ref_labels": [], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2709856105933327, "ci_lo": -0.05816572037999203, "effect": 0.10640994510667033, "n_groups": 2.0, "p_value": 0.20506341936935923, "se": 0.08396871921362546, "t_stat": 1.2672569750165175}, + "-2": {"ci_hi": 0.11697447968922686, "ci_lo": -0.23656565158958748, "effect": -0.05979558595018031, "n_groups": 2.0, "p_value": 0.5073354765114999, "se": 0.09019046627067992, "t_stat": -0.6629923141845347}, + "-3": {"ci_hi": 0.22082500062751104, "ci_lo": -0.04774980495688662, "effect": 0.08653759783531222, "n_groups": 2.0, "p_value": 0.20657423263505748, "se": 0.0685152400000412, "t_stat": 1.2630415924290739}, + "-4": {"ci_hi": 0.12174279226139513, "ci_lo": -0.2793102104847449, "effect": -0.07878370911167488, "n_groups": 1.0, "p_value": 0.4412767413768576, "se": 0.10231131947056042, "t_stat": -0.770039029106105}, + "-5": {"ci_hi": 0.17622133798852047, "ci_lo": -0.23226670424785045, "effect": -0.028022683129665008, "n_groups": 1.0, "p_value": 0.7879982256940536, "se": 0.10420804807090144, "t_stat": -0.2689109300905323}, + "0": {"ci_hi": 2.2847017505459717, "ci_lo": 1.9934056143098826, "effect": 2.139053682427927, "n_groups": 2.0, "p_value": 3.312798938201983e-182, "se": 0.07431160432890503, "t_stat": 28.784921301933164}, + "1": {"ci_hi": 2.37539394539002, "ci_lo": 2.0428676038131157, "effect": 2.209130774601568, "n_groups": 2.0, "p_value": 1.6597989190426364e-149, "se": 0.0848297071272302, "t_stat": 26.04194744287217}, + "2": {"ci_hi": 2.788829854510437, "ci_lo": 2.360697993352844, "effect": 2.5747639239316404, "n_groups": 1.0, "p_value": 7.081667471367201e-123, "se": 0.10921931845040075, "t_stat": 23.57425371685418}, + "3": {"ci_hi": 2.9748255058745325, "ci_lo": 2.5278939160075815, "effect": 2.751359710941057, "n_groups": 1.0, "p_value": 1.1677819938852978e-128, "se": 0.11401525573742445, "t_stat": 24.131504974013303} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.5958288199018664, "ci_lo": 2.30773927722724, "df_used": null, "effect": 2.4517840485645532, "p_value": 5.127222339006657e-244, "se": 0.07349358073593183, "t_stat": 33.360519708163416}, + "6": {"ci_hi": 2.28138179947517, "ci_lo": 1.9401526064706776, "df_used": null, "effect": 2.110767202972924, "p_value": 6.976023946500123e-130, "se": 0.08704986308321609, "t_stat": 24.24779463415258} + }, + "overall": {"att": 2.3344742536810323, "ci_hi": 2.4645714278546444, "ci_lo": 2.2043770795074202, "p_value": 5.800930872836493e-271, "se": 0.0663773289712474, "t_stat": 35.169752833716075}, + "ref_labels": [], + "reference_event_times": null + }, + "default_balance_e1": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2709856105933327, "ci_lo": -0.05816572037999203, "effect": 0.10640994510667033, "n_groups": 2.0, "p_value": 0.20506341936935923, "se": 0.08396871921362546, "t_stat": 1.2672569750165175}, + "-2": {"ci_hi": 0.11697447968922686, "ci_lo": -0.23656565158958748, "effect": -0.05979558595018031, "n_groups": 2.0, "p_value": 0.5073354765114999, "se": 0.09019046627067992, "t_stat": -0.6629923141845347}, + "-3": {"ci_hi": 0.22082500062751104, "ci_lo": -0.04774980495688662, "effect": 0.08653759783531222, "n_groups": 2.0, "p_value": 0.20657423263505748, "se": 0.0685152400000412, "t_stat": 1.2630415924290739}, + "-4": {"ci_hi": 0.12174279226139513, "ci_lo": -0.2793102104847449, "effect": -0.07878370911167488, "n_groups": 1.0, "p_value": 0.4412767413768576, "se": 0.10231131947056042, "t_stat": -0.770039029106105}, + "-5": {"ci_hi": 0.17622133798852047, "ci_lo": -0.23226670424785045, "effect": -0.028022683129665008, "n_groups": 1.0, "p_value": 0.7879982256940536, "se": 0.10420804807090144, "t_stat": -0.2689109300905323}, + "0": {"ci_hi": 2.2847017505459717, "ci_lo": 1.9934056143098826, "effect": 2.139053682427927, "n_groups": 2.0, "p_value": 3.312798938201983e-182, "se": 0.07431160432890503, "t_stat": 28.784921301933164}, + "1": {"ci_hi": 2.37539394539002, "ci_lo": 2.0428676038131157, "effect": 2.209130774601568, "n_groups": 2.0, "p_value": 1.6597989190426364e-149, "se": 0.0848297071272302, "t_stat": 26.04194744287217}, + "2": {"ci_hi": 2.788829854510437, "ci_lo": 2.360697993352844, "effect": 2.5747639239316404, "n_groups": 1.0, "p_value": 7.081667471367201e-123, "se": 0.10921931845040075, "t_stat": 23.57425371685418}, + "3": {"ci_hi": 2.9748255058745325, "ci_lo": 2.5278939160075815, "effect": 2.751359710941057, "n_groups": 1.0, "p_value": 1.1677819938852978e-128, "se": 0.11401525573742445, "t_stat": 24.131504974013303} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": [], + "reference_event_times": null + }, + "fractional_balance_e1": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.3146093598929288, "ci_lo": -0.2622394722841087, "effect": 0.026184943804410075, "n_groups": 1.0, "p_value": 0.8587719858017362, "se": 0.14715801839399795, "t_stat": 0.17793759449996824}, + "0": {"ci_hi": 0.4744597998730013, "ci_lo": 0.015147611354018375, "effect": 0.24480370561350984, "n_groups": 2.0, "p_value": 0.036686223159691944, "se": 0.11717362975594932, "t_stat": 2.0892389023314375}, + "1": {"ci_hi": 0.986702075354783, "ci_lo": 0.28499042836241567, "effect": 0.6358462518585993, "n_groups": 2.0, "p_value": 0.00038233197115598855, "se": 0.1790113625881341, "t_stat": 3.551988223907005}, + "2": {"ci_hi": 1.3936424795222435, "ci_lo": 0.21450892977573222, "effect": 0.8040757046489879, "n_groups": 1.0, "p_value": 0.007515822291562017, "se": 0.300804902296003, "t_stat": 2.673080453515176} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": [], + "reference_event_times": null + }, + "replicate_dropped_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.20617267536550088, "ci_lo": 0.019700274428429754, "effect": 0.11293647489696532, "n_groups": 2.0, "p_value": 0.02643895774079647, "se": 0.03627046576615062, "t_stat": 3.1137310346415012}, + "-2": {"ci_hi": 0.23657244811883715, "ci_lo": -0.08741004099020665, "effect": 0.07458120356431526, "n_groups": 2.0, "p_value": 0.28980705716909955, "se": 0.06301734584319237, "t_stat": 1.1835027731872036}, + "-3": {"ci_hi": 0.3583062568665918, "ci_lo": -0.3811413817874023, "effect": -0.01141756246040529, "n_groups": 2.0, "p_value": 0.9398072229709629, "se": 0.14382884613960428, "t_stat": -0.07938298030509879}, + "-4": {"ci_hi": 0.3736092816398969, "ci_lo": -0.41179827395487517, "effect": -0.019094496157489163, "n_groups": 1.0, "p_value": 0.9054014622833809, "se": 0.152768440340347, "t_stat": -0.12498979576507596}, + "-5": {"ci_hi": 0.3112478647728595, "ci_lo": -0.7005741843534062, "effect": -0.19466315979027338, "n_groups": 1.0, "p_value": 0.36803175093580826, "se": 0.1968079823601107, "t_stat": -0.9891019533653221}, + "0": {"ci_hi": 2.1955202928848925, "ci_lo": 1.881770568724981, "effect": 2.038645430804937, "n_groups": 2.0, "p_value": 4.5190200876562725e-07, "se": 0.061026986149664196, "t_stat": 33.40563838111404}, + "1": {"ci_hi": 2.287849440452601, "ci_lo": 1.9200243174820395, "effect": 2.1039368789673203, "n_groups": 2.0, "p_value": 8.524607433909252e-07, "se": 0.07154511050209594, "t_stat": 29.407137178237843}, + "2": {"ci_hi": 2.8375838230738695, "ci_lo": 2.167432286298155, "effect": 2.5025080546860123, "n_groups": 1.0, "p_value": 7.070366059304265e-06, "se": 0.13035016576506445, "t_stat": 19.19834961465555}, + "3": {"ci_hi": 2.843426362334893, "ci_lo": 1.9739071738830476, "effect": 2.4086667681089704, "n_groups": 1.0, "p_value": 3.074938920934596e-05, "se": 0.1691288673244295, "t_stat": 14.241606452011371} + }, + "event_study_df": 5.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.4108530500509575, "ci_lo": 2.124965132058029, "df_used": 5.0, "effect": 2.2679090910544932, "p_value": 1.671298822719448e-07, "se": 0.055607628208840944, "t_stat": 40.784136351528886}, + "6": {"ci_hi": 2.2022388351368507, "ci_lo": 1.9225928722300827, "df_used": 5.0, "effect": 2.0624158536834667, "p_value": 2.4039593777510486e-07, "se": 0.054393514929188244, "t_stat": 37.91657620156384} + }, + "overall": {"att": 2.1990812313146457, "ci_hi": 2.305846433891284, "ci_lo": 2.0923160287380074, "p_value": 4.543919004683007e-08, "se": 0.04153347740053949, "t_stat": 52.9471975126764}, + "ref_labels": [], + "reference_event_times": null + }, + "replicate_healthy_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.21440107352506832, "ci_lo": 0.011471876268862327, "effect": 0.11293647489696532, "n_groups": 2.0, "p_value": 0.03381540320096352, "se": 0.04290939609528135, "t_stat": 2.631975398725891}, + "-2": {"ci_hi": 0.25918791131931473, "ci_lo": -0.1100255041906842, "effect": 0.07458120356431526, "n_groups": 2.0, "p_value": 0.3712400237445083, "se": 0.07807020825006361, "t_stat": 0.9553093969651925}, + "-3": {"ci_hi": 0.33438379567865784, "ci_lo": -0.35721892059946836, "effect": -0.01141756246040529, "n_groups": 2.0, "p_value": 0.9399537290613854, "se": 0.1462394534379554, "t_stat": -0.0780744333487911}, + "-4": {"ci_hi": 0.35986326946781366, "ci_lo": -0.39805226178279196, "effect": -0.019094496157489163, "n_groups": 1.0, "p_value": 0.9085069131605498, "se": 0.16026130382873355, "t_stat": -0.11914601779288454}, + "-5": {"ci_hi": 0.3165847199281572, "ci_lo": -0.7059110395087039, "effect": -0.19466315979027338, "n_groups": 1.0, "p_value": 0.39784652278287136, "se": 0.21620681568078304, "t_stat": -0.900356259248008}, + "0": {"ci_hi": 2.1887082667531965, "ci_lo": 1.8885825948566772, "effect": 2.038645430804937, "n_groups": 2.0, "p_value": 7.325303476526665e-09, "se": 0.06346159896109461, "t_stat": 32.12407919401364}, + "1": {"ci_hi": 2.285494910650649, "ci_lo": 1.9223788472839918, "effect": 2.1039368789673203, "n_groups": 2.0, "p_value": 2.211976615022427e-08, "se": 0.07678092261848078, "t_stat": 27.40181814982402}, + "2": {"ci_hi": 2.865220931618041, "ci_lo": 2.139795177753984, "effect": 2.5025080546860123, "n_groups": 1.0, "p_value": 7.918962813834602e-07, "se": 0.15339133762487184, "t_stat": 16.31453309838169}, + "3": {"ci_hi": 2.905419287328675, "ci_lo": 1.911914248889266, "effect": 2.4086667681089704, "n_groups": 1.0, "p_value": 8.626843458268939e-06, "se": 0.21007672524930657, "t_stat": 11.465652681183544} + }, + "event_study_df": 7.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.432853428159171, "ci_lo": 2.1029647539498155, "df_used": 7.0, "effect": 2.2679090910544932, "p_value": 6.737582497082923e-09, "se": 0.0697549883426819, "t_stat": 32.51250046682035}, + "6": {"ci_hi": 2.2143113368727203, "ci_lo": 1.9105203704942129, "df_used": 7.0, "effect": 2.0624158536834667, "p_value": 7.353191232871137e-09, "se": 0.06423662579242291, "t_stat": 32.106540906243254} + }, + "overall": {"att": 2.1990812313146457, "ci_hi": 2.311107203157329, "ci_lo": 2.0870552594719625, "p_value": 5.63118374871591e-10, "se": 0.04737580263216192, "t_stat": 46.41781477326908}, + "ref_labels": [], + "reference_event_times": null + }, + "replicate_undefined_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.11293647489696532, "n_groups": 2.0, "p_value": null, "se": null, "t_stat": null}, + "-2": {"ci_hi": null, "ci_lo": null, "effect": 0.07458120356431526, "n_groups": 2.0, "p_value": null, "se": null, "t_stat": null}, + "-3": {"ci_hi": null, "ci_lo": null, "effect": -0.01141756246040529, "n_groups": 2.0, "p_value": null, "se": null, "t_stat": null}, + "-4": {"ci_hi": null, "ci_lo": null, "effect": -0.019094496157489163, "n_groups": 1.0, "p_value": null, "se": null, "t_stat": null}, + "-5": {"ci_hi": null, "ci_lo": null, "effect": -0.19466315979027338, "n_groups": 1.0, "p_value": null, "se": null, "t_stat": null}, + "0": {"ci_hi": null, "ci_lo": null, "effect": 2.038645430804937, "n_groups": 2.0, "p_value": null, "se": null, "t_stat": null}, + "1": {"ci_hi": null, "ci_lo": null, "effect": 2.1039368789673203, "n_groups": 2.0, "p_value": null, "se": null, "t_stat": null}, + "2": {"ci_hi": null, "ci_lo": null, "effect": 2.5025080546860123, "n_groups": 1.0, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": 2.4086667681089704, "n_groups": 1.0, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": null, "ci_lo": null, "df_used": null, "effect": 2.2679090910544932, "p_value": null, "se": null, "t_stat": null}, + "6": {"ci_hi": null, "ci_lo": null, "df_used": null, "effect": 2.0624158536834667, "p_value": null, "se": null, "t_stat": null} + }, + "overall": {"att": 2.1990812313146457, "ci_hi": null, "ci_lo": null, "p_value": null, "se": null, "t_stat": null}, + "ref_labels": [], + "reference_event_times": null + }, + "survey_weights_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.267519566015824, "ci_lo": -0.04161819618559991, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.15053966189644433, "se": 0.07806115385315175, "t_stat": 1.4469512598749732}, + "-2": {"ci_hi": 0.2248611026673304, "ci_lo": -0.07569358301803729, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.32773009565878386, "se": 0.0758938196145844, "t_stat": 0.982738254622171}, + "-3": {"ci_hi": 0.1439935104903139, "ci_lo": -0.1667986856240219, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.884723845236879, "se": 0.07847891912160686, "t_stat": -0.1452949109707428}, + "-4": {"ci_hi": 0.17608191819562358, "ci_lo": -0.21421518029160125, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 0.8469269695135784, "se": 0.09855490198443846, "t_stat": -0.19346202638402896}, + "-5": {"ci_hi": 0.009591345137165197, "ci_lo": -0.3989750884023271, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.061578894730832306, "se": 0.10316813772812161, "t_stat": -1.8871317823498113}, + "0": {"ci_hi": 2.1725825069769638, "ci_lo": 1.9046604942557408, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 1.569930461666579e-57, "se": 0.06765366128920351, "t_stat": 30.13320287133795}, + "1": {"ci_hi": 2.278861855573475, "ci_lo": 1.9289048047812756, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 4.408466512313737e-47, "se": 0.08836853508076307, "t_stat": 23.80805937605012}, + "2": {"ci_hi": 2.7256707971138354, "ci_lo": 2.279310831814035, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 3.9311032336517034e-44, "se": 0.11271147748831938, "t_stat": 22.202626300620324}, + "3": {"ci_hi": 2.6498379553287457, "ci_lo": 2.1673650365431456, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 2.097656359849327e-39, "se": 0.12183045020155887, "t_stat": 19.77011077239807} + }, + "event_study_df": 119.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.4016034347275013, "ci_lo": 2.1340989258805103, "df_used": 119.0, "effect": 2.267851180304006, "p_value": 1.5568170738855175e-62, "se": 0.0675482363358473, "t_stat": 33.57380300839143}, + "6": {"ci_hi": 2.226677589780442, "ci_lo": 1.8981477747502113, "df_used": 119.0, "effect": 2.0624126822653266, "p_value": 6.014113516928942e-49, "se": 0.08295788988636275, "t_stat": 24.86095879596814} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.319421649289587, "ci_lo": 2.0786616605986086, "p_value": 4.806697219094109e-66, "se": 0.06079491028547976, "t_stat": 36.171476273554376}, + "ref_labels": [], + "reference_event_times": null + } + }, + "ImputationDiD": { + "clustered_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.2159054453557054, "ci_lo": 1.89618861628084, "effect": 2.056047030818273, "n_obs": 84.0, "p_value": 3.238610970500986e-140, "se": 0.08156191429963794, "t_stat": 25.20842047999111}, + "1": {"ci_hi": 2.352040303841366, "ci_lo": 2.097017136469497, "effect": 2.2245287201554316, "n_obs": 84.0, "p_value": 3.0786199047882803e-256, "se": 0.06505812591033798, "t_stat": 34.192941911994815}, + "2": {"ci_hi": 2.4433114745425133, "ci_lo": 1.872173603228359, "effect": 2.157742538885436, "n_obs": 38.0, "p_value": 1.2741818175899908e-49, "se": 0.14570111385189144, "t_stat": 14.809375727071183}, + "3": {"ci_hi": 2.876346949872234, "ci_lo": 2.4301070282791555, "effect": 2.6532269890756948, "n_obs": 38.0, "p_value": 3.775737968071735e-120, "se": 0.11383880650689567, "t_stat": 23.30687636746243} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3210630691403344, "ci_lo": 2.0951075824372807, "df_used": null, "effect": 2.2080853257888076, "n_obs": 152.0, "p_value": 0.0, "se": 0.057642764990928914, "t_stat": 38.306374202144674}, + "6": {"ci_hi": 2.409225957453245, "ci_lo": 2.085474816990568, "df_used": null, "effect": 2.2473503872219065, "n_obs": 92.0, "p_value": 4.87045717484717e-163, "se": 0.08259109427938083, "t_stat": 27.210565580107176} + }, + "overall": {"att": 2.2228901850176808, "ci_hi": 2.336664859712262, "ci_lo": 2.1091155103230994, "p_value": 0.0, "se": 0.05804937008640027, "t_stat": 38.293097439457945}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "covariates_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.0667601467972876, "ci_lo": 1.755278610253442, "effect": 1.9110193785253646, "n_obs": 84.0, "p_value": 8.394800705547965e-128, "se": 0.07946103576411921, "t_stat": 24.049766783788755}, + "1": {"ci_hi": 2.413042058669716, "ci_lo": 2.1103369121060034, "effect": 2.2616894853878597, "n_obs": 84.0, "p_value": 1.4701123052632405e-188, "se": 0.07722211962857788, "t_stat": 29.2881041891379}, + "2": {"ci_hi": 2.7352531776680205, "ci_lo": 2.261379550962076, "effect": 2.4983163643150483, "n_obs": 49.0, "p_value": 6.963867534441308e-95, "se": 0.12088835061353145, "t_stat": 20.666311945159446}, + "3": {"ci_hi": 2.978777223475571, "ci_lo": 2.4680388232186536, "effect": 2.7234080233471123, "n_obs": 49.0, "p_value": 5.11183802574655e-97, "se": 0.1302928023896247, "t_stat": 20.902213885945084} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.446604794900199, "ci_lo": 2.1667825297071155, "df_used": null, "effect": 2.3066936623036574, "n_obs": 196.0, "p_value": 4.5015632591896784e-229, "se": 0.07138454262432534, "t_stat": 32.31362949879877}, + "6": {"ci_hi": 2.3885787481502128, "ci_lo": 2.0188521590680706, "df_used": null, "effect": 2.2037154536091417, "n_obs": 70.0, "p_value": 9.860446633121675e-121, "se": 0.09431974056628045, "t_stat": 23.36430783607324} + }, + "overall": {"att": 2.2795941336998373, "ci_hi": 2.408043187198349, "ci_lo": 2.1511450802013257, "p_value": 4.303191349062899e-265, "se": 0.06553643562417542, "t_stat": 34.78361482416246}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.2939030232819024, "ci_lo": 1.9979777038935498, "effect": 2.145940363587726, "n_obs": 84.0, "p_value": 9.689747789053283e-178, "se": 0.07549254009833184, "t_stat": 28.42585983717807}, + "1": {"ci_hi": 2.4089072341547855, "ci_lo": 2.075976074529496, "effect": 2.242441654342141, "n_obs": 84.0, "p_value": 1.2830604132067007e-153, "se": 0.08493297893517628, "t_stat": 26.402484434858316}, + "2": {"ci_hi": 2.8153981515675794, "ci_lo": 2.3811395559002286, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 1.2081848360326175e-121, "se": 0.11078228964734226, "t_stat": 23.453828784412007}, + "3": {"ci_hi": 2.995785182944395, "ci_lo": 2.5370755701019942, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 1.4718445943547193e-123, "se": 0.11701990864644539, "t_stat": 23.640681389364833} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.606436721671823, "ci_lo": 2.3152673072750503, "df_used": null, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 1.109545391138316e-240, "se": 0.07427927673505222, "t_stat": 33.129725040957034}, + "6": {"ci_hi": 2.3271589354063664, "ci_lo": 1.9750975197798244, "df_used": null, "effect": 2.1511282275930954, "n_obs": 86.0, "p_value": 8.993663607114609e-127, "se": 0.08981323595830264, "t_stat": 23.95112707654576} + }, + "overall": {"att": 2.3543070317865995, "ci_hi": 2.4824381082496836, "ci_lo": 2.2261759553235154, "p_value": 5.2770465230089974e-284, "se": 0.06537419946170739, "t_stat": 36.01278564283794}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_balance_e1": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.357542634758938, "ci_lo": 1.9364833434338276, "effect": 2.1470129890963827, "n_obs": 41.0, "p_value": 7.003712394613622e-89, "se": 0.10741505829861472, "t_stat": 19.988007483342507}, + "1": {"ci_hi": 2.5747666322558524, "ci_lo": 2.0886250448246804, "effect": 2.3316958385402664, "n_obs": 41.0, "p_value": 7.372829612721875e-79, "se": 0.12401798993904861, "t_stat": 18.801271006619523}, + "2": {"ci_hi": 2.8153981515675794, "ci_lo": 2.3811395559002286, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 1.2081848360326175e-121, "se": 0.11078228964734226, "t_stat": 23.453828784412007}, + "3": {"ci_hi": 2.995785182944395, "ci_lo": 2.5370755701019942, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 1.4718445943547193e-123, "se": 0.11701990864644539, "t_stat": 23.640681389364833} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "pretrends_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.059815163957579826, "ci_lo": -0.3559461519303694, "effect": -0.14806549398639476, "n_obs": 84.0, "p_value": 0.1623681759350742, "se": 0.10584141425966392, "t_stat": -1.3989372215222033}, + "-3": {"ci_hi": 0.1584251692403737, "ci_lo": -0.20715293281084962, "effect": -0.024363881785237962, "n_obs": 84.0, "p_value": 0.7935758227509604, "se": 0.09306614604301801, "t_stat": -0.261791025213145}, + "-4": {"ci_hi": 0.06957272135423254, "ci_lo": -0.34423198704465785, "effect": -0.13732963284521266, "n_obs": 84.0, "p_value": 0.19287575347672523, "se": 0.10534331572120142, "t_stat": -1.3036387919349843}, + "-5": {"ci_hi": 0.1567750477585551, "ci_lo": -0.43065589815781424, "effect": -0.13694042519962957, "n_obs": 43.0, "p_value": 0.3601949931340407, "se": 0.14954378803351004, "t_stat": -0.9157212546263955}, + "-6": {"ci_hi": 0.07789021576946764, "ci_lo": -0.4705808158646776, "effect": -0.19634530004760498, "n_obs": 43.0, "p_value": 0.16019426463331482, "se": 0.13962566369272308, "t_stat": -1.4062264404322253}, + "0": {"ci_hi": 2.2939030232819024, "ci_lo": 1.9979777038935498, "effect": 2.145940363587726, "n_obs": 84.0, "p_value": 9.689747789053283e-178, "se": 0.07549254009833184, "t_stat": 28.42585983717807}, + "1": {"ci_hi": 2.4089072341547855, "ci_lo": 2.075976074529496, "effect": 2.242441654342141, "n_obs": 84.0, "p_value": 1.2830604132067007e-153, "se": 0.08493297893517628, "t_stat": 26.402484434858316}, + "2": {"ci_hi": 2.8153981515675794, "ci_lo": 2.3811395559002286, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 1.2081848360326175e-121, "se": 0.11078228964734226, "t_stat": 23.453828784412007}, + "3": {"ci_hi": 2.995785182944395, "ci_lo": 2.5370755701019942, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 1.4718445943547193e-123, "se": 0.11701990864644539, "t_stat": 23.640681389364833} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.606436721671823, "ci_lo": 2.3152673072750503, "df_used": null, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 1.109545391138316e-240, "se": 0.07427927673505222, "t_stat": 33.129725040957034}, + "6": {"ci_hi": 2.3271589354063664, "ci_lo": 1.9750975197798244, "df_used": null, "effect": 2.1511282275930954, "n_obs": 86.0, "p_value": 8.993663607114609e-127, "se": 0.08981323595830264, "t_stat": 23.95112707654576} + }, + "overall": {"att": 2.3543070317865995, "ci_hi": 2.4824381082496836, "ci_lo": 2.2261759553235154, "p_value": 5.2770465230089974e-284, "se": 0.06537419946170739, "t_stat": 36.01278564283794}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_cohort_zero_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.1680508884019085, "ci_lo": 1.9043438037087796, "effect": 2.036197346055344, "n_obs": 84.0, "p_value": 2.293215235007041e-08, "se": 0.05388569362842653, "t_stat": 37.787345934453754}, + "1": {"ci_hi": 2.299918954264144, "ci_lo": 1.9564364620909478, "effect": 2.128177708177546, "n_obs": 84.0, "p_value": 8.53842772143812e-08, "se": 0.07018693624220074, "t_stat": 30.32156441240918}, + "2": {"ci_hi": 2.8036175732664543, "ci_lo": 2.1763872908836377, "effect": 2.490002432075046, "n_obs": 46.0, "p_value": 1.2044634532663274e-06, "se": 0.12816773152030753, "t_stat": 19.42768591235086}, + "3": {"ci_hi": 2.92045633505207, "ci_lo": 1.9500889838677034, "effect": 2.4352726594598866, "n_obs": 46.0, "p_value": 1.775233893645019e-05, "se": 0.19828408422851526, "t_stat": 12.281735414796694} + }, + "event_study_df": 6.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_cohort_zero_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.4032310731914537, "ci_lo": 2.0854758976995678, "df_used": 6.0, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": 3.90581008847123e-08, "se": 0.06492983703994071, "t_stat": 34.56582655620908}, + "6": {"ci_hi": 2.309922293726208, "ci_lo": 1.9658814992440419, "df_used": 6.0, "effect": 2.137901896485125, "n_obs": 76.0, "p_value": 8.390230560937787e-08, "se": 0.07030101928706174, "t_stat": 30.410681355207412} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": 2.3062345655767293, "ci_lo": 2.1111626617160617, "p_value": 2.3202656603532574e-09, "se": 0.039860835969507374, "t_stat": 55.41024316037926}, + "ref_labels": null, + "reference_event_times": null + }, + "replicate_dropped_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.136782065574815, "ci_lo": 1.9356126265358726, "effect": 2.036197346055344, "n_obs": 84.0, "p_value": 4.9543946201583643e-08, "se": 0.03912916450472498, "t_stat": 52.03784368586419}, + "1": {"ci_hi": 2.27405473304184, "ci_lo": 1.9823006833132517, "effect": 2.128177708177546, "n_obs": 84.0, "p_value": 2.539445057626956e-07, "se": 0.05674864065480501, "t_stat": 37.50182706794668}, + "2": {"ci_hi": 2.7526664230984723, "ci_lo": 2.2273384410516197, "effect": 2.490002432075046, "n_obs": 46.0, "p_value": 2.1694339963138596e-06, "se": 0.10218075432654233, "t_stat": 24.368604914753956}, + "3": {"ci_hi": 2.8897497837648305, "ci_lo": 1.9807955351549427, "effect": 2.4352726594598866, "n_obs": 46.0, "p_value": 3.620355289148721e-05, "se": 0.17679932146273952, "t_stat": 13.77421948971179} + }, + "event_study_df": 5.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_dropped_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.404655956514118, "ci_lo": 2.0840510143769033, "df_used": 5.0, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": 3.117495193273171e-07, "se": 0.062360384270328646, "t_stat": 35.990052205521515}, + "6": {"ci_hi": 2.30022339985521, "ci_lo": 1.97558039311504, "df_used": 5.0, "effect": 2.137901896485125, "n_obs": 76.0, "p_value": 4.2270488084337436e-07, "se": 0.0631458221324841, "t_stat": 33.856585032651346} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": 2.3075450626660454, "ci_lo": 2.1098521646267456, "p_value": 3.025940744574244e-08, "se": 0.038452947752655994, "t_stat": 57.438993438255665}, + "ref_labels": null, + "reference_event_times": null + }, + "replicate_none_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.1688072307535076, "ci_lo": 1.9035874613571804, "effect": 2.036197346055344, "n_obs": 84.0, "p_value": 3.123033441217508e-09, "se": 0.05608074289555267, "t_stat": 36.308316204862884}, + "1": {"ci_hi": 2.2945995849020826, "ci_lo": 1.961755831453009, "effect": 2.128177708177546, "n_obs": 84.0, "p_value": 1.1157136251777773e-08, "se": 0.07037984010036138, "t_stat": 30.23845614230968}, + "2": {"ci_hi": 2.7944818811565653, "ci_lo": 2.1855229829935268, "effect": 2.490002432075046, "n_obs": 46.0, "p_value": 2.465383218809266e-07, "se": 0.12876441103757838, "t_stat": 19.33766024331341}, + "3": {"ci_hi": 2.907120306474585, "ci_lo": 1.9634250124451882, "effect": 2.4352726594598866, "n_obs": 46.0, "p_value": 5.677230931463324e-06, "se": 0.19954445053875566, "t_stat": 12.204161292808823} + }, + "event_study_df": 7.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_none_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.399306089281152, "ci_lo": 2.0894008816098695, "df_used": 7.0, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": 4.689798509195468e-09, "se": 0.06552948263609615, "t_stat": 34.24952243113293}, + "6": {"ci_hi": 2.305457193114026, "ci_lo": 1.970346599856224, "df_used": 7.0, "effect": 2.137901896485125, "n_obs": 76.0, "p_value": 1.13313568324369e-08, "se": 0.07085916357156435, "t_stat": 30.171142146293427} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": 2.3052116853616416, "ci_lo": 2.1121855419311495, "p_value": 1.9291321481698958e-10, "se": 0.040815394517854514, "t_stat": 54.11435169835759}, + "ref_labels": null, + "reference_event_times": null + }, + "replicate_undefined_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": null, "ci_lo": null, "effect": 2.036197346055344, "n_obs": 84.0, "p_value": null, "se": null, "t_stat": null}, + "1": {"ci_hi": null, "ci_lo": null, "effect": 2.128177708177546, "n_obs": 84.0, "p_value": null, "se": null, "t_stat": null}, + "2": {"ci_hi": null, "ci_lo": null, "effect": 2.490002432075046, "n_obs": 46.0, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": 2.4352726594598866, "n_obs": 46.0, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_undefined_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": null, "ci_lo": null, "df_used": 0.0, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": null, "se": null, "t_stat": null}, + "6": {"ci_hi": null, "ci_lo": null, "df_used": 0.0, "effect": 2.137901896485125, "n_obs": 76.0, "p_value": null, "se": null, "t_stat": null} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": null, "ci_lo": null, "p_value": null, "se": null, "t_stat": null}, + "ref_labels": null, + "reference_event_times": null + }, + "survey_tsl_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.188071726851769, "ci_lo": 1.8843229652589164, "effect": 2.0361973460553426, "n_obs": 84.0, "p_value": 7.96612226199571e-52, "se": 0.07670036375547899, "t_stat": 26.547427500432022}, + "1": {"ci_hi": 2.3070154714381226, "ci_lo": 1.9493399449169722, "effect": 2.1281777081775473, "n_obs": 84.0, "p_value": 1.218116737632538e-46, "se": 0.09031754680000065, "t_stat": 23.563280708788383}, + "2": {"ci_hi": 2.730075258232533, "ci_lo": 2.2499296059175635, "effect": 2.4900024320750482, "n_obs": 46.0, "p_value": 6.258218704259847e-41, "se": 0.12124278629169719, "t_stat": 20.537324390453783}, + "3": {"ci_hi": 2.6907567582364753, "ci_lo": 2.1797885606833076, "effect": 2.4352726594598915, "n_obs": 46.0, "p_value": 1.39128774786627e-37, "se": 0.12902586471230448, "t_stat": 18.874298303599378} + }, + "event_study_df": 119.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3845208310087664, "ci_lo": 2.104186139882255, "df_used": 119.0, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": 7.18482203623246e-60, "se": 0.07078801793274278, "t_stat": 31.70527373118314}, + "6": {"ci_hi": 2.3194403426048797, "ci_lo": 1.9563634503653775, "df_used": 119.0, "effect": 2.1379018964851286, "n_obs": 76.0, "p_value": 3.3848134601287466e-46, "se": 0.09168145924264394, "t_stat": 23.318803105292666} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": 2.3344309010229924, "ci_lo": 2.0829663262697986, "p_value": 3.3908506819933194e-64, "se": 0.0634979522354065, "t_stat": 34.783777049345915}, + "ref_labels": ["-1"], + "reference_event_times": null + } + }, + "TwoStageDiD": { + "always_treated_survey_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.0165930246514527, "ci_lo": 1.6462320409397084, "effect": 1.8314125327955806, "n_obs": 92.0, "p_value": 3.726531470166178e-40, "se": 0.09358129359870074, "t_stat": 19.5702844272395}, + "1": {"ci_hi": 2.290954642175793, "ci_lo": 1.8910905755993215, "effect": 2.0910226088875574, "n_obs": 92.0, "p_value": 1.651106610777188e-42, "se": 0.10103601151191301, "t_stat": 20.69581506234545}, + "2": {"ci_hi": 2.3066580186901473, "ci_lo": 1.5741092476348801, "effect": 1.9403836331625137, "n_obs": 47.0, "p_value": 6.599380151986617e-19, "se": 0.1850974174775541, "t_stat": 10.483040009987254}, + "3": {"ci_hi": 2.4035758014866477, "ci_lo": 1.7651064004311579, "effect": 2.084341100958903, "n_obs": 47.0, "p_value": 6.725552089887526e-25, "se": 0.16132582831798364, "t_stat": 12.92007065880692}, + "4": {"ci_hi": 0.37185836366550307, "ci_lo": -1.2391289797745204, "effect": -0.4336353080545087, "n_obs": 8.0, "p_value": 0.28876490638278013, "se": 0.40705767130046416, "t_stat": -1.065292067016289}, + "5": {"ci_hi": 0.057220117025725936, "ci_lo": -1.015056008861297, "effect": -0.4789179459177856, "n_obs": 8.0, "p_value": 0.07952484892385321, "se": 0.27093833143506935, "t_stat": -1.767627132643499}, + "6": {"ci_hi": 0.4471006489812237, "ci_lo": -0.9422623230482301, "effect": -0.2475808370335032, "n_obs": 8.0, "p_value": 0.48195180478065625, "se": 0.3510585345616383, "t_stat": -0.7052409004744857} + }, + "event_study_df": 127.0, + "event_study_vcov": [ + [0.008757458511606227, 0.004146487434122724, 0.010074864365616481, 0.004444937450505918, 0.005881320769607086, 0.007543776458122666, 0.0028848518114763513], + [0.004146487434122724, 0.010208275622235417, 0.00847297102264899, 0.011193162986745902, 0.009181937942576346, 0.006636350563387816, 0.007460826990613974], + [0.010074864365616481, 0.00847297102264899, 0.034261053956859945, 0.019024783300514207, 0.0029261362869892333, 0.012062865344136897, 0.0029535986866108003], + [0.004444937450505918, 0.0111931629867459, 0.019024783300514207, 0.026026022882483534, 1.7937653228008368e-05, 0.0038278508794065565, 0.004667851231119962], + [0.005881320769607085, 0.009181937942576344, 0.0029261362869892333, 1.793765322800837e-05, 0.1656959477645567, -0.00026043277378823753, 0.05269359949418312], + [0.007543776458122667, 0.0066363505633878165, 0.012062865344136899, 0.0038278508794065565, -0.00026043277378823753, 0.07340757944081949, 0.03423566208614829], + [0.0028848518114763517, 0.007460826990613973, 0.0029535986866108003, 0.004667851231119962, 0.05269359949418312, 0.03423566208614829, 0.12324209468856501] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + "group": { + "1": {"ci_hi": 0.1945177330957572, "ci_lo": -0.654284393016846, "effect": -0.2298833299605444, "n_obs": 56.0, "p_value": 0.2858164948174116, "se": 0.21447183819116233, "t_stat": -1.0718578807332528}, + "4": {"ci_hi": 2.374582231477566, "ci_lo": 2.0975847464345883, "effect": 2.2360834889560772, "n_obs": 156.0, "p_value": 1.4680535172538918e-62, "se": 0.06999058786949273, "t_stat": 31.948345585060213}, + "6": {"ci_hi": 2.320447443894367, "ci_lo": 1.940860802557352, "effect": 2.1306541232258596, "n_obs": 90.0, "p_value": 1.4158709788296335e-45, "se": 0.095912394910235, "t_stat": 22.214585770900122} + }, + "overall": {"att": 1.8215502043857208, "ci_hi": 2.090378408033829, "ci_lo": 1.5527220007376128, "p_value": 4.3728256907716846e-26, "se": 0.13585281473809396, "t_stat": 13.408262522182008}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "always_treated_survey_psu_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.0165930246514527, "ci_lo": 1.6462320409397084, "effect": 1.8314125327955806, "n_obs": 92.0, "p_value": 3.726531470166178e-40, "se": 0.09358129359870074, "t_stat": 19.5702844272395}, + "1": {"ci_hi": 2.290954642175793, "ci_lo": 1.8910905755993215, "effect": 2.0910226088875574, "n_obs": 92.0, "p_value": 1.651106610777188e-42, "se": 0.10103601151191301, "t_stat": 20.69581506234545}, + "2": {"ci_hi": 2.3066580186901473, "ci_lo": 1.5741092476348801, "effect": 1.9403836331625137, "n_obs": 47.0, "p_value": 6.599380151986617e-19, "se": 0.1850974174775541, "t_stat": 10.483040009987254}, + "3": {"ci_hi": 2.4035758014866477, "ci_lo": 1.7651064004311579, "effect": 2.084341100958903, "n_obs": 47.0, "p_value": 6.725552089887526e-25, "se": 0.16132582831798364, "t_stat": 12.92007065880692}, + "4": {"ci_hi": 0.37185836366550307, "ci_lo": -1.2391289797745204, "effect": -0.4336353080545087, "n_obs": 8.0, "p_value": 0.28876490638278013, "se": 0.40705767130046416, "t_stat": -1.065292067016289}, + "5": {"ci_hi": 0.057220117025725936, "ci_lo": -1.015056008861297, "effect": -0.4789179459177856, "n_obs": 8.0, "p_value": 0.07952484892385321, "se": 0.27093833143506935, "t_stat": -1.767627132643499}, + "6": {"ci_hi": 0.4471006489812237, "ci_lo": -0.9422623230482301, "effect": -0.2475808370335032, "n_obs": 8.0, "p_value": 0.48195180478065625, "se": 0.3510585345616383, "t_stat": -0.7052409004744857} + }, + "event_study_df": 127.0, + "event_study_vcov": [ + [0.008757458511606227, 0.004146487434122724, 0.010074864365616481, 0.004444937450505918, 0.005881320769607086, 0.007543776458122666, 0.0028848518114763513], + [0.004146487434122724, 0.010208275622235417, 0.00847297102264899, 0.011193162986745902, 0.009181937942576346, 0.006636350563387816, 0.007460826990613974], + [0.010074864365616481, 0.00847297102264899, 0.034261053956859945, 0.019024783300514207, 0.0029261362869892333, 0.012062865344136897, 0.0029535986866108003], + [0.004444937450505918, 0.0111931629867459, 0.019024783300514207, 0.026026022882483534, 1.7937653228008368e-05, 0.0038278508794065565, 0.004667851231119962], + [0.005881320769607085, 0.009181937942576344, 0.0029261362869892333, 1.793765322800837e-05, 0.1656959477645567, -0.00026043277378823753, 0.05269359949418312], + [0.007543776458122667, 0.0066363505633878165, 0.012062865344136899, 0.0038278508794065565, -0.00026043277378823753, 0.07340757944081949, 0.03423566208614829], + [0.0028848518114763517, 0.007460826990613973, 0.0029535986866108003, 0.004667851231119962, 0.05269359949418312, 0.03423566208614829, 0.12324209468856501] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + "group": { + "1": {"ci_hi": 0.1945177330957572, "ci_lo": -0.654284393016846, "effect": -0.2298833299605444, "n_obs": 56.0, "p_value": 0.2858164948174116, "se": 0.21447183819116233, "t_stat": -1.0718578807332528}, + "4": {"ci_hi": 2.374582231477566, "ci_lo": 2.0975847464345883, "effect": 2.2360834889560772, "n_obs": 156.0, "p_value": 1.4680535172538918e-62, "se": 0.06999058786949273, "t_stat": 31.948345585060213}, + "6": {"ci_hi": 2.320447443894367, "ci_lo": 1.940860802557352, "effect": 2.1306541232258596, "n_obs": 90.0, "p_value": 1.4158709788296335e-45, "se": 0.095912394910235, "t_stat": 22.214585770900122} + }, + "overall": {"att": 1.8215502043857208, "ci_hi": 2.090378408033829, "ci_lo": 1.5527220007376128, "p_value": 4.3728256907716846e-26, "se": 0.13585281473809396, "t_stat": 13.408262522182008}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "clustered_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.218829131988498, "ci_lo": 1.893264929648046, "effect": 2.056047030818272, "n_obs": 84.0, "p_value": 2.694464189754571e-135, "se": 0.08305361856351977, "t_stat": 24.75565865015018}, + "1": {"ci_hi": 2.350509982450415, "ci_lo": 2.0985474578604464, "effect": 2.2245287201554307, "n_obs": 84.0, "p_value": 1.8958681893332686e-262, "se": 0.06427733534325543, "t_stat": 34.60829090496591}, + "2": {"ci_hi": 2.443311474542479, "ci_lo": 1.8721736032283969, "effect": 2.157742538885438, "n_obs": 38.0, "p_value": 1.2741818175541948e-49, "se": 0.14570111385187298, "t_stat": 14.809375727073071}, + "3": {"ci_hi": 2.8763469498721994, "ci_lo": 2.4301070282791946, "effect": 2.653226989075697, "n_obs": 38.0, "p_value": 3.775737967728134e-120, "se": 0.11383880650687672, "t_stat": 23.306876367466327} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.006897903556494635, 0.0007346537576831481, 0.005338947079466995, -0.0007721350063547473], + [0.0007346537576831481, 0.004131575838829314, 0.0014031116885452183, 0.0010808021797016122], + [0.005338947079466994, 0.0014031116885452183, 0.02122881457767645, 0.005982112982436549], + [-0.0007721350063547474, 0.0010808021797016122, 0.005982112982436549, 0.012959273866910119] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0], + "group": { + "4": {"ci_hi": 2.3210630691402483, "ci_lo": 2.0951075824373695, "effect": 2.208085325788809, "n_obs": 152.0, "p_value": 0.0, "se": 0.05764276499088422, "t_stat": 38.306374202174396}, + "6": {"ci_hi": 2.4092259574532466, "ci_lo": 2.085474816990569, "effect": 2.247350387221908, "n_obs": 92.0, "p_value": 4.87045717484717e-163, "se": 0.08259109427938087, "t_stat": 27.210565580107176} + }, + "overall": {"att": 2.222890185017679, "ci_hi": 2.3363632254275863, "ci_lo": 2.1094171446077716, "p_value": 0.0, "se": 0.057895472215289887, "t_stat": 38.39488823498404}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "covariates_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.0681011373082723, "ci_lo": 1.7539376197424565, "effect": 1.9110193785253644, "n_obs": 84.0, "p_value": 1.1560288875298786e-125, "se": 0.08014522716843206, "t_stat": 23.844456445412444}, + "1": {"ci_hi": 2.4141502384231557, "ci_lo": 2.109228732352562, "effect": 2.261689485387859, "n_obs": 84.0, "p_value": 7.387292341585388e-186, "se": 0.07778752785147476, "t_stat": 29.075219998072992}, + "2": {"ci_hi": 2.735253177668089, "ci_lo": 2.261379550962006, "effect": 2.4983163643150474, "n_obs": 49.0, "p_value": 6.963867535311036e-95, "se": 0.12088835061356668, "t_stat": 20.666311945153417}, + "3": {"ci_hi": 2.9787772234755887, "ci_lo": 2.4680388232186323, "effect": 2.7234080233471105, "n_obs": 49.0, "p_value": 5.111838025917359e-97, "se": 0.13029280238963456, "t_stat": 20.90221388594349} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.00642325743787958, 0.0013599605974883074, 0.004559300805275063, 0.001693740094331948], + [0.0013599605974883074, 0.006050899489243962, 0.0024190372773471706, 0.004527070665223844], + [0.004559300805275063, 0.0024190372773471706, 0.014613993314068627, 0.0031251713472249323], + [0.001693740094331948, 0.004527070665223844, 0.0031251713472249323, 0.01697621435454436] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0], + "group": { + "4": {"ci_hi": 2.44660479490025, "ci_lo": 2.1667825297070613, "effect": 2.3066936623036556, "n_obs": 196.0, "p_value": 4.501563260946723e-229, "se": 0.07138454262435194, "t_stat": 32.3136294987867}, + "6": {"ci_hi": 2.388578748150253, "ci_lo": 2.0188521590680284, "effect": 2.203715453609141, "n_obs": 70.0, "p_value": 9.860446634319966e-121, "se": 0.09431974056630137, "t_stat": 23.364307836068054} + }, + "overall": {"att": 2.279594133699835, "ci_hi": 2.408400908621343, "ci_lo": 2.1507873587783273, "p_value": 1.2366723273465232e-263, "se": 0.06571894990801841, "t_stat": 34.68701397223178}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.293908957498144, "ci_lo": 1.9979717696773065, "effect": 2.145940363587725, "n_obs": 84.0, "p_value": 1.0009287652680278e-177, "se": 0.07549556781531501, "t_stat": 28.42471983040573}, + "1": {"ci_hi": 2.410429895674018, "ci_lo": 2.0744534130102625, "effect": 2.2424416543421404, "n_obs": 84.0, "p_value": 6.97981762805575e-151, "se": 0.08570986133263034, "t_stat": 26.16316978555684}, + "2": {"ci_hi": 2.815398151567543, "ci_lo": 2.3811395559002624, "effect": 2.5982688537339027, "n_obs": 41.0, "p_value": 1.208184835925906e-121, "se": 0.11078228964732444, "t_stat": 23.453828784415766}, + "3": {"ci_hi": 2.9957851829443607, "ci_lo": 2.5370755701020267, "effect": 2.7664303765231937, "n_obs": 41.0, "p_value": 1.4718445942354515e-123, "se": 0.11701990864642843, "t_stat": 23.640681389368254} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.005699580759756827, 0.0021237246279570804, 0.004694454408243992, 0.0023631068662294725], + [0.0021237246279570804, 0.007346180329658721, 0.0020270336522953784, 0.003804898371103219], + [0.004694454408243992, 0.002027033652295379, 0.012272715699503688, 0.004183516038185585], + [0.0023631068662294725, 0.003804898371103219, 0.004183516038185585, 0.013693659019618455] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0], + "group": { + "4": {"ci_hi": 2.6064367216717703, "ci_lo": 2.3152673072751013, "effect": 2.460852014473436, "n_obs": 164.0, "p_value": 1.1095453907055126e-240, "se": 0.07427927673502582, "t_stat": 33.12972504096879}, + "6": {"ci_hi": 2.327158935406365, "ci_lo": 1.9750975197798237, "effect": 2.1511282275930945, "n_obs": 86.0, "p_value": 8.993663607111023e-127, "se": 0.08981323595830254, "t_stat": 23.951127076545774} + }, + "overall": {"att": 2.3543070317865986, "ci_hi": 2.486708263656467, "ci_lo": 2.22190579991673, "p_value": 4.066833864881585e-266, "se": 0.06755289021340831, "t_stat": 34.85131464174277}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_balance_e1": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.357542634758937, "ci_lo": 1.9364833434338267, "effect": 2.147012989096382, "n_obs": 41.0, "p_value": 7.003712394616416e-89, "se": 0.1074150582986148, "t_stat": 19.988007483342486}, + "1": {"ci_hi": 2.574766632255852, "ci_lo": 2.088625044824679, "effect": 2.3316958385402655, "n_obs": 41.0, "p_value": 7.37282961272776e-79, "se": 0.12401798993904885, "t_stat": 18.80127100661948}, + "2": {"ci_hi": 2.815398151567543, "ci_lo": 2.3811395559002624, "effect": 2.5982688537339027, "n_obs": 41.0, "p_value": 1.208184835925906e-121, "se": 0.11078228964732444, "t_stat": 23.453828784415766}, + "3": {"ci_hi": 2.9957851829443607, "ci_lo": 2.5370755701020267, "effect": 2.7664303765231937, "n_obs": 41.0, "p_value": 1.4718445942354515e-123, "se": 0.11701990864642843, "t_stat": 23.640681389368254} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.011537994749294815, 0.003854389507149603, 0.0028535048676717094, 0.0034123441254676657], + [0.003854389507149603, 0.01538046182852202, 0.0025509113328664066, 0.000842206098417954], + [0.0028535048676717094, 0.0025509113328664066, 0.012272715699503688, 0.004183516038185585], + [0.0034123441254676657, 0.000842206098417954, 0.004183516038185585, 0.013693659019618455] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0], + "ref_labels": ["-1"], + "reference_event_times": null + }, + "pretrends_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.04063576079673298, "ci_lo": -0.10034972823002453, "effect": -0.02985698371664577, "n_obs": 84.0, "p_value": 0.4064615116607474, "se": 0.03596634686627741, "t_stat": -0.8301366782579782}, + "-3": {"ci_hi": 0.10859632307117377, "ci_lo": -0.03940259393571609, "effect": 0.03459686456772884, "n_obs": 84.0, "p_value": 0.3594889860225441, "se": 0.037755519533595114, "t_stat": 0.9163392530446924}, + "-4": {"ci_hi": 0.08462802785008987, "ci_lo": -0.08967148307729171, "effect": -0.002521727613600925, "n_obs": 84.0, "p_value": 0.9547740812498333, "se": 0.04446497800526793, "t_stat": -0.05671266976230522}, + "-5": {"ci_hi": 0.07422880954615485, "ci_lo": -0.14253409272093076, "effect": -0.034152641587387955, "n_obs": 43.0, "p_value": 0.5368295343838174, "se": 0.05529767484935533, "t_stat": -0.6176144237606424}, + "-6": {"ci_hi": 0.07322299031555263, "ci_lo": -0.14057126632090916, "effect": -0.03367413800267827, "n_obs": 43.0, "p_value": 0.5369597523913643, "se": 0.05454035337456291, "t_stat": -0.6174169384531997}, + "0": {"ci_hi": 2.293908957498144, "ci_lo": 1.9979717696773065, "effect": 2.145940363587725, "n_obs": 84.0, "p_value": 1.0009287652680278e-177, "se": 0.075495567815315, "t_stat": 28.424719830405735}, + "1": {"ci_hi": 2.410429895674018, "ci_lo": 2.0744534130102625, "effect": 2.2424416543421404, "n_obs": 84.0, "p_value": 6.979817628056545e-151, "se": 0.08570986133263035, "t_stat": 26.163169785556832}, + "2": {"ci_hi": 2.815398151567543, "ci_lo": 2.3811395559002624, "effect": 2.5982688537339027, "n_obs": 41.0, "p_value": 1.208184835925906e-121, "se": 0.11078228964732444, "t_stat": 23.453828784415766}, + "3": {"ci_hi": 2.9957851829443607, "ci_lo": 2.5370755701020267, "effect": 2.7664303765231937, "n_obs": 41.0, "p_value": 1.4718445942354515e-123, "se": 0.11701990864642843, "t_stat": 23.640681389368254} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.0029746501462221955, -0.0002650826832603199, -0.0016144965708846215, -0.00012485970044849535, 0.0003483893209992448, 0.000275170184157467, -0.00015995093636165776, 0.0003080120706126748, -0.00011370693707910455], + [-0.0002650826832603199, 0.0030578328437450247, -0.0002912316450706456, -0.0009561997675826662, -0.0001954228732153961, 0.00017584929105873346, -0.000361734091322721, 0.0005037134940451455, -0.0002361806666820629], + [-0.0016144965708846217, -0.0002912316450706456, 0.0019771342690089607, 8.951576021501633e-05, -0.0008341845900340859, -0.0001766436323657939, -6.271618395012136e-05, -1.8816121884652606e-05, -0.00028962160307614306], + [-0.00012485970044849538, -0.0009561997675826663, 8.951576021501633e-05, 0.0014254792552516823, -0.00024764321482010634, -0.00016385353845332678, 5.5604351051812014e-05, -0.0004789980568244183, -0.0003673350995655875], + [0.00034838932099924487, -0.00019542287321539612, -0.0008341845900340859, -0.00024764321482010634, 0.0012935781069053829, 0.00022222923115049502, 9.52174900419265e-05, -0.00024031674802400514, 0.0006254291031602148], + [0.00027517018415746693, 0.00017584929105873346, -0.0001766436323657939, -0.00016385353845332678, 0.00022222923115049502, 0.005699580759756826, 0.00212372462795708, 0.004694454408243992, 0.0023631068662294716], + [-0.00015995093636165776, -0.000361734091322721, -6.271618395012136e-05, 5.5604351051812014e-05, 9.52174900419265e-05, 0.00212372462795708, 0.007346180329658723, 0.0020270336522953806, 0.003804898371103219], + [0.0003080120706126748, 0.0005037134940451455, -1.8816121884652603e-05, -0.00047899805682441834, -0.00024031674802400517, 0.004694454408243992, 0.0020270336522953806, 0.012272715699503688, 0.004183516038185584], + [-0.00011370693707910455, -0.00023618066668206286, -0.00028962160307614306, -0.00036733509956558743, 0.0006254291031602148, 0.0023631068662294716, 0.003804898371103219, 0.004183516038185584, 0.013693659019618455] + ], + "event_study_vcov_index": [-6.0, -5.0, -4.0, -3.0, -2.0, 0.0, 1.0, 2.0, 3.0], + "group": { + "4": {"ci_hi": 2.6064367216717703, "ci_lo": 2.3152673072751013, "effect": 2.460852014473436, "n_obs": 164.0, "p_value": 1.1095453907055126e-240, "se": 0.07427927673502582, "t_stat": 33.12972504096879}, + "6": {"ci_hi": 2.327158935406365, "ci_lo": 1.9750975197798237, "effect": 2.1511282275930945, "n_obs": 86.0, "p_value": 8.993663607111023e-127, "se": 0.08981323595830254, "t_stat": 23.951127076545774} + }, + "overall": {"att": 2.3543070317865986, "ci_hi": 2.486708263656467, "ci_lo": 2.22190579991673, "p_value": 4.066833864881585e-266, "se": 0.06755289021340831, "t_stat": 34.85131464174277}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "prop5_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.154880923793283, "ci_lo": 1.6520643539647772, "effect": 1.9034726388790302, "n_obs": 36.0, "p_value": 8.15311783634353e-50, "se": 0.12827189014559934, "t_stat": 14.83935908887309}, + "1": {"ci_hi": 2.531707036369357, "ci_lo": 2.072970733295254, "effect": 2.3023388848323054, "n_obs": 36.0, "p_value": 3.629398566410162e-86, "se": 0.11702671750413689, "t_stat": 19.673617563022887}, + "2": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 36.0, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 36.0, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.016453677801524708, 0.0030754834831884597], + [0.0030754834831884597, 0.013695252609793058] + ], + "event_study_vcov_index": [0.0, 1.0], + "group": { + "4": {"ci_hi": 2.2896171440554287, "ci_lo": 1.9161943796559084, "effect": 2.1029057618556686, "n_obs": 72.0, "p_value": 5.51874900161238e-108, "se": 0.09526265975931998, "t_stat": 22.074816797774027}, + "6": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 0.0, "p_value": null, "se": null, "t_stat": null} + }, + "overall": {"att": 2.1029057618556686, "ci_hi": 2.2896171440554287, "ci_lo": 1.9161943796559084, "p_value": 5.51874900161238e-108, "se": 0.09526265975931998, "t_stat": 22.074816797774027}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_cohort_zero_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.168050888401908, "ci_lo": 1.9043438037087792, "effect": 2.0361973460553435, "n_obs": 84.0, "p_value": 2.2932152350070424e-08, "se": 0.05388569362842653, "t_stat": 37.78734593445375}, + "1": {"ci_hi": 2.2999189542641423, "ci_lo": 1.9564364620909487, "effect": 2.1281777081775455, "n_obs": 84.0, "p_value": 8.538427721437762e-08, "se": 0.07018693624220022, "t_stat": 30.321564412409394}, + "2": {"ci_hi": 2.803617573266457, "ci_lo": 2.1763872908836404, "effect": 2.4900024320750487, "n_obs": 46.0, "p_value": 1.2044634532663213e-06, "se": 0.12816773152030758, "t_stat": 19.427685912350874}, + "3": {"ci_hi": 2.9204563350520716, "ci_lo": 1.9500889838677062, "effect": 2.435272659459889, "n_obs": 46.0, "p_value": 1.775233893644997e-05, "se": 0.198284084228515, "t_stat": 12.28173541479672} + }, + "event_study_df": 6.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_cohort_zero_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.4032310731914563, "ci_lo": 2.0854758976995686, "effect": 2.2443534854455125, "n_obs": 184.0, "p_value": 3.905810088471317e-08, "se": 0.06492983703994101, "t_stat": 34.56582655620895}, + "6": {"ci_hi": 2.3099222937262094, "ci_lo": 1.9658814992440428, "effect": 2.137901896485126, "n_obs": 76.0, "p_value": 8.390230560937804e-08, "se": 0.0703010192870618, "t_stat": 30.410681355207398} + }, + "overall": {"att": 2.2086986136463946, "ci_hi": 2.3062345655767267, "ci_lo": 2.1111626617160626, "p_value": 2.3202656603530585e-09, "se": 0.039860835969506785, "t_stat": 55.410243160380055}, + "ref_labels": null, + "reference_event_times": null + }, + "replicate_dropped_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.1367820655748115, "ci_lo": 1.9356126265358757, "effect": 2.0361973460553435, "n_obs": 84.0, "p_value": 4.954394620157524e-08, "se": 0.039129164504723646, "t_stat": 52.03784368586595}, + "1": {"ci_hi": 2.274054733041839, "ci_lo": 1.9823006833132524, "effect": 2.1281777081775455, "n_obs": 84.0, "p_value": 2.539445057626865e-07, "se": 0.056748640654804595, "t_stat": 37.50182706794695}, + "2": {"ci_hi": 2.7526664230984763, "ci_lo": 2.227338441051621, "effect": 2.4900024320750487, "n_obs": 46.0, "p_value": 2.16943399631389e-06, "se": 0.10218075432654272, "t_stat": 24.368604914753888}, + "3": {"ci_hi": 2.8897497837648327, "ci_lo": 1.980795535154945, "effect": 2.435272659459889, "n_obs": 46.0, "p_value": 3.620355289148702e-05, "se": 0.17679932146273947, "t_stat": 13.774219489711806} + }, + "event_study_df": 5.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_dropped_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.4046559565141195, "ci_lo": 2.0840510143769055, "effect": 2.2443534854455125, "n_obs": 184.0, "p_value": 3.117495193273137e-07, "se": 0.062360384270328556, "t_stat": 35.99005220552159}, + "6": {"ci_hi": 2.300223399855212, "ci_lo": 1.9755803931150402, "effect": 2.137901896485126, "n_obs": 76.0, "p_value": 4.227048808433817e-07, "se": 0.06314582213248435, "t_stat": 33.856585032651225} + }, + "overall": {"att": 2.2086986136463946, "ci_hi": 2.3075450626660428, "ci_lo": 2.1098521646267465, "p_value": 3.025940744573983e-08, "se": 0.038452947752655314, "t_stat": 57.43899343825666}, + "ref_labels": null, + "reference_event_times": null + }, + "replicate_none_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.1688072307535067, "ci_lo": 1.9035874613571806, "effect": 2.0361973460553435, "n_obs": 84.0, "p_value": 3.123033441217411e-09, "se": 0.056080742895552406, "t_stat": 36.30831620486305}, + "1": {"ci_hi": 2.2945995849020813, "ci_lo": 1.9617558314530097, "effect": 2.1281777081775455, "n_obs": 84.0, "p_value": 1.1157136251777283e-08, "se": 0.07037984010036093, "t_stat": 30.23845614230987}, + "2": {"ci_hi": 2.7944818811565684, "ci_lo": 2.185522982993529, "effect": 2.4900024320750487, "n_obs": 46.0, "p_value": 2.4653832188092595e-07, "se": 0.1287644110375785, "t_stat": 19.337660243313415}, + "3": {"ci_hi": 2.907120306474587, "ci_lo": 1.9634250124451906, "effect": 2.435272659459889, "n_obs": 46.0, "p_value": 5.677230931463275e-06, "se": 0.19954445053875558, "t_stat": 12.20416129280884} + }, + "event_study_df": 7.0, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_none_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.399306089281154, "ci_lo": 2.089400881609871, "effect": 2.2443534854455125, "n_obs": 184.0, "p_value": 4.689798509195582e-09, "se": 0.06552948263609643, "t_stat": 34.24952243113281}, + "6": {"ci_hi": 2.305457193114027, "ci_lo": 1.9703465998562248, "effect": 2.137901896485126, "n_obs": 76.0, "p_value": 1.1331356832436942e-08, "se": 0.07085916357156442, "t_stat": 30.17114214629341} + }, + "overall": {"att": 2.2086986136463946, "ci_hi": 2.305211685361639, "ci_lo": 2.1121855419311504, "p_value": 1.929132148169655e-10, "se": 0.04081539451785377, "t_stat": 54.114351698358554}, + "ref_labels": null, + "reference_event_times": null + }, + "replicate_undefined_event_study": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": null, "ci_lo": null, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": null, "t_stat": null}, + "0": {"ci_hi": null, "ci_lo": null, "effect": 2.0361973460553435, "n_obs": 84.0, "p_value": null, "se": null, "t_stat": null}, + "1": {"ci_hi": null, "ci_lo": null, "effect": 2.1281777081775455, "n_obs": 84.0, "p_value": null, "se": null, "t_stat": null}, + "2": {"ci_hi": null, "ci_lo": null, "effect": 2.4900024320750487, "n_obs": 46.0, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": 2.435272659459889, "n_obs": 46.0, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "replicate_undefined_group": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": null, "ci_lo": null, "effect": 2.2443534854455125, "n_obs": 184.0, "p_value": null, "se": null, "t_stat": null}, + "6": {"ci_hi": null, "ci_lo": null, "effect": 2.137901896485126, "n_obs": 76.0, "p_value": null, "se": null, "t_stat": null} + }, + "overall": {"att": 2.2086986136463946, "ci_hi": null, "ci_lo": null, "p_value": null, "se": null, "t_stat": null}, + "ref_labels": null, + "reference_event_times": null + }, + "survey_psu_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.190082439558195, "ci_lo": 1.8823122525524882, "effect": 2.0361973460553418, "n_obs": 84.0, "p_value": 3.037332996371183e-51, "se": 0.07771582400086065, "t_stat": 26.200550174090573}, + "1": {"ci_hi": 2.307570690322095, "ci_lo": 1.948784726033002, "effect": 2.1281777081775486, "n_obs": 84.0, "p_value": 1.6511071384703697e-46, "se": 0.09059794623367204, "t_stat": 23.490352669678735}, + "2": {"ci_hi": 2.729101569325406, "ci_lo": 2.2509032948246905, "effect": 2.4900024320750482, "n_obs": 46.0, "p_value": 4.286564449946556e-41, "se": 0.120751049021924, "t_stat": 20.62095900817354}, + "3": {"ci_hi": 2.6896939894444376, "ci_lo": 2.180851329475347, "effect": 2.4352726594598924, "n_obs": 46.0, "p_value": 9.578962482444616e-38, "se": 0.12848913987095953, "t_stat": 18.953140023395086} + }, + "event_study_df": 119.0, + "event_study_vcov": [ + [0.006039749300132747, 0.0009122105474724166, 0.00347327195903601, 0.0006267600776265658], + [0.0009122105474724166, 0.00820798786175933, 0.0019989898586376753, 0.0056241547555801975], + [0.0034732719590360103, 0.0019989898586376753, 0.014580815839895095, 0.0007538373004725811], + [0.0006267600776265658, 0.005624154755580198, 0.0007538373004725811, 0.016509459064779] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0], + "group": { + "4": {"ci_hi": 2.3839857957507595, "ci_lo": 2.1047211751402655, "effect": 2.2443534854455125, "n_obs": 184.0, "p_value": 4.7801766523344554e-60, "se": 0.07051781173538364, "t_stat": 31.82676022147984}, + "6": {"ci_hi": 2.3187371940696404, "ci_lo": 1.9570665989006186, "effect": 2.1379018964851295, "n_obs": 76.0, "p_value": 2.315113512572155e-46, "se": 0.09132635163236602, "t_stat": 23.409474464623834} + }, + "overall": {"att": 2.2086986136463964, "ci_hi": 2.335225426756549, "ci_lo": 2.0821718005362437, "p_value": 6.710965133892381e-64, "se": 0.06389920761802759, "t_stat": 34.56535215349473}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "survey_tsl_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.190082439558195, "ci_lo": 1.8823122525524882, "effect": 2.0361973460553418, "n_obs": 84.0, "p_value": 3.037332996371183e-51, "se": 0.07771582400086065, "t_stat": 26.200550174090573}, + "1": {"ci_hi": 2.307570690322095, "ci_lo": 1.948784726033002, "effect": 2.1281777081775486, "n_obs": 84.0, "p_value": 1.6511071384703697e-46, "se": 0.09059794623367204, "t_stat": 23.490352669678735}, + "2": {"ci_hi": 2.729101569325406, "ci_lo": 2.2509032948246905, "effect": 2.4900024320750482, "n_obs": 46.0, "p_value": 4.286564449946556e-41, "se": 0.120751049021924, "t_stat": 20.62095900817354}, + "3": {"ci_hi": 2.6896939894444376, "ci_lo": 2.180851329475347, "effect": 2.4352726594598924, "n_obs": 46.0, "p_value": 9.578962482444616e-38, "se": 0.12848913987095953, "t_stat": 18.953140023395086} + }, + "event_study_df": 119.0, + "event_study_vcov": [ + [0.006039749300132747, 0.0009122105474724166, 0.00347327195903601, 0.0006267600776265658], + [0.0009122105474724166, 0.00820798786175933, 0.0019989898586376753, 0.0056241547555801975], + [0.0034732719590360103, 0.0019989898586376753, 0.014580815839895095, 0.0007538373004725811], + [0.0006267600776265658, 0.005624154755580198, 0.0007538373004725811, 0.016509459064779] + ], + "event_study_vcov_index": [0.0, 1.0, 2.0, 3.0], + "group": { + "4": {"ci_hi": 2.3839857957507595, "ci_lo": 2.1047211751402655, "effect": 2.2443534854455125, "n_obs": 184.0, "p_value": 4.7801766523344554e-60, "se": 0.07051781173538364, "t_stat": 31.82676022147984}, + "6": {"ci_hi": 2.3187371940696404, "ci_lo": 1.9570665989006186, "effect": 2.1379018964851295, "n_obs": 76.0, "p_value": 2.315113512572155e-46, "se": 0.09132635163236602, "t_stat": 23.409474464623834} + }, + "overall": {"att": 2.2086986136463964, "ci_hi": 2.335225426756549, "ci_lo": 2.0821718005362437, "p_value": 6.710965133892381e-64, "se": 0.06389920761802759, "t_stat": 34.56535215349473}, + "ref_labels": ["-1"], + "reference_event_times": null + } + } + }, + "boot": { + "numpy": { + "CallawaySantAnna": { + "cband_all": { + "cband_crit_value": 2.3503719520172743, + "es": { + "-1": {"cband_hi": 0.19516957587855796, "cband_lo": -0.06918691038138083, "ci_hi": 0.1441877963462948, "ci_lo": -0.04499070699452153, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.28, "se": 0.05623715983188262, "t_stat": 1.1201016007368991}, + "-2": {"cband_hi": 0.14471235336700228, "cband_lo": -0.13470662540196537, "ci_hi": 0.11684252371326793, "ci_lo": -0.08577173934805524, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 0.8, "se": 0.059441438307062056, "t_stat": 0.08416458492600216}, + "-3": {"cband_hi": 0.16676857819771282, "cband_lo": -0.16642315009713507, "ci_hi": 0.11297371250678216, "ci_lo": -0.12884471501435063, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 1.0, "se": 0.07088063827703452, "t_stat": 0.0024366886993007564}, + "-4": {"cband_hi": 0.25510748253343274, "cband_lo": -0.2999410363083401, "ci_hi": 0.19786249324652264, "ci_lo": -0.21858723977972763, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.72, "se": 0.11807674065489644, "t_stat": -0.18984921808581534}, + "0": {"cband_hi": 1.5699282098576621, "cband_lo": 1.337909484150053, "ci_hi": 1.5427104461156507, "ci_lo": 1.358846153875199, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04935787408211546, "t_stat": 29.45667482730332}, + "1": {"cband_hi": 1.932886183298838, "cband_lo": 1.6567777789083131, "ci_hi": 1.9009139876482957, "ci_lo": 1.6975457252715178, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.0587371722491725, "t_stat": 30.557003552871265}, + "2": {"cband_hi": 2.263798782225143, "cband_lo": 1.9589640647426647, "ci_hi": 2.2013194843407495, "ci_lo": 1.9706079514860055, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.0648481865223172, "t_stat": 32.55883528458088}, + "3": {"cband_hi": 2.566577676798864, "cband_lo": 2.150917062149906, "ci_hi": 2.4832311040359816, "ci_lo": 2.193152316015116, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08842443305456527, "t_stat": 26.675289713408066}, + "4": {"cband_hi": 2.929965671350734, "cband_lo": 2.40694840053334, "ci_hi": 2.913130037505706, "ci_lo": 2.491481851299588, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11126266001610917, "t_stat": 23.98340139949633} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.013942116683683675, -0.0005956656548591118, -0.001138670143573453, 0.00016573192314804966, -0.00036394232929632265, 0.001269797110119199, 0.0012012584106901372, 6.16355732146296e-05, -0.0002465972045182631], + [-0.0005956656548591118, 0.005024064882559809, -0.0003493077020280771, 0.00032985948607312923, 0.0005874188794070946, -0.0001857389502446375, -0.0002403711303387929, -0.000765648781189367, 0.0007099296659948749], + [-0.001138670143573453, -0.0003493077020280771, 0.003533284588012265, -0.0005107115845477108, -0.00016970527996405257, -0.0011997563139342984, -0.0011668723069694197, -0.0009233392984879418, -0.00044113983051368414], + [0.00016573192314804966, 0.00032985948607312923, -0.0005107115845477108, 0.0031626181459567114, -0.0013462637052243924, -0.000974192687644395, 0.0003130365717833466, -0.0005445313498226878, -0.001536297124972345], + [-0.00036394232929632265, 0.0005874188794070946, -0.00016970527996405257, -0.0013462637052243924, 0.002436199733905964, 0.001597621457378677, 0.001548773254731629, 0.0019157607759756133, 0.0020191941883636647], + [0.001269797110119199, -0.0001857389502446375, -0.0011997563139342984, -0.000974192687644395, 0.001597621457378677, 0.0034500554038289595, 0.0024401552989299293, 0.003301352865965306, 0.0032879518727625284], + [0.0012012584106901372, -0.0002403711303387929, -0.0011668723069694197, 0.0003130365717833466, 0.001548773254731629, 0.0024401552989299293, 0.0042052872952332435, 0.003388475246944632, 0.0025449206742504116], + [6.16355732146296e-05, -0.000765648781189367, -0.0009233392984879418, -0.0005445313498226878, 0.0019157607759756133, 0.003301352865965306, 0.003388475246944632, 0.007818880361021295, 0.004611790191592932], + [-0.0002465972045182631, 0.0007099296659948749, -0.00044113983051368414, -0.001536297124972345, 0.0020191941883636647, 0.0032879518727625284, 0.0025449206742504116, 0.004611790191592932, 0.012379379513860297] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.292975262873179, "ci_lo": 1.9642542132504837, "df_used": null, "effect": 2.11762432932182, "p_value": 0.0196078431372549, "se": 0.08903611464777396, "t_stat": 23.783880706150782}, + "4": {"ci_hi": 2.033444610661108, "ci_lo": 1.6962644258188386, "df_used": null, "effect": 1.886783657115476, "p_value": 0.0196078431372549, "se": 0.09410568127127582, "t_stat": 20.049625395904606}, + "5": {"ci_hi": 2.062784103433676, "ci_lo": 1.724497263090421, "df_used": null, "effect": 1.9068267927553257, "p_value": 0.0196078431372549, "se": 0.09649166150111434, "t_stat": 19.761570721147802}, + "6": {"ci_hi": 1.6131610200020865, "ci_lo": 1.1990178634324093, "df_used": null, "effect": 1.4296813522176333, "p_value": 0.0196078431372549, "se": 0.11413158539632841, "t_stat": 12.526605560178487} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 1.9983178748675663, "ci_lo": 1.801026843282318, "p_value": 0.0196078431372549, "se": 0.05791140759360639, "t_stat": 32.950699387052914}, + "ref_labels": [], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": 2.3503719520172743, + "es": { + "-1": {"cband_hi": 0.19516957587855796, "cband_lo": -0.06918691038138083, "ci_hi": 0.1441877963462948, "ci_lo": -0.04499070699452153, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.28, "se": 0.05623715983188262, "t_stat": 1.1201016007368991}, + "-2": {"cband_hi": 0.14471235336700228, "cband_lo": -0.13470662540196537, "ci_hi": 0.11684252371326793, "ci_lo": -0.08577173934805524, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 0.8, "se": 0.059441438307062056, "t_stat": 0.08416458492600216}, + "-3": {"cband_hi": 0.16676857819771282, "cband_lo": -0.16642315009713507, "ci_hi": 0.11297371250678216, "ci_lo": -0.12884471501435063, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 1.0, "se": 0.07088063827703452, "t_stat": 0.0024366886993007564}, + "-4": {"cband_hi": 0.25510748253343274, "cband_lo": -0.2999410363083401, "ci_hi": 0.19786249324652264, "ci_lo": -0.21858723977972763, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.72, "se": 0.11807674065489644, "t_stat": -0.18984921808581534}, + "0": {"cband_hi": 1.5699282098576621, "cband_lo": 1.337909484150053, "ci_hi": 1.5427104461156507, "ci_lo": 1.358846153875199, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04935787408211546, "t_stat": 29.45667482730332}, + "1": {"cband_hi": 1.932886183298838, "cband_lo": 1.6567777789083131, "ci_hi": 1.9009139876482957, "ci_lo": 1.6975457252715178, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.0587371722491725, "t_stat": 30.557003552871265}, + "2": {"cband_hi": 2.263798782225143, "cband_lo": 1.9589640647426647, "ci_hi": 2.2013194843407495, "ci_lo": 1.9706079514860055, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.0648481865223172, "t_stat": 32.55883528458088}, + "3": {"cband_hi": 2.566577676798864, "cband_lo": 2.150917062149906, "ci_hi": 2.4832311040359816, "ci_lo": 2.193152316015116, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08842443305456527, "t_stat": 26.675289713408066}, + "4": {"cband_hi": 2.929965671350734, "cband_lo": 2.40694840053334, "ci_hi": 2.913130037505706, "ci_lo": 2.491481851299588, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11126266001610917, "t_stat": 23.98340139949633} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.013942116683683675, -0.0005956656548591118, -0.001138670143573453, 0.00016573192314804966, -0.00036394232929632265, 0.001269797110119199, 0.0012012584106901372, 6.16355732146296e-05, -0.0002465972045182631], + [-0.0005956656548591118, 0.005024064882559809, -0.0003493077020280771, 0.00032985948607312923, 0.0005874188794070946, -0.0001857389502446375, -0.0002403711303387929, -0.000765648781189367, 0.0007099296659948749], + [-0.001138670143573453, -0.0003493077020280771, 0.003533284588012265, -0.0005107115845477108, -0.00016970527996405257, -0.0011997563139342984, -0.0011668723069694197, -0.0009233392984879418, -0.00044113983051368414], + [0.00016573192314804966, 0.00032985948607312923, -0.0005107115845477108, 0.0031626181459567114, -0.0013462637052243924, -0.000974192687644395, 0.0003130365717833466, -0.0005445313498226878, -0.001536297124972345], + [-0.00036394232929632265, 0.0005874188794070946, -0.00016970527996405257, -0.0013462637052243924, 0.002436199733905964, 0.001597621457378677, 0.001548773254731629, 0.0019157607759756133, 0.0020191941883636647], + [0.001269797110119199, -0.0001857389502446375, -0.0011997563139342984, -0.000974192687644395, 0.001597621457378677, 0.0034500554038289595, 0.0024401552989299293, 0.003301352865965306, 0.0032879518727625284], + [0.0012012584106901372, -0.0002403711303387929, -0.0011668723069694197, 0.0003130365717833466, 0.001548773254731629, 0.0024401552989299293, 0.0042052872952332435, 0.003388475246944632, 0.0025449206742504116], + [6.16355732146296e-05, -0.000765648781189367, -0.0009233392984879418, -0.0005445313498226878, 0.0019157607759756133, 0.003301352865965306, 0.003388475246944632, 0.007818880361021295, 0.004611790191592932], + [-0.0002465972045182631, 0.0007099296659948749, -0.00044113983051368414, -0.001536297124972345, 0.0020191941883636647, 0.0032879518727625284, 0.0025449206742504116, 0.004611790191592932, 0.012379379513860297] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.292975262873179, "ci_lo": 1.9642542132504837, "df_used": null, "effect": 2.11762432932182, "p_value": 0.0196078431372549, "se": 0.08903611464777396, "t_stat": 23.783880706150782}, + "4": {"ci_hi": 2.033444610661108, "ci_lo": 1.6962644258188386, "df_used": null, "effect": 1.886783657115476, "p_value": 0.0196078431372549, "se": 0.09410568127127582, "t_stat": 20.049625395904606}, + "5": {"ci_hi": 2.062784103433676, "ci_lo": 1.724497263090421, "df_used": null, "effect": 1.9068267927553257, "p_value": 0.0196078431372549, "se": 0.09649166150111434, "t_stat": 19.761570721147802}, + "6": {"ci_hi": 1.6131610200020865, "ci_lo": 1.1990178634324093, "df_used": null, "effect": 1.4296813522176333, "p_value": 0.0196078431372549, "se": 0.11413158539632841, "t_stat": 12.526605560178487} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 1.9983178748675663, "ci_lo": 1.801026843282318, "p_value": 0.0196078431372549, "se": 0.05791140759360639, "t_stat": 32.950699387052914}, + "ref_labels": [], + "reference_event_times": null + }, + "default_plain_overall": {"att": 1.9082213826980177, "ci_hi": 1.9983178748675663, "ci_lo": 1.801026843282318, "p_value": 0.0196078431372549, "se": 0.05791140759360639, "t_stat": 32.950699387052914}, + "fpc_survey_all": { + "cband_crit_value": 2.088430467966986, + "es": { + "-1": {"cband_hi": 0.1680898380363563, "cband_lo": -0.04244970971575161, "ci_hi": 0.1382512355352203, "ci_lo": -0.004304935650038981, "effect": 0.06282006416030234, "n_groups": 4.0, "p_value": 0.16, "se": 0.05040616649235653, "t_stat": 1.2462773611206521}, + "-2": {"cband_hi": 0.12644579297121583, "cband_lo": -0.05599467709909608, "ci_hi": 0.1126929742459523, "ci_lo": -0.04178813080220816, "effect": 0.03522555793605988, "n_groups": 3.0, "p_value": 0.4, "se": 0.04367884707406882, "t_stat": 0.8064672100050124}, + "-3": {"cband_hi": 0.084382476292053, "cband_lo": -0.13817638231902168, "ci_hi": 0.06812095240947186, "ci_lo": -0.12717048917620993, "effect": -0.026896953013484336, "n_groups": 2.0, "p_value": 0.52, "se": 0.053283760705648, "t_stat": -0.5047870618980784}, + "-4": {"cband_hi": 0.13054432431757218, "cband_lo": -0.24507762336466782, "ci_hi": 0.11648391002864777, "ci_lo": -0.1969679972901584, "effect": -0.05726664952354782, "n_groups": 1.0, "p_value": 0.68, "se": 0.08992924434010362, "t_stat": -0.6367967388558381}, + "0": {"cband_hi": 1.5499074637592198, "cband_lo": 1.3424231027205973, "ci_hi": 1.5325110400407835, "ci_lo": 1.3551332565277572, "effect": 1.4461652832399086, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04967471127746023, "t_stat": 29.112706366068043}, + "1": {"cband_hi": 1.9295689961703417, "cband_lo": 1.6231504786611988, "ci_hi": 1.8966684056520522, "ci_lo": 1.6684750050075896, "effect": 1.7763597374157702, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.07336095747717915, "t_stat": 24.21396609999745}, + "2": {"cband_hi": 2.1775682200067967, "cband_lo": 2.039085855270554, "ci_hi": 2.166416215314656, "ci_lo": 2.054673396729145, "effect": 2.1083270376386753, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.033154650552252, "t_stat": 63.590687958418826}, + "3": {"cband_hi": 2.4929968582044553, "cband_lo": 2.2052294441099494, "ci_hi": 2.463269709510683, "ci_lo": 2.245927265731861, "effect": 2.3491131511572023, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06889561766799857, "t_stat": 34.09669919032231}, + "4": {"cband_hi": 2.73816627886742, "cband_lo": 2.5084910281686184, "ci_hi": 2.7151721269528073, "ci_lo": 2.5178407106549985, "effect": 2.623328653518019, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.05498752633176786, "t_stat": 47.707704429003336} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.008087268987582056, 0.0012504979959297412, -0.00033679532126120777, 0.0008540821145543527, -9.378998303289014e-05, -0.003619896340762503, 0.0009207063876911761, 0.0010087147073243098, -0.0031027198990377673], + [0.0012504979959297412, 0.002839159154936758, -0.0011485864644969002, -0.0017002456430631136, 0.002188585485463668, 0.002261812177285801, 0.0007587768349911242, 0.0021438451478425686, -0.0012997855131507599], + [-0.00033679532126120777, -0.0011485864644969002, 0.0019078416817198904, 0.0003298159850136896, -0.0007713292978956817, -0.0010590592495394783, -0.0009450294356736527, 0.0005351797980297642, 0.0009377973975823789], + [0.0008540821145543527, -0.0017002456430631136, 0.0003298159850136896, 0.0025407816204551657, -0.0018414222593918696, -0.0022346077571933345, -6.240452939681602e-05, -0.0011206436004738737, 0.0004497046105360568], + [-9.378998303289014e-05, 0.002188585485463668, -0.0007713292978956817, -0.0018414222593918696, 0.0024675769404990337, 0.002272965989810194, -6.847429920527389e-05, 0.0016709767884699556, -0.0015218876971304534], + [-0.003619896340762503, 0.002261812177285801, -0.0010590592495394783, -0.0022346077571933345, 0.002272965989810194, 0.0053818300819684865, 0.000647706749481896, 0.0027141582170014415, 0.0008691634366920787], + [0.0009207063876911761, 0.0007587768349911242, -0.0009450294356736527, -6.240452939681602e-05, -6.847429920527389e-05, 0.000647706749481896, 0.0010992308532419444, 0.0003888799534489396, 8.950338846201122e-05], + [0.0010087147073243098, 0.0021438451478425686, 0.0005351797980297642, -0.0011206436004738737, 0.0016709767884699556, 0.0027141582170014415, 0.0003888799534489396, 0.004746606133855038, -0.00013185407154647736], + [-0.0031027198990377673, -0.0012997855131507599, 0.0009377973975823789, 0.0004497046105360568, -0.0015218876971304534, 0.0008691634366920787, 8.950338846201122e-05, -0.00013185407154647736, 0.0030236280520868626] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.2138910900883313, "ci_lo": 1.9724022721909245, "df_used": null, "effect": 2.0958253131634734, "p_value": 0.0196078431372549, "se": 0.06024332172521127, "t_stat": 34.789338521590686}, + "4": {"ci_hi": 2.109710173914765, "ci_lo": 1.6921902463094525, "df_used": null, "effect": 1.8950605918145964, "p_value": 0.0196078431372549, "se": 0.11104974005314198, "t_stat": 17.064970984243008}, + "5": {"ci_hi": 1.9798713065489306, "ci_lo": 1.7663446700880652, "df_used": null, "effect": 1.8773932029725373, "p_value": 0.0196078431372549, "se": 0.0604292234772768, "t_stat": 31.067637393660924}, + "6": {"ci_hi": 1.617281075270017, "ci_lo": 1.1833931347179263, "df_used": null, "effect": 1.4055713190956884, "p_value": 0.0196078431372549, "se": 0.12198111640796012, "t_stat": 11.522859935097012} + }, + "overall": {"att": 1.8839037596672263, "ci_hi": 1.946808722478177, "ci_lo": 1.821311697690987, "p_value": 0.0196078431372549, "se": 0.03936130944534419, "t_stat": 47.861816240721176}, + "ref_labels": [], + "reference_event_times": null + }, + "psu_cluster_all": { + "cband_crit_value": 2.184238347643952, + "es": { + "-1": {"cband_hi": 0.16924033717529835, "cband_lo": -0.04325767167812124, "ci_hi": 0.1430755649635514, "ci_lo": -0.004080448907026893, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.2, "se": 0.04864350291318538, "t_stat": 1.2949588120948017}, + "-2": {"cband_hi": 0.07867996117518301, "cband_lo": -0.06867423321014614, "ci_hi": 0.06232135050962796, "ci_lo": -0.05301588287510689, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 1.0, "se": 0.033731253401048024, "t_stat": 0.14831538938197314}, + "-3": {"cband_hi": 0.1382755485278721, "cband_lo": -0.13793012042729436, "ci_hi": 0.11671752917977099, "ci_lo": -0.12035677448406452, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 1.0, "se": 0.06322699838437919, "t_stat": 0.0027316503187275335}, + "-4": {"cband_hi": 0.18150832578975928, "cband_lo": -0.22634187956466667, "ci_hi": 0.15338066620495105, "ci_lo": -0.15830189042523726, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.88, "se": 0.09336211082328931, "t_stat": -0.24010572050885762}, + "0": {"cband_hi": 1.5396199870355747, "cband_lo": 1.3682177069721404, "ci_hi": 1.5219792287715384, "ci_lo": 1.379747292227955, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.03923616675083074, "t_stat": 37.055578243332704}, + "1": {"cband_hi": 1.9229669213968006, "cband_lo": 1.6666970408103505, "ci_hi": 1.8851667154854796, "ci_lo": 1.7053286756526582, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.05866344230767625, "t_stat": 30.595408494613988}, + "2": {"cband_hi": 2.1912487113615735, "cband_lo": 2.031514135606234, "ci_hi": 2.181268953455749, "ci_lo": 2.0584923931500803, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.036565280507880125, "t_stat": 57.7427930035675}, + "3": {"cband_hi": 2.5173383147053787, "cband_lo": 2.2001564242433913, "ci_hi": 2.4793686547888076, "ci_lo": 2.247837108064675, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.0726069778062725, "t_stat": 32.486510811232435}, + "4": {"cband_hi": 2.7605766781134897, "cband_lo": 2.5763373937705842, "ci_hi": 2.7454892957996955, "ci_lo": 2.5914247760843785, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.04217472066215604, "t_stat": 63.27148097358901} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.008716483737380154, 0.0031368017555590353, -0.0007823413227427847, -0.00023605926554679427, -0.00032309081590136275, -0.001862680353307813, 0.0018895848756440704, 0.0017763102898080416, 0.0009039814905905331], + [0.0031368017555590353, 0.003997653324698289, -0.0006871169622722578, -0.00210486850624056, 0.0018007024961598083, 0.0013319777692532098, 0.0011398162164730165, 0.0026535796916937124, 0.00016929699112616897], + [-0.0007823413227427847, -0.0006871169622722578, 0.0011377974560057137, -0.00012668458994523498, -0.0002519225106441739, -0.00020728224428964083, -0.00029917936528020187, 0.0007137784249973768, 0.0006227283010727826], + [-0.00023605926554679427, -0.00210486850624056, -0.00012668458994523498, 0.002366190375665075, -0.00118349564869507, -0.0012038059781428819, -0.00038597925611077516, -0.001525797373927881, 0.00012599006177169128], + [-0.00032309081590136275, 0.0018007024961598083, -0.0002519225106441739, -0.00118349564869507, 0.001539476781298995, 0.0012462435729113925, -3.528562811855558e-05, 0.001724610439409343, -0.0005502158100394466], + [-0.001862680353307813, 0.0013319777692532098, -0.00020728224428964083, -0.0012038059781428819, 0.0012462435729113925, 0.0034413994633860586, 0.001017304189394465, 0.0020452457324781777, 0.00029107073980289027], + [0.0018895848756440704, 0.0011398162164730165, -0.00029917936528020187, -0.00038597925611077516, -3.528562811855558e-05, 0.001017304189394465, 0.0013370197386199582, 0.0009787841764979373, 0.0008105559416001281], + [0.0017763102898080416, 0.0026535796916937124, 0.0007137784249973768, -0.001525797373927881, 0.001724610439409343, 0.0020452457324781777, 0.0009787841764979373, 0.005271773226160545, 0.00042660473113220114], + [0.0009039814905905331, 0.00016929699112616897, 0.0006227283010727826, 0.00012599006177169128, -0.0005502158100394466, 0.00029107073980289027, 0.0008105559416001281, 0.00042660473113220114, 0.0017787070629308912] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.2215490997241383, "ci_lo": 1.9825486444060603, "df_used": null, "effect": 2.11762432932182, "p_value": 0.0196078431372549, "se": 0.06176690040916711, "t_stat": 34.28412815430081}, + "4": {"ci_hi": 2.063272787234798, "ci_lo": 1.7036433202687882, "df_used": null, "effect": 1.886783657115476, "p_value": 0.0196078431372549, "se": 0.10201104531625374, "t_stat": 18.49587612072875}, + "5": {"ci_hi": 2.031294855380226, "ci_lo": 1.7782393777825032, "df_used": null, "effect": 1.9068267927553257, "p_value": 0.0196078431372549, "se": 0.07305278237293848, "t_stat": 26.10204198685917}, + "6": {"ci_hi": 1.6236682371990814, "ci_lo": 1.2055309289654856, "df_used": null, "effect": 1.4296813522176333, "p_value": 0.0196078431372549, "se": 0.11716959048196342, "t_stat": 12.201812316120643} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 1.9692574622530463, "ci_lo": 1.8495002921441916, "p_value": 0.0196078431372549, "se": 0.03590486646829275, "t_stat": 53.146594609484204}, + "ref_labels": [], + "reference_event_times": null + }, + "rcs_all": { + "cband_crit_value": 2.5032485415667294, + "es": { + "-1": {"cband_hi": 0.2419169907745826, "cband_lo": -0.15646344000598159, "ci_hi": 0.23193889058623263, "ci_lo": -0.08402686276846599, "effect": 0.0427267753843005, "n_groups": 3.0, "p_value": 0.64, "se": 0.07957268808218829, "t_stat": 0.5369527712846558}, + "-2": {"cband_hi": 0.19779678418162824, "cband_lo": -0.3314738333839149, "ci_hi": 0.11386727825091214, "ci_lo": -0.22434894519365428, "effect": -0.06683852460114334, "n_groups": 2.0, "p_value": 0.64, "se": 0.105716753406013, "t_stat": -0.6322415553610983}, + "-3": {"cband_hi": 0.491145550468881, "cband_lo": -0.6387807206669726, "ci_hi": 0.36915052164562834, "ci_lo": -0.3670896236757313, "effect": -0.07381758509904579, "n_groups": 1.0, "p_value": 0.92, "se": 0.22569198630765144, "t_stat": -0.327072247033271}, + "0": {"cband_hi": 1.657639070468162, "cband_lo": 1.133037942183543, "ci_hi": 1.548736127069461, "ci_lo": 1.1986332058122362, "effect": 1.3953385063258525, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.10478406749740532, "t_stat": 13.316323174421571}, + "1": {"cband_hi": 1.766573477304478, "cband_lo": 1.1795852482808868, "ci_hi": 1.6664518857903967, "ci_lo": 1.2426487189195357, "effect": 1.4730793627926824, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.11724529531857991, "t_stat": 12.56408079138714}, + "2": {"cband_hi": 1.7384342878692394, "cband_lo": 1.0502726599064736, "ci_hi": 1.6495729142412845, "ci_lo": 1.158638363374429, "effect": 1.3943534738878565, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.13745371594864894, "t_stat": 10.144167178490614}, + "3": {"cband_hi": 1.8604617335165599, "cband_lo": 0.7828707770711971, "ci_hi": 1.6620909587710624, "ci_lo": 0.8615093309488864, "effect": 1.3216662552938785, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.2152385067946393, "t_stat": 6.140473073226113} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.050936872683493124, 0.006271729161590834, -0.00029642641591662484, -0.012391849701166996, -0.011110341814922759, -0.01802232526930492, -0.032892167402762934], + [0.006271729161590834, 0.011176031950707762, 0.0010839480175273068, -0.004267302185366262, -0.0042471889830368315, -0.0047063656234199855, -0.002716853018081402], + [-0.00029642641591662484, 0.0010839480175273068, 0.006331812688625228, -0.001098338641256804, -0.002038660128725046, -0.002813048940940501, -0.0007371819034420659], + [-0.012391849701166996, -0.004267302185366262, -0.001098338641256804, 0.010979700801300793, 0.008092148643336971, 0.010332059884831603, 0.012491802075782881], + [-0.011110341814922759, -0.0042471889830368315, -0.002038660128725046, 0.008092148643336971, 0.013746459274341018, 0.011313358963193566, 0.015590395459648031], + [-0.01802232526930492, -0.0047063656234199855, -0.002813048940940501, 0.010332059884831603, 0.011313358963193566, 0.018893524028091867, 0.02107896462553038], + [-0.032892167402762934, -0.002716853018081402, -0.0007371819034420659, 0.012491802075782881, 0.015590395459648031, 0.02107896462553038, 0.04632761480718597] + ], + "event_study_vcov_index": [-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], + "group": { + "3": {"ci_hi": 1.6707426375710521, "ci_lo": 0.901309278339554, "df_used": null, "effect": 1.3058492644714335, "p_value": 0.0196078431372549, "se": 0.2056024701717729, "t_stat": 6.35133062059248}, + "4": {"ci_hi": 1.7003828761577662, "ci_lo": 1.211679454332908, "df_used": null, "effect": 1.4404583188885236, "p_value": 0.0196078431372549, "se": 0.1330452432884848, "t_stat": 10.826830657636872}, + "5": {"ci_hi": 1.9674823459264268, "ci_lo": 1.2919913297471537, "df_used": null, "effect": 1.5854273979369462, "p_value": 0.0196078431372549, "se": 0.20086157978043698, "t_stat": 7.89313416567761} + }, + "overall": {"att": 1.4128474233805774, "ci_hi": 1.591319564054138, "ci_lo": 1.2050694782025257, "p_value": 0.0196078431372549, "se": 0.11125728722649807, "t_stat": 12.698920300872487}, + "ref_labels": [], + "reference_event_times": null + }, + "stratified_survey_all": { + "cband_crit_value": 2.4385334214792573, + "es": { + "-1": {"cband_hi": 0.20863066827218424, "cband_lo": -0.06936405467022061, "ci_hi": 0.18121772004419043, "ci_lo": -0.04343260960903114, "effect": 0.06963330680098181, "n_groups": 4.0, "p_value": 0.16, "se": 0.05700039222217597, "t_stat": 1.221628555283713}, + "-2": {"cband_hi": 0.19850133416910964, "cband_lo": -0.1946240760020796, "ci_hi": 0.16118426417141604, "ci_lo": -0.15522783972859977, "effect": 0.001938629083515029, "n_groups": 3.0, "p_value": 0.88, "se": 0.08060693503489332, "t_stat": 0.02405040065939526}, + "-3": {"cband_hi": 0.17689858770985728, "cband_lo": -0.16901862231076187, "ci_hi": 0.13440593977912665, "ci_lo": -0.12929571759247466, "effect": 0.0039399826995477095, "n_groups": 2.0, "p_value": 1.0, "se": 0.07092730552177129, "t_stat": 0.05554958940796536}, + "-4": {"cband_hi": 0.2909088521650203, "cband_lo": -0.26569578531593013, "ci_hi": 0.19435867258874057, "ci_lo": -0.212428598208563, "effect": 0.012606533424545097, "n_groups": 1.0, "p_value": 0.92, "se": 0.11412692411312211, "t_stat": 0.11046064302976881}, + "0": {"cband_hi": 1.533860107960097, "cband_lo": 1.3696470244733665, "ci_hi": 1.4976640759526723, "ci_lo": 1.3908911347703121, "effect": 1.4517535662167318, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.03367045988385837, "t_stat": 43.11653512379565}, + "1": {"cband_hi": 1.8861103310888025, "cband_lo": 1.6657497352750505, "ci_hi": 1.84894275994513, "ci_lo": 1.6848815996561057, "effect": 1.7759300331819265, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.045183017356407104, "t_stat": 39.305255316909324}, + "2": {"cband_hi": 2.2830101412365114, "cband_lo": 1.934336642405787, "ci_hi": 2.2532482613613998, "ci_lo": 1.9689431103907475, "effect": 2.1086733918211493, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.07149245849154959, "t_stat": 29.495046558937325}, + "3": {"cband_hi": 2.558832763930004, "cband_lo": 2.0846790217486126, "ci_hi": 2.494965560845798, "ci_lo": 2.1720816183825393, "effect": 2.3217558928393083, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.09722108747924431, "t_stat": 23.881196487695934}, + "4": {"cband_hi": 2.902900940587747, "cband_lo": 2.4325932522829046, "ci_hi": 2.8763288469837063, "ci_lo": 2.514209781713328, "effect": 2.667747096435326, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.09643248769162777, "t_stat": 27.664401907438698} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.013024954807522332, 0.002369297928261079, -0.0015222581491149403, -0.00043509722725348026, 0.000466631111583956, -0.001965308128820638, 0.0017065348615906647, 6.931227285932411e-05, -0.0038179835016279337], + [0.002369297928261079, 0.005030682668578689, -0.0019876890512482022, -0.0016408360456225315, 0.0011277152001795507, 0.0004735862835932727, 0.00030447170826796485, 0.0013180341684036964, -0.0017646704877972482], + [-0.0015222581491149403, -0.0019876890512482022, 0.006497477975719511, -0.0007376533689779569, -0.0009826008711669055, -0.0005728700411489965, -0.001910074212906339, -0.0021802547089884677, 0.0033412775067301254], + [-0.00043509722725348026, -0.0016408360456225315, -0.0007376533689779569, 0.003249044713481899, -0.00042347399388194685, 4.085591130641185e-05, 0.0021645588948692733, 0.0020216649628543946, -0.0005527979436737165], + [0.000466631111583956, 0.0011277152001795507, -0.0009826008711669055, -0.00042347399388194685, 0.0011336998687905158, 0.00040647671199136047, 0.0003272692170020557, 0.0013488591934123879, 0.00023155567709411704], + [-0.001965308128820638, 0.0004735862835932727, -0.0005728700411489965, 4.085591130641185e-05, 0.00040647671199136047, 0.0020415050574293853, 0.0010414737398037912, 0.0025351769689276097, 0.0015249602164618796], + [0.0017065348615906647, 0.00030447170826796485, -0.001910074212906339, 0.0021645588948692733, 0.0003272692170020557, 0.0010414737398037912, 0.005111171621165941, 0.0042776781765350436, 0.0005787036378496957], + [6.931227285932411e-05, 0.0013180341684036964, -0.0021802547089884677, 0.0020216649628543946, 0.0013488591934123879, 0.0025351769689276097, 0.0042776781765350436, 0.009451939850646874, 0.0028236024819365277], + [-0.0038179835016279337, -0.0017646704877972482, 0.0033412775067301254, -0.0005527979436737165, 0.00023155567709411704, 0.0015249602164618796, 0.0005787036378496957, 0.0028236024819365277, 0.009299224682395938] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.30062747087495, "ci_lo": 1.9457647932554265, "df_used": null, "effect": 2.1242700768040943, "p_value": 0.0196078431372549, "se": 0.09624435241511488, "t_stat": 22.071633540031844}, + "4": {"ci_hi": 2.013229590875692, "ci_lo": 1.5954562636154388, "df_used": null, "effect": 1.8300828461356922, "p_value": 0.0196078431372549, "se": 0.10911131754759941, "t_stat": 16.772621642455423}, + "5": {"ci_hi": 2.0973034921814397, "ci_lo": 1.6390768741483428, "df_used": null, "effect": 1.9033747125067557, "p_value": 0.0196078431372549, "se": 0.11729583583619467, "t_stat": 16.22712945380982}, + "6": {"ci_hi": 1.5956134547819238, "ci_lo": 1.1898156451856987, "df_used": null, "effect": 1.4106784908973298, "p_value": 0.0196078431372549, "se": 0.12670562914080272, "t_stat": 11.13351080345216} + }, + "overall": {"att": 1.899842807641477, "ci_hi": 1.9667359583706414, "ci_lo": 1.8187567819914907, "p_value": 0.0196078431372549, "se": 0.041664313181091844, "t_stat": 45.59880297039593}, + "ref_labels": [], + "reference_event_times": null + }, + "unbalanced_all": { + "cband_crit_value": 2.5177496571907962, + "es": { + "-1": {"cband_hi": 0.22050705455531838, "cband_lo": -0.15116363887654516, "ci_hi": 0.1673838498206694, "ci_lo": -0.08372178324924352, "effect": 0.034671707839386626, "n_groups": 4.0, "p_value": 0.56, "se": 0.07381009711794753, "t_stat": 0.46974207043762195}, + "-2": {"cband_hi": 0.19644317393909194, "cband_lo": -0.22385455517765013, "ci_hi": 0.1532551010186349, "ci_lo": -0.136445594937331, "effect": -0.013705690619279079, "n_groups": 3.0, "p_value": 1.0, "se": 0.08346694198058066, "t_stat": -0.16420501690918343}, + "-3": {"cband_hi": 0.3019755206734568, "cband_lo": -0.16230611603577472, "ci_hi": 0.2262966213034, "ci_lo": -0.12339440243092054, "effect": 0.06983470231884105, "n_groups": 2.0, "p_value": 0.56, "se": 0.09220170785908442, "t_stat": 0.7574122425755089}, + "-4": {"cband_hi": 0.3855985578172657, "cband_lo": -0.33193668692164213, "ci_hi": 0.24903118044479772, "ci_lo": -0.21415407896096336, "effect": 0.026830935447811788, "n_groups": 1.0, "p_value": 0.92, "se": 0.14249535149167786, "t_stat": 0.18829340864062358}, + "0": {"cband_hi": 1.6160986458623754, "cband_lo": 1.3226464203624289, "ci_hi": 1.5533437652616269, "ci_lo": 1.3328242426852523, "effect": 1.4693725331124021, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.05827668860202898, "t_stat": 25.213727278616236}, + "1": {"cband_hi": 2.0026374532587554, "cband_lo": 1.6104289344965246, "ci_hi": 1.9560069804332327, "ci_lo": 1.679920780506253, "effect": 1.8065331938776399, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.07788870462995935, "t_stat": 23.19377633073088}, + "2": {"cband_hi": 2.3237941997910823, "cband_lo": 1.9256097448490155, "ci_hi": 2.2397125729668375, "ci_lo": 1.976612206038197, "effect": 2.124701972320049, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.0790754660227707, "t_stat": 26.86929434861903}, + "3": {"cband_hi": 2.657043231852401, "cband_lo": 2.052901147060518, "ci_hi": 2.499949011235328, "ci_lo": 2.1202298798541785, "effect": 2.3549721894564595, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.11997659955318205, "t_stat": 19.628595894756714}, + "4": {"cband_hi": 3.1445778532965556, "cband_lo": 2.4255649433944115, "ci_hi": 3.0238294581560754, "ci_lo": 2.541619663560865, "effect": 2.7850713983454836, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.14278880107253983, "t_stat": 19.504830753012673} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.020304925196736826, 0.0017540022322210527, -0.004004770687209634, -0.0006210746026122286, 0.00038603970963394614, 0.001676404305205262, 0.002298445370755396, -0.00048789649618640146, -0.0008164032462456036], + [0.0017540022322210527, 0.00850115493213195, -0.00234635465800927, 0.0022002212134620875, -0.0007564885230661014, -0.00115855150813362, -0.001976682213885571, -0.004304514805019216, -0.0012561464592894774], + [-0.004004770687209634, -0.00234635465800927, 0.006966730403589619, -0.0007817429377851379, -0.0001649190632926541, -0.0017289409414762728, -0.0013544981053104552, -0.0006282662761213152, -5.13819377154175e-06], + [-0.0006210746026122286, 0.0022002212134620875, -0.0007817429377851379, 0.005447930436560847, -0.0022089685635942736, -0.002196961266991716, -0.0005263916105925329, -0.0027037973912000584, -0.004541758136959235], + [0.00038603970963394614, -0.0007564885230661014, -0.0001649190632926541, -0.0022089685635942736, 0.003396172434417855, 0.0030071594513221217, 0.0026128786125534935, 0.0038047876205627765, 0.0037793082490690535], + [0.001676404305205262, -0.00115855150813362, -0.0017289409414762728, -0.002196961266991716, 0.0030071594513221217, 0.00606665030893305, 0.004097353492153848, 0.0065785365818059275, 0.004664109517868452], + [0.002298445370755396, -0.001976682213885571, -0.0013544981053104552, -0.0005263916105925329, 0.0026128786125534935, 0.004097353492153848, 0.006252929326718362, 0.006488130123813102, 0.004919491487570636], + [-0.00048789649618640146, -0.004304514805019216, -0.0006282662761213152, -0.0027037973912000584, 0.0038047876205627765, 0.0065785365818059275, 0.006488130123813102, 0.014394384440344597, 0.007789225857104804], + [-0.0008164032462456036, -0.0012561464592894774, -5.13819377154175e-06, -0.004541758136959235, 0.0037793082490690535, 0.004664109517868452, 0.004919491487570636, 0.007789225857104804, 0.02038864171173334] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.3826719529803944, "ci_lo": 2.0447215654749873, "df_used": null, "effect": 2.210550101699499, "p_value": 0.0196078431372549, "se": 0.11509586811905738, "t_stat": 19.206163851276255}, + "4": {"ci_hi": 2.0071972563896847, "ci_lo": 1.5988187554903945, "df_used": null, "effect": 1.805270195115015, "p_value": 0.0196078431372549, "se": 0.10934663208529208, "t_stat": 16.509609493110645}, + "5": {"ci_hi": 2.1089299077475627, "ci_lo": 1.7169735976858742, "df_used": null, "effect": 1.9435648441208038, "p_value": 0.0196078431372549, "se": 0.11023753196499957, "t_stat": 17.630699902986635}, + "6": {"ci_hi": 1.683744070082475, "ci_lo": 1.0862179296985843, "df_used": null, "effect": 1.4341093904293774, "p_value": 0.0196078431372549, "se": 0.15519903127929793, "t_stat": 9.240453233554904} + }, + "overall": {"att": 1.926624614441337, "ci_hi": 2.045370778962362, "ci_lo": 1.7955127117608987, "p_value": 0.0196078431372549, "se": 0.07207914710145592, "t_stat": 26.72929261676046}, + "ref_labels": [], + "reference_event_times": null + } + }, + "ContinuousDiD": { + "default_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.09710063057106542, "ci_lo": -0.5124216763917435, "effect": -0.23880903293700426, "p_value": 0.5333333333333333, "se": 0.21454099569378204, "t_stat": -1.1131160837803717}, + "-2": {"ci_hi": 0.031124731733537636, "ci_lo": -0.996368373344308, "effect": -0.4167762389538105, "p_value": 0.06666666666666667, "se": 0.29660876771575784, "t_stat": -1.4051379605649754}, + "0": {"ci_hi": 4.981600139743175, "ci_lo": 4.100072507765267, "effect": 4.537644146658957, "p_value": 0.03225806451612903, "se": 0.2675254958160739, "t_stat": 16.961539059359886}, + "1": {"ci_hi": 4.823392079878206, "ci_lo": 3.8264605856879523, "effect": 4.466341388503657, "p_value": 0.03225806451612903, "se": 0.307478923835168, "t_stat": 14.525683038028173}, + "2": {"ci_hi": 5.280577805151575, "ci_lo": 4.265318902151936, "effect": 4.879913354176228, "p_value": 0.03225806451612903, "se": 0.27636217673802127, "t_stat": 17.65767447548426}, + "3": {"ci_hi": 4.532441654446876, "ci_lo": 3.5195431546392206, "effect": 4.010249427241352, "p_value": 0.03225806451612903, "se": 0.3320130299147904, "t_stat": 12.078590494688004} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "overall": {"att": 4.641957051061234, "ci_hi": 4.967273156162676, "ci_lo": 4.134387805824363, "p_value": 0.03225806451612903, "se": 0.2543921845714363, "t_stat": 18.247247095587237}, + "overall_acrt": 1.5251919678360473, + "overall_acrt_se": 0.2016199172383717, + "ref_labels": [], + "reference_event_times": null + }, + "default_plain": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "overall": {"att": 4.641957051061234, "ci_hi": 4.967273156162676, "ci_lo": 4.134387805824363, "p_value": 0.03225806451612903, "se": 0.2543921845714363, "t_stat": 18.247247095587237}, + "overall_acrt": 1.5251919678360473, + "overall_acrt_se": 0.2016199172383717, + "ref_labels": null, + "reference_event_times": null + } + }, + "EfficientDiD": { + "alpha10_mammen_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.24221705483423295, "ci_lo": -0.04129297809532719, "effect": 0.10640994510667033, "n_groups": 2.0, "p_value": 0.2, "se": 0.09082719459361532, "t_stat": 1.1715648114287391}, + "-2": {"ci_hi": 0.07350223941743948, "ci_lo": -0.1902421854499715, "effect": -0.05979558595018031, "n_groups": 2.0, "p_value": 0.24, "se": 0.08192102171168368, "t_stat": -0.7299174827265635}, + "-3": {"ci_hi": 0.17679335900182375, "ci_lo": 0.024128481365125642, "effect": 0.08653759783531222, "n_groups": 2.0, "p_value": 0.08, "se": 0.06142251800108509, "t_stat": 1.4088904306036987}, + "-4": {"ci_hi": 0.04295046077535563, "ci_lo": -0.23595913327069593, "effect": -0.07878370911167488, "n_groups": 1.0, "p_value": 0.28, "se": 0.09704138659811155, "t_stat": -0.8118567950595219}, + "-5": {"ci_hi": 0.1305463006831535, "ci_lo": -0.2133265906073057, "effect": -0.028022683129665008, "n_groups": 1.0, "p_value": 0.64, "se": 0.11255225363536349, "t_stat": -0.24897487366579382}, + "0": {"ci_hi": 2.2644433037147746, "ci_lo": 2.0348533336006214, "effect": 2.139053682427927, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07536489596461503, "t_stat": 28.38262635474539}, + "1": {"ci_hi": 2.324792506944098, "ci_lo": 2.0765065157978606, "effect": 2.209130774601568, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.0766862633712802, "t_stat": 28.807385801365175}, + "2": {"ci_hi": 2.778095086042005, "ci_lo": 2.4449931602862764, "effect": 2.5747639239316404, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10722388143564181, "t_stat": 24.012970706316686}, + "3": {"ci_hi": 2.920274100055838, "ci_lo": 2.5944714579901587, "effect": 2.751359710941057, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11644797948889085, "t_stat": 23.627371836052657} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.5815049985405913, "ci_lo": 2.323931970325787, "df_used": null, "effect": 2.4517840485645532, "p_value": 0.0196078431372549, "se": 0.0770935109059595, "t_stat": 31.8027291759393}, + "6": {"ci_hi": 2.2480367206776597, "ci_lo": 2.008422938749132, "df_used": null, "effect": 2.110767202972924, "p_value": 0.0196078431372549, "se": 0.07667452497406933, "t_stat": 27.52892442029171} + }, + "overall": {"att": 2.3344742536810323, "ci_hi": 2.4390179672865395, "ci_lo": 2.247063651855358, "p_value": 0.0196078431372549, "se": 0.06213463477137176, "t_stat": 37.571223557857465}, + "ref_labels": [], + "reference_event_times": null + }, + "anticipation1_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.26046246717193844, "ci_lo": -0.035037422749307844, "effect": 0.10636878782275372, "n_groups": 2.0, "p_value": 0.2, "se": 0.08469459663515397, "t_stat": 1.2559099641381786}, + "-2": {"ci_hi": 0.11504467609532483, "ci_lo": -0.16690330777288512, "effect": -0.05582061177659424, "n_groups": 2.0, "p_value": 0.76, "se": 0.09792101497351809, "t_stat": -0.5700575284242146}, + "-3": {"ci_hi": 0.2460847195096275, "ci_lo": 0.005482769995357208, "effect": 0.11313385945021384, "n_groups": 2.0, "p_value": 0.04, "se": 0.0657284233179768, "t_stat": 1.721231907585886}, + "-4": {"ci_hi": 0.06780309592192169, "ci_lo": -0.2915040695369847, "effect": -0.06989062805983277, "n_groups": 1.0, "p_value": 0.4, "se": 0.11615091360581879, "t_stat": -0.6017225856442293}, + "-5": {"ci_hi": 0.1376560616667031, "ci_lo": -0.2099639479249415, "effect": -0.01002045534608281, "n_groups": 1.0, "p_value": 0.84, "se": 0.10226942795050302, "t_stat": -0.09798094647534912}, + "0": {"ci_hi": 2.30323724583583, "ci_lo": 2.058568933064277, "effect": 2.1545153502792065, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07167567462643351, "t_stat": 30.05922666941506}, + "1": {"ci_hi": 2.4271422731656633, "ci_lo": 2.0464201237314903, "effect": 2.2643905584831225, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.1064573161554432, "t_stat": 21.270408087094573}, + "2": {"ci_hi": 2.854269856478038, "ci_lo": 2.4477637737234748, "effect": 2.6089274980620862, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11448261115449816, "t_stat": 22.78885388577703}, + "3": {"ci_hi": 3.0079620006980665, "ci_lo": 2.566294678619044, "effect": 2.780266413016901, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.12637370824079672, "t_stat": 22.000354755114788} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.177965766913026, "ci_lo": 1.8735255869194996, "df_used": null, "effect": 2.0005610412926904, "p_value": 0.0196078431372549, "se": 0.07869695411466296, "t_stat": 25.421073328680993}, + "6": {"ci_hi": 1.627408627798855, "ci_lo": 1.3126134663223246, "df_used": null, "effect": 1.4803489255998747, "p_value": 0.0196078431372549, "se": 0.08667732540955839, "t_stat": 17.07884869087837} + }, + "overall": {"att": 1.799640793016124, "ci_hi": 1.90869692058219, "ci_lo": 1.6873909387690198, "p_value": 0.0196078431372549, "se": 0.06975829384512622, "t_stat": 25.798234071085428}, + "ref_labels": [], + "reference_event_times": null + }, + "clustered_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.06140673774260856, "ci_lo": -0.31404606857946105, "effect": -0.134313461987274, "n_groups": 2.0, "p_value": 0.16, "se": 0.09928418226921563, "t_stat": -1.352818333368292}, + "-2": {"ci_hi": 0.22981016540940377, "ci_lo": -0.01702825129935269, "effect": 0.11002324972857475, "n_groups": 2.0, "p_value": 0.12, "se": 0.07021359998656145, "t_stat": 1.5669791856511086}, + "-3": {"ci_hi": 0.2672323393524546, "ci_lo": 0.03181730364093322, "effect": 0.1436002497880893, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06403693269280925, "t_stat": 2.2424598391829953}, + "-4": {"ci_hi": 0.01870147771689904, "ci_lo": -0.2769337196011382, "effect": -0.11660705660378556, "n_groups": 1.0, "p_value": 0.16, "se": 0.07870932472850917, "t_stat": -1.4814897346152625}, + "-5": {"ci_hi": -0.06367160729172608, "ci_lo": -0.45358108626115884, "effect": -0.2390395706166469, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.09994447698608666, "t_stat": -2.391723663228772}, + "0": {"ci_hi": 2.1966367267700275, "ci_lo": 1.8485635956736268, "effect": 2.0272407859305153, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.09115942957587225, "t_stat": 22.238410171744626}, + "1": {"ci_hi": 2.3862123030442994, "ci_lo": 2.1466698825817487, "effect": 2.246187711623196, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06303311366536404, "t_stat": 35.63504293232225}, + "2": {"ci_hi": 2.4384632300589923, "ci_lo": 1.9699706047649748, "effect": 2.15004585834593, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.14006235458773447, "t_stat": 15.350633399491725}, + "3": {"ci_hi": 2.8740652257063366, "ci_lo": 2.414588824667343, "effect": 2.621212494601111, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.12398718288794852, "t_stat": 21.140995654123305} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3514262384213946, "ci_lo": 2.140349256781182, "df_used": null, "effect": 2.217677302443109, "p_value": 0.0196078431372549, "se": 0.05839383988637902, "t_stat": 37.9779323770829}, + "6": {"ci_hi": 2.3647658441654524, "ci_lo": 2.065501633683991, "df_used": null, "effect": 2.2085745786428994, "p_value": 0.0196078431372549, "se": 0.08472807309705341, "t_stat": 26.06662110813077} + }, + "overall": {"att": 2.214245127895489, "ci_hi": 2.3260154344745962, "ci_lo": 2.1347163378870384, "p_value": 0.0196078431372549, "se": 0.05709102466386134, "t_stat": 38.78446990455065}, + "ref_labels": [], + "reference_event_times": null + }, + "covariates_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.19609860855034558, "ci_lo": -0.07165932912426154, "effect": 0.06932893163207215, "n_groups": 2.0, "p_value": 0.28, "se": 0.07156396940758114, "t_stat": 0.9687686723638863}, + "-2": {"ci_hi": 0.07115569655129643, "ci_lo": -0.243749789355843, "effect": -0.10624754187517575, "n_groups": 2.0, "p_value": 0.24, "se": 0.08937053874462333, "t_stat": -1.1888430277765083}, + "-3": {"ci_hi": 0.23750639262808998, "ci_lo": -0.011733046520941523, "effect": 0.09562758875698771, "n_groups": 2.0, "p_value": 0.12, "se": 0.06931665526518704, "t_stat": 1.3795759243019163}, + "-4": {"ci_hi": 0.12708259565666488, "ci_lo": -0.22854021563494797, "effect": -0.04229093032752835, "n_groups": 1.0, "p_value": 0.6, "se": 0.10506348005531412, "t_stat": -0.40252740824178773}, + "-5": {"ci_hi": 0.1328916210763672, "ci_lo": -0.17817597032137267, "effect": -0.014115097686006502, "n_groups": 1.0, "p_value": 0.84, "se": 0.0835274102193946, "t_stat": -0.1689876131551491}, + "0": {"ci_hi": 2.4142927058174184, "ci_lo": 1.959709345317566, "effect": 2.185839855043282, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.12908138565044447, "t_stat": 16.933811517662118}, + "1": {"ci_hi": 2.438568790562177, "ci_lo": 1.9568721147045078, "effect": 2.2153173261739316, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.14113354405002382, "t_stat": 15.696603816515283}, + "2": {"ci_hi": 3.171004839685526, "ci_lo": 1.8960038116454507, "effect": 2.596118485574259, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.33333619207756904, "t_stat": 7.788288662546817}, + "3": {"ci_hi": 3.4171867110421728, "ci_lo": 2.110229055160581, "effect": 2.7894803980786036, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.34389062504969653, "t_stat": 8.111533711264995} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 3.0693647202237115, "ci_lo": 1.8183670058978807, "df_used": null, "effect": 2.4535373408629084, "p_value": 0.0196078431372549, "se": 0.313409697166902, "t_stat": 7.828530396608345}, + "6": {"ci_hi": 2.7424177469934587, "ci_lo": 1.8260541600056384, "df_used": null, "effect": 2.187518994773213, "p_value": 0.0196078431372549, "se": 0.24079152630011888, "t_stat": 9.084700896188194} + }, + "overall": {"att": 2.362027029808053, "ci_hi": 2.681845812043713, "ci_lo": 2.068871804142213, "p_value": 0.0196078431372549, "se": 0.17339479227354118, "t_stat": 13.622248966288485}, + "ref_labels": [], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2605749520355526, "ci_lo": -0.035048967394950475, "effect": 0.10640994510667033, "n_groups": 2.0, "p_value": 0.2, "se": 0.08472980304307906, "t_stat": 1.255873863563314}, + "-2": {"ci_hi": 0.12435262021680903, "ci_lo": -0.1875069894306046, "effect": -0.05979558595018031, "n_groups": 2.0, "p_value": 0.6, "se": 0.08985729450712923, "t_stat": -0.6654505488748735}, + "-3": {"ci_hi": 0.23435645131917457, "ci_lo": -0.02967738704445458, "effect": 0.08653759783531222, "n_groups": 2.0, "p_value": 0.16, "se": 0.07113622041795355, "t_stat": 1.216505421947771}, + "-4": {"ci_hi": 0.057629806978680434, "ci_lo": -0.29349999008931327, "effect": -0.07878370911167488, "n_groups": 1.0, "p_value": 0.4, "se": 0.11240838715227047, "t_stat": -0.7008703808280161}, + "-5": {"ci_hi": 0.14032826360085635, "ci_lo": -0.22157050663066102, "effect": -0.028022683129665008, "n_groups": 1.0, "p_value": 0.76, "se": 0.10205268372250821, "t_stat": -0.2745903596799235}, + "0": {"ci_hi": 2.2730956766537713, "ci_lo": 2.0240927701086133, "effect": 2.139053682427927, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.0710813088208181, "t_stat": 30.09305424890329}, + "1": {"ci_hi": 2.333395710107787, "ci_lo": 2.041687404210498, "effect": 2.209130774601568, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08918558875238361, "t_stat": 24.770041948537628}, + "2": {"ci_hi": 2.804114042639948, "ci_lo": 2.3877126006806453, "effect": 2.5747639239316404, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11166693464763619, "t_stat": 23.05753204434491}, + "3": {"ci_hi": 2.971445529708958, "ci_lo": 2.550115501326433, "effect": 2.751359710941057, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.1201103620587699, "t_stat": 22.906930457796964} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.5871780715675565, "ci_lo": 2.3249228067217893, "df_used": null, "effect": 2.4517840485645532, "p_value": 0.0196078431372549, "se": 0.07790950399807581, "t_stat": 31.46964006631472}, + "6": {"ci_hi": 2.2518896489681466, "ci_lo": 1.9261714280339786, "df_used": null, "effect": 2.110767202972924, "p_value": 0.0196078431372549, "se": 0.08591637300079445, "t_stat": 24.56769448302253} + }, + "overall": {"att": 2.3344742536810323, "ci_hi": 2.4413523584679644, "ci_lo": 2.237210144978274, "p_value": 0.0196078431372549, "se": 0.0666262699770178, "t_stat": 35.03834530263049}, + "ref_labels": [], + "reference_event_times": null + }, + "default_plain_overall": {"att": 2.3344742536810323, "ci_hi": 2.4413523584679644, "ci_lo": 2.237210144978274, "p_value": 0.0196078431372549, "se": 0.0666262699770178, "t_stat": 35.03834530263049}, + "fpc_survey_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2654898985976349, "ci_lo": -0.029546895280431588, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.12, "se": 0.07619477432858715, "t_stat": 1.4823941131187868}, + "-2": {"ci_hi": 0.20550821973543668, "ci_lo": -0.08469002677251986, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.36, "se": 0.07455090278162223, "t_stat": 1.0004407329998481}, + "-3": {"ci_hi": 0.11127020711596734, "ci_lo": -0.12138927171243204, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.8, "se": 0.06283516651493215, "t_stat": -0.18146824778675938}, + "-4": {"ci_hi": 0.1391680776765107, "ci_lo": -0.16028687555616358, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 0.92, "se": 0.09395987110355748, "t_stat": -0.20292312903425136}, + "-5": {"ci_hi": 0.020511715721675726, "ci_lo": -0.4111304161866572, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.08, "se": 0.11999664016599068, "t_stat": -1.6224776907358804}, + "0": {"ci_hi": 2.1436625890052734, "ci_lo": 1.9442280433715498, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.05731270458326901, "t_stat": 35.57015002937894}, + "1": {"ci_hi": 2.279218055191279, "ci_lo": 1.9607415627755733, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08951432301375273, "t_stat": 23.503314992999954}, + "2": {"ci_hi": 2.726121676654111, "ci_lo": 2.3213404164151057, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10559337299624555, "t_stat": 23.699316950060048}, + "3": {"ci_hi": 2.601274114600099, "ci_lo": 2.220020301209764, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11048622106660316, "t_stat": 21.80001698568363} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.39003271662755, "ci_lo": 2.1711750250233814, "df_used": null, "effect": 2.267851180304006, "p_value": 0.0196078431372549, "se": 0.055309240471839265, "t_stat": 41.00311559076071}, + "6": {"ci_hi": 2.148749361253044, "ci_lo": 1.9495825949116592, "df_used": null, "effect": 2.0624126822653266, "p_value": 0.0196078431372549, "se": 0.05485191290632769, "t_stat": 37.59964918247013} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.287984327865623, "ci_lo": 2.097667311536792, "p_value": 0.0196078431372549, "se": 0.048238828046649765, "t_stat": 45.58654809809011}, + "ref_labels": [], + "reference_event_times": null + }, + "fractional_balance_e1_es": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.23119857226080878, "ci_lo": -0.23028584066490967, "effect": 0.026184943804410075, "n_groups": 1.0, "p_value": 0.84, "se": 0.1382116969754096, "t_stat": 0.1894553382776919}, + "0": {"ci_hi": 0.4581907251744048, "ci_lo": 0.07950931065579732, "effect": 0.24480370561350984, "n_groups": 2.0, "p_value": 0.04, "se": 0.1108351331638737, "t_stat": 2.2087193710640363}, + "1": {"ci_hi": 0.9191250916004011, "ci_lo": 0.3604761016581456, "effect": 0.6358462518585993, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.14731426401160125, "t_stat": 4.316257194269561}, + "2": {"ci_hi": 1.4696357627539216, "ci_lo": 0.2636018803591971, "effect": 0.8040757046489879, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.32158781006538734, "t_stat": 2.5003301726066605} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": [], + "reference_event_times": null + }, + "stratified_psu_survey_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2341681818279641, "ci_lo": -0.012605995579570668, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.12, "se": 0.07382329655029328, "t_stat": 1.530014103856262}, + "-2": {"ci_hi": 0.245876271222739, "ci_lo": -0.0681235255341213, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.32, "se": 0.08444394222328314, "t_stat": 0.8832339876723816}, + "-3": {"ci_hi": 0.1285795168194135, "ci_lo": -0.13635567927805417, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.8, "se": 0.06358199239982865, "t_stat": -0.17933674514554412}, + "-4": {"ci_hi": 0.14356637440823122, "ci_lo": -0.15041804226742786, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 0.76, "se": 0.08503290448100671, "t_stat": -0.224226505778685}, + "-5": {"ci_hi": 0.044372534115678805, "ci_lo": -0.3867279606592377, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.12, "se": 0.12606335987951608, "t_stat": -1.5443969748121575}, + "0": {"ci_hi": 2.1537261129444243, "ci_lo": 1.9388589224111425, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.059276136953055, "t_stat": 34.39194261648464}, + "1": {"ci_hi": 2.2583816766098894, "ci_lo": 1.9380914210633695, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08633551187403303, "t_stat": 24.36869006170978}, + "2": {"ci_hi": 2.705681821089329, "ci_lo": 2.3565597898157056, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10069712926475424, "t_stat": 24.851659950348264}, + "3": {"ci_hi": 2.609072967054054, "ci_lo": 2.2184296329305195, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10884064212889258, "t_stat": 22.1296149014226} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3752330135384976, "ci_lo": 2.158980257271318, "df_used": null, "effect": 2.267851180304006, "p_value": 0.0196078431372549, "se": 0.056314729245228005, "t_stat": 40.27101276521149}, + "6": {"ci_hi": 2.1375228715477235, "ci_lo": 1.9471671672818898, "df_used": null, "effect": 2.0624126822653266, "p_value": 0.0196078431372549, "se": 0.05147453168670459, "t_stat": 40.066662380107275} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.2961195691978564, "ci_lo": 2.1074255353198432, "p_value": 0.0196078431372549, "se": 0.04788565840007229, "t_stat": 45.92276118606689}, + "ref_labels": [], + "reference_event_times": null + }, + "survey_weights_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.25431157023796336, "ci_lo": -0.014375068126058226, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.12, "se": 0.07544541841931844, "t_stat": 1.497117880470129}, + "-2": {"ci_hi": 0.19985573302070553, "ci_lo": -0.06532168096359618, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.44, "se": 0.0754114840868331, "t_stat": 0.9890238963970865}, + "-3": {"ci_hi": 0.10911411454532711, "ci_lo": -0.11105482725807203, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.92, "se": 0.0670354829586452, "t_stat": -0.17009779095480454}, + "-4": {"ci_hi": 0.11309086737544662, "ci_lo": -0.2088589463899141, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 0.84, "se": 0.08749542398036779, "t_stat": -0.21791575125422546}, + "-5": {"ci_hi": 0.002106435862849542, "ci_lo": -0.341026255826165, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.08, "se": 0.09832228590036135, "t_stat": -1.9801398009590563}, + "0": {"ci_hi": 2.135715223163936, "ci_lo": 1.9145153281112819, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06391514657548718, "t_stat": 31.895749440371418}, + "1": {"ci_hi": 2.2482942628486886, "ci_lo": 1.9606836228520357, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.09033760899954524, "t_stat": 23.289119044405595}, + "2": {"ci_hi": 2.7319703078942243, "ci_lo": 2.3309800447594795, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11904925455042854, "t_stat": 21.020634055326195}, + "3": {"ci_hi": 2.5960693757767013, "ci_lo": 2.154538806967436, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.12304990563388465, "t_stat": 19.57418401524301} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.4007652054016155, "ci_lo": 2.1392241661065134, "df_used": null, "effect": 2.267851180304006, "p_value": 0.0196078431372549, "se": 0.06805052453928004, "t_stat": 33.32599117579115}, + "6": {"ci_hi": 2.2291417762758665, "ci_lo": 1.9221679802557796, "df_used": null, "effect": 2.0624126822653266, "p_value": 0.0196078431372549, "se": 0.08080094432595937, "t_stat": 25.52461112267872} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.2927633157503373, "ci_lo": 2.1091408555163262, "p_value": 0.0196078431372549, "se": 0.056641695596933314, "t_stat": 38.82372573364767}, + "ref_labels": [], + "reference_event_times": null + } + }, + "ImputationDiD": { + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.3060480256636122, "ci_lo": 2.027751271589125, "effect": 2.145940363587726, "n_obs": 84.0, "p_value": 0.05, "se": 0.08156915520915543, "t_stat": 26.308233278685016}, + "1": {"ci_hi": 2.339046004176374, "ci_lo": 2.1207335731351606, "effect": 2.242441654342141, "n_obs": 84.0, "p_value": 0.05, "se": 0.06987184726545864, "t_stat": 32.093636308520765}, + "2": {"ci_hi": 2.7961219749685484, "ci_lo": 2.4322016495426557, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 0.05, "se": 0.11100400667818662, "t_stat": 23.40698260799346}, + "3": {"ci_hi": 2.9324719817762177, "ci_lo": 2.53139383275232, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 0.05, "se": 0.12354164101325206, "t_stat": 22.39269572456501} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.571627829224781, "ci_lo": 2.3050703901774883, "df_used": null, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 0.05, "se": 0.08100030104413747, "t_stat": 30.38077615455412}, + "6": {"ci_hi": 2.3255341366205866, "ci_lo": 2.0032444014190465, "df_used": null, "effect": 2.1511282275930954, "n_obs": 86.0, "p_value": 0.05, "se": 0.10039252979248592, "t_stat": 21.427174233377084} + }, + "overall": {"att": 2.3543070317865995, "ci_hi": 2.4758495125487974, "ci_lo": 2.2339808544115445, "p_value": 0.05, "se": 0.0674211743735065, "t_stat": 34.91940111787398}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_plain_overall": {"att": 2.3543070317865995, "ci_hi": 2.4758495125487974, "ci_lo": 2.2339808544115445, "p_value": 0.05, "se": 0.0674211743735065, "t_stat": 34.91940111787398}, + "pretrends_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.059815163957579826, "ci_lo": -0.3559461519303694, "effect": -0.14806549398639476, "n_obs": 84.0, "p_value": 0.1623681759350742, "se": 0.10584141425966392, "t_stat": -1.3989372215222033}, + "-3": {"ci_hi": 0.1584251692403737, "ci_lo": -0.20715293281084962, "effect": -0.024363881785237962, "n_obs": 84.0, "p_value": 0.7935758227509604, "se": 0.09306614604301801, "t_stat": -0.261791025213145}, + "-4": {"ci_hi": 0.06957272135423254, "ci_lo": -0.34423198704465785, "effect": -0.13732963284521266, "n_obs": 84.0, "p_value": 0.19287575347672523, "se": 0.10534331572120142, "t_stat": -1.3036387919349843}, + "-5": {"ci_hi": 0.1567750477585551, "ci_lo": -0.43065589815781424, "effect": -0.13694042519962957, "n_obs": 43.0, "p_value": 0.3601949931340407, "se": 0.14954378803351004, "t_stat": -0.9157212546263955}, + "-6": {"ci_hi": 0.07789021576946764, "ci_lo": -0.4705808158646776, "effect": -0.19634530004760498, "n_obs": 43.0, "p_value": 0.16019426463331482, "se": 0.13962566369272308, "t_stat": -1.4062264404322253}, + "0": {"ci_hi": 2.3060480256636122, "ci_lo": 2.027751271589125, "effect": 2.145940363587726, "n_obs": 84.0, "p_value": 0.05, "se": 0.08156915520915543, "t_stat": 26.308233278685016}, + "1": {"ci_hi": 2.339046004176374, "ci_lo": 2.1207335731351606, "effect": 2.242441654342141, "n_obs": 84.0, "p_value": 0.05, "se": 0.06987184726545864, "t_stat": 32.093636308520765}, + "2": {"ci_hi": 2.7961219749685484, "ci_lo": 2.4322016495426557, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 0.05, "se": 0.11100400667818662, "t_stat": 23.40698260799346}, + "3": {"ci_hi": 2.9324719817762177, "ci_lo": 2.53139383275232, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 0.05, "se": 0.12354164101325206, "t_stat": 22.39269572456501} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.571627829224781, "ci_lo": 2.3050703901774883, "df_used": null, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 0.05, "se": 0.08100030104413747, "t_stat": 30.38077615455412}, + "6": {"ci_hi": 2.3255341366205866, "ci_lo": 2.0032444014190465, "df_used": null, "effect": 2.1511282275930954, "n_obs": 86.0, "p_value": 0.05, "se": 0.10039252979248592, "t_stat": 21.427174233377084} + }, + "overall": {"att": 2.3543070317865995, "ci_hi": 2.4758495125487974, "ci_lo": 2.2339808544115445, "p_value": 0.05, "se": 0.0674211743735065, "t_stat": 34.91940111787398}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "pretrends_plain_overall": {"att": 2.3543070317865995, "ci_hi": 2.4758495125487974, "ci_lo": 2.2339808544115445, "p_value": 0.05, "se": 0.0674211743735065, "t_stat": 34.91940111787398}, + "survey_tsl_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.0614800976250596, "ci_lo": 1.9232838880591523, "effect": 2.0361973460553426, "n_obs": 84.0, "p_value": 0.05, "se": 0.05110820464469081, "t_stat": 39.84090930626861}, + "1": {"ci_hi": 2.2616110022002758, "ci_lo": 1.9480020790996089, "effect": 2.1281777081775473, "n_obs": 84.0, "p_value": 0.05, "se": 0.09267938062775614, "t_stat": 22.962795972119267}, + "2": {"ci_hi": 2.673975059553055, "ci_lo": 2.367430710487471, "effect": 2.4900024320750482, "n_obs": 46.0, "p_value": 0.05, "se": 0.08887132991490315, "t_stat": 28.01806200559052}, + "3": {"ci_hi": 2.816791739465061, "ci_lo": 2.308734840770423, "effect": 2.4352726594598915, "n_obs": 46.0, "p_value": 0.05, "se": 0.1529122151542249, "t_stat": 15.925952396960001} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.420472976901457, "ci_lo": 2.1267973389224313, "df_used": null, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": 0.05, "se": 0.07975913859456671, "t_stat": 28.13913897508415}, + "6": {"ci_hi": 2.271924746304451, "ci_lo": 1.9563874258342548, "df_used": null, "effect": 2.1379018964851286, "n_obs": 76.0, "p_value": 0.05, "se": 0.09539976421054529, "t_stat": 22.409928516875826} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": 2.320700304079089, "ci_lo": 2.0966707213213933, "p_value": 0.05, "se": 0.06301094530299538, "t_stat": 35.05261828759455}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "survey_tsl_plain_overall": {"att": 2.2086986136463955, "ci_hi": 2.320700304079089, "ci_lo": 2.0966707213213933, "p_value": 0.05, "se": 0.06301094530299538, "t_stat": 35.05261828759455} + }, + "TwoStageDiD": { + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.2642399550189514, "ci_lo": 1.9858728804243493, "effect": 2.145940363587725, "n_obs": 84.0, "p_value": 0.05, "se": 0.08154313850706789, "t_stat": 26.316627037868088}, + "1": {"ci_hi": 2.3485697900520406, "ci_lo": 2.147903099025333, "effect": 2.2424416543421404, "n_obs": 84.0, "p_value": 0.05, "se": 0.06615023187989777, "t_stat": 33.899225907681064}, + "2": {"ci_hi": 2.764336057924927, "ci_lo": 2.4004157324983204, "effect": 2.5982688537339027, "n_obs": 41.0, "p_value": 0.05, "se": 0.11100400667836488, "t_stat": 23.406982607955857}, + "3": {"ci_hi": 3.001466920294402, "ci_lo": 2.6003887712697877, "effect": 2.7664303765231937, "n_obs": 41.0, "p_value": 0.05, "se": 0.12354164101342824, "t_stat": 22.39269572453307} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.616633638769783, "ci_lo": 2.350076199721541, "effect": 2.460852014473436, "n_obs": 164.0, "p_value": 0.05, "se": 0.08100030104425085, "t_stat": 30.380776154511583}, + "6": {"ci_hi": 2.299012053766804, "ci_lo": 1.976722318564882, "effect": 2.1511282275930945, "n_obs": 86.0, "p_value": 0.05, "se": 0.10039252979263526, "t_stat": 21.4271742333452} + }, + "overall": {"att": 2.3543070317865986, "ci_hi": 2.4633962373267533, "ci_lo": 2.2360788433223795, "p_value": 0.05, "se": 0.06315261378178635, "t_stat": 37.27964514535417}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_plain_overall": {"att": 2.3543070317865986, "ci_hi": 2.4633962373267533, "ci_lo": 2.2360788433223795, "p_value": 0.05, "se": 0.06315261378178635, "t_stat": 37.27964514535417}, + "pretrends_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.01924623551227377, "ci_lo": -0.09396845261566486, "effect": -0.02985698371664577, "n_obs": 84.0, "p_value": 0.5263157894736842, "se": 0.037194572669387, "t_stat": -0.8027242033948563}, + "-3": {"ci_hi": 0.10529958766448326, "ci_lo": -0.030845886551268072, "effect": 0.03459686456772884, "n_obs": 84.0, "p_value": 0.631578947368421, "se": 0.045456175254620056, "t_stat": 0.7611037306578604}, + "-4": {"ci_hi": 0.08700029442668869, "ci_lo": -0.044211008350224276, "effect": -0.002521727613600925, "n_obs": 84.0, "p_value": 1.0, "se": 0.03906518771601502, "t_stat": -0.06455178538837858}, + "-5": {"ci_hi": 0.09276364317560559, "ci_lo": -0.14996338707692644, "effect": -0.034152641587387955, "n_obs": 43.0, "p_value": 0.631578947368421, "se": 0.06710147786684861, "t_stat": -0.5089700357294369}, + "-6": {"ci_hi": 0.03240640435441551, "ci_lo": -0.13906560665947076, "effect": -0.03367413800267827, "n_obs": 43.0, "p_value": 0.10526315789473684, "se": 0.05013672637303242, "t_stat": -0.6716461252801487}, + "0": {"ci_hi": 2.2642399550189514, "ci_lo": 1.9858728804243493, "effect": 2.145940363587725, "n_obs": 84.0, "p_value": 0.05, "se": 0.08154313850706789, "t_stat": 26.316627037868088}, + "1": {"ci_hi": 2.3485697900520406, "ci_lo": 2.147903099025333, "effect": 2.2424416543421404, "n_obs": 84.0, "p_value": 0.05, "se": 0.06615023187989783, "t_stat": 33.899225907681036}, + "2": {"ci_hi": 2.764336057924927, "ci_lo": 2.4004157324983204, "effect": 2.5982688537339027, "n_obs": 41.0, "p_value": 0.05, "se": 0.11100400667836488, "t_stat": 23.406982607955857}, + "3": {"ci_hi": 3.001466920294402, "ci_lo": 2.6003887712697877, "effect": 2.7664303765231937, "n_obs": 41.0, "p_value": 0.05, "se": 0.12354164101342824, "t_stat": 22.39269572453307} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.616633638769783, "ci_lo": 2.350076199721541, "effect": 2.460852014473436, "n_obs": 164.0, "p_value": 0.05, "se": 0.08100030104425085, "t_stat": 30.380776154511583}, + "6": {"ci_hi": 2.299012053766804, "ci_lo": 1.976722318564882, "effect": 2.1511282275930945, "n_obs": 86.0, "p_value": 0.05, "se": 0.10039252979263526, "t_stat": 21.4271742333452} + }, + "overall": {"att": 2.3543070317865986, "ci_hi": 2.4633962373267533, "ci_lo": 2.2360788433223795, "p_value": 0.05, "se": 0.06315261378178635, "t_stat": 37.27964514535417}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "prop5_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.139071141854362, "ci_lo": 1.734124092890948, "effect": 1.9034726388790302, "n_obs": 36.0, "p_value": 0.05, "se": 0.11122136576199927, "t_stat": 17.114271397747796}, + "1": {"ci_hi": 2.496262490086056, "ci_lo": 2.1246248208374805, "effect": 2.3023388848323054, "n_obs": 36.0, "p_value": 0.05, "se": 0.1098242533981663, "t_stat": 20.9638473615223}, + "2": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 36.0, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 36.0, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.211593169391866, "ci_lo": 1.9643843614584682, "effect": 2.1029057618556686, "n_obs": 72.0, "p_value": 0.05, "se": 0.07324916315889564, "t_stat": 28.708939067248362}, + "6": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 0.0, "p_value": null, "se": null, "t_stat": null} + }, + "overall": {"att": 2.1029057618556686, "ci_hi": 2.211593169391866, "ci_lo": 1.9643843614584684, "p_value": 0.05, "se": 0.07324916315889562, "t_stat": 28.70893906724837}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "prop5_plain_overall": {"att": 2.1029057618556686, "ci_hi": 2.211593169391866, "ci_lo": 1.9643843614584684, "p_value": 0.05, "se": 0.07324916315889562, "t_stat": 28.70893906724837} + } + }, + "rust": { + "CallawaySantAnna": { + "cband_all": { + "cband_crit_value": 2.6040884451808886, + "es": { + "-1": {"cband_hi": 0.17975452776202336, "cband_lo": -0.05377186226484623, "ci_hi": 0.1383020908309624, "ci_lo": -0.018122331027180152, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.12, "se": 0.04483841369885731, "t_stat": 1.4048519461827855}, + "-2": {"cband_hi": 0.18928382576292235, "cband_lo": -0.17927809779788545, "ci_hi": 0.1523371721407983, "ci_lo": -0.14238567774290867, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 1.0, "se": 0.07076601492604186, "t_stat": 0.07069585574017383}, + "-3": {"cband_hi": 0.21134585666275663, "cband_lo": -0.21100042856217888, "ci_hi": 0.12434521944668774, "ci_lo": -0.156788135544127, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 0.96, "se": 0.08109292255540075, "t_stat": 0.002129828903020242}, + "-4": {"cband_hi": 0.20155862551695183, "cband_lo": -0.24639217929185922, "ci_hi": 0.11632440673805822, "ci_lo": -0.20400189129157686, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.84, "se": 0.08600913798411615, "t_stat": -0.26063250269516175}, + "0": {"cband_hi": 1.5673872303415597, "cband_lo": 1.3404504636661554, "ci_hi": 1.5415374421679076, "ci_lo": 1.368208423073332, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04357316801112729, "t_stat": 33.36729720071237}, + "1": {"cband_hi": 1.915820850854157, "cband_lo": 1.6738431113529941, "ci_hi": 1.8620294053554955, "ci_lo": 1.705826580818813, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04646112153927904, "t_stat": 38.63083631302773}, + "2": {"cband_hi": 2.2749242055141607, "cband_lo": 1.9478386414536466, "ci_hi": 2.2334184090136393, "ci_lo": 2.017323275290894, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.06280231469591921, "t_stat": 33.61948414969962}, + "3": {"cband_hi": 2.5660352098040353, "cband_lo": 2.1514595291447347, "ci_hi": 2.4874425791723134, "ci_lo": 2.2203312249189118, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07960092166349271, "t_stat": 29.6321615401116}, + "4": {"cband_hi": 2.951878071538104, "cband_lo": 2.38503600034597, "ci_hi": 2.8351030413642038, "ci_lo": 2.502427034472139, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10883694680976154, "t_stat": 24.517933607658904} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.0073975718167707295, 0.00016446371867586494, -0.0002088961399349469, -0.00021236165833216837, -0.0001633312780568242, 0.0014476594611222385, 0.000666349669921231, 0.001179347445298157, 0.0013592959379791015], + [0.00016446371867586494, 0.0065760620885762225, -0.002522081555470615, 0.0019071768396986158, -0.0001407528669437694, -0.0005392420997478858, -0.0010878301732690446, -0.0017264698348421908, 0.0006937139554424842], + [-0.0002088961399349469, -0.002522081555470615, 0.005007828868512777, -0.0014681134471507247, 7.209808136776544e-06, 0.00034075358171953713, 0.0002868594739125574, 0.0017837388184845857, 0.0021586787024569817], + [-0.00021236165833216837, 0.0019071768396986158, -0.0014681134471507247, 0.002010483343029875, -0.0004594846748256816, -0.00048820978703640553, -5.365828426266825e-05, -0.0004304779916969216, 7.423688070610573e-05], + [-0.0001633312780568242, -0.0001407528669437694, 7.209808136776544e-06, -0.0004594846748256816, 0.001898620970525926, 0.0002966210423376534, 0.0011423083319685183, 0.0008730765307925354, 0.0008774114921566766], + [0.0014476594611222385, -0.0005392420997478858, 0.00034075358171953713, -0.00048820978703640553, 0.0002966210423376534, 0.002158635814687659, 0.0010768991364652517, 0.0019264693682071116, 0.0017553877425780344], + [0.000666349669921231, -0.0010878301732690446, 0.0002868594739125574, -5.365828426266825e-05, 0.0011423083319685183, 0.0010768991364652517, 0.003944130731165269, 0.002688437695846199, 0.0025195292692069414], + [0.001179347445298157, -0.0017264698348421908, 0.0017837388184845857, -0.0004304779916969216, 0.0008730765307925354, 0.0019264693682071116, 0.002688437695846199, 0.006336306729677503, 0.003978413514594727], + [0.0013592959379791015, 0.0006937139554424842, 0.0021586787024569817, 7.423688070610573e-05, 0.0008774114921566766, 0.0017553877425780344, 0.0025195292692069414, 0.003978413514594727, 0.011845480990870861] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.2951208963267224, "ci_lo": 1.9775561783486255, "df_used": null, "effect": 2.11762432932182, "p_value": 0.0196078431372549, "se": 0.08591158728014282, "t_stat": 24.648879113554422}, + "4": {"ci_hi": 2.022216284914674, "ci_lo": 1.7653129426768104, "df_used": null, "effect": 1.886783657115476, "p_value": 0.0196078431372549, "se": 0.0725752691070692, "t_stat": 25.997611587659875}, + "5": {"ci_hi": 2.088078317762701, "ci_lo": 1.7350726741746059, "df_used": null, "effect": 1.9068267927553257, "p_value": 0.0196078431372549, "se": 0.0949065559582727, "t_stat": 20.09162352908154}, + "6": {"ci_hi": 1.6145137520317316, "ci_lo": 1.2614122062588298, "df_used": null, "effect": 1.4296813522176333, "p_value": 0.0196078431372549, "se": 0.08946080880594207, "t_stat": 15.981091287905643} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 1.976801238969689, "ci_lo": 1.8119756021686342, "p_value": 0.0196078431372549, "se": 0.04722597436226548, "t_stat": 40.40618342906495}, + "ref_labels": [], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": 2.6040884451808886, + "es": { + "-1": {"cband_hi": 0.17975452776202336, "cband_lo": -0.05377186226484623, "ci_hi": 0.1383020908309624, "ci_lo": -0.018122331027180152, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.12, "se": 0.04483841369885731, "t_stat": 1.4048519461827855}, + "-2": {"cband_hi": 0.18928382576292235, "cband_lo": -0.17927809779788545, "ci_hi": 0.1523371721407983, "ci_lo": -0.14238567774290867, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 1.0, "se": 0.07076601492604186, "t_stat": 0.07069585574017383}, + "-3": {"cband_hi": 0.21134585666275663, "cband_lo": -0.21100042856217888, "ci_hi": 0.12434521944668774, "ci_lo": -0.156788135544127, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 0.96, "se": 0.08109292255540075, "t_stat": 0.002129828903020242}, + "-4": {"cband_hi": 0.20155862551695183, "cband_lo": -0.24639217929185922, "ci_hi": 0.11632440673805822, "ci_lo": -0.20400189129157686, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 0.84, "se": 0.08600913798411615, "t_stat": -0.26063250269516175}, + "0": {"cband_hi": 1.5673872303415597, "cband_lo": 1.3404504636661554, "ci_hi": 1.5415374421679076, "ci_lo": 1.368208423073332, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04357316801112729, "t_stat": 33.36729720071237}, + "1": {"cband_hi": 1.915820850854157, "cband_lo": 1.6738431113529941, "ci_hi": 1.8620294053554955, "ci_lo": 1.705826580818813, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.04646112153927904, "t_stat": 38.63083631302773}, + "2": {"cband_hi": 2.2749242055141607, "cband_lo": 1.9478386414536466, "ci_hi": 2.2334184090136393, "ci_lo": 2.017323275290894, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.06280231469591921, "t_stat": 33.61948414969962}, + "3": {"cband_hi": 2.5660352098040353, "cband_lo": 2.1514595291447347, "ci_hi": 2.4874425791723134, "ci_lo": 2.2203312249189118, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07960092166349271, "t_stat": 29.6321615401116}, + "4": {"cband_hi": 2.951878071538104, "cband_lo": 2.38503600034597, "ci_hi": 2.8351030413642038, "ci_lo": 2.502427034472139, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10883694680976154, "t_stat": 24.517933607658904} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.0073975718167707295, 0.00016446371867586494, -0.0002088961399349469, -0.00021236165833216837, -0.0001633312780568242, 0.0014476594611222385, 0.000666349669921231, 0.001179347445298157, 0.0013592959379791015], + [0.00016446371867586494, 0.0065760620885762225, -0.002522081555470615, 0.0019071768396986158, -0.0001407528669437694, -0.0005392420997478858, -0.0010878301732690446, -0.0017264698348421908, 0.0006937139554424842], + [-0.0002088961399349469, -0.002522081555470615, 0.005007828868512777, -0.0014681134471507247, 7.209808136776544e-06, 0.00034075358171953713, 0.0002868594739125574, 0.0017837388184845857, 0.0021586787024569817], + [-0.00021236165833216837, 0.0019071768396986158, -0.0014681134471507247, 0.002010483343029875, -0.0004594846748256816, -0.00048820978703640553, -5.365828426266825e-05, -0.0004304779916969216, 7.423688070610573e-05], + [-0.0001633312780568242, -0.0001407528669437694, 7.209808136776544e-06, -0.0004594846748256816, 0.001898620970525926, 0.0002966210423376534, 0.0011423083319685183, 0.0008730765307925354, 0.0008774114921566766], + [0.0014476594611222385, -0.0005392420997478858, 0.00034075358171953713, -0.00048820978703640553, 0.0002966210423376534, 0.002158635814687659, 0.0010768991364652517, 0.0019264693682071116, 0.0017553877425780344], + [0.000666349669921231, -0.0010878301732690446, 0.0002868594739125574, -5.365828426266825e-05, 0.0011423083319685183, 0.0010768991364652517, 0.003944130731165269, 0.002688437695846199, 0.0025195292692069414], + [0.001179347445298157, -0.0017264698348421908, 0.0017837388184845857, -0.0004304779916969216, 0.0008730765307925354, 0.0019264693682071116, 0.002688437695846199, 0.006336306729677503, 0.003978413514594727], + [0.0013592959379791015, 0.0006937139554424842, 0.0021586787024569817, 7.423688070610573e-05, 0.0008774114921566766, 0.0017553877425780344, 0.0025195292692069414, 0.003978413514594727, 0.011845480990870861] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.2951208963267224, "ci_lo": 1.9775561783486255, "df_used": null, "effect": 2.11762432932182, "p_value": 0.0196078431372549, "se": 0.08591158728014282, "t_stat": 24.648879113554422}, + "4": {"ci_hi": 2.022216284914674, "ci_lo": 1.7653129426768104, "df_used": null, "effect": 1.886783657115476, "p_value": 0.0196078431372549, "se": 0.0725752691070692, "t_stat": 25.997611587659875}, + "5": {"ci_hi": 2.088078317762701, "ci_lo": 1.7350726741746059, "df_used": null, "effect": 1.9068267927553257, "p_value": 0.0196078431372549, "se": 0.0949065559582727, "t_stat": 20.09162352908154}, + "6": {"ci_hi": 1.6145137520317316, "ci_lo": 1.2614122062588298, "df_used": null, "effect": 1.4296813522176333, "p_value": 0.0196078431372549, "se": 0.08946080880594207, "t_stat": 15.981091287905643} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 1.976801238969689, "ci_lo": 1.8119756021686342, "p_value": 0.0196078431372549, "se": 0.04722597436226548, "t_stat": 40.40618342906495}, + "ref_labels": [], + "reference_event_times": null + }, + "default_plain_overall": {"att": 1.9082213826980177, "ci_hi": 1.976801238969689, "ci_lo": 1.8119756021686342, "p_value": 0.0196078431372549, "se": 0.04722597436226548, "t_stat": 40.40618342906495}, + "fpc_survey_all": { + "cband_crit_value": 2.198655409564874, + "es": { + "-1": {"cband_hi": 0.15613290365037302, "cband_lo": -0.030492775329768337, "ci_hi": 0.13635677518287867, "ci_lo": -0.0034747661219304577, "effect": 0.06282006416030234, "n_groups": 4.0, "p_value": 0.08, "se": 0.042440865942034, "t_stat": 1.480178661908133}, + "-2": {"cband_hi": 0.1331217297050343, "cband_lo": -0.06267061383291456, "ci_hi": 0.1236898400496428, "ci_lo": -0.039397666932691464, "effect": 0.03522555793605988, "n_groups": 3.0, "p_value": 0.48, "se": 0.04452547286086482, "t_stat": 0.7911327083742439}, + "-3": {"cband_hi": 0.05606546039292498, "cband_lo": -0.10985936641989366, "ci_hi": 0.0456125734991019, "ci_lo": -0.10030920605569461, "effect": -0.026896953013484336, "n_groups": 2.0, "p_value": 0.84, "se": 0.037733249624063665, "t_stat": -0.712818357323015}, + "-4": {"cband_hi": 0.09006636298469048, "cband_lo": -0.20459966203178612, "ci_hi": 0.08016707226558367, "ci_lo": -0.15720742075367447, "effect": -0.05726664952354782, "n_groups": 1.0, "p_value": 0.64, "se": 0.06701050645193933, "t_stat": -0.854592101383685}, + "0": {"cband_hi": 1.540688396180239, "cband_lo": 1.3516421702995782, "ci_hi": 1.5254636548092535, "ci_lo": 1.3594838237318418, "effect": 1.4461652832399086, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.042991326666799966, "t_stat": 33.63853584813211}, + "1": {"cband_hi": 1.9053288739883822, "cband_lo": 1.6473906008431582, "ci_hi": 1.8678019689221705, "ci_lo": 1.6712801664795045, "effect": 1.7763597374157702, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.058658185367090186, "t_stat": 30.283237135603343}, + "2": {"cband_hi": 2.187011567931592, "cband_lo": 2.0296425073457587, "ci_hi": 2.182888602868014, "ci_lo": 2.0637171252569777, "effect": 2.1083270376386753, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.03578756814306293, "t_stat": 58.91227448622697}, + "3": {"cband_hi": 2.4956865673451394, "cband_lo": 2.2025397349692653, "ci_hi": 2.4750827750499718, "ci_lo": 2.2409175813818676, "effect": 2.3491131511572023, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06666502424631637, "t_stat": 35.23756539077542}, + "4": {"cband_hi": 2.7231135567158957, "cband_lo": 2.5235437503201426, "ci_hi": 2.7088339552183904, "ci_lo": 2.545841021599899, "effect": 2.623328653518019, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.045384512172202844, "t_stat": 57.80228822476489} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.004490407974945402, -6.417775983372599e-05, 0.00015343241396781004, 0.0006475744216073651, -0.0006684763526359291, -0.0025606661484777484, 0.00047070889712017624, 0.00023632612207851472, -0.0014006990469745107], + [-6.417775983372599e-05, 0.0014237981271919009, -0.0003785763491996777, -0.0006501575822976633, 0.0012664822107358792, 0.001371215356313856, 5.217779862370023e-05, 0.0012146970397161733, -0.0005063486262898457], + [0.00015343241396781004, -0.0003785763491996777, 0.001982517733483609, -7.18959336827405e-05, 1.524309481110808e-05, -0.0006259237640079817, -0.0010968408603891121, 0.0014900128632927044, 0.0003674191353703922], + [0.0006475744216073651, -0.0006501575822976633, -7.18959336827405e-05, 0.001801227101909701, -0.0009586825832075483, -0.0008642862099551052, 0.00021293782695345156, -0.0001051936566265261, 0.0002728616470639717], + [-0.0006684763526359291, 0.0012664822107358792, 1.524309481110808e-05, -0.0009586825832075483, 0.0018482541685715054, 0.001319626698752786, -0.0007396979465342897, 0.001157374942932415, -0.0010696472786354513], + [-0.0025606661484777484, 0.001371215356313856, -0.0006259237640079817, -0.0008642862099551052, 0.001319626698752786, 0.0034407827105599133, 0.0003370466356638977, 0.001825612916804522, 0.0007665618456507005], + [0.00047070889712017624, 5.217779862370023e-05, -0.0010968408603891121, 0.00021293782695345156, -0.0007396979465342897, 0.0003370466356638977, 0.0012807500335943723, -0.00048248887029781075, 0.0005318388975436898], + [0.00023632612207851472, 0.0012146970397161733, 0.0014900128632927044, -0.0001051936566265261, 0.001157374942932415, 0.001825612916804522, -0.00048248887029781075, 0.004444225457761949, 0.0003166322604017601], + [-0.0014006990469745107, -0.0005063486262898457, 0.0003674191353703922, 0.0002728616470639717, -0.0010696472786354513, 0.0007665618456507005, 0.0005318388975436898, 0.0003166322604017601, 0.0020597539451088277] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.2278169032160604, "ci_lo": 1.9909472116786369, "df_used": null, "effect": 2.0958253131634734, "p_value": 0.0196078431372549, "se": 0.07327648485625686, "t_stat": 28.601608241371817}, + "4": {"ci_hi": 2.070754112102948, "ci_lo": 1.7095143475834085, "df_used": null, "effect": 1.8950605918145964, "p_value": 0.0196078431372549, "se": 0.09800045384239431, "t_stat": 19.337263425965958}, + "5": {"ci_hi": 1.9815385073499994, "ci_lo": 1.7932969810131167, "df_used": null, "effect": 1.8773932029725373, "p_value": 0.0196078431372549, "se": 0.05823008862621429, "t_stat": 32.240947030387375}, + "6": {"ci_hi": 1.5723934009228542, "ci_lo": 1.226768891860088, "df_used": null, "effect": 1.4055713190956884, "p_value": 0.0196078431372549, "se": 0.10725945860194185, "t_stat": 13.104404379962457} + }, + "overall": {"att": 1.8839037596672263, "ci_hi": 1.9415322152544534, "ci_lo": 1.821311697690987, "p_value": 0.0196078431372549, "se": 0.03290109841000663, "t_stat": 57.25960076440034}, + "ref_labels": [], + "reference_event_times": null + }, + "psu_cluster_all": { + "cband_crit_value": 2.173244570172544, + "es": { + "-1": {"cband_hi": 0.1476703093792444, "cband_lo": -0.0216876438820673, "ci_hi": 0.1295408113758293, "ci_lo": -0.000990485259906229, "effect": 0.06299133274858856, "n_groups": 4.0, "p_value": 0.12, "se": 0.03896431068682381, "t_stat": 1.6166417841927778}, + "-2": {"cband_hi": 0.07997565193095012, "cband_lo": -0.06996992396591324, "ci_hi": 0.07490562557228783, "ci_lo": -0.05231562254459107, "effect": 0.005002863982518443, "n_groups": 3.0, "p_value": 0.84, "se": 0.03449809054048585, "t_stat": 0.14501857651069824}, + "-3": {"cband_hi": 0.0951601165421576, "cband_lo": -0.09481468844157985, "ci_hi": 0.08657415168941751, "ci_lo": -0.07281551397813087, "effect": 0.00017271405028887465, "n_groups": 2.0, "p_value": 0.76, "se": 0.04370764514751657, "t_stat": 0.003951575284048175}, + "-4": {"cband_hi": 0.12804736703498143, "cband_lo": -0.17288092080988882, "ci_hi": 0.11362479644795043, "ci_lo": -0.1330264678220342, "effect": -0.022416776887453693, "n_groups": 1.0, "p_value": 1.0, "se": 0.06923479574620038, "t_stat": -0.3237790571323225}, + "0": {"cband_hi": 1.5379537681884359, "cband_lo": 1.3698839258192792, "ci_hi": 1.5219792287715384, "ci_lo": 1.3838318976643895, "effect": 1.4539188470038575, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.03866795405263867, "t_stat": 37.60009761635277}, + "1": {"cband_hi": 1.9022693035728484, "cband_lo": 1.6873946586343027, "ci_hi": 1.8794723320847229, "ci_lo": 1.709002763572227, "effect": 1.7948319811035756, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.049436369906928145, "t_stat": 36.30590159598355}, + "2": {"cband_hi": 2.18865729103232, "cband_lo": 2.0341055559354873, "ci_hi": 2.184767486987823, "ci_lo": 2.048554398875639, "effect": 2.1113814234839037, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.03555783302487729, "t_stat": 59.37879909629813}, + "3": {"cband_hi": 2.5162189152480705, "cband_lo": 2.2012758237006995, "ci_hi": 2.4845168016464587, "ci_lo": 2.2438787818826396, "effect": 2.358747369474385, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07245919209230238, "t_stat": 32.55276937768899}, + "4": {"cband_hi": 2.754900041074348, "cband_lo": 2.582014030809726, "ci_hi": 2.7418858589334234, "ci_lo": 2.595383187627167, "effect": 2.668457035942037, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.0397760133943176, "t_stat": 67.08709114431394} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.004793456942018086, 0.000683526581101642, -0.00044390461494074764, 0.0002914120288491085, -0.0010722763760465984, -0.00120052797668387, 0.001345577150299979, 9.063620376320222e-05, 0.0006114258906619185], + [0.000683526581101642, 0.001910358244341229, -0.00015382509249728876, -0.0007796170405323712, 0.001193907161242858, 0.0008317351031483285, 0.00022694576205742388, 0.0015478722532630876, 0.00022780825387102343], + [-0.00044390461494074764, -0.00015382509249728876, 0.0011901182509395593, -0.0003417271075439688, 0.00014784849399063048, -0.0001379076013106159, -0.0003474141068648317, 0.0014460767050406297, 0.0005457412417765935], + [0.0002914120288491085, -0.0007796170405323712, -0.0003417271075439688, 0.0015182175072993313, -0.00048728647974170347, -0.000301484707588978, 7.413997135648223e-05, -0.0005260436744299985, 0.00020186511559477864], + [-0.0010722763760465984, 0.001193907161242858, 0.00014784849399063048, -0.00048728647974170347, 0.0014952106706169757, 0.000802204903291636, -0.0005855444791999376, 0.001689066959047903, -0.00031336691159789036], + [-0.00120052797668387, 0.0008317351031483285, -0.0001379076013106159, -0.000301484707588978, 0.000802204903291636, 0.002443954669574631, 0.0007865546434961742, 0.0015271581767485184, 0.0005644782212722514], + [0.001345577150299979, 0.00022694576205742388, -0.0003474141068648317, 7.413997135648223e-05, -0.0005855444791999376, 0.0007865546434961742, 0.001264359489425054, 4.1087071235788835e-05, 0.0007490868256899574], + [9.063620376320222e-05, 0.0015478722532630876, 0.0014460767050406297, -0.0005260436744299985, 0.001689066959047903, 0.0015271581767485184, 4.1087071235788835e-05, 0.005250334518669174, 0.0007537524504042382], + [0.0006114258906619185, 0.00022780825387102343, 0.0005457412417765935, 0.00020186511559477864, -0.00031336691159789036, 0.0005644782212722514, 0.0007490868256899574, 0.0007537524504042382, 0.0015821312415449328] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.281728332848565, "ci_lo": 1.990452897959769, "df_used": null, "effect": 2.11762432932182, "p_value": 0.0196078431372549, "se": 0.08096399175052113, "t_stat": 26.15513740783649}, + "4": {"ci_hi": 2.0660689853785597, "ci_lo": 1.705836182804786, "df_used": null, "effect": 1.886783657115476, "p_value": 0.0196078431372549, "se": 0.09289407427562829, "t_stat": 20.311130412012652}, + "5": {"ci_hi": 2.0496023680956625, "ci_lo": 1.8243748527668917, "df_used": null, "effect": 1.9068267927553257, "p_value": 0.0196078431372549, "se": 0.07235224620319872, "t_stat": 26.354769793878553}, + "6": {"ci_hi": 1.6058486209245848, "ci_lo": 1.2501188418433753, "df_used": null, "effect": 1.4296813522176333, "p_value": 0.0196078431372549, "se": 0.10729732732104728, "t_stat": 13.324482425735027} + }, + "overall": {"att": 1.9082213826980177, "ci_hi": 1.9692574622530463, "ci_lo": 1.8573773397271987, "p_value": 0.0196078431372549, "se": 0.03243141563709646, "t_stat": 58.838670628836546}, + "ref_labels": [], + "reference_event_times": null + }, + "rcs_all": { + "cband_crit_value": 2.4452393997457427, + "es": { + "-1": {"cband_hi": 0.2507264429429815, "cband_lo": -0.16527289217438054, "ci_hi": 0.16902928728039007, "ci_lo": -0.13052477619374098, "effect": 0.0427267753843005, "n_groups": 3.0, "p_value": 0.72, "se": 0.08506310980442609, "t_stat": 0.50229500758362}, + "-2": {"cband_hi": 0.25364731571011445, "cband_lo": -0.3873243649124012, "ci_hi": 0.14688714618504903, "ci_lo": -0.27211488118346533, "effect": -0.06683852460114334, "n_groups": 2.0, "p_value": 0.72, "se": 0.13106522017622574, "t_stat": -0.5099638524337318}, + "-3": {"cband_hi": 0.39986313843821575, "cband_lo": -0.5474983086363073, "ci_hi": 0.22373234474557005, "ci_lo": -0.45850937168174594, "effect": -0.07381758509904579, "n_groups": 1.0, "p_value": 0.6, "se": 0.1937154797957677, "t_stat": -0.38106188094452204}, + "0": {"cband_hi": 1.6253289822404993, "cband_lo": 1.1653480304112058, "ci_hi": 1.5795781956762522, "ci_lo": 1.2624587716570248, "effect": 1.3953385063258525, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.09405642487952767, "t_stat": 14.835121663544774}, + "1": {"cband_hi": 1.7341037228891292, "cband_lo": 1.2120550026962356, "ci_hi": 1.7029405730802565, "ci_lo": 1.2567321638510929, "effect": 1.4730793627926824, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.10674797736515625, "t_stat": 13.799599759662634}, + "2": {"cband_hi": 1.6948030532645186, "cband_lo": 1.0939038945111943, "ci_hi": 1.6212880574707522, "ci_lo": 1.1924590081103401, "effect": 1.3943534738878565, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.12287123273406401, "t_stat": 11.348087285050044}, + "3": {"cband_hi": 1.8729196534506194, "cband_lo": 0.7704128571371376, "ci_hi": 1.7927103856142033, "ci_lo": 0.9610926778938671, "effect": 1.3216662552938785, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.22543943885987627, "t_stat": 5.862622183491909} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.03752568711250447, -5.234594404186521e-05, 0.003605670704404306, -0.007455040552475199, -0.002959482678697484, -0.005718553980854703, -0.021797951765049485], + [-5.234594404186521e-05, 0.017178091939842537, -0.0029981009071407146, 0.0016273223104292345, 0.0012250488846789153, 0.0005679305341129964, 0.0026903694142308], + [0.003605670704404306, -0.0029981009071407146, 0.007235732649599848, -0.002003260888891906, -0.001993977259471703, -0.001239559390323457, -0.002252107721451391], + [-0.007455040552475199, 0.0016273223104292345, -0.002003260888891906, 0.008846611061118234, 0.004731122586219354, 0.00748431413589406, 0.013697188894851145], + [-0.002959482678697484, 0.0012250488846789153, -0.001993977259471703, 0.004731122586219354, 0.011395130671551908, 0.007043602160257294, 0.012537612209613093], + [-0.005718553980854703, 0.0005679305341129964, -0.001239559390323457, 0.00748431413589406, 0.007043602160257294, 0.015097339833588524, 0.020777977366497617], + [-0.021797951765049485, 0.0026903694142308, -0.002252107721451391, 0.013697188894851145, 0.012537612209613093, 0.020777977366497617, 0.05082294059345589] + ], + "event_study_vcov_index": [-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], + "group": { + "3": {"ci_hi": 1.6912789713370406, "ci_lo": 0.9844593371229816, "df_used": null, "effect": 1.3058492644714335, "p_value": 0.0196078431372549, "se": 0.20971059375438392, "t_stat": 6.2269112928117645}, + "4": {"ci_hi": 1.6362404763811278, "ci_lo": 1.2165860081176543, "df_used": null, "effect": 1.4404583188885236, "p_value": 0.0196078431372549, "se": 0.1258042601819472, "t_stat": 11.449996341977837}, + "5": {"ci_hi": 2.0006089428422285, "ci_lo": 1.2921271728479413, "df_used": null, "effect": 1.5854273979369462, "p_value": 0.0196078431372549, "se": 0.20217622576196564, "t_stat": 7.841809253099651} + }, + "overall": {"att": 1.4128474233805774, "ci_hi": 1.6083558959574078, "ci_lo": 1.269160775280555, "p_value": 0.0196078431372549, "se": 0.0985627282957861, "t_stat": 14.334499945462463}, + "ref_labels": [], + "reference_event_times": null + }, + "stratified_survey_all": { + "cband_crit_value": 2.4385334214792573, + "es": { + "-1": {"cband_hi": 0.20863066827218424, "cband_lo": -0.06936405467022061, "ci_hi": 0.18121772004419043, "ci_lo": -0.04343260960903114, "effect": 0.06963330680098181, "n_groups": 4.0, "p_value": 0.16, "se": 0.05700039222217597, "t_stat": 1.221628555283713}, + "-2": {"cband_hi": 0.19850133416910964, "cband_lo": -0.1946240760020796, "ci_hi": 0.16118426417141604, "ci_lo": -0.15522783972859977, "effect": 0.001938629083515029, "n_groups": 3.0, "p_value": 0.88, "se": 0.08060693503489332, "t_stat": 0.02405040065939526}, + "-3": {"cband_hi": 0.17689858770985728, "cband_lo": -0.16901862231076187, "ci_hi": 0.13440593977912665, "ci_lo": -0.12929571759247466, "effect": 0.0039399826995477095, "n_groups": 2.0, "p_value": 1.0, "se": 0.07092730552177129, "t_stat": 0.05554958940796536}, + "-4": {"cband_hi": 0.2909088521650203, "cband_lo": -0.26569578531593013, "ci_hi": 0.19435867258874057, "ci_lo": -0.212428598208563, "effect": 0.012606533424545097, "n_groups": 1.0, "p_value": 0.92, "se": 0.11412692411312211, "t_stat": 0.11046064302976881}, + "0": {"cband_hi": 1.533860107960097, "cband_lo": 1.3696470244733665, "ci_hi": 1.4976640759526723, "ci_lo": 1.3908911347703121, "effect": 1.4517535662167318, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.03367045988385837, "t_stat": 43.11653512379565}, + "1": {"cband_hi": 1.8861103310888025, "cband_lo": 1.6657497352750505, "ci_hi": 1.84894275994513, "ci_lo": 1.6848815996561057, "effect": 1.7759300331819265, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.045183017356407104, "t_stat": 39.305255316909324}, + "2": {"cband_hi": 2.2830101412365114, "cband_lo": 1.934336642405787, "ci_hi": 2.2532482613613998, "ci_lo": 1.9689431103907475, "effect": 2.1086733918211493, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.07149245849154959, "t_stat": 29.495046558937325}, + "3": {"cband_hi": 2.558832763930004, "cband_lo": 2.0846790217486126, "ci_hi": 2.494965560845798, "ci_lo": 2.1720816183825393, "effect": 2.3217558928393083, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.09722108747924431, "t_stat": 23.881196487695934}, + "4": {"cband_hi": 2.902900940587747, "cband_lo": 2.4325932522829046, "ci_hi": 2.8763288469837063, "ci_lo": 2.514209781713328, "effect": 2.667747096435326, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.09643248769162777, "t_stat": 27.664401907438698} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.013024954807522332, 0.002369297928261079, -0.0015222581491149403, -0.00043509722725348026, 0.000466631111583956, -0.001965308128820638, 0.0017065348615906647, 6.931227285932411e-05, -0.0038179835016279337], + [0.002369297928261079, 0.005030682668578689, -0.0019876890512482022, -0.0016408360456225315, 0.0011277152001795507, 0.0004735862835932727, 0.00030447170826796485, 0.0013180341684036964, -0.0017646704877972482], + [-0.0015222581491149403, -0.0019876890512482022, 0.006497477975719511, -0.0007376533689779569, -0.0009826008711669055, -0.0005728700411489965, -0.001910074212906339, -0.0021802547089884677, 0.0033412775067301254], + [-0.00043509722725348026, -0.0016408360456225315, -0.0007376533689779569, 0.003249044713481899, -0.00042347399388194685, 4.085591130641185e-05, 0.0021645588948692733, 0.0020216649628543946, -0.0005527979436737165], + [0.000466631111583956, 0.0011277152001795507, -0.0009826008711669055, -0.00042347399388194685, 0.0011336998687905158, 0.00040647671199136047, 0.0003272692170020557, 0.0013488591934123879, 0.00023155567709411704], + [-0.001965308128820638, 0.0004735862835932727, -0.0005728700411489965, 4.085591130641185e-05, 0.00040647671199136047, 0.0020415050574293853, 0.0010414737398037912, 0.0025351769689276097, 0.0015249602164618796], + [0.0017065348615906647, 0.00030447170826796485, -0.001910074212906339, 0.0021645588948692733, 0.0003272692170020557, 0.0010414737398037912, 0.005111171621165941, 0.0042776781765350436, 0.0005787036378496957], + [6.931227285932411e-05, 0.0013180341684036964, -0.0021802547089884677, 0.0020216649628543946, 0.0013488591934123879, 0.0025351769689276097, 0.0042776781765350436, 0.009451939850646874, 0.0028236024819365277], + [-0.0038179835016279337, -0.0017646704877972482, 0.0033412775067301254, -0.0005527979436737165, 0.00023155567709411704, 0.0015249602164618796, 0.0005787036378496957, 0.0028236024819365277, 0.009299224682395938] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.30062747087495, "ci_lo": 1.9457647932554265, "df_used": null, "effect": 2.1242700768040943, "p_value": 0.0196078431372549, "se": 0.09624435241511488, "t_stat": 22.071633540031844}, + "4": {"ci_hi": 2.013229590875692, "ci_lo": 1.5954562636154388, "df_used": null, "effect": 1.8300828461356922, "p_value": 0.0196078431372549, "se": 0.10911131754759941, "t_stat": 16.772621642455423}, + "5": {"ci_hi": 2.0973034921814397, "ci_lo": 1.6390768741483428, "df_used": null, "effect": 1.9033747125067557, "p_value": 0.0196078431372549, "se": 0.11729583583619467, "t_stat": 16.22712945380982}, + "6": {"ci_hi": 1.5956134547819238, "ci_lo": 1.1898156451856987, "df_used": null, "effect": 1.4106784908973298, "p_value": 0.0196078431372549, "se": 0.12670562914080272, "t_stat": 11.13351080345216} + }, + "overall": {"att": 1.899842807641477, "ci_hi": 1.9667359583706414, "ci_lo": 1.8187567819914907, "p_value": 0.0196078431372549, "se": 0.041664313181091844, "t_stat": 45.59880297039593}, + "ref_labels": [], + "reference_event_times": null + }, + "unbalanced_all": { + "cband_crit_value": 2.755728699682932, + "es": { + "-1": {"cband_hi": 0.20525710922254087, "cband_lo": -0.1359136935437676, "ci_hi": 0.13691134268219624, "ci_lo": -0.08669784170876578, "effect": 0.034671707839386626, "n_groups": 4.0, "p_value": 0.72, "se": 0.06190210284589386, "t_stat": 0.5601054931155137}, + "-2": {"cband_hi": 0.22814256459166515, "cband_lo": -0.2555539458302233, "ci_hi": 0.13120930443889592, "ci_lo": -0.24073015705660175, "effect": -0.013705690619279079, "n_groups": 3.0, "p_value": 0.8, "se": 0.08776199748501032, "t_stat": -0.15616885453889082}, + "-3": {"cband_hi": 0.35397241977100247, "cband_lo": -0.2143030151333204, "ci_hi": 0.23123321930977367, "ci_lo": -0.18559560430677494, "effect": 0.06983470231884105, "n_groups": 2.0, "p_value": 0.56, "se": 0.10310801548964298, "t_stat": 0.6772965417597028}, + "-4": {"cband_hi": 0.31575212791905233, "cband_lo": -0.26209025702342875, "ci_hi": 0.15151748123285266, "ci_lo": -0.2104782295788583, "effect": 0.026830935447811788, "n_groups": 1.0, "p_value": 0.76, "se": 0.10484384493454786, "t_stat": 0.25591331054829075}, + "0": {"cband_hi": 1.609622798050211, "cband_lo": 1.3291222681745933, "ci_hi": 1.5566061046103468, "ci_lo": 1.3729485386084475, "effect": 1.4693725331124021, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.05089407565916985, "t_stat": 28.871190095927357}, + "1": {"cband_hi": 1.976392528898735, "cband_lo": 1.6366738588565448, "ci_hi": 1.9133840254379209, "ci_lo": 1.678770438508927, "effect": 1.8065331938776399, "n_groups": 4.0, "p_value": 0.0196078431372549, "se": 0.06163862757630629, "t_stat": 29.30845907691926}, + "2": {"cband_hi": 2.3422363378299025, "cband_lo": 1.9071676068101953, "ci_hi": 2.262184506175058, "ci_lo": 1.993275149519106, "effect": 2.124701972320049, "n_groups": 3.0, "p_value": 0.0196078431372549, "se": 0.07893896287210081, "t_stat": 26.91575737779267}, + "3": {"cband_hi": 2.6436316317302166, "cband_lo": 2.0663127471827023, "ci_hi": 2.542133038811411, "ci_lo": 2.1665668146783403, "effect": 2.3549721894564595, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.10474886091180523, "t_stat": 22.482079222219525}, + "4": {"cband_hi": 3.282820949540758, "cband_lo": 2.287321847150209, "ci_hi": 3.145661878384702, "ci_lo": 2.4993118458064583, "effect": 2.7850713983454836, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.18062356836960936, "t_stat": 15.41920261837814} + }, + "event_study_df": null, + "event_study_vcov": [ + [0.010992231820659516, 0.0016075789425968922, 0.00015433328912239436, 0.00014774097999867386, -0.0004760000388592188, 0.0009080244169362376, 0.0008874077145557655, 0.0003795834449845045, 0.0014305408231968895], + [0.0016075789425968922, 0.010631262858212455, -0.004289547098481048, 0.0029615395450929365, -0.0009257175339638762, -0.0012808583554109424, -0.002300735058698825, -0.003343107718915638, 0.0013822688455910805], + [0.00015433328912239436, -0.004289547098481048, 0.007702168202558956, -0.0016355726313933405, 0.0002989414069434767, -0.00020714241665072944, 0.0005271777759488719, 0.0030431728539327754, 0.003138312682748947], + [0.00014774097999867386, 0.0029615395450929365, -0.0016355726313933405, 0.0038318703367436198, -0.0011909324235682942, -0.0011219824052899102, -0.0006501918993496128, -0.002671800912934509, -0.0027826023199812393], + [-0.0004760000388592188, -0.0009257175339638762, 0.0002989414069434767, -0.0011909324235682942, 0.0025902069372013047, 0.0008594305828633683, 0.0020409652730726284, 0.0026615563427213247, 0.002969899726819766], + [0.0009080244169362376, -0.0012808583554109424, -0.00020714241665072944, -0.0011219824052899102, 0.0008594305828633683, 0.003799320409490585, 0.0025550266311452335, 0.00397286055987585, 0.004311886563245176], + [0.0008874077145557655, -0.002300735058698825, 0.0005271777759488719, -0.0006501918993496128, 0.0020409652730726284, 0.0025550266311452335, 0.006231359859322909, 0.005471910294786851, 0.005364393907811711], + [0.0003795834449845045, -0.003343107718915638, 0.0030431728539327754, -0.002671800912934509, 0.0026615563427213247, 0.00397286055987585, 0.005471910294786851, 0.010972323862320716, 0.01063802713222148], + [0.0014305408231968895, 0.0013822688455910805, 0.003138312682748947, -0.0027826023199812393, 0.002969899726819766, 0.004311886563245176, 0.005364393907811711, 0.01063802713222148, 0.032624873450570954] + ], + "event_study_vcov_index": [-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0], + "group": { + "3": {"ci_hi": 2.4664580859435525, "ci_lo": 2.0064568111355765, "df_used": null, "effect": 2.210550101699499, "p_value": 0.0196078431372549, "se": 0.12248921625668134, "t_stat": 18.046895630936177}, + "4": {"ci_hi": 1.9654821641037383, "ci_lo": 1.6442998031178537, "df_used": null, "effect": 1.805270195115015, "p_value": 0.0196078431372549, "se": 0.08822763824970717, "t_stat": 20.461504251146682}, + "5": {"ci_hi": 2.124898005505914, "ci_lo": 1.7300458820046116, "df_used": null, "effect": 1.9435648441208038, "p_value": 0.0196078431372549, "se": 0.10818921841362256, "t_stat": 17.964496579412216}, + "6": {"ci_hi": 1.6405170282134223, "ci_lo": 1.1834676861987437, "df_used": null, "effect": 1.4341093904293774, "p_value": 0.0196078431372549, "se": 0.11957427443410792, "t_stat": 11.993460944809257} + }, + "overall": {"att": 1.926624614441337, "ci_hi": 2.0447032937774394, "ci_lo": 1.7932241220216163, "p_value": 0.0196078431372549, "se": 0.06637891367966088, "t_stat": 29.024648154669528}, + "ref_labels": [], + "reference_event_times": null + } + }, + "ContinuousDiD": { + "default_eventstudy": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.24868843189268658, "ci_lo": -0.5776295436330691, "effect": -0.23880903293700426, "p_value": 0.4666666666666667, "se": 0.2417569258465497, "t_stat": -0.9878063765940813}, + "-2": {"ci_hi": 0.020162307092176122, "ci_lo": -0.915836085693537, "effect": -0.4167762389538105, "p_value": 0.13333333333333333, "se": 0.254468135843636, "t_stat": -1.637832719495806}, + "0": {"ci_hi": 4.981169730623575, "ci_lo": 4.1360750485967746, "effect": 4.537644146658957, "p_value": 0.03225806451612903, "se": 0.24097042409197178, "t_stat": 18.830709883828163}, + "1": {"ci_hi": 4.907211255187454, "ci_lo": 4.046594982396809, "effect": 4.466341388503657, "p_value": 0.03225806451612903, "se": 0.26531715243641507, "t_stat": 16.833971522342654}, + "2": {"ci_hi": 5.380108465443674, "ci_lo": 4.45128715797091, "effect": 4.879913354176228, "p_value": 0.03225806451612903, "se": 0.2606991744649117, "t_stat": 18.71856082472187}, + "3": {"ci_hi": 4.850856481392669, "ci_lo": 3.3032538263056765, "effect": 4.010249427241352, "p_value": 0.03225806451612903, "se": 0.4513864331348498, "t_stat": 8.884293219427148} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "overall": {"att": 4.641957051061234, "ci_hi": 5.023688056681055, "ci_lo": 4.283074795160602, "p_value": 0.03225806451612903, "se": 0.22673492373259108, "t_stat": 20.473057148160873}, + "overall_acrt": 1.5251919678360484, + "overall_acrt_se": 0.16600191982346912, + "ref_labels": [], + "reference_event_times": null + }, + "default_plain": { + "cband_crit_value": null, + "es": null, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "overall": {"att": 4.641957051061234, "ci_hi": 5.023688056681055, "ci_lo": 4.283074795160602, "p_value": 0.03225806451612903, "se": 0.22673492373259108, "t_stat": 20.473057148160873}, + "overall_acrt": 1.5251919678360484, + "overall_acrt_se": 0.16600191982346912, + "ref_labels": null, + "reference_event_times": null + } + }, + "EfficientDiD": { + "alpha10_mammen_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2069653867525924, "ci_lo": 0.00894165204302786, "effect": 0.10640994510667033, "n_groups": 2.0, "p_value": 0.08, "se": 0.06323549188239748, "t_stat": 1.6827566598923078}, + "-2": {"ci_hi": 0.0644847391397063, "ci_lo": -0.20431571585166933, "effect": -0.05979558595018031, "n_groups": 2.0, "p_value": 0.48, "se": 0.09290434403313289, "t_stat": -0.6436252962386252}, + "-3": {"ci_hi": 0.19449862099655296, "ci_lo": -0.01535738133276679, "effect": 0.08653759783531222, "n_groups": 2.0, "p_value": 0.24, "se": 0.07054801635725064, "t_stat": 1.226648207897035}, + "-4": {"ci_hi": 0.07338620391720051, "ci_lo": -0.2517278572873782, "effect": -0.07878370911167488, "n_groups": 1.0, "p_value": 0.32, "se": 0.10448868717891932, "t_stat": -0.7539927167117242}, + "-5": {"ci_hi": 0.1538244991549865, "ci_lo": -0.1807234481916718, "effect": -0.028022683129665008, "n_groups": 1.0, "p_value": 0.88, "se": 0.10650636942441988, "t_stat": -0.26310804960402623}, + "0": {"ci_hi": 2.252349225994819, "ci_lo": 2.028462484038432, "effect": 2.139053682427927, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07293976731850607, "t_stat": 29.326302524209073}, + "1": {"ci_hi": 2.3606205765284227, "ci_lo": 2.06002696925978, "effect": 2.209130774601568, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08917139394334885, "t_stat": 24.773984984523654}, + "2": {"ci_hi": 2.786349853824884, "ci_lo": 2.3960295095787334, "effect": 2.5747639239316404, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.12185677690000059, "t_stat": 21.12942742646616}, + "3": {"ci_hi": 2.9255673415712624, "ci_lo": 2.6059274108171437, "effect": 2.751359710941057, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10248724900442287, "t_stat": 26.84587339077002} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.571062556876526, "ci_lo": 2.3339334073541944, "df_used": null, "effect": 2.4517840485645532, "p_value": 0.0196078431372549, "se": 0.07473191357087457, "t_stat": 32.80772472444881}, + "6": {"ci_hi": 2.2883707135526605, "ci_lo": 2.007579026967107, "df_used": null, "effect": 2.110767202972924, "p_value": 0.0196078431372549, "se": 0.0913298169742138, "t_stat": 23.111479612062304} + }, + "overall": {"att": 2.3344742536810323, "ci_hi": 2.435133409660847, "ci_lo": 2.2468867126953773, "p_value": 0.0196078431372549, "se": 0.06575894298879394, "t_stat": 35.50048324345562}, + "ref_labels": [], + "reference_event_times": null + }, + "anticipation1_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2949562897915877, "ci_lo": -0.03272754387065147, "effect": 0.10636878782275372, "n_groups": 2.0, "p_value": 0.28, "se": 0.08818178441308307, "t_stat": 1.2062444475434355}, + "-2": {"ci_hi": 0.1165576098145125, "ci_lo": -0.24358971251437297, "effect": -0.05582061177659424, "n_groups": 2.0, "p_value": 0.44, "se": 0.10322978613391538, "t_stat": -0.5407413292921159}, + "-3": {"ci_hi": 0.24843460261652606, "ci_lo": -0.03856295605404653, "effect": 0.11313385945021384, "n_groups": 2.0, "p_value": 0.12, "se": 0.08163682666623681, "t_stat": 1.3858189259701286}, + "-4": {"ci_hi": 0.11340238732434535, "ci_lo": -0.33843768992588696, "effect": -0.06989062805983277, "n_groups": 1.0, "p_value": 0.36, "se": 0.12182823496963649, "t_stat": -0.5736816927311782}, + "-5": {"ci_hi": 0.22451587535870757, "ci_lo": -0.2559681449873954, "effect": -0.01002045534608281, "n_groups": 1.0, "p_value": 1.0, "se": 0.11599128869339971, "t_stat": -0.08638972339181371}, + "0": {"ci_hi": 2.250026641919012, "ci_lo": 2.0397012675080433, "effect": 2.1545153502792065, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06045674905367468, "t_stat": 35.63730078119791}, + "1": {"ci_hi": 2.455769965763932, "ci_lo": 2.1016475084408643, "effect": 2.2643905584831225, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.10797376176398864, "t_stat": 20.97167424279128}, + "2": {"ci_hi": 2.7855272241749023, "ci_lo": 2.4196970593676044, "effect": 2.6089274980620862, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.1048655231156337, "t_stat": 24.878791623298913}, + "3": {"ci_hi": 2.9455036097149803, "ci_lo": 2.5929868072018896, "effect": 2.780266413016901, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.1091121383234863, "t_stat": 25.480816852605393} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.121746455283096, "ci_lo": 1.8688586012419277, "df_used": null, "effect": 2.0005610412926904, "p_value": 0.0196078431372549, "se": 0.06586812198376915, "t_stat": 30.372219232023305}, + "6": {"ci_hi": 1.6646739348193995, "ci_lo": 1.3195927489498467, "df_used": null, "effect": 1.4803489255998747, "p_value": 0.0196078431372549, "se": 0.09247728117866298, "t_stat": 16.007703802838783} + }, + "overall": {"att": 1.799640793016124, "ci_hi": 1.8949615449655917, "ci_lo": 1.7155665306739123, "p_value": 0.0196078431372549, "se": 0.05952551365574474, "t_stat": 30.233099766665227}, + "ref_labels": [], + "reference_event_times": null + }, + "clustered_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0769575250137137, "ci_lo": -0.29293613062148505, "effect": -0.134313461987274, "n_groups": 2.0, "p_value": 0.16, "se": 0.1043324660537642, "t_stat": -1.2873601772056276}, + "-2": {"ci_hi": 0.2646284088636564, "ci_lo": 0.015814777229796153, "effect": 0.11002324972857475, "n_groups": 2.0, "p_value": 0.04, "se": 0.07018613850266579, "t_stat": 1.5675922920933436}, + "-3": {"ci_hi": 0.2673279156349315, "ci_lo": 0.02795626875005316, "effect": 0.1436002497880893, "n_groups": 2.0, "p_value": 0.04, "se": 0.06047988466803218, "t_stat": 2.374347282179789}, + "-4": {"ci_hi": 0.0451249932540678, "ci_lo": -0.2840699228037753, "effect": -0.11660705660378556, "n_groups": 1.0, "p_value": 0.24, "se": 0.09875410149944121, "t_stat": -1.1807819101513002}, + "-5": {"ci_hi": -0.041127672281621945, "ci_lo": -0.39171971574337977, "effect": -0.2390395706166469, "n_groups": 1.0, "p_value": 0.04, "se": 0.09287400502700738, "t_stat": -2.573804915025848}, + "0": {"ci_hi": 2.181469232542371, "ci_lo": 1.9046962579883426, "effect": 2.0272407859305153, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.078864917490311, "t_stat": 25.70522927611727}, + "1": {"ci_hi": 2.356905496747323, "ci_lo": 2.123916236454315, "effect": 2.246187711623196, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.061886193380769006, "t_stat": 36.29545766053198}, + "2": {"ci_hi": 2.3895220659536145, "ci_lo": 1.9249456780507133, "effect": 2.15004585834593, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.14606900131940634, "t_stat": 14.719384940850421}, + "3": {"ci_hi": 2.8078556385784514, "ci_lo": 2.3843688726893153, "effect": 2.621212494601111, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10861079734652748, "t_stat": 24.13399550173652} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.29691690476249, "ci_lo": 2.1247494088206818, "df_used": null, "effect": 2.217677302443109, "p_value": 0.0196078431372549, "se": 0.05473518261226614, "t_stat": 40.51648677510995}, + "6": {"ci_hi": 2.337424933705111, "ci_lo": 2.06444227383698, "df_used": null, "effect": 2.2085745786428994, "p_value": 0.0196078431372549, "se": 0.07523875477035036, "t_stat": 29.354214930644247} + }, + "overall": {"att": 2.214245127895489, "ci_hi": 2.298379525627497, "ci_lo": 2.117371772658049, "p_value": 0.0196078431372549, "se": 0.052215366360726184, "t_stat": 42.40600578378656}, + "ref_labels": [], + "reference_event_times": null + }, + "covariates_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.22869389364679932, "ci_lo": -0.06421193609870868, "effect": 0.0693289316320718, "n_groups": 2.0, "p_value": 0.48, "se": 0.08470361812183318, "t_stat": 0.8184884325998064}, + "-2": {"ci_hi": 0.044318292153904655, "ci_lo": -0.2630938472786741, "effect": -0.10624754187517556, "n_groups": 2.0, "p_value": 0.16, "se": 0.07891163191389268, "t_stat": -1.346411667054503}, + "-3": {"ci_hi": 0.2036476451989844, "ci_lo": -0.04006554333396963, "effect": 0.09562758875698614, "n_groups": 2.0, "p_value": 0.12, "se": 0.06512710399536871, "t_stat": 1.4683224478058532}, + "-4": {"ci_hi": 0.1193023235589903, "ci_lo": -0.20864531566497174, "effect": -0.042290930327525664, "n_groups": 1.0, "p_value": 0.52, "se": 0.09732707945690357, "t_stat": -0.4345237786185918}, + "-5": {"ci_hi": 0.16654883810323243, "ci_lo": -0.16477263553637858, "effect": -0.014115097686006972, "n_groups": 1.0, "p_value": 1.0, "se": 0.09251980977425811, "t_stat": -0.1525629778146629}, + "0": {"ci_hi": 2.417721130488818, "ci_lo": 1.9071556181200007, "effect": 2.1858398550432816, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.12272100639659014, "t_stat": 17.81145640200695}, + "1": {"ci_hi": 2.45565560251713, "ci_lo": 1.9673809563829046, "effect": 2.215317326173932, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.132221880016068, "t_stat": 16.75454414885585}, + "2": {"ci_hi": 3.095216831831508, "ci_lo": 1.9955082568539393, "effect": 2.596118485574259, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.30047925004459836, "t_stat": 8.639926002174635}, + "3": {"ci_hi": 3.3553086285271725, "ci_lo": 2.0879563593119768, "effect": 2.7894803980786036, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.33292877179859975, "t_stat": 8.378610184421243} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.960797297588587, "ci_lo": 1.9177806667472521, "df_used": null, "effect": 2.4535373408629084, "p_value": 0.0196078431372549, "se": 0.2833965364376829, "t_stat": 8.65761230431419}, + "6": {"ci_hi": 2.7991018272683843, "ci_lo": 1.8289167409471196, "df_used": null, "effect": 2.1875189947732125, "p_value": 0.0196078431372549, "se": 0.2692205562046164, "t_stat": 8.125378780922757} + }, + "overall": {"att": 2.3620270298080532, "ci_hi": 2.611868520150393, "ci_lo": 2.067039033845001, "p_value": 0.0196078431372549, "se": 0.14718727631299472, "t_stat": 16.047766416882308}, + "ref_labels": [], + "reference_event_times": null + }, + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2950846771617092, "ci_lo": -0.03273380191629378, "effect": 0.10640994510667033, "n_groups": 2.0, "p_value": 0.28, "se": 0.08821742950973835, "t_stat": 1.2062235966071049}, + "-2": {"ci_hi": 0.1189650046330156, "ci_lo": -0.2144256270211968, "effect": -0.05979558595018031, "n_groups": 2.0, "p_value": 0.48, "se": 0.08905494890157543, "t_stat": -0.6714459632812443}, + "-3": {"ci_hi": 0.22427582199968346, "ci_lo": -0.06368449705087242, "effect": 0.08653759783531222, "n_groups": 2.0, "p_value": 0.28, "se": 0.07610892325690728, "t_stat": 1.137023020851349}, + "-4": {"ci_hi": 0.09005630431861018, "ci_lo": -0.300037559708117, "effect": -0.07878370911167488, "n_groups": 1.0, "p_value": 0.32, "se": 0.10955965253134237, "t_stat": -0.7190941856002763}, + "-5": {"ci_hi": 0.16955739764998407, "ci_lo": -0.23109205841272243, "effect": -0.028022683129665008, "n_groups": 1.0, "p_value": 0.88, "se": 0.10932803058821132, "t_stat": -0.2563174602057329}, + "0": {"ci_hi": 2.2397550957454255, "ci_lo": 2.021497183359017, "effect": 2.139053682427927, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.06192403077775023, "t_stat": 34.543191965412326}, + "1": {"ci_hi": 2.388555283000411, "ci_lo": 2.0611375558370506, "effect": 2.209130774601568, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.09429774255024284, "t_stat": 23.427186217363786}, + "2": {"ci_hi": 2.756755023497587, "ci_lo": 2.3504191325757886, "effect": 2.5747639239316404, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11182754285798713, "t_stat": 23.024416508922172}, + "3": {"ci_hi": 2.9334178487858735, "ci_lo": 2.5825189620206497, "effect": 2.751359710941057, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.11112905360609585, "t_stat": 24.75823937719681} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.5888250563955078, "ci_lo": 2.323746289846125, "df_used": null, "effect": 2.4517840485645532, "p_value": 0.0196078431372549, "se": 0.07087923642548828, "t_stat": 34.591005380567104}, + "6": {"ci_hi": 2.280987570553098, "ci_lo": 1.9645913278054095, "df_used": null, "effect": 2.110767202972924, "p_value": 0.0196078431372549, "se": 0.09779588543005005, "t_stat": 21.5833947787372} + }, + "overall": {"att": 2.3344742536810323, "ci_hi": 2.455323387619704, "ci_lo": 2.229831791181562, "p_value": 0.0196078431372549, "se": 0.06149222602398572, "t_stat": 37.96372980172234}, + "ref_labels": [], + "reference_event_times": null + }, + "default_plain_overall": {"att": 2.3344742536810323, "ci_hi": 2.455323387619704, "ci_lo": 2.229831791181562, "p_value": 0.0196078431372549, "se": 0.06149222602398572, "t_stat": 37.96372980172234}, + "fpc_survey_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2080668951510884, "ci_lo": -0.013233877441019162, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.12, "se": 0.06049144734858437, "t_stat": 1.8672174309903553}, + "-2": {"ci_hi": 0.2213478334561589, "ci_lo": -0.02478114852587132, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.24, "se": 0.07549122642190782, "t_stat": 0.9879791779750724}, + "-3": {"ci_hi": 0.12463313496574259, "ci_lo": -0.14211839192105763, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.92, "se": 0.07182468884336507, "t_stat": -0.15875582269100671}, + "-4": {"ci_hi": 0.14522912159412443, "ci_lo": -0.1796174496115894, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 1.0, "se": 0.09490407338283532, "t_stat": -0.2009042432886477}, + "-5": {"ci_hi": -0.027699444417426694, "ci_lo": -0.401554211356682, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.04, "se": 0.1114273305151771, "t_stat": -1.7472542035462537}, + "0": {"ci_hi": 2.1299191962105595, "ci_lo": 1.9526175242434007, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.046257362293041754, "t_stat": 44.071287240755865}, + "1": {"ci_hi": 2.2438057468317743, "ci_lo": 1.9419279285817606, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.09038666642982485, "t_stat": 23.27647885776168}, + "2": {"ci_hi": 2.709613408523424, "ci_lo": 2.3611436845330642, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.101589730491553, "t_stat": 24.633304984227838}, + "3": {"ci_hi": 2.642733028186197, "ci_lo": 2.149379949465344, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.13837582575695007, "t_stat": 17.40623033510585} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.367998933184762, "ci_lo": 2.1755528896298806, "df_used": null, "effect": 2.267851180304006, "p_value": 0.0196078431372549, "se": 0.05374577284541157, "t_stat": 42.19589858400592}, + "6": {"ci_hi": 2.1719747586846045, "ci_lo": 1.955613462138116, "df_used": null, "effect": 2.0624126822653266, "p_value": 0.0196078431372549, "se": 0.054029961584698215, "t_stat": 38.17164813327243} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.2855534394791994, "ci_lo": 2.1042555276218216, "p_value": 0.0196078431372549, "se": 0.050056493921646114, "t_stat": 43.93119618777691}, + "ref_labels": [], + "reference_event_times": null + }, + "fractional_balance_e1_es": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2777064924348559, "ci_lo": -0.3165727867106564, "effect": 0.026184943804410075, "n_groups": 1.0, "p_value": 0.72, "se": 0.14117302462526216, "t_stat": 0.1854812126744249}, + "0": {"ci_hi": 0.5053580355957054, "ci_lo": 0.04843947527034796, "effect": 0.24480370561350984, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.11921519605698168, "t_stat": 2.0534605797779353}, + "1": {"ci_hi": 1.0047566550455853, "ci_lo": 0.34004991430453557, "effect": 0.6358462518585993, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.20053945804803613, "t_stat": 3.170679017723745}, + "2": {"ci_hi": 1.4023682848122243, "ci_lo": 0.25916510539259974, "effect": 0.8040757046489879, "n_groups": 1.0, "p_value": 0.04, "se": 0.3079850596961206, "t_stat": 2.6107620461925807} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "ref_labels": [], + "reference_event_times": null + }, + "stratified_psu_survey_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.2341681818279641, "ci_lo": -0.012605995579570668, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.12, "se": 0.07382329655029328, "t_stat": 1.530014103856262}, + "-2": {"ci_hi": 0.245876271222739, "ci_lo": -0.0681235255341213, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.32, "se": 0.08444394222328314, "t_stat": 0.8832339876723816}, + "-3": {"ci_hi": 0.1285795168194135, "ci_lo": -0.13635567927805417, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.8, "se": 0.06358199239982865, "t_stat": -0.17933674514554412}, + "-4": {"ci_hi": 0.14356637440823122, "ci_lo": -0.15041804226742786, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 0.76, "se": 0.08503290448100671, "t_stat": -0.224226505778685}, + "-5": {"ci_hi": 0.044372534115678805, "ci_lo": -0.3867279606592377, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.12, "se": 0.12606335987951608, "t_stat": -1.5443969748121575}, + "0": {"ci_hi": 2.1537261129444243, "ci_lo": 1.9388589224111425, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.059276136953055, "t_stat": 34.39194261648464}, + "1": {"ci_hi": 2.2583816766098894, "ci_lo": 1.9380914210633695, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.08633551187403303, "t_stat": 24.36869006170978}, + "2": {"ci_hi": 2.705681821089329, "ci_lo": 2.3565597898157056, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10069712926475424, "t_stat": 24.851659950348264}, + "3": {"ci_hi": 2.609072967054054, "ci_lo": 2.2184296329305195, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.10884064212889258, "t_stat": 22.1296149014226} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3752330135384976, "ci_lo": 2.158980257271318, "df_used": null, "effect": 2.267851180304006, "p_value": 0.0196078431372549, "se": 0.056314729245228005, "t_stat": 40.27101276521149}, + "6": {"ci_hi": 2.1375228715477235, "ci_lo": 1.9471671672818898, "df_used": null, "effect": 2.0624126822653266, "p_value": 0.0196078431372549, "se": 0.05147453168670459, "t_stat": 40.066662380107275} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.2961195691978564, "ci_lo": 2.1074255353198432, "p_value": 0.0196078431372549, "se": 0.04788565840007229, "t_stat": 45.92276118606689}, + "ref_labels": [], + "reference_event_times": null + }, + "survey_weights_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.24981359224310373, "ci_lo": -0.030781329527225308, "effect": 0.11295068491511205, "n_groups": 2.0, "p_value": 0.08, "se": 0.07832729282730308, "t_stat": 1.442034836620066}, + "-2": {"ci_hi": 0.20845260426605097, "ci_lo": -0.07588254636234743, "effect": 0.07458375982464656, "n_groups": 2.0, "p_value": 0.4, "se": 0.07701222714392936, "t_stat": 0.9684664707236138}, + "-3": {"ci_hi": 0.12831232059427325, "ci_lo": -0.1508515979629058, "effect": -0.011402587566853993, "n_groups": 2.0, "p_value": 0.72, "se": 0.06781076188021756, "t_stat": -0.16815306672111688}, + "-4": {"ci_hi": 0.14163107640343256, "ci_lo": -0.22976314548174362, "effect": -0.019066631047988822, "n_groups": 1.0, "p_value": 0.92, "se": 0.10694078013209038, "t_stat": -0.17829149015406687}, + "-5": {"ci_hi": 0.0383954617068058, "ci_lo": -0.4107733308581979, "effect": -0.19469187163258095, "n_groups": 1.0, "p_value": 0.08, "se": 0.12065722044259501, "t_stat": -1.6135948674966314}, + "0": {"ci_hi": 2.194297955513491, "ci_lo": 1.9085141138216501, "effect": 2.0386215006163524, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.0720997774366947, "t_stat": 28.275004072048766}, + "1": {"ci_hi": 2.2336126970295407, "ci_lo": 1.9587053965594128, "effect": 2.1038833301773754, "n_groups": 2.0, "p_value": 0.0196078431372549, "se": 0.07867884018729762, "t_stat": 26.740141633621064}, + "2": {"ci_hi": 2.739783179581819, "ci_lo": 2.300041840945201, "effect": 2.5024908144639353, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.106637540713229, "t_stat": 23.46725925716596}, + "3": {"ci_hi": 2.641148083707181, "ci_lo": 2.1651279087896564, "effect": 2.4086014959359456, "n_groups": 1.0, "p_value": 0.0196078431372549, "se": 0.13476397816153352, "t_stat": 17.87273965041978} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3913814803057063, "ci_lo": 2.1511502302633945, "df_used": null, "effect": 2.267851180304006, "p_value": 0.0196078431372549, "se": 0.06878951579779898, "t_stat": 32.967977082004246}, + "6": {"ci_hi": 2.2246075931989067, "ci_lo": 1.925620434041747, "df_used": null, "effect": 2.0624126822653266, "p_value": 0.0196078431372549, "se": 0.08507845584431388, "t_stat": 24.24130365082508} + }, + "overall": {"att": 2.199041654944098, "ci_hi": 2.307007867079546, "ci_lo": 2.1001498623329025, "p_value": 0.0196078431372549, "se": 0.06172586389268372, "t_stat": 35.625935649395544}, + "ref_labels": [], + "reference_event_times": null + } + }, + "ImputationDiD": { + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.244194579906093, "ci_lo": 2.0759929420176784, "effect": 2.145940363587726, "n_obs": 84.0, "p_value": 0.05, "se": 0.05530889329763085, "t_stat": 38.79919187751398}, + "1": {"ci_hi": 2.3576847763043283, "ci_lo": 2.1606791601654245, "effect": 2.242441654342141, "n_obs": 84.0, "p_value": 0.05, "se": 0.060823150057492385, "t_stat": 36.86822619713873}, + "2": {"ci_hi": 2.7879575301476263, "ci_lo": 2.474642715050153, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 0.05, "se": 0.09662278671372519, "t_stat": 26.89084989270778}, + "3": {"ci_hi": 2.8626691734943193, "ci_lo": 2.598801947853138, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 0.05, "se": 0.09451615379064816, "t_stat": 29.269392221046104} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.5409735465284364, "ci_lo": 2.378212505064488, "df_used": null, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 0.05, "se": 0.05575674613997778, "t_stat": 44.13550260439243}, + "6": {"ci_hi": 2.3603208389178514, "ci_lo": 2.027066634034936, "df_used": null, "effect": 2.1511282275930954, "n_obs": 86.0, "p_value": 0.05, "se": 0.09332024816422878, "t_stat": 23.05103415292523} + }, + "overall": {"att": 2.3543070317865995, "ci_hi": 2.4498889778888646, "ci_lo": 2.291396781980644, "p_value": 0.05, "se": 0.04875538870512177, "t_stat": 48.288139922865156}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_plain_overall": {"att": 2.3543070317865995, "ci_hi": 2.4498889778888646, "ci_lo": 2.291396781980644, "p_value": 0.05, "se": 0.04875538870512177, "t_stat": 48.288139922865156}, + "pretrends_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.05981516395757938, "ci_lo": -0.3559461519303688, "effect": -0.1480654939863947, "n_obs": 84.0, "p_value": 0.16236817593507333, "se": 0.10584141425966366, "t_stat": -1.3989372215222062}, + "-3": {"ci_hi": 0.1584251692403736, "ci_lo": -0.20715293281084962, "effect": -0.024363881785238024, "n_obs": 84.0, "p_value": 0.79357582275096, "se": 0.09306614604301798, "t_stat": -0.26179102521314573}, + "-4": {"ci_hi": 0.06957272135423231, "ci_lo": -0.34423198704465807, "effect": -0.13732963284521288, "n_obs": 84.0, "p_value": 0.1928757534767246, "se": 0.10534331572120142, "t_stat": -1.3036387919349863}, + "-5": {"ci_hi": 0.15677504775855503, "ci_lo": -0.4306558981578147, "effect": -0.13694042519962982, "n_obs": 43.0, "p_value": 0.36019499313404013, "se": 0.14954378803351012, "t_stat": -0.9157212546263966}, + "-6": {"ci_hi": 0.07789021576946761, "ci_lo": -0.47058081586467804, "effect": -0.19634530004760523, "n_obs": 43.0, "p_value": 0.16019426463331468, "se": 0.1396256636927232, "t_stat": -1.406226440432226}, + "0": {"ci_hi": 2.244194579906093, "ci_lo": 2.0759929420176784, "effect": 2.145940363587726, "n_obs": 84.0, "p_value": 0.05, "se": 0.05530889329763085, "t_stat": 38.79919187751398}, + "1": {"ci_hi": 2.3576847763043283, "ci_lo": 2.1606791601654245, "effect": 2.242441654342141, "n_obs": 84.0, "p_value": 0.05, "se": 0.060823150057492385, "t_stat": 36.86822619713873}, + "2": {"ci_hi": 2.7879575301476263, "ci_lo": 2.474642715050153, "effect": 2.598268853733904, "n_obs": 41.0, "p_value": 0.05, "se": 0.09662278671372519, "t_stat": 26.89084989270778}, + "3": {"ci_hi": 2.8626691734943193, "ci_lo": 2.598801947853138, "effect": 2.7664303765231946, "n_obs": 41.0, "p_value": 0.05, "se": 0.09451615379064816, "t_stat": 29.269392221046104} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.5409735465284364, "ci_lo": 2.378212505064488, "df_used": null, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 0.05, "se": 0.05575674613997778, "t_stat": 44.13550260439243}, + "6": {"ci_hi": 2.3603208389178514, "ci_lo": 2.027066634034936, "df_used": null, "effect": 2.1511282275930954, "n_obs": 86.0, "p_value": 0.05, "se": 0.09332024816422878, "t_stat": 23.05103415292523} + }, + "overall": {"att": 2.3543070317865995, "ci_hi": 2.4498889778888646, "ci_lo": 2.291396781980644, "p_value": 0.05, "se": 0.04875538870512177, "t_stat": 48.288139922865156}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "pretrends_plain_overall": {"att": 2.3543070317865995, "ci_hi": 2.4498889778888646, "ci_lo": 2.291396781980644, "p_value": 0.05, "se": 0.04875538870512177, "t_stat": 48.288139922865156}, + "survey_tsl_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.220784930763723, "ci_lo": 1.9227633105976678, "effect": 2.0361973460553426, "n_obs": 84.0, "p_value": 0.05, "se": 0.08990414546895434, "t_stat": 22.64853678808928}, + "1": {"ci_hi": 2.232753678345635, "ci_lo": 2.005171985889536, "effect": 2.1281777081775473, "n_obs": 84.0, "p_value": 0.05, "se": 0.06525565374864616, "t_stat": 32.61292448888691}, + "2": {"ci_hi": 2.7159073842435433, "ci_lo": 2.2524410941180917, "effect": 2.4900024320750482, "n_obs": 46.0, "p_value": 0.05, "se": 0.13297834548487006, "t_stat": 18.72487150442366}, + "3": {"ci_hi": 2.7088409830699383, "ci_lo": 2.151233032203804, "effect": 2.4352726594598915, "n_obs": 46.0, "p_value": 0.05, "se": 0.16212690788794695, "t_stat": 15.02078027136011} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.3334490124802647, "ci_lo": 2.1252162505371825, "df_used": null, "effect": 2.2443534854455107, "n_obs": 184.0, "p_value": 0.05, "se": 0.06379681143793288, "t_stat": 35.17971251006822}, + "6": {"ci_hi": 2.3556685282151895, "ci_lo": 1.987448116972624, "df_used": null, "effect": 2.1379018964851286, "n_obs": 76.0, "p_value": 0.05, "se": 0.10055826903197541, "t_stat": 21.260329131215663} + }, + "overall": {"att": 2.2086986136463955, "ci_hi": 2.323535613491389, "ci_lo": 2.120735084887383, "p_value": 0.05, "se": 0.05917362183752092, "t_stat": 37.32572969271757}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "survey_tsl_plain_overall": {"att": 2.2086986136463955, "ci_hi": 2.323535613491389, "ci_lo": 2.120735084887383, "p_value": 0.05, "se": 0.05917362183752092, "t_stat": 37.32572969271757} + }, + "TwoStageDiD": { + "default_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.215861652531097, "ci_lo": 2.047714596195685, "effect": 2.1459403635877266, "n_obs": 84.0, "p_value": 0.05, "se": 0.055304727578956686, "t_stat": 38.80211434951994}, + "1": {"ci_hi": 2.321940670314704, "ci_lo": 2.1294141287480315, "effect": 2.2424416543421417, "n_obs": 84.0, "p_value": 0.05, "se": 0.061290033810934993, "t_stat": 36.5873783209116}, + "2": {"ci_hi": 2.7218949924181612, "ci_lo": 2.408580177320143, "effect": 2.5982688537339036, "n_obs": 41.0, "p_value": 0.05, "se": 0.0966227867138525, "t_stat": 26.89084989267235}, + "3": {"ci_hi": 2.9340588051934278, "ci_lo": 2.6701915795511035, "effect": 2.7664303765231955, "n_obs": 41.0, "p_value": 0.05, "se": 0.09451615379091943, "t_stat": 29.269392220962107} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.543491523882159, "ci_lo": 2.3807304824183935, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 0.05, "se": 0.05575674613993602, "t_stat": 44.13550260442548}, + "6": {"ci_hi": 2.2751898211508865, "ci_lo": 1.9419356162672947, "effect": 2.151128227593096, "n_obs": 86.0, "p_value": 0.05, "se": 0.09332024816436411, "t_stat": 23.051034152891805} + }, + "overall": {"att": 2.3543070317865986, "ci_hi": 2.4137285528561563, "ci_lo": 2.2737514743334115, "p_value": 0.05, "se": 0.04883129244003266, "t_stat": 48.21308046838631}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "default_plain_overall": {"att": 2.3543070317865986, "ci_hi": 2.4137285528561563, "ci_lo": 2.2737514743334115, "p_value": 0.05, "se": 0.04883129244003266, "t_stat": 48.21308046838631}, + "pretrends_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "-2": {"ci_hi": 0.04085242646287475, "ci_lo": -0.09948469861005711, "effect": -0.029856983716645812, "n_obs": 84.0, "p_value": 0.42105263157894735, "se": 0.04090615773293422, "t_stat": -0.729889712731623}, + "-3": {"ci_hi": 0.07969999083423256, "ci_lo": -0.03999085796309322, "effect": 0.034596864567728876, "n_obs": 84.0, "p_value": 0.631578947368421, "se": 0.036877661987001153, "t_stat": 0.9381523313469214}, + "-4": {"ci_hi": 0.063284196345179, "ci_lo": -0.09643479922002418, "effect": -0.002521727613600901, "n_obs": 84.0, "p_value": 0.8421052631578947, "se": 0.04654795670678167, "t_stat": -0.054174829401985446}, + "-5": {"ci_hi": 0.07196494227817381, "ci_lo": -0.11758544269811656, "effect": -0.034152641587387955, "n_obs": 43.0, "p_value": 0.8421052631578947, "se": 0.05419133038898293, "t_stat": -0.6302233464696627}, + "-6": {"ci_hi": 0.07262749167373701, "ci_lo": -0.10048039306864671, "effect": -0.03367413800267828, "n_obs": 43.0, "p_value": 0.631578947368421, "se": 0.05355222200691118, "t_stat": -0.6288093517078052}, + "0": {"ci_hi": 2.215861652531097, "ci_lo": 2.047714596195685, "effect": 2.1459403635877266, "n_obs": 84.0, "p_value": 0.05, "se": 0.055304727578956686, "t_stat": 38.80211434951994}, + "1": {"ci_hi": 2.321940670314704, "ci_lo": 2.1294141287480315, "effect": 2.2424416543421417, "n_obs": 84.0, "p_value": 0.05, "se": 0.06129003381093498, "t_stat": 36.587378320911604}, + "2": {"ci_hi": 2.7218949924181612, "ci_lo": 2.408580177320143, "effect": 2.5982688537339036, "n_obs": 41.0, "p_value": 0.05, "se": 0.0966227867138525, "t_stat": 26.89084989267235}, + "3": {"ci_hi": 2.9340588051934278, "ci_lo": 2.6701915795511035, "effect": 2.7664303765231955, "n_obs": 41.0, "p_value": 0.05, "se": 0.09451615379091943, "t_stat": 29.269392220962107} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.543491523882159, "ci_lo": 2.3807304824183935, "effect": 2.4608520144734367, "n_obs": 164.0, "p_value": 0.05, "se": 0.05575674613993602, "t_stat": 44.13550260442548}, + "6": {"ci_hi": 2.2751898211508865, "ci_lo": 1.9419356162672947, "effect": 2.151128227593096, "n_obs": 86.0, "p_value": 0.05, "se": 0.09332024816436411, "t_stat": 23.051034152891805} + }, + "overall": {"att": 2.3543070317865986, "ci_hi": 2.4137285528561563, "ci_lo": 2.2737514743334115, "p_value": 0.05, "se": 0.04883129244003266, "t_stat": 48.21308046838631}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "prop5_all": { + "cband_crit_value": null, + "es": { + "-1": {"ci_hi": 0.0, "ci_lo": 0.0, "effect": 0.0, "n_obs": 0.0, "p_value": null, "se": 0.0, "t_stat": null}, + "0": {"ci_hi": 2.071166212018816, "ci_lo": 1.7061026470953098, "effect": 1.9034726388790313, "n_obs": 36.0, "p_value": 0.05, "se": 0.10828021901063675, "t_stat": 17.579135471567955}, + "1": {"ci_hi": 2.4653739293382753, "ci_lo": 2.162460266870133, "effect": 2.3023388848323068, "n_obs": 36.0, "p_value": 0.05, "se": 0.0937291628451595, "t_stat": 24.563741048618652}, + "2": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 36.0, "p_value": null, "se": null, "t_stat": null}, + "3": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 36.0, "p_value": null, "se": null, "t_stat": null} + }, + "event_study_df": null, + "event_study_vcov": null, + "event_study_vcov_index": null, + "group": { + "4": {"ci_hi": 2.2241133871066556, "ci_lo": 1.965041675328376, "effect": 2.1029057618556686, "n_obs": 72.0, "p_value": 0.05, "se": 0.07944228111318641, "t_stat": 26.47086327820228}, + "6": {"ci_hi": null, "ci_lo": null, "effect": null, "n_obs": 0.0, "p_value": null, "se": null, "t_stat": null} + }, + "overall": {"att": 2.1029057618556677, "ci_hi": 2.2241133871066547, "ci_lo": 1.9650416753283753, "p_value": 0.05, "se": 0.07944228111318632, "t_stat": 26.470863278202295}, + "ref_labels": ["-1"], + "reference_event_times": null + }, + "prop5_plain_overall": {"att": 2.1029057618556677, "ci_hi": 2.2241133871066547, "ci_lo": 1.9650416753283753, "p_value": 0.05, "se": 0.07944228111318632, "t_stat": 26.470863278202295} + } + } + }, + "provenance": { + "backends": ["numpy", "rust"], + "captured_at_head": "8724fecd98eca407e862c0c0adacb212bc964f16", + "captured_with": "tests/_capture_v4_aggregate_oracles.py, once per DIFF_DIFF_BACKEND (python -> 'numpy', rust -> 'rust'), on a tree whose diff_diff/ is byte-identical to captured_at_head (the 4.0 PR 3a branch: tests/benchmarks only); the analytic sections of the two runs agreed to rtol 1e-12 / atol 1e-13 and are stored once. Every arm of the original capture reproduced exactly (0 differences); this re-capture added ref_labels (reference-row membership) to every event-study bundle, the CS bootstrap arms fpc_survey_all / rcs_all / unbalanced_all nine ContinuousDiD analytic design arms, the EfficientDiD replicate analytic arms and the EfficientDiD covariates / anticipation / alpha-Mammen bootstrap arms. A further re-capture (same tree, same checks, 0 differences) added the TwoStageDiD clustered_all analytic arm.", + "diff_diff_version": "3.13.0", + "numpy": "2.5.4" + } +} diff --git a/tests/helpers/legacy_surface.py b/tests/helpers/legacy_surface.py new file mode 100644 index 000000000..134398550 --- /dev/null +++ b/tests/helpers/legacy_surface.py @@ -0,0 +1,180 @@ +"""Test helpers bridging the post-fit aggregation containers and the 3.x dict shapes. + +The post-fit ``results.aggregate(...)`` container is the canonical route (the 3.9 +fit-time ``fit(aggregate=...)`` keyword is deprecated and removed at the 4.0 cut), so a +test that used to read ``results.event_study_effects[h]["effect"]`` after a +fit-time aggregation reads the post-fit container instead:: + + es = results.aggregate("event_study") # EventStudyResults + grp = results.aggregate("group") # AggregationResult + +Three helpers keep dict-iteration tests short and let the few tests whose +SUBJECT is the native (legacy-pickle) route build a faithful stand-in: + +``es_dict(results, balance_e=None)`` + ``aggregate("event_study")`` as the legacy ``event_study_effects`` mapping + ``{event_time: {"effect", "se", "t_stat", "p_value", "conf_int", }}``. + The count key follows the container's ``n_kind`` (``"groups"`` -> ``n_groups``, + ``"obs"`` -> ``n_obs``, ...); a NaN count becomes 0 (the legacy zero-count + reference sentinel); when the container has NO ``n_kind`` (ContinuousDiD) no + count key is emitted - ``_from_relative_dict`` rejects a zero-count row with a + finite nonzero effect. ``cband_conf_int`` is added where the band is finite. +``group_dict(results)`` + ``aggregate("group")`` as the legacy ``group_effects`` mapping (label keys; + ``df_used`` from the ``df`` column). +``legacy_standin(results, balance_e=None)`` + A COPY of ``results`` whose legacy fields hold the container's surface + (``event_study_effects`` + ``event_study_vcov``/``_index`` + the surface's + ``reference_event_times`` + ``cband_crit_value`` + ``event_study_df``), i.e. + what a 3.x ``fit(aggregate="event_study", balance_e=...)`` stored. Use it ONLY + where a test exercises the native route that plain fits do not + populate (plotter / HonestDiD / PreTrendsPower raw-field branches, + ``build_event_study_surface(native)`` fixtures). + +``load_oracles()`` / ``boot_oracles()`` load ``tests/fixtures/v4_aggregate_oracles.json`` +(see ``tests/fixtures/README.md``); ``boot_oracles`` returns the active weight +backend's section or skips the test when that backend was never captured. +""" + +from __future__ import annotations + +import dataclasses +import json +import math +import os +from functools import lru_cache +from typing import Any, Dict, Optional + +import numpy as np +import pytest + +_COUNT_KEY_BY_N_KIND = { + "groups": "n_groups", + "obs": "n_obs", + "units": "n_units", + "cells": "n_cells", + "switcher_cells": "n_switcher_cells", + "clusters": "n_clusters", +} + +_FIXTURE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "fixtures") + + +def _label(x: Any) -> Any: + """Container labels back to the legacy dict-key type (int where integral).""" + if isinstance(x, (np.integer, int)): + return int(x) + if isinstance(x, (np.floating, float)): + f = float(x) + return int(f) if f.is_integer() else f + return x + + +def _count_key(n_kind: Optional[str]) -> Optional[str]: + if n_kind is None: + return None + return _COUNT_KEY_BY_N_KIND.get(n_kind, f"n_{n_kind}") + + +def es_dict(results: Any, *, balance_e: Optional[int] = None) -> Dict[Any, Dict[str, Any]]: + es = results.aggregate("event_study", balance_e=balance_e) + return es_dict_from_container(es) + + +def es_dict_from_container(es: Any) -> Dict[Any, Dict[str, Any]]: + """The legacy ``event_study_effects`` shape of an ``EventStudyResults``.""" + key = _count_key(es.n_kind) + out: Dict[Any, Dict[str, Any]] = {} + for i, e in enumerate(es.event_time): + row: Dict[str, Any] = { + "effect": float(es.att[i]), + "se": float(es.se[i]), + "t_stat": float(es.t_stat[i]), + "p_value": float(es.p_value[i]), + "conf_int": (float(es.conf_int_lower[i]), float(es.conf_int_upper[i])), + } + if key is not None: + n = float(es.n[i]) + row[key] = 0 if math.isnan(n) else (int(n) if float(n).is_integer() else n) + if es.cband_lower is not None and es.cband_upper is not None: + lo, hi = float(es.cband_lower[i]), float(es.cband_upper[i]) + if math.isfinite(lo) and math.isfinite(hi): + row["cband_conf_int"] = (lo, hi) + out[_label(e)] = row + return out + + +def group_dict(results: Any) -> Dict[Any, Dict[str, Any]]: + """``aggregate("group")`` as the legacy ``group_effects`` mapping.""" + grp = results.aggregate("group") + key = _count_key(grp.n_kind) + out: Dict[Any, Dict[str, Any]] = {} + for i, g in enumerate(grp.label): + row: Dict[str, Any] = { + "effect": float(grp.att[i]), + "se": float(grp.se[i]), + "t_stat": float(grp.t_stat[i]), + "p_value": float(grp.p_value[i]), + "conf_int": (float(grp.conf_int_lower[i]), float(grp.conf_int_upper[i])), + } + if key is not None: + n = float(grp.n[i]) + row[key] = 0 if math.isnan(n) else (int(n) if float(n).is_integer() else n) + df = float(grp.df[i]) + row["df_used"] = None if math.isnan(df) else df + out[_label(g)] = row + return out + + +def legacy_standin(results: Any, *, balance_e: Optional[int] = None) -> Any: + """A copy of ``results`` carrying the container's surface in the 3.x fit-time fields.""" + es = results.aggregate("event_study", balance_e=balance_e) + names = {f.name for f in dataclasses.fields(results)} + updates: Dict[str, Any] = {} + if "event_study_effects" in names: + updates["event_study_effects"] = es_dict_from_container(es) + if "event_study_vcov" in names: + updates["event_study_vcov"] = None if es.vcov is None else np.asarray(es.vcov).copy() + if "event_study_vcov_index" in names: + # The 3.x field is a plain list of event-time labels (consumers call + # ``.index(t)`` on it), not an array. + updates["event_study_vcov_index"] = ( + None if es.vcov_index is None else [_label(x) for x in es.vcov_index] + ) + if "reference_event_times" in names: + updates["reference_event_times"] = es.reference_event_times + if "cband_crit_value" in names: + updates["cband_crit_value"] = es.cband_crit_value + if "event_study_df" in names: + df = es.df + if df is None: + updates["event_study_df"] = None + else: + arr = np.asarray(df, dtype=float).ravel() + finite = arr[np.isfinite(arr)] + updates["event_study_df"] = ( + float(finite[0]) if finite.size and np.all(finite == finite[0]) else None + ) + return dataclasses.replace(results, **updates) + + +@lru_cache(maxsize=1) +def load_oracles() -> Dict[str, Any]: + with open(os.path.join(_FIXTURE, "v4_aggregate_oracles.json")) as fh: + return json.load(fh) + + +def boot_oracles() -> Dict[str, Any]: + """The bootstrapped oracles for the ACTIVE weight backend, or skip.""" + from diff_diff.bootstrap_chunking import effective_weight_backend + + backend = effective_weight_backend() + boot = load_oracles()["boot"] + if backend not in boot: + pytest.skip(f"no bootstrapped aggregation oracle captured for weight backend {backend!r}") + return boot[backend] + + +def oracle_value(x: Any) -> float: + """JSON ``null`` back to NaN.""" + return float("nan") if x is None else float(x) diff --git a/tests/test_aggregate_contract.py b/tests/test_aggregate_contract.py index 700672237..56c43b499 100644 --- a/tests/test_aggregate_contract.py +++ b/tests/test_aggregate_contract.py @@ -2,31 +2,372 @@ The ``test_ref`` for every aggregate-postfit ledger row this file pins: M-020/M-023/M-021/M-022 (the CS / EfficientDiD / Imputation / TwoStage -``fit(aggregate=)`` shims), M-024/M-026 (the Stacked / dCDH shims + view -relays), M-025 (the ContinuousDiD shim + MIXED view/recompute aggregate() - -'simple'/'dose' views, 'event_study' pruned-IF-payload kit), -M-117/M-120/M-118/M-119 (``balance_e`` moves onto ``aggregate()``), -M-027/M-139 (the HAD fit + pretest-workflow mode-selector shims with -panel-shape inference, and the two PURE-VIEW results classes) -and M-122 (``AggregationResult``). - -The headline gate is NUMERICAL INERTNESS: for every supported type, -``fit(aggregate=T)`` and ``fit(); .aggregate(T)`` must agree to 1e-14. The -refactor that enabled post-fit aggregation touched the influence-function -path, so drift here means a regression, not a design change. +fit-time ``aggregate=`` shims, deprecated since 3.9), M-024/M-026 (the Stacked / +dCDH shims + view relays), M-025 (the ContinuousDiD shim + MIXED +view/recompute aggregate() - 'simple'/'dose' views, 'event_study' +pruned-IF-payload kit on analytic fits), M-117/M-120/M-118/M-119 (``balance_e`` +lives on ``aggregate()`` only), M-027/M-139 (the HAD fit + pretest-workflow +mode selectors with panel-shape inference, and the two PURE-VIEW results +classes) and M-122 (``AggregationResult``). + +The headline gate is ORACLE PARITY: the deprecated fit-time route is scheduled +for removal at the 4.0 cut (PRs 3b/3c), so it no longer serves as this file's +numerical reference; every aggregation surface is pinned against +``tests/fixtures/v4_aggregate_oracles.json`` - captured on the unmodified tree by +``tests/_capture_v4_aggregate_oracles.py`` (see ``tests/fixtures/README.md``). +Analytic surfaces must match at ``rtol=1e-12, atol=1e-13``; bootstrapped +surfaces (the CS/EfficientDiD replays and the plain-fit overall quintets) match +the backend-matched oracle at the replay-parity tolerances (``rtol=1e-13, +atol=1e-13`` on att/se/t/CI, ``atol=2/n_bootstrap`` on the count-statistic +p-values). Drift here means a regression, not a design change. + +The ``Test*Shim`` classes pin the deprecated fit-time keywords' 3.9 FutureWarning +behaviour until the removals land (CS/EfficientDiD/Stacked/dCDH/HAD in PR 3b; +Imputation/TwoStage/ContinuousDiD, together with the bootstrapped-fit relay, in +PR 3c). """ import copy +import dataclasses +import inspect +import math import pickle import warnings import numpy as np import pandas as pd import pytest +from legacy_surface import boot_oracles, load_oracles, oracle_value from diff_diff import CallawaySantAnna, StaggeredTripleDifference from diff_diff.aggregation import AGGREGATION_SCHEMA, AggregationResult -from diff_diff.results_base import EventStudyResults +from diff_diff.results_base import EventStudyResults, build_event_study_surface +from diff_diff.utils import safe_inference +from tests.conftest import assert_nan_inference + +# --------------------------------------------------------------------------- # +# Removal-pin helper (shared wording with tests/test_v4_rename_mechanical.py) +# --------------------------------------------------------------------------- # + + +# --------------------------------------------------------------------------- # +# Oracle comparison helpers (tests/fixtures/v4_aggregate_oracles.json) +# --------------------------------------------------------------------------- # + +#: Analytic surfaces: the kit recompute re-executes the fit-time arithmetic, +#: so only BLAS summation-order noise separates it from the oracle. +ANALYTIC_TOL = dict(rtol=1e-12, atol=1e-13) +#: Confidence-interval bounds additionally cross scipy's ``t.ppf``: its +#: quantile inverter moved between scipy 1.13 (the Python 3.9 CI leg) and +#: 1.14+, shifting the two-sided critical value by up to 3.4e-11 relative for +#: df in the 100-130 range (the survey arms), while ``t.sf`` (p-values) agrees +#: to 1e-15. att/se/t_stat/p_value keep the arithmetic tolerance; CI bounds use +#: at least this rtol. +CI_RTOL = 1e-9 +#: Bootstrapped surfaces (replays + relays): the replayed weight stream is +#: bit-identical and only GEMM tile reassociation differs (~1 ULP), with one +#: order of headroom for quantile interpolation. p-values are COUNT statistics +#: (a draw within a ULP of the point estimate can flip one count), hence the +#: additional ``atol=2/n_bootstrap`` applied by the callers. +BOOT_TOL = dict(rtol=1e-13, atol=1e-13) + + +def _oracle_key(label): + """The oracle's stringified label for a container label (int / float).""" + if isinstance(label, str): + return label + f = float(label) + return str(int(f)) if f.is_integer() else repr(f) + + +def _oracle_row(rows, label): + """Oracle row for a container label, tolerating the int/float spellings.""" + key = _oracle_key(label) + if key in rows: + return rows[key] + alt = repr(float(label)) + if alt in rows: + return rows[alt] + raise AssertionError(f"label {label!r} ({key}) has no oracle row; oracle keys: {sorted(rows)}") + + +def _close(got, want, *, rtol, atol, what): + g, w = float(got), oracle_value(want) + if math.isnan(w) or math.isnan(g): + assert math.isnan(w) and math.isnan(g), f"{what}: got {g}, oracle {w}" + return + np.testing.assert_allclose(g, w, rtol=rtol, atol=atol, err_msg=what) + + +def _assert_es_matches_oracle(es, orow, *, rtol, atol, p_atol=None, name=""): + """An ``EventStudyResults`` against an oracle ES bundle. + + Non-reference rows are compared column-for-column (att/se/t/p/CI and the + count where the oracle row carries one); the container's reference rows + (normalized to att=0 / NaN inference) are skipped but their oracle entry + (the legacy zero-count sentinel, or a genuine estimate the container + anchors) must exist. The joint vcov, its index, the scalar df and the + sup-t band are compared wherever the oracle bundle recorded them. + """ + rows = orow["es"] + assert rows is not None, f"{name}: oracle stored no event-study surface" + p_atol = atol if p_atol is None else p_atol + ci_rtol = max(rtol, CI_RTOL) + ref = np.asarray(es.is_reference, dtype=bool) + labels = list(es.event_time) + non_ref = {_oracle_key(e) for e, r in zip(labels, ref) if not r} + ref_keys = {_oracle_key(e) for e, r in zip(labels, ref) if r} + # Reference MEMBERSHIP is pinned by the oracle, not read off the container: + # a surface that marks an estimated horizon as reference must FAIL here + # rather than have that row skipped below. + expected_ref = orow.get("ref_labels") + if expected_ref is not None: + assert ref_keys == set(expected_ref), ( + f"{name}: reference set drifted: container {sorted(ref_keys)} vs oracle " + f"{sorted(expected_ref)}" + ) + assert set(rows) - ref_keys == non_ref, ( + f"{name}: event-time set drifted: container {sorted(non_ref)} vs oracle " + f"{sorted(set(rows) - ref_keys)}" + ) + # Reference rows are normalized: att 0 with NaN inference. + for i, is_ref in enumerate(ref): + if is_ref: + assert float(es.att[i]) == 0.0, f"{name}/es[{labels[i]}]: reference att != 0" + assert math.isnan(float(es.se[i])), f"{name}/es[{labels[i]}]: reference se not NaN" + compared = 0 + for i, e in enumerate(labels): + r = _oracle_row(rows, e) + if ref[i]: + continue + tag = f"{name}/es[{e}]" + _close(es.att[i], r["effect"], rtol=rtol, atol=atol, what=f"{tag}.att") + _close(es.se[i], r["se"], rtol=rtol, atol=atol, what=f"{tag}.se") + _close(es.t_stat[i], r["t_stat"], rtol=rtol, atol=atol, what=f"{tag}.t_stat") + _close(es.p_value[i], r["p_value"], rtol=rtol, atol=p_atol, what=f"{tag}.p_value") + _close(es.conf_int_lower[i], r["ci_lo"], rtol=ci_rtol, atol=atol, what=f"{tag}.ci_lo") + _close(es.conf_int_upper[i], r["ci_hi"], rtol=ci_rtol, atol=atol, what=f"{tag}.ci_hi") + for count_key in ("n_obs", "n_groups", "n_units"): + if count_key in r: + _close(es.n[i], r[count_key], rtol=0, atol=0, what=f"{tag}.{count_key}") + if "cband_lo" in r: + assert es.cband_lower is not None and es.cband_upper is not None, tag + _close( + es.cband_lower[i], r["cband_lo"], rtol=ci_rtol, atol=atol, what=f"{tag}.cband_lo" + ) + _close( + es.cband_upper[i], r["cband_hi"], rtol=ci_rtol, atol=atol, what=f"{tag}.cband_hi" + ) + compared += 1 + assert compared > 0, f"{name}: no non-reference event times compared" + if orow.get("event_study_vcov") is not None: + assert es.vcov is not None, f"{name}: oracle carried a joint vcov, container has none" + assert [float(x) for x in es.vcov_index] == [ + float(x) for x in orow["event_study_vcov_index"] + ], f"{name}: vcov index drifted" + want = np.array( + [[oracle_value(x) for x in row] for row in orow["event_study_vcov"]], dtype=float + ) + np.testing.assert_allclose( + np.asarray(es.vcov, dtype=float), want, rtol=rtol, atol=atol, equal_nan=True + ) + if orow.get("event_study_df") is not None: + df_col = np.asarray(es.df, dtype=float) + finite = df_col[np.isfinite(df_col)] + assert finite.size > 0, f"{name}: oracle df {orow['event_study_df']} but no finite df row" + assert np.all(finite == float(orow["event_study_df"])), (name, finite) + if orow.get("cband_crit_value") is not None: + assert es.cband_crit_value is not None, name + _close( + es.cband_crit_value, + orow["cband_crit_value"], + rtol=rtol, + atol=atol, + what=f"{name}.cband_crit_value", + ) + return es + + +def _assert_group_matches_oracle(agg, orow, *, rtol, atol, p_atol=None, name=""): + rows = orow["group"] + assert rows is not None, f"{name}: oracle stored no group surface" + p_atol = atol if p_atol is None else p_atol + ci_rtol = max(rtol, CI_RTOL) + labels = list(agg.label) + assert len(labels) == len(rows), f"{name}: group label set drifted: {labels} vs {sorted(rows)}" + for i, g in enumerate(labels): + r = _oracle_row(rows, g) + tag = f"{name}/group[{g}]" + _close(agg.att[i], r["effect"], rtol=rtol, atol=atol, what=f"{tag}.att") + _close(agg.se[i], r["se"], rtol=rtol, atol=atol, what=f"{tag}.se") + _close(agg.t_stat[i], r["t_stat"], rtol=rtol, atol=atol, what=f"{tag}.t_stat") + _close(agg.p_value[i], r["p_value"], rtol=rtol, atol=p_atol, what=f"{tag}.p_value") + _close(agg.conf_int_lower[i], r["ci_lo"], rtol=ci_rtol, atol=atol, what=f"{tag}.ci_lo") + _close(agg.conf_int_upper[i], r["ci_hi"], rtol=ci_rtol, atol=atol, what=f"{tag}.ci_hi") + for count_key in ("n_obs", "n_groups", "n_units"): + if count_key in r: + _close(agg.n[i], r[count_key], rtol=0, atol=0, what=f"{tag}.{count_key}") + if "df_used" in r: + # df is per-row PROVENANCE: the legacy row's df_used (None -> NaN) + # - except that the container NaNs df wherever p is non-finite + # (AggregationResult.__post_init__; the legacy dict kept e.g. the + # replicate-undefined 0 sentinel beside NaN inference). + if np.isfinite(agg.p_value[i]): + _close(agg.df[i], r["df_used"], rtol=0, atol=0, what=f"{tag}.df") + else: + assert np.isnan(agg.df[i]), f"{tag}.df must be NaN beside non-finite p" + return agg + + +def _assert_overall_matches_oracle(res, o, *, rtol, atol, p_atol=None, name=""): + """The stored overall quintet against an oracle ``overall`` record.""" + p_atol = atol if p_atol is None else p_atol + _close(res.overall_att, o["att"], rtol=rtol, atol=atol, what=f"{name}.overall_att") + _close(res.overall_se, o["se"], rtol=rtol, atol=atol, what=f"{name}.overall_se") + _close(res.overall_t_stat, o["t_stat"], rtol=rtol, atol=atol, what=f"{name}.overall_t") + _close(res.overall_p_value, o["p_value"], rtol=rtol, atol=p_atol, what=f"{name}.overall_p") + ci = res.overall_conf_int + ci_rtol = max(rtol, CI_RTOL) + _close(ci[0], o["ci_lo"], rtol=ci_rtol, atol=atol, what=f"{name}.overall_ci_lo") + _close(ci[1], o["ci_hi"], rtol=ci_rtol, atol=atol, what=f"{name}.overall_ci_hi") + + +def _assert_simple_relays_stored_overall(res): + """(ii) aggregate('simple') is a view of the stored overall quintet.""" + s = res.aggregate("simple") + for got, want in ( + (s.att[0], res.overall_att), + (s.se[0], res.overall_se), + (s.t_stat[0], res.overall_t_stat), + (s.p_value[0], res.overall_p_value), + (s.conf_int_lower[0], res.overall_conf_int[0]), + (s.conf_int_upper[0], res.overall_conf_int[1]), + ): + assert (float(got) == float(want)) or (np.isnan(got) and np.isnan(want)) + return s + + +def _assert_analytic_inference_contract(surface, *, alpha): + """(iv) structural SE/p contract on an ANALYTIC surface. + + Every row is NaN-CONSISTENT (se/t/p/CI NaN together - a finite effect with + NaN inference is valid; a NaN effect with n > 0 is valid too) and, where + se AND the per-row df are finite, p/t are exactly ``safe_inference(att, + se, df)``. Reference rows are skipped (normalized by the container). + """ + ref = ( + np.asarray(surface.is_reference, dtype=bool) + if hasattr(surface, "is_reference") + else np.zeros(len(surface.att), dtype=bool) + ) + df_col = np.asarray(surface.df, dtype=float) + checked = 0 + for i in range(len(surface.att)): + if ref[i]: + continue + se = float(surface.se[i]) + att = float(surface.att[i]) + ci = (float(surface.conf_int_lower[i]), float(surface.conf_int_upper[i])) + if not (np.isfinite(se) and se > 0): + assert_nan_inference( + { + "se": se, + "t_stat": float(surface.t_stat[i]), + "p_value": float(surface.p_value[i]), + "conf_int": ci, + } + ) + continue + if not np.isfinite(att): + # A NaN effect beside a finite se: the inference triple is NaN too. + assert np.isnan(surface.t_stat[i]) and np.isnan(surface.p_value[i]) + assert np.isnan(ci[0]) and np.isnan(ci[1]) + continue + assert np.isfinite(surface.p_value[i]) and 0.0 <= float(surface.p_value[i]) <= 1.0 + assert ci[0] <= att <= ci[1] + if np.isfinite(df_col[i]): + t, p, ci_want = safe_inference(att, se, alpha=alpha, df=float(df_col[i])) + np.testing.assert_allclose(t, surface.t_stat[i], rtol=1e-12) + np.testing.assert_allclose(p, surface.p_value[i], rtol=1e-12) + np.testing.assert_allclose(ci_want, ci, rtol=1e-12) + checked += 1 + return checked + + +def _assert_bootstrap_p_values_in_unit_interval(surface): + """(iv) on BOOTSTRAP arms: percentile p-values are counts in [0, 1]; no + ``safe_inference`` identity holds (and none is asserted).""" + p = np.asarray(surface.p_value, dtype=float) + finite = p[np.isfinite(p)] + assert finite.size > 0 + assert np.all((finite >= 0.0) & (finite <= 1.0)) + + +def _assert_att_from_cells(res, es, *, bucket=False, retained=None, name=""): + """(iii) CS/EfficientDiD point estimates from the PUBLIC ``group_time_effects``: + ``att[e]`` is the cohort-size (``n_treated``) weighted mean of the finite + cells at horizon ``e`` (``t - g == e``; ``int(t - g) == e`` when + ``bucket``), over the ``retained`` cohorts when a balance_e filter applies. + Non-survey arms only (survey arms weight by survey mass).""" + gte = res.group_time_effects + ref = np.asarray(es.is_reference, dtype=bool) + compared = 0 + for i, e in enumerate(es.event_time): + if ref[i]: + continue + cells = [ + (float(c["effect"]), float(c["n_treated"])) + for (g, t), c in gte.items() + if (retained is None or g in retained) + and ((int(t - g) == int(e)) if bucket else (t - g == e)) + and np.isfinite(c["effect"]) + ] + if not cells: + assert np.isnan(es.att[i]), (name, e) + continue + w = np.array([c[1] for c in cells]) + eff = np.array([c[0] for c in cells]) + np.testing.assert_allclose( + es.att[i], float(np.sum(w / w.sum() * eff)), rtol=1e-12, err_msg=f"{name}/att[{e}]" + ) + compared += 1 + assert compared > 0, name + + +def _assert_post_att_is_tau_mean(res, es, *, name=""): + """(iii) Imputation/TwoStage point estimates from the PUBLIC + ``treatment_effects``: at POST horizons ``e >= 0`` the event-study att is + the plain mean of ``tau_hat`` over ``rel_time == e`` (non-survey arms; + leads come from the lead regression and are oracle-only).""" + te = res.treatment_effects + compared = 0 + for i, e in enumerate(es.event_time): + if e < 0 or bool(es.is_reference[i]): + continue + sel = te.loc[te["rel_time"] == e, "tau_hat"] + finite = sel[np.isfinite(sel)] + if finite.empty or not np.isfinite(es.att[i]): + continue + np.testing.assert_allclose( + es.att[i], float(finite.mean()), rtol=1e-12, err_msg=f"{name}/att[{e}]" + ) + compared += 1 + assert compared > 0, name + + +def _frame_equal(a, b): + """Two container frames are identical (NaN-aware, dtype-tolerant).""" + assert list(a.columns) == list(b.columns) + assert a.shape == b.shape + for col in a.columns: + x, y = a[col].to_numpy(), b[col].to_numpy() + if x.dtype.kind in "fc" and y.dtype.kind in "fc": + np.testing.assert_allclose( + x.astype(float), y.astype(float), rtol=0, atol=0, equal_nan=True, err_msg=col + ) + else: + assert list(x) == list(y), col + FIT_KW = dict(outcome="y", unit="unit", time="time", first_treat="first_treat") @@ -61,7 +402,7 @@ def panel(): @pytest.fixture(scope="module") def fitted(panel): - """A plain fit - no aggregate= argument, so no deprecation warning.""" + """A plain fit - the canonical route (no fit-time aggregation keywords).""" return CallawaySantAnna().fit(panel, **FIT_KW) @@ -73,9 +414,8 @@ def fit_time(panel): return CallawaySantAnna().fit(panel, aggregate="all", **FIT_KW) -# --------------------------------------------------------------------------- # -# Numerical inertness (the headline gate) -# --------------------------------------------------------------------------- # +def _cs_oracle(arm): + return load_oracles()["analytic"]["CallawaySantAnna"][arm] class TestInertness: @@ -138,6 +478,91 @@ def test_balance_e_matches_fit_time(self, panel, fitted, balance_e): assert np.allclose(row["se"], got["se"], rtol=1e-14, atol=1e-14, equal_nan=True) +# --------------------------------------------------------------------------- # +# Oracle parity (the headline gate) - CallawaySantAnna, default panel +# --------------------------------------------------------------------------- # + + +class TestOracleParity: + def test_mismarked_reference_row_fails_the_oracle_check(self, fitted): + """Reference membership is pinned by the oracle: marking an ESTIMATED + horizon as reference (which would otherwise skip its numeric + comparison) must fail, not pass.""" + es = fitted.aggregate("event_study") + flags = np.asarray(es.is_reference, dtype=bool).copy() + estimated = [i for i, f in enumerate(flags) if not f] + flags[estimated[0]] = True + att = np.asarray(es.att, dtype=float).copy() + att[estimated[0]] = 0.0 + tampered = dataclasses.replace(es, is_reference=flags, att=att) + with pytest.raises(AssertionError, match="reference set drifted"): + _assert_es_matches_oracle(tampered, _cs_oracle("default_all"), **ANALYTIC_TOL) + + def test_simple_relays_stored_overall_and_matches_oracle(self, fitted): + _assert_simple_relays_stored_overall(fitted) + _assert_overall_matches_oracle( + fitted, _cs_oracle("default_all")["overall"], **ANALYTIC_TOL, name="cs" + ) + + def test_group_matches_oracle(self, fitted): + _assert_group_matches_oracle( + fitted.aggregate("group"), _cs_oracle("default_all"), **ANALYTIC_TOL, name="cs" + ) + + def test_event_study_matches_oracle(self, fitted): + _assert_es_matches_oracle( + fitted.aggregate("event_study"), _cs_oracle("default_all"), **ANALYTIC_TOL, name="cs" + ) + + def test_balance_e1_matches_oracle(self, fitted): + """M-117: balance_e on aggregate() reproduces the pre-removal + fit(balance_e=1) surface.""" + _assert_es_matches_oracle( + fitted.aggregate("event_study", balance_e=1), + _cs_oracle("default_balance_e1"), + **ANALYTIC_TOL, + name="cs/be1", + ) + + @pytest.mark.parametrize("balance_e", [0, 1, 2]) + def test_balance_e_retains_cohorts_at_the_exact_anchor(self, fitted, balance_e): + """(v) CS semantics AS IMPLEMENTED: a cohort is retained iff it has a + FINITE effect at the EXACT anchor ``t - g == balance_e``; EVERY horizon + of a retained cohort stays (no lower bound on event_time) and the + per-horizon att is the cohort-size weighted mean over retained + cohorts only.""" + es = fitted.aggregate("event_study", balance_e=balance_e) + gte = fitted.group_time_effects + retained = { + g for (g, t), c in gte.items() if t - g == balance_e and np.isfinite(c["effect"]) + } + assert retained, "fixture reaches no cohort at the anchor" + horizons = sorted({int(t - g) for (g, t) in gte if g in retained}) + assert sorted(int(e) for e in es.event_time) == horizons + assert min(horizons) < 0 # pre-treatment horizons of retained cohorts stay + _assert_att_from_cells(fitted, es, retained=retained, name=f"cs/be{balance_e}") + if balance_e == 2: + # Discriminating arm: the last cohort never reaches e=2 and drops out. + assert max(gte, key=lambda k: k[0])[0] not in retained + + def test_point_estimates_from_public_cells(self, fitted): + """(iii) att[e] is reconstructible from group_time_effects alone.""" + _assert_att_from_cells(fitted, fitted.aggregate("event_study"), name="cs") + + def test_analytic_inference_contract(self, fitted): + """(iv) NaN-consistency on every row; p/t == safe_inference where se + and df are finite (the plain panel has no df, so the identity leg is + exercised by the survey arm below).""" + for level in ("event_study", "group", "simple"): + _assert_analytic_inference_contract(fitted.aggregate(level), alpha=fitted.alpha) + + def test_legacy_fields_none_after_analytic_fit(self, fitted): + """A plain CS fit materializes no fit-time surface.""" + assert fitted.event_study_effects is None + assert fitted.group_effects is None + assert fitted.event_study_vcov is None and fitted.event_study_df is None + + # --------------------------------------------------------------------------- # # Immutability # --------------------------------------------------------------------------- # @@ -238,15 +663,20 @@ def test_no_raw_unit_identifiers_are_retained(self): assert np.isfinite(np.asarray(res.aggregate(level).att)).any() def test_identifier_minimization_is_numerically_inert(self, panel, fitted): - """Canonical codes must not perturb any aggregate: compared at - atol=rtol=0, since substituting positions for labels should not change - a single floating-point operation.""" - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - native = CallawaySantAnna().fit(panel, aggregate="all", **FIT_KW) - got = fitted.aggregate("simple") - assert got.att[0] == native.overall_att - assert got.se[0] == native.overall_se + """Canonical codes must not perturb any aggregate: a fit on the same + panel with STRING unit identifiers is compared at atol=rtol=0, since + substituting positions for labels should not change a single + floating-point operation.""" + labelled = panel.copy() + labelled["unit"] = labelled["unit"].map(lambda u: f"unit-{u:03d}@example.invalid") + res = CallawaySantAnna().fit(labelled, **FIT_KW) + got, want = res.aggregate("simple"), fitted.aggregate("simple") + assert got.att[0] == want.att[0] + assert got.se[0] == want.se[0] + _frame_equal( + res.aggregate("event_study").to_dataframe(), + fitted.aggregate("event_study").to_dataframe(), + ) # --------------------------------------------------------------------------- # @@ -363,63 +793,30 @@ def _boot_fit_time(data, *, seed=42, n_bootstrap=50, aggregate="all", fit_kwargs ) -def _assert_es_replay_parity(es, ref, n_bootstrap=50): - """Replayed EventStudyResults vs the native fit-time bootstrap surface. +_CS_NBOOT = 50 - Tolerance note (reconciling with this file's 1e-14 analytical-parity - convention): the replayed weight stream is bit-identical, and only the - fused GEMM's tile-boundary/column-count reassociation differs between - the fit-time and replay passes — ~1 ULP relative — so 1e-13 is the same - contract with one order of headroom for quantile-interpolation - arithmetic; any real desynchronization is O(1), not O(1e-13). p-values - are COUNT statistics (a draw within a ULP of the point estimate can - flip one count), hence the additional 2/n_bootstrap atol. - """ - ref_es = ref.event_study_effects - p_atol = 2.0 / n_bootstrap - for i, e in enumerate(es.event_time): - if bool(es.is_reference[i]): - continue - r = ref_es[int(e)] - assert float(es.att[i]) == float(r["effect"]) # same analytical point path - np.testing.assert_allclose(es.se[i], r["se"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(es.t_stat[i], r["t_stat"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(es.p_value[i], r["p_value"], rtol=1e-13, atol=p_atol) - np.testing.assert_allclose( - [es.conf_int_lower[i], es.conf_int_upper[i]], - list(r["conf_int"]), - rtol=1e-13, - atol=1e-13, - ) - cb = r.get("cband_conf_int") - if cb is not None: - np.testing.assert_allclose( - [es.cband_lower[i], es.cband_upper[i]], list(cb), rtol=1e-13, atol=1e-13 - ) - # Percentile provenance: no analytical df, and the joint covariance is - # the fit-time one of the same bootstrap draws (never the analytical one). - assert ref.event_study_vcov is not None - assert list(es.vcov_index) == list(ref.event_study_vcov_index) - np.testing.assert_allclose(es.vcov, ref.event_study_vcov, rtol=1e-13, atol=1e-13) + +def _cs_boot_oracle(arm): + """The backend-matched bootstrapped CS oracle (skips when uncaptured).""" + return boot_oracles()["CallawaySantAnna"][arm] + + +def _assert_es_replay_parity(es, orow, n_bootstrap=_CS_NBOOT): + """Replayed EventStudyResults vs the pre-removal fit-time bootstrap surface + (oracle). Tolerances per ``BOOT_TOL``; the percentile provenance pins - + no analytical df, the joint covariance of the same draws - ride the + oracle bundle (vcov / vcov_index / cband_crit_value).""" + _assert_es_matches_oracle(es, orow, **BOOT_TOL, p_atol=2.0 / n_bootstrap, name="cs/boot") + assert es.vcov is not None assert np.all(np.isnan(np.asarray(es.df, dtype=float))) - if ref.cband_crit_value is not None: - np.testing.assert_allclose(es.cband_crit_value, ref.cband_crit_value, rtol=1e-13) - - -def _assert_group_replay_parity(agg, ref, n_bootstrap=50): - gdf = agg.to_dataframe() - p_atol = 2.0 / n_bootstrap - for _, row in gdf.iterrows(): - r = ref.group_effects[row["label"]] - assert float(row["att"]) == float(r["effect"]) - if np.isnan(r["se"]): - assert np.isnan(row["se"]) - continue - np.testing.assert_allclose(row["se"], r["se"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(row["t_stat"], r["t_stat"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(row["p_value"], r["p_value"], rtol=1e-13, atol=p_atol) - # df cleared: percentile inference never used the analytical df. - assert np.isnan(row["df"]) + return es + + +def _assert_group_replay_parity(agg, orow, n_bootstrap=_CS_NBOOT): + _assert_group_matches_oracle(agg, orow, **BOOT_TOL, p_atol=2.0 / n_bootstrap, name="cs/boot") + # df cleared: percentile inference never used the analytical df. + assert np.all(np.isnan(np.asarray(agg.df, dtype=float))) + return agg class TestBootstrapReplay: @@ -435,30 +832,85 @@ class TestBootstrapReplay: by the convention, not pinned individually. """ - def test_event_study_parity_with_fit_time(self, panel): - ref = _boot_fit_time(panel, aggregate="event_study") + def test_event_study_matches_oracle(self, panel): plain = _boot_fit(panel) with warnings.catch_warnings(): warnings.simplefilter("ignore") es = plain.aggregate("event_study") assert isinstance(es, EventStudyResults) - _assert_es_replay_parity(es, ref) + _assert_es_replay_parity(es, _cs_boot_oracle("default_all")) + # (vi) the joint covariance is the covariance of the same draws. + non_ref = ~np.asarray(es.is_reference, dtype=bool) + np.testing.assert_allclose( + np.diag(np.asarray(es.vcov)), np.asarray(es.se)[non_ref] ** 2, rtol=1e-12 + ) + # The CS multiplier bootstrap always carries the sup-t band (cband= + # only changes how the per-horizon interval is formed - pinned by the + # cband_all oracle below), so the band is present on every replay. + assert es.cband_lower is not None and es.cband_upper is not None - def test_group_parity_with_fit_time(self, panel): - ref = _boot_fit_time(panel, aggregate="group") + def test_group_matches_oracle(self, panel): plain = _boot_fit(panel) with warnings.catch_warnings(): warnings.simplefilter("ignore") agg = plain.aggregate("group") - _assert_group_replay_parity(agg, ref) + _assert_group_replay_parity(agg, _cs_boot_oracle("default_all")) + + def test_plain_overall_matches_oracle(self, panel): + """The headline bootstrap of a PLAIN fit is untouched by the removal.""" + plain = _boot_fit(panel) + _assert_overall_matches_oracle( + plain, + _cs_boot_oracle("default_plain_overall"), + **BOOT_TOL, + p_atol=2.0 / _CS_NBOOT, + name="cs/boot", + ) + # ... and equals the overall of the pre-removal fit(aggregate="all"). + _assert_overall_matches_oracle( + plain, _cs_boot_oracle("default_all")["overall"], **BOOT_TOL, p_atol=2.0 / _CS_NBOOT + ) - def test_balance_e_parity_with_fit_time(self, panel): - ref = _boot_fit_time(panel, aggregate="event_study", fit_kwargs={"balance_e": 1}) + def test_cband_replay_matches_oracle(self, panel): + """(vi) cband=True: the sup-t band and its critical value replay + (the backend-matched cband_all oracle pins the cband=True numbers).""" + plain = _boot_fit(panel, cband=True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + es = plain.aggregate("event_study") + _assert_es_replay_parity(es, _cs_boot_oracle("cband_all")) + assert es.cband_lower is not None and es.cband_upper is not None + assert es.cband_crit_value is not None + + def test_balance_e_replay_retains_anchor_cohorts(self, panel, fitted): + """(v) on the replay route: balance_e keeps the analytical anchor rule + (same labels and point estimates as the analytic container) while + the inference is percentile (df NaN) and deterministic.""" plain = _boot_fit(panel) with warnings.catch_warnings(): warnings.simplefilter("ignore") es = plain.aggregate("event_study", balance_e=1) - _assert_es_replay_parity(es, ref) + again = _boot_fit(panel).aggregate("event_study", balance_e=1) + ana = fitted.aggregate("event_study", balance_e=1) + assert list(es.event_time) == list(ana.event_time) + np.testing.assert_array_equal(np.asarray(es.att), np.asarray(ana.att)) + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) + np.testing.assert_array_equal(np.asarray(es.se), np.asarray(again.se)) + _assert_bootstrap_p_values_in_unit_interval(es) + + def test_same_seed_fits_replay_identically(self, panel): + """(vi) determinism: two same-seed fits replay the same surface.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + a = _boot_fit(panel).aggregate("event_study").to_dataframe() + b = _boot_fit(panel).aggregate("event_study").to_dataframe() + _frame_equal(a, b) + + def test_legacy_fields_none_after_bootstrapped_fit(self, panel): + """CS stores NO fit-time surface after ANY fit (the replay serves).""" + plain = _boot_fit(panel) + assert plain.event_study_effects is None and plain.group_effects is None + assert plain.event_study_vcov is None and plain.event_study_df is None def test_seedless_fit_replays_and_is_idempotent(self, panel): """seed=None fits replay too — the spec captures the actual RNG @@ -516,8 +968,6 @@ def test_backend_mismatch_fails_closed(self, panel): """A different weight backend regenerates different draws from the same RNG state — the replay must refuse, deterministically under either installed backend.""" - import dataclasses - from diff_diff.bootstrap_chunking import effective_weight_backend plain = _boot_fit(panel) @@ -544,9 +994,9 @@ def test_no_post_treatment_cells_group_returns_zero_rows(self): """A fit whose every treated cohort's onset lies beyond the observed panel has NO post-treatment cells: overall inference is NaN, only pre-treatment cells exist, and the group aggregation is empty. The - bootstrapped group path must return the supported zero-row result on - BOTH routes (they share `_run_multiplier_bootstrap`, whose group - stats block would otherwise np.column_stack an empty list).""" + bootstrapped group path must return the supported zero-row result + (`_run_multiplier_bootstrap`'s group stats block would otherwise + np.column_stack an empty list).""" rng = np.random.default_rng(0) rows = [] for u in range(30): @@ -568,24 +1018,44 @@ def test_no_post_treatment_cells_group_returns_zero_rows(self): agg = plain.aggregate("group") assert len(agg.to_dataframe()) == 0 assert np.isnan(plain.overall_att) # unchanged by the replay - # The deprecated fit-time route rides the same helper — it must - # survive too (the crash was reachable there pre-replay). - ref = _boot_fit_time(data, n_bootstrap=25, seed=1, aggregate="group") - assert ref.group_effects in (None, {}) + assert plain.group_effects is None # nothing is stored at fit any more class TestBootstrapReplayDesigns: - """Replay parity on the engine branches the plain panel never reaches.""" + """Replay parity on the engine branches the plain panel never reaches. + + Every arm pins its numbers against its backend-matched oracle; the + replay contract - percentile provenance (NaN df, joint vcov), p in [0, 1] + and same-seed determinism - is asserted alongside.""" + + @staticmethod + def _assert_replay_contract(plain, levels=("event_study", "group")): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + first = {lv: plain.aggregate(lv) for lv in levels} + again = {lv: plain.aggregate(lv) for lv in levels} + for lv, surface in first.items(): + assert np.all(np.isnan(np.asarray(surface.df, dtype=float))), lv + _assert_bootstrap_p_values_in_unit_interval(surface) + np.testing.assert_array_equal(np.asarray(surface.se), np.asarray(again[lv].se)) + if "event_study" in first: + es = first["event_study"] + assert es.vcov is not None + non_ref = ~np.asarray(es.is_reference, dtype=bool) + np.testing.assert_allclose( + np.diag(np.asarray(es.vcov)), np.asarray(es.se)[non_ref] ** 2, rtol=1e-12 + ) + return first def test_bare_cluster_psu_expansion(self, panel): d = panel.copy() d["psu"] = d["unit"] % 7 - ref = _boot_fit_time(d, cluster="psu") plain = _boot_fit(d, cluster="psu") + orow = _cs_boot_oracle("psu_cluster_all") with warnings.catch_warnings(): warnings.simplefilter("ignore") - _assert_es_replay_parity(plain.aggregate("event_study"), ref) - _assert_group_replay_parity(plain.aggregate("group"), ref) + _assert_es_replay_parity(plain.aggregate("event_study"), orow) + _assert_group_replay_parity(plain.aggregate("group"), orow) def test_stratified_survey_is_portable_and_matches(self, panel): from diff_diff import SurveyDesign @@ -597,18 +1067,18 @@ def test_stratified_survey_is_portable_and_matches(self, panel): d["psu"] = d["unit"] % 7 d["stratum"] = d["unit"] % 2 sd = SurveyDesign(weights="w", strata="stratum", psu="psu", nest=True) - ref = _boot_fit_time(d, fit_kwargs={"survey_design": sd}) plain = _boot_fit(d, fit_kwargs={"survey_design": sd}) # The stratified survey generator draws through NumPy regardless of # the installed backend — provably backend-independent, stamped # "portable" so the artifact replays anywhere. assert plain._aggregation_kit.bootstrap.backend == "portable" + orow = _cs_boot_oracle("stratified_survey_all") with warnings.catch_warnings(): warnings.simplefilter("ignore") - _assert_es_replay_parity(plain.aggregate("event_study"), ref) - _assert_group_replay_parity(plain.aggregate("group"), ref) + _assert_es_replay_parity(plain.aggregate("event_study"), orow) + _assert_group_replay_parity(plain.aggregate("group"), orow) - def test_fpc_survey_matches(self, panel): + def test_fpc_survey_replay_contract(self, panel): from diff_diff import SurveyDesign d = panel.copy() @@ -618,28 +1088,35 @@ def test_fpc_survey_matches(self, panel): d["psu"] = d["unit"] % 7 d["fpc"] = 100 # non-census: the fpc_scale branch sd = SurveyDesign(weights="w", psu="psu", fpc="fpc") - ref = _boot_fit_time(d, fit_kwargs={"survey_design": sd}) plain = _boot_fit(d, fit_kwargs={"survey_design": sd}) + es = self._assert_replay_contract(plain, levels=("event_study",))["event_study"] + assert es.df_survey == plain.survey_metadata.df_survey + orow = _cs_boot_oracle("fpc_survey_all") with warnings.catch_warnings(): warnings.simplefilter("ignore") - _assert_es_replay_parity(plain.aggregate("event_study"), ref) + _assert_es_replay_parity(es, orow) + _assert_group_replay_parity(plain.aggregate("group"), orow) - def test_repeated_cross_sections_match(self): + def test_repeated_cross_sections_replay_contract(self): data = _rcs() - ref = _boot_fit_time(data, panel=False) plain = _boot_fit(data, panel=False) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - _assert_es_replay_parity(plain.aggregate("event_study"), ref) - _assert_group_replay_parity(plain.aggregate("group"), ref) - - def test_unbalanced_panel_matches(self, panel): + surfaces = self._assert_replay_contract(plain) + # Same analytical point path as the analytic RCS fit. + ana = CallawaySantAnna(panel=False).fit(data, **FIT_KW).aggregate("event_study") + np.testing.assert_array_equal(np.asarray(surfaces["event_study"].att), np.asarray(ana.att)) + orow = _cs_boot_oracle("rcs_all") + _assert_es_replay_parity(surfaces["event_study"], orow) + _assert_group_replay_parity(surfaces["group"], orow) + + def test_unbalanced_panel_replay_contract(self, panel): thinned = panel.drop(panel.index[::13]).reset_index(drop=True) - ref = _boot_fit_time(thinned, allow_unbalanced_panel=True) plain = _boot_fit(thinned, allow_unbalanced_panel=True) + es = self._assert_replay_contract(plain, levels=("event_study",))["event_study"] + orow = _cs_boot_oracle("unbalanced_all") with warnings.catch_warnings(): warnings.simplefilter("ignore") - _assert_es_replay_parity(plain.aggregate("event_study"), ref) + _assert_es_replay_parity(es, orow) + _assert_group_replay_parity(plain.aggregate("group"), orow) def test_single_psu_nan_surfaces_and_warning(self, panel): from diff_diff import SurveyDesign @@ -769,7 +1246,7 @@ def test_non_estimable_row_keeps_point_estimate(self): # --------------------------------------------------------------------------- # -# Deprecation shim (M-020 / M-117) +# Fit-time keyword shim (M-020 / M-117; removal lands in PR 3b) # --------------------------------------------------------------------------- # @@ -817,6 +1294,84 @@ def _rcs(seed=17, n_per_cell=6, n_periods=6): return pd.DataFrame(rows) +class TestOracleParityAcrossDesigns: + """The oracle gate on designs the balanced-panel fixture does not reach. + + The refactor made the aggregators pure by threading bookkeeping that the + panel path and the repeated-cross-section path populate differently (RCS + makes several kit keys observation-length rather than unit-length), and + ``anticipation`` shifts which ``(g, t)`` cells are eligible - so each + design arm carries its own oracle, captured from the pre-removal fit-time + route on exactly these panels. + """ + + @pytest.mark.parametrize("level", ["simple", "group", "event_study"]) + def test_repeated_cross_sections_match_oracle(self, level): + post = CallawaySantAnna(panel=False).fit(_rcs(), **FIT_KW) + _assert_level_matches_oracle(post, _cs_oracle("rcs_all"), level, "cs/rcs") + + @pytest.mark.parametrize("level", ["simple", "group", "event_study", "total"]) + def test_anticipation_matches_oracle(self, panel, level): + post = CallawaySantAnna(anticipation=1).fit(panel, **FIT_KW) + _assert_level_matches_oracle(post, _cs_oracle("anticipation1_all"), level, "cs/antic") + + @pytest.mark.parametrize("level", ["simple", "group", "event_study", "total"]) + def test_universal_base_period_matches_oracle(self, panel, level): + """REGISTRY gives universal bases their own reference-cell and + VCV-index semantics (a zero reference cell per cohort), which the + default 'varying' fixture never exercises.""" + post = CallawaySantAnna(base_period="universal").fit(panel, **FIT_KW) + _assert_level_matches_oracle(post, _cs_oracle("universal_all"), level, "cs/universal") + + @pytest.mark.parametrize("level", ["simple", "group", "event_study", "total"]) + def test_not_yet_treated_with_anticipation_matches_oracle(self, panel, level): + """The not-yet-treated control group interacts with anticipation in + picking each (g, t) comparison set - a different code path from the + never-treated default.""" + post = CallawaySantAnna(control_group="not_yet_treated", anticipation=1).fit( + panel, **FIT_KW + ) + _assert_level_matches_oracle(post, _cs_oracle("nyt_anticipation1_all"), level, "cs/nyt") + + @pytest.mark.parametrize("level", ["simple", "group", "event_study"]) + def test_unbalanced_panel_matches_oracle(self, panel, level): + """allow_unbalanced_panel changes which units survive into the + influence vectors, so the retained bookkeeping must agree with what + the fit actually used.""" + thinned = panel.drop(panel.index[::13]).reset_index(drop=True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + post = CallawaySantAnna(allow_unbalanced_panel=True).fit(thinned, **FIT_KW) + _assert_level_matches_oracle(post, _cs_oracle("unbalanced_all"), level, "cs/unbalanced") + + @pytest.mark.parametrize("level", ["simple", "group", "event_study"]) + def test_survey_numbers_match_oracle(self, survey_fit, level): + """Survey parity on the ESTIMATES, not just the df metadata.""" + _assert_level_matches_oracle(survey_fit, _cs_oracle("survey_all"), level, "cs/survey") + + def test_survey_inference_contract_with_finite_df(self, survey_fit): + """(iv) the identity leg: on the explicit-survey arm every row carries + the finite survey df and p/t/CI are exactly safe_inference(att, se, df).""" + for level in ("event_study", "group", "simple"): + checked = _assert_analytic_inference_contract( + survey_fit.aggregate(level), alpha=survey_fit.alpha + ) + assert checked > 0, level + + @pytest.mark.parametrize( + "ctor", + [ + {"anticipation": 1}, + {"base_period": "universal"}, + {"control_group": "not_yet_treated", "anticipation": 1}, + ], + ) + def test_point_estimates_from_public_cells_across_designs(self, panel, ctor): + """(iii) on the non-survey design arms.""" + post = CallawaySantAnna(**ctor).fit(panel, **FIT_KW) + _assert_att_from_cells(post, post.aggregate("event_study"), name=str(ctor)) + + class TestInertnessAcrossDesigns: """The inertness gate on designs the balanced-panel fixture does not reach. @@ -937,6 +1492,35 @@ def _assert_level_matches(post, native, level): assert compared > 0, f"no {level} rows compared" +def _assert_level_matches_oracle(post, orow, level, name=""): + """Compare a post-fit aggregation against the pre-removal oracle arm.""" + if level == "total": + # Identity form (total == n x overall on the SAME fit) AND the + # independent-mass form (n pinned against a frame-derived oracle from + # the fit's PUBLIC cells) - the identity alone passes for ANY C. + got = post.aggregate("total") + antic = getattr(post, "anticipation", 0) + n_exp = sum( + d["n_treated"] + for (g, t), d in post.group_time_effects.items() + if t >= g - antic and np.isfinite(d["effect"]) + ) + assert got.n[0] == float(n_exp) + o = orow["overall"] + assert np.allclose(got.att[0], got.n[0] * oracle_value(o["att"]), rtol=1e-12) + assert np.allclose(got.se[0], got.n[0] * oracle_value(o["se"]), rtol=1e-12) + _close(got.t_stat[0], o["t_stat"], **ANALYTIC_TOL, what=f"{name}/total.t") + return + if level == "simple": + _assert_simple_relays_stored_overall(post) + _assert_overall_matches_oracle(post, orow["overall"], **ANALYTIC_TOL, name=name) + return + if level == "group": + _assert_group_matches_oracle(post.aggregate("group"), orow, **ANALYTIC_TOL, name=name) + return + _assert_es_matches_oracle(post.aggregate("event_study"), orow, **ANALYTIC_TOL, name=name) + + def _survey_panel(): rng = np.random.default_rng(3) rows = [] @@ -1001,6 +1585,8 @@ def test_bootstrap_clears_group_df_provenance(self, survey): the retained analytical df described inference that no longer exists. Leaving it finite would claim a t-reference governed percentile-bootstrap numbers - the same false provenance the event-study path already clears. + On a plain fit the CS surface is the REPLAY container (nothing is stored on + the results), so the pin reads the replayed df column. """ from diff_diff import SurveyDesign @@ -1031,13 +1617,15 @@ def test_bootstrap_clears_group_df_provenance(self, survey): with warnings.catch_warnings(): warnings.simplefilter("ignore") res = CallawaySantAnna(n_bootstrap=50, seed=1, **ctor).fit( - pd.DataFrame(rows), aggregate="all", **fit_extra, **FIT_KW + pd.DataFrame(rows), **fit_extra, **FIT_KW ) - assert res.group_effects, "fixture produced no group rows" - for g, eff in res.group_effects.items(): - assert eff.get("df_used") is None, f"group {g} kept analytical df on a bootstrap fit" + grp = res.aggregate("group") + es = res.aggregate("event_study") + assert res.group_effects is None and res.event_study_df is None # nothing stored + assert len(grp.label) > 0, "fixture produced no group rows" + assert np.all(np.isnan(np.asarray(grp.df, dtype=float))) # The event-study path's equivalent clearing must not have regressed. - assert res.event_study_df is None + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) @pytest.mark.parametrize("survey", [False, True]) def test_sddd_bootstrap_clears_group_df_provenance(self, survey): @@ -1263,7 +1851,7 @@ def test_staggered_triple_diff_overall_att_es_still_works(): # --------------------------------------------------------------------------- # -# dCDH (row M-026): fit(aggregate=) shim + the VIEW-based aggregate() +# dCDH (row M-026): fit(aggregate=) shim pin + the VIEW-based aggregate() # --------------------------------------------------------------------------- # DCDH_KW = dict(outcome="outcome", unit="unit", time="period", treatment="treat") @@ -1491,7 +2079,6 @@ def test_mixin_hooks_are_not_dataclass_fields(self): # dynamically for EVERY dataclass that mixes AggregationMixin in, # so later 2(b) waves are enrolled automatically. import dataclasses - import inspect import diff_diff from diff_diff.aggregation import AggregationMixin @@ -1528,7 +2115,7 @@ def test_mixin_hooks_are_not_dataclass_fields(self): # --------------------------------------------------------------------------- # -# StackedDiD (row M-024): fit(aggregate=) shim + the VIEW-based aggregate() +# StackedDiD (row M-024): fit(aggregate=) shim pin + the VIEW-based aggregate() # --------------------------------------------------------------------------- # STACKED_KW = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") @@ -1847,7 +2434,7 @@ def test_unsupported_types_fail_closed(self, stacked_fitted, bad): # --------------------------------------------------------------------------- # -# EfficientDiD (rows M-023/M-120): fit(aggregate=/balance_e=) shim + the +# EfficientDiD (rows M-023/M-120): fit(aggregate=/balance_e=) shim pins + the # lazy-KIT recompute aggregate() (the CallawaySantAnna class) # --------------------------------------------------------------------------- # @@ -1932,6 +2519,10 @@ def efficient_fit_time(efficient_panel): return _fit_efficient(efficient_panel, aggregate="all") +def _ed_oracle(arm): + return load_oracles()["analytic"]["EfficientDiD"][arm] + + class TestEfficientShim: def test_plain_fit_does_not_warn(self, efficient_panel): from diff_diff import EfficientDiD @@ -1987,49 +2578,25 @@ def test_warn_and_still_work(self, efficient_panel, efficient_fitted, efficient_ class TestEfficientAggregate: - def _assert_es_matches_fit_time(self, res, fit_time, balance_e=None, skip_reference=False): - es = res.aggregate("event_study", balance_e=balance_e) - native = fit_time.event_study_effects - ref_marked = set() - if skip_reference: - ref_marked = {int(e) for e, m in zip(es.event_time, es.is_reference) if m} - assert sorted(int(e) for e in es.event_time) == sorted(native) - for e, row in native.items(): - i = list(es.event_time).index(e) - if int(e) in ref_marked: - # The container's reference normalization NaNs se/t/p/CI/n - # and forces att 0.0 on the marked row - compare the NATIVE - # dict instead (its anchor must be an exact mechanical zero). - assert row["effect"] == 0.0 and row["se"] == 0.0 - assert es.att[i] == 0.0 - assert np.isnan(es.se[i]) and np.isnan(es.n[i]) - continue - np.testing.assert_allclose(row["effect"], es.att[i], rtol=1e-14) - np.testing.assert_allclose(row["se"], es.se[i], rtol=1e-14) - if np.isfinite(row["p_value"]): - np.testing.assert_allclose(row["p_value"], es.p_value[i], rtol=1e-14) - return es - - def test_event_study_inertness(self, efficient_fitted, efficient_fit_time): - self._assert_es_matches_fit_time(efficient_fitted, efficient_fit_time) + def test_event_study_matches_oracle(self, efficient_fitted): + _assert_es_matches_oracle( + efficient_fitted.aggregate("event_study"), + _ed_oracle("default_all"), + **ANALYTIC_TOL, + name="edid", + ) - def test_group_inertness(self, efficient_fitted, efficient_fit_time): + def test_group_matches_oracle(self, efficient_fitted): grp = efficient_fitted.aggregate("group") - for i, g in enumerate(grp.label): - row = efficient_fit_time.group_effects[g] - np.testing.assert_allclose(row["effect"], grp.att[i], rtol=1e-14) - np.testing.assert_allclose(row["se"], grp.se[i], rtol=1e-14) + _assert_group_matches_oracle(grp, _ed_oracle("default_all"), **ANALYTIC_TOL, name="edid") assert grp.n_kind == "cells" assert grp.weight is None def test_simple_relay_bit_exact(self, efficient_fitted): - simple = efficient_fitted.aggregate("simple") - assert simple.att[0] == efficient_fitted.overall_att - assert simple.se[0] == efficient_fitted.overall_se - assert simple.t_stat[0] == efficient_fitted.overall_t_stat - assert simple.p_value[0] == efficient_fitted.overall_p_value - assert simple.conf_int_lower[0] == efficient_fitted.overall_conf_int[0] - assert simple.conf_int_upper[0] == efficient_fitted.overall_conf_int[1] + simple = _assert_simple_relays_stored_overall(efficient_fitted) + _assert_overall_matches_oracle( + efficient_fitted, _ed_oracle("default_all")["overall"], **ANALYTIC_TOL, name="edid" + ) assert simple.target[0] == "att" # Disjoint treated+control total (last_cohort trimming reassigns # BEFORE the counts, so a true total exists - unlike StackedDiD). @@ -2040,26 +2607,71 @@ def test_simple_relay_bit_exact(self, efficient_fitted): # Plain fit: the kit's post-overall df snapshot is None -> NaN df. assert np.isnan(simple.df[0]) + def test_balance_e1_matches_oracle(self, efficient_fitted): + _assert_es_matches_oracle( + efficient_fitted.aggregate("event_study", balance_e=1), + _ed_oracle("default_balance_e1"), + **ANALYTIC_TOL, + name="edid/be1", + ) + @pytest.mark.parametrize("balance_e", [0, 1, 2]) - def test_balance_e_inertness(self, efficient_panel, efficient_fitted, balance_e): - fit_time = _fit_efficient(efficient_panel, aggregate="event_study", balance_e=balance_e) - self._assert_es_matches_fit_time(efficient_fitted, fit_time, balance_e=balance_e) + def test_balance_e_retains_cohorts_present_in_the_anchor_bucket( + self, efficient_fitted, balance_e + ): + """(v) EfficientDiD semantics AS IMPLEMENTED: a cohort is retained iff + it has a finite cell in the truncation BUCKET ``int(t - g) == + balance_e``; every bucket of a retained cohort stays (no lower bound) + and the per-bucket att pools the retained cohorts' cells.""" + es = efficient_fitted.aggregate("event_study", balance_e=balance_e) + gte = efficient_fitted.group_time_effects + retained = { + g for (g, t), c in gte.items() if int(t - g) == balance_e and np.isfinite(c["effect"]) + } + assert retained + buckets = sorted({int(t - g) for (g, t) in gte if g in retained}) + assert sorted(int(e) for e in es.event_time) == buckets + assert min(buckets) < 0 + _assert_att_from_cells( + efficient_fitted, es, bucket=True, retained=retained, name=f"edid/be{balance_e}" + ) + if balance_e == 2: + assert max(gte, key=lambda k: k[0])[0] not in retained - def test_cluster_inertness(self): - d = _efficient_clustered_panel() - plain = _fit_efficient(d, est_kw={"cluster": "cl"}) - fit_time = _fit_efficient(d, est_kw={"cluster": "cl"}, aggregate="all") - self._assert_es_matches_fit_time(plain, fit_time) - grp = plain.aggregate("group") - for i, g in enumerate(grp.label): - np.testing.assert_allclose(fit_time.group_effects[g]["se"], grp.se[i], rtol=1e-14) + def test_point_estimates_from_public_cells(self, efficient_fitted): + """(iii) bucketed cohort-size weighting from group_time_effects.""" + _assert_att_from_cells( + efficient_fitted, efficient_fitted.aggregate("event_study"), bucket=True, name="edid" + ) + + def test_analytic_inference_contract(self, efficient_fitted): + """(iv) on the plain panel (no df) and the group/simple levels.""" + for level in ("event_study", "group", "simple"): + _assert_analytic_inference_contract( + efficient_fitted.aggregate(level), alpha=efficient_fitted.alpha + ) - def test_survey_tsl_inertness_and_df(self): + def test_cluster_matches_oracle(self): + plain = _fit_efficient(_efficient_clustered_panel(), est_kw={"cluster": "cl"}) + orow = _ed_oracle("clustered_all") + _assert_es_matches_oracle( + plain.aggregate("event_study"), orow, **ANALYTIC_TOL, name="edid/cluster" + ) + _assert_group_matches_oracle( + plain.aggregate("group"), orow, **ANALYTIC_TOL, name="edid/cluster" + ) + + def test_survey_tsl_matches_oracle_and_df(self): d, _ = _efficient_survey_panel() sd = _efficient_survey_design() plain = _fit_efficient(d, survey_design=sd) - fit_time = _fit_efficient(d, survey_design=sd, aggregate="all") - es = self._assert_es_matches_fit_time(plain, fit_time) + orow = _ed_oracle("survey_weights_all") + es = _assert_es_matches_oracle( + plain.aggregate("event_study"), orow, **ANALYTIC_TOL, name="edid/tsl" + ) + _assert_group_matches_oracle( + plain.aggregate("group"), orow, **ANALYTIC_TOL, name="edid/tsl" + ) # TSL survey: the post-overall snapshot is the finite survey df and # is provenance-exact on every row. assert es.df_survey == plain._aggregation_kit.bookkeeping["df_survey"] @@ -2068,8 +2680,6 @@ def test_survey_tsl_inertness_and_df(self): grp = plain.aggregate("group") # Non-circular oracle: safe_inference(att, se, df=row.df) reproduces # each stored group row's inference (a stale df would mismatch p). - from diff_diff.utils import safe_inference - for i in range(len(grp.label)): t, p, ci = safe_inference( float(grp.att[i]), @@ -2079,30 +2689,24 @@ def test_survey_tsl_inertness_and_df(self): ) np.testing.assert_allclose(t, grp.t_stat[i], rtol=1e-12) np.testing.assert_allclose(p, grp.p_value[i], rtol=1e-12) + assert _assert_analytic_inference_contract(es, alpha=plain.alpha) > 0 - def test_replicate_inertness_healthy_and_dropped(self): - from diff_diff.utils import safe_inference - + def test_replicate_df_contract_healthy_and_dropped(self): for degenerate, expected_df in ((None, 7.0), ("dropped", 5.0)): d, rep_cols = _efficient_survey_panel(replicate=True, degenerate=degenerate) sd = _efficient_survey_design(rep_cols) plain = _fit_efficient(d, survey_design=sd) - fit_time = _fit_efficient(d, survey_design=sd, aggregate="all") - # Inertness against the SAME object's stored fit-time surface. - self._assert_es_matches_fit_time(plain, fit_time) + es = plain.aggregate("event_study") grp = plain.aggregate("group") - # Cross-FIT df contrast: all group rows share n_valid - 1. + orow = _ed_oracle(f"replicate_{'healthy' if degenerate is None else degenerate}_all") + _assert_es_matches_oracle(es, orow, **ANALYTIC_TOL, name=f"edid/rep/{degenerate}") + _assert_group_matches_oracle(grp, orow, **ANALYTIC_TOL, name=f"edid/rep/{degenerate}") + # Cross-FIT df contrast: all rows share n_valid - 1. assert set(np.unique(grp.df[np.isfinite(grp.df)])) <= {expected_df} - for i in range(len(grp.label)): - if not np.isfinite(grp.df[i]): - continue - t, p, ci = safe_inference( - float(grp.att[i]), - float(grp.se[i]), - alpha=plain.alpha, - df=float(grp.df[i]), - ) - np.testing.assert_allclose(p, grp.p_value[i], rtol=1e-12) + es_df = np.asarray(es.df, dtype=float) + assert set(np.unique(es_df[np.isfinite(es_df)])) <= {expected_df} + assert _assert_analytic_inference_contract(grp, alpha=plain.alpha) > 0 + assert _assert_analytic_inference_contract(es, alpha=plain.alpha) > 0 def test_replicate_undefined_df_degenerate(self): # n_valid <= 1: the working df degenerates to None mid-fit. The @@ -2110,31 +2714,40 @@ def test_replicate_undefined_df_degenerate(self): # df_survey resolves to the 0.0 replicate-undefined sentinel (the # shared resolver ladder), NOT the raw survey_metadata value the # is-not-None guard leaves finite - this arm is the discriminator - # for the carrier's metadata copy. + # for the carrier's metadata copy. The per-row df column is NaN + # (the 0 sentinel NaNs the inference triple). d, rep_cols = _efficient_survey_panel(replicate=True, degenerate="undefined") sd = _efficient_survey_design(rep_cols) plain = _fit_efficient(d, survey_design=sd) assert plain._aggregation_kit.bookkeeping["df_survey"] is None es = plain.aggregate("event_study") assert es.df_survey == 0.0 - fit_time = _fit_efficient(d, survey_design=sd, aggregate="all") - self._assert_es_matches_fit_time(plain, fit_time) + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) + _assert_analytic_inference_contract(es, alpha=plain.alpha) - def test_pt_post_inertness_and_reference(self, efficient_panel): + def test_pt_post_reference_anchor(self, efficient_panel): plain = _fit_efficient(efficient_panel, est_kw={"pt_assumption": "post"}) - fit_time = _fit_efficient( - efficient_panel, est_kw={"pt_assumption": "post"}, aggregate="all" - ) - es = self._assert_es_matches_fit_time(plain, fit_time, skip_reference=True) + es = plain.aggregate("event_study") # Exactly one is_reference row, at the materialized mechanical anchor. marked = [int(e) for e, m in zip(es.event_time, es.is_reference) if m] assert marked == [-1] - # The NATIVE anchor is an exact mechanical zero BEFORE marking (the - # explicit-reference branch has no value check; post_init would - # silently rewrite a nonzero anchor, so the native pin is the guard). - assert fit_time.event_study_effects[-1]["effect"] == 0.0 - assert fit_time.event_study_effects[-1]["se"] == 0.0 - assert fit_time.event_study_effects[-1]["n_groups"] > 0 + i = list(es.event_time).index(-1) + assert es.att[i] == 0.0 and np.isnan(es.se[i]) and np.isnan(es.n[i]) + # The PUBLIC anchor cells are exact mechanical zeros BEFORE marking + # (the explicit-reference branch has no value check; post_init would + # silently rewrite a nonzero anchor, so the cell pin is the guard). + anchors = [c for (g, t), c in plain.group_time_effects.items() if t == g - 1] + assert anchors and all(c["effect"] == 0.0 and c["se"] == 0.0 for c in anchors) + _assert_att_from_cells(plain, es, bucket=True, name="edid/pt_post") + + def test_legacy_fields_none_after_every_fit(self, efficient_panel, efficient_fitted): + """A plain EfficientDiD fit stores no fit-time surface (analytic OR + bootstrapped) - the kit recompute / replay serves both.""" + assert efficient_fitted.event_study_effects is None + assert efficient_fitted.group_effects is None + boot = _fit_efficient(efficient_panel, est_kw={"n_bootstrap": 20, "seed": 1}) + assert boot.event_study_effects is None and boot.group_effects is None + assert boot.event_study_df is None def test_pt_post_absent_anchor_not_synthesized(self): # Single cohort baselined at the panel's first period: the anchor @@ -2249,14 +2862,15 @@ def test_bootstrap_survey_simple_relay_df_nan(self): assert res.survey_metadata is not None _assert_bootstrap_simple_relay(res) - def test_bootstrap_fit_time_group_rows_clear_df_used(self, efficient_panel): - res = _fit_efficient( - efficient_panel, - est_kw={"n_bootstrap": 20, "seed": 1}, - aggregate="group", - ) - for row in res.group_effects.values(): - assert row.get("df_used") is None + def test_bootstrap_group_rows_df_is_nan(self, efficient_panel): + """Percentile inference publishes no analytical df: the replayed + group rows carry a NaN df column (nothing is stored at fit).""" + res = _fit_efficient(efficient_panel, est_kw={"n_bootstrap": 20, "seed": 1}) + assert res.group_effects is None + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + grp = res.aggregate("group") + assert np.all(np.isnan(np.asarray(grp.df, dtype=float))) def test_weights_rejected(self, efficient_fitted): with pytest.raises(ValueError, match="does not accept a weights selector"): @@ -2341,9 +2955,8 @@ def test_zero_row_balance_e_surface(self, efficient_fitted): # EfficientDiD bootstrap REPLAY (the CS BootstrapReplaySpec mechanism): # post-fit aggregate('event_study'/'group') on bootstrapped fits replays the # fit-time multiplier bootstrap from the kit-retained RNG state. The parity -# REFERENCE is always the NATIVE fit-time surface (the kit attaches -# unconditionally, so a fit-time-aggregated result's own aggregate() would -# ALSO replay - replay-vs-replay proves nothing). +# REFERENCE is the pre-removal fit-time surface captured in the oracle file +# (backend-matched: the seeded multiplier stream differs per weight backend). # Tolerances: se/ci/t at 1e-13 (the replayed draws are bit-identical; only # GEMM tile-boundary reassociation differs - ~1 ULP, with headroom for # quantile interpolation); percentile p-values are COUNT statistics, so a @@ -2369,50 +2982,42 @@ def _efficient_boot_fit_time(data, *, aggregate="all", **fit_kw): ) -def _assert_edid_es_replay_parity(es, fit_time, n_boot=_EDID_NBOOT): - df = es.to_dataframe() - assert es.vcov is None - assert np.all(np.isnan(df["df"].to_numpy(dtype=float))) - assert len(df) == len(fit_time.event_study_effects) - p_atol = 2.0 / n_boot - for _, row in df.iterrows(): - ref = fit_time.event_study_effects[int(row["event_time"])] - assert row["att"] == ref["effect"] - np.testing.assert_allclose(row["se"], ref["se"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(row["t_stat"], ref["t_stat"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(row["p_value"], ref["p_value"], rtol=1e-13, atol=p_atol) - np.testing.assert_allclose( - [row["conf_int_lower"], row["conf_int_upper"]], - list(ref["conf_int"]), - rtol=1e-13, - atol=1e-13, - ) +def _ed_boot_oracle(arm): + return boot_oracles()["EfficientDiD"][arm] -def _assert_edid_group_replay_parity(grp, fit_time, n_boot=_EDID_NBOOT): - df = grp.to_dataframe() - assert np.all(np.isnan(df["df"].to_numpy(dtype=float))) - assert len(df) == len(fit_time.group_effects) - p_atol = 2.0 / n_boot - for _, row in df.iterrows(): - ref = fit_time.group_effects[float(row["label"])] - assert row["att"] == ref["effect"] - np.testing.assert_allclose(row["se"], ref["se"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(row["t_stat"], ref["t_stat"], rtol=1e-13, atol=1e-13) - np.testing.assert_allclose(row["p_value"], ref["p_value"], rtol=1e-13, atol=p_atol) - np.testing.assert_allclose( - [row["conf_int_lower"], row["conf_int_upper"]], - list(ref["conf_int"]), - rtol=1e-13, - atol=1e-13, - ) +def _assert_edid_es_replay_parity(es, orow, n_boot=_EDID_NBOOT): + # EfficientDiD's replay container carries no joint vcov and no sup-t band + # (the kit hard-codes cband=False); percentile inference publishes no df. + assert es.vcov is None and es.vcov_index is None + assert es.cband_lower is None and es.cband_upper is None + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) + return _assert_es_matches_oracle(es, orow, **BOOT_TOL, p_atol=2.0 / n_boot, name="edid/boot") + + +def _assert_edid_group_replay_parity(grp, orow, n_boot=_EDID_NBOOT): + assert np.all(np.isnan(np.asarray(grp.df, dtype=float))) + return _assert_group_matches_oracle( + grp, orow, **BOOT_TOL, p_atol=2.0 / n_boot, name="edid/boot" + ) + + +def _assert_edid_replay_contract(res, levels=("event_study", "group")): + """Structural replay pins asserted ALONGSIDE the oracle comparison: NaN df, + no vcov, p in [0, 1], same-fit idempotency.""" + first = {lv: res.aggregate(lv) for lv in levels} + again = {lv: res.aggregate(lv) for lv in levels} + for lv, surface in first.items(): + assert np.all(np.isnan(np.asarray(surface.df, dtype=float))), lv + _assert_bootstrap_p_values_in_unit_interval(surface) + np.testing.assert_array_equal(np.asarray(surface.se), np.asarray(again[lv].se)) + if "event_study" in first: + assert first["event_study"].vcov is None + return first def _efficient_fractional_panel(cohorts=(3.0, 2.25), seed=0, n_units=90): """0.5-spaced panel; cohorts may sit ON the grid or OFF it (e.g. 2.25).""" - import numpy as np - import pandas as pd - rng = np.random.default_rng(seed) periods = np.arange(1.0, 5.0, 0.5) per_cohort = n_units // (len(cohorts) + 2) @@ -2427,20 +3032,48 @@ def _efficient_fractional_panel(cohorts=(3.0, 2.25), seed=0, n_units=90): class TestEfficientBootstrapReplay: - def test_event_study_parity_with_fit_time(self, efficient_panel): + def test_event_study_matches_oracle(self, efficient_panel): res = _efficient_boot_fit(efficient_panel) - ftime = _efficient_boot_fit_time(efficient_panel) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) + _assert_edid_es_replay_parity(res.aggregate("event_study"), _ed_boot_oracle("default_all")) - def test_group_parity_with_fit_time(self, efficient_panel): + def test_group_matches_oracle(self, efficient_panel): res = _efficient_boot_fit(efficient_panel) - ftime = _efficient_boot_fit_time(efficient_panel) - _assert_edid_group_replay_parity(res.aggregate("group"), ftime) + _assert_edid_group_replay_parity(res.aggregate("group"), _ed_boot_oracle("default_all")) - def test_balance_e_parity_with_fit_time(self, efficient_panel): + def test_plain_overall_matches_oracle(self, efficient_panel): res = _efficient_boot_fit(efficient_panel) - ftime = _efficient_boot_fit_time(efficient_panel, aggregate="event_study", balance_e=1) - _assert_edid_es_replay_parity(res.aggregate("event_study", balance_e=1), ftime) + _assert_overall_matches_oracle( + res, + _ed_boot_oracle("default_plain_overall"), + **BOOT_TOL, + p_atol=2.0 / _EDID_NBOOT, + name="edid/boot", + ) + _assert_overall_matches_oracle( + res, _ed_boot_oracle("default_all")["overall"], **BOOT_TOL, p_atol=2.0 / _EDID_NBOOT + ) + + def test_same_seed_fits_replay_identically(self, efficient_panel): + """(vi) determinism across two same-seed fits; nothing stored at fit.""" + a = _efficient_boot_fit(efficient_panel) + b = _efficient_boot_fit(efficient_panel) + assert a.event_study_effects is None and a.group_effects is None + _frame_equal( + a.aggregate("event_study").to_dataframe(), b.aggregate("event_study").to_dataframe() + ) + _frame_equal(a.aggregate("group").to_dataframe(), b.aggregate("group").to_dataframe()) + + def test_balance_e_replay_keeps_the_analytic_anchor_rule( + self, efficient_panel, efficient_fitted + ): + res = _efficient_boot_fit(efficient_panel) + es = res.aggregate("event_study", balance_e=1) + ana = efficient_fitted.aggregate("event_study", balance_e=1) + assert list(es.event_time) == list(ana.event_time) + np.testing.assert_array_equal(np.asarray(es.att), np.asarray(ana.att)) + np.testing.assert_array_equal(np.asarray(es.n), np.asarray(ana.n)) + _assert_edid_replay_contract(res, levels=("event_study",)) + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) def test_balance_e_empty_anchor_replay_warns_zero_rows(self, efficient_panel): # The replay re-emits the fit-time anchor warning and returns the @@ -2451,33 +3084,39 @@ def test_balance_e_empty_anchor_replay_warns_zero_rows(self, efficient_panel): es = res.aggregate("event_study", balance_e=99) assert len(es.event_time) == 0 - def test_pt_post_parity_with_fit_time(self, efficient_panel): + def test_pt_post_reference_row_survives_replay(self, efficient_panel): # PT-Post: the bootstrap prep DOES key the finite effect=0.0 anchor # at e=-1 and the override NaNs its inference (zero-SE draws); the - # is_reference marking is label-based and survives. Parity holds - # row-for-row, the anchor included. + # is_reference marking is label-based and survives. res = _efficient_boot_fit(efficient_panel, est_kw={"pt_assumption": "post"}) - ftime = _efficient_boot_fit_time(efficient_panel, est_kw={"pt_assumption": "post"}) es = res.aggregate("event_study") df = es.to_dataframe() anchor = df[df["is_reference"]] assert len(anchor) == 1 assert float(anchor["att"].iloc[0]) == 0.0 assert np.isnan(float(anchor["se"].iloc[0])) - # Non-reference rows hit full parity; the reference row's fit-time - # DICT entry is also NaN'd by the override (same applier), so the - # container/dict split is exercised on the fractional fixture below. - _assert_edid_es_replay_parity(es, ftime) + _assert_edid_replay_contract(res, levels=("event_study",)) + non_ref = df[~df["is_reference"]] + assert np.all(np.isfinite(non_ref["se"].to_numpy(dtype=float))) - def test_covariates_parity_with_fit_time(self, efficient_panel): + def test_covariates_replay_contract(self, efficient_panel): d = efficient_panel.copy() rng = np.random.default_rng(3) xmap = {u: rng.normal() for u in d["unit"].unique()} d["x1"] = d["unit"].map(xmap) res = _efficient_boot_fit(d, covariates=["x1"]) - ftime = _efficient_boot_fit_time(d, covariates=["x1"]) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) - _assert_edid_group_replay_parity(res.aggregate("group"), ftime) + surfaces = _assert_edid_replay_contract(res) + orow = _ed_boot_oracle("covariates_all") + _assert_edid_es_replay_parity(surfaces["event_study"], orow) + _assert_edid_group_replay_parity(surfaces["group"], orow) + # Same analytical point path as the analytic covariate fit. + ana = _fit_efficient(d, covariates=["x1"]) + np.testing.assert_array_equal( + np.asarray(surfaces["event_study"].att), np.asarray(ana.aggregate("event_study").att) + ) + np.testing.assert_array_equal( + np.asarray(surfaces["group"].att), np.asarray(ana.aggregate("group").att) + ) def test_seedless_fit_replays_and_is_idempotent(self, efficient_panel): res = _fit_efficient(efficient_panel, est_kw={"n_bootstrap": _EDID_NBOOT}) @@ -2499,8 +3138,6 @@ def test_set_params_and_mutation_immunity(self, efficient_panel): np.testing.assert_array_equal(before["se"].to_numpy(), after["se"].to_numpy()) def test_pickle_round_trip_replays(self, efficient_panel): - import pickle - res = _efficient_boot_fit(efficient_panel) before = res.aggregate("group").to_dataframe() res2 = pickle.loads(pickle.dumps(res)) @@ -2523,8 +3160,6 @@ def test_legacy_kit_without_spec_fails_closed(self, efficient_panel): res.aggregate(level) def test_backend_mismatch_fails_closed(self, efficient_panel): - import dataclasses as dc - from diff_diff.bootstrap_chunking import effective_weight_backend res = _efficient_boot_fit(efficient_panel) @@ -2532,13 +3167,13 @@ def test_backend_mismatch_fails_closed(self, efficient_panel): current = effective_weight_backend() other = "numpy" if current == "rust" else "rust" assert kit.bootstrap.backend == current # plain fits stamp the live backend - object.__setattr__(kit, "bootstrap", dc.replace(kit.bootstrap, backend=other)) + object.__setattr__(kit, "bootstrap", dataclasses.replace(kit.bootstrap, backend=other)) for level in ("event_study", "group"): with pytest.raises(NotImplementedError, match="weight backend"): res.aggregate(level) # None (unknown) also fails closed - a permissive default on a # safety discriminator would disarm the guard. - object.__setattr__(kit, "bootstrap", dc.replace(kit.bootstrap, backend=None)) + object.__setattr__(kit, "bootstrap", dataclasses.replace(kit.bootstrap, backend=None)) with pytest.raises(NotImplementedError, match="weight backend"): res.aggregate("event_study") @@ -2549,21 +3184,22 @@ def test_low_bootstrap_warning_re_emitted_on_replay(self, efficient_panel): class TestEfficientBootstrapReplayDesigns: - def test_cluster_parity(self): + def test_cluster_matches_oracle(self): d = _efficient_clustered_panel() res = _efficient_boot_fit(d, est_kw={"cluster": "cl"}) - ftime = _efficient_boot_fit_time(d, est_kw={"cluster": "cl"}) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) - _assert_edid_group_replay_parity(res.aggregate("group"), ftime) + orow = _ed_boot_oracle("clustered_all") + _assert_edid_es_replay_parity(res.aggregate("event_study"), orow) + _assert_edid_group_replay_parity(res.aggregate("group"), orow) - def test_weights_only_survey_parity(self): + def test_weights_only_survey_matches_oracle(self): # Weights-only design: unit weight PATH + survey EIF scaling. d, _ = _efficient_survey_panel() sd = _efficient_survey_design() res = _efficient_boot_fit(d, survey_design=sd) - ftime = _efficient_boot_fit_time(d, survey_design=sd) assert res._aggregation_kit.bootstrap.backend != "portable" - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) + orow = _ed_boot_oracle("survey_weights_all") + _assert_edid_es_replay_parity(res.aggregate("event_study"), orow) + _assert_edid_group_replay_parity(res.aggregate("group"), orow) @staticmethod def _psu_survey_frame(strata=False, fpc=None): @@ -2582,45 +3218,37 @@ def _psu_survey_frame(strata=False, fpc=None): kw["fpc"] = "fpc_col" return d, SurveyDesign(**kw) - def test_stratified_survey_portable_stamp_and_parity(self): + def test_stratified_survey_portable_stamp_and_matches_oracle(self): d, sd = self._psu_survey_frame(strata=True) res = _efficient_boot_fit(d, survey_design=sd) assert res._aggregation_kit.bootstrap.backend == "portable" - ftime = _efficient_boot_fit_time(d, survey_design=sd) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) - _assert_edid_group_replay_parity(res.aggregate("group"), ftime) + orow = _ed_boot_oracle("stratified_psu_survey_all") + _assert_edid_es_replay_parity(res.aggregate("event_study"), orow) + _assert_edid_group_replay_parity(res.aggregate("group"), orow) - def test_fpc_parity(self): + def test_fpc_matches_oracle(self): # Non-census FPC (population 10x the PSU count): fpc_scale on top of # the backend-dependent generator. d, sd = self._psu_survey_frame(fpc=200.0) res = _efficient_boot_fit(d, survey_design=sd) assert res._aggregation_kit.bootstrap.backend != "portable" - ftime = _efficient_boot_fit_time(d, survey_design=sd) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) + orow = _ed_boot_oracle("fpc_survey_all") + _assert_edid_es_replay_parity(res.aggregate("event_study"), orow) + _assert_edid_group_replay_parity(res.aggregate("group"), orow) def test_census_fpc_portable(self): # Census FPC (fpc == n_psu): every weight block is zeroed, so every # bootstrap distribution is EXACTLY CONSTANT at the original effect # - the discarded draws' backend is irrelevant (stamped portable), # and the constant-distribution guard must NaN ALL inference fields - # on BOTH routes (a tiny-positive roundoff np.std at a non-zero - # constant level must never leak a huge finite t - CI review P0). + # (a tiny-positive roundoff np.std at a non-zero constant level must + # never leak a huge finite t - CI review P0). d, sd = self._psu_survey_frame(fpc=20.0) # 120 units // 6 = 20 PSUs with warnings.catch_warnings(): warnings.simplefilter("ignore") res = _efficient_boot_fit(d, survey_design=sd) - ftime = _efficient_boot_fit_time(d, survey_design=sd) assert res._aggregation_kit.bootstrap.backend == "portable" - # Fit-time surfaces: full-NaN inference beside finite effects. - for surface in (ftime.event_study_effects, ftime.group_effects): - assert surface - for row in surface.values(): - assert np.isfinite(row["effect"]) - assert np.isnan(row["se"]) and np.isnan(row["t_stat"]) - assert np.isnan(row["p_value"]) - assert np.isnan(row["conf_int"][0]) and np.isnan(row["conf_int"][1]) - # Replayed surfaces reproduce the same degenerate state. + assert res.event_study_effects is None and res.group_effects is None with warnings.catch_warnings(): warnings.simplefilter("ignore") es_df = res.aggregate("event_study").to_dataframe() @@ -2658,72 +3286,121 @@ def test_single_psu_nan_surfaces_and_warning(self): grp = res.aggregate("group").to_dataframe() assert np.all(np.isnan(grp["se"].to_numpy(dtype=float))) - def test_anticipation_parity(self, efficient_panel): + def test_anticipation_wiring(self, efficient_panel): # Pins the replay host's anticipation wiring (shifts the engine's - # post-treatment mask and the group prep's inclusion rule); a - # defaulted host attribute would pass every anticipation=0 arm. + # post-treatment mask and the group prep's inclusion rule): the + # replayed surface carries the analytic anticipation=1 container's + # labels / point estimates / anticipation, which an anticipation=0 + # host could not reproduce. res = _efficient_boot_fit(efficient_panel, est_kw={"anticipation": 1}) - ftime = _efficient_boot_fit_time(efficient_panel, est_kw={"anticipation": 1}) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) - _assert_edid_group_replay_parity(res.aggregate("group"), ftime) + orow = _ed_boot_oracle("anticipation1_all") + _assert_edid_es_replay_parity(res.aggregate("event_study"), orow) + _assert_edid_group_replay_parity(res.aggregate("group"), orow) + ana = _fit_efficient(efficient_panel, est_kw={"anticipation": 1}) + surfaces = _assert_edid_replay_contract(res) + es, ana_es = surfaces["event_study"], ana.aggregate("event_study") + assert es.anticipation == ana_es.anticipation == 1 + assert list(es.event_time) == list(ana_es.event_time) + np.testing.assert_array_equal(np.asarray(es.att), np.asarray(ana_es.att)) + np.testing.assert_array_equal( + np.asarray(surfaces["group"].att), np.asarray(ana.aggregate("group").att) + ) + # ...and differs from the anticipation=0 replay on the same draws. + base = _efficient_boot_fit(efficient_panel).aggregate("event_study") + assert list(base.event_time) != list(es.event_time) or not np.array_equal( + np.asarray(base.att), np.asarray(es.att) + ) - def test_alpha_and_mammen_parity(self, efficient_panel): + def test_alpha_and_mammen_wiring(self, efficient_panel): # Pins the host/spec wiring of alpha and weight_type - a host that # hard-codes the defaults passes every other arm. - kw = {"alpha": 0.10, "bootstrap_weights": "mammen"} - res = _efficient_boot_fit(efficient_panel, est_kw=kw) - ftime = _efficient_boot_fit_time(efficient_panel, est_kw=kw) - _assert_edid_es_replay_parity(res.aggregate("event_study"), ftime) - _assert_edid_group_replay_parity(res.aggregate("group"), ftime) + res = _efficient_boot_fit( + efficient_panel, est_kw={"alpha": 0.10, "bootstrap_weights": "mammen"} + ) + surfaces = _assert_edid_replay_contract(res) + es = surfaces["event_study"] + assert es.alpha == 0.10 and surfaces["group"].alpha == 0.10 + orow = _ed_boot_oracle("alpha10_mammen_all") + _assert_edid_es_replay_parity(es, orow) + _assert_edid_group_replay_parity(surfaces["group"], orow) + # alpha wiring: the 90% interval sits inside the 95% one on the SAME + # mammen draws (same seed); weight-type wiring: mammen != rademacher. + wide = _efficient_boot_fit( + efficient_panel, est_kw={"alpha": 0.05, "bootstrap_weights": "mammen"} + ).aggregate("event_study") + rade = _efficient_boot_fit(efficient_panel).aggregate("event_study") + non_ref = ~np.asarray(es.is_reference, dtype=bool) + np.testing.assert_array_equal(np.asarray(es.se)[non_ref], np.asarray(wide.se)[non_ref]) + assert np.all( + np.asarray(es.conf_int_lower)[non_ref] >= np.asarray(wide.conf_int_lower)[non_ref] + ) + assert np.all( + np.asarray(es.conf_int_upper)[non_ref] <= np.asarray(wide.conf_int_upper)[non_ref] + ) + assert not np.allclose(np.asarray(es.se)[non_ref], np.asarray(rade.se)[non_ref]) class TestEfficientFractionalPeriods: """Decision-10 pins: int(t - g) truncation-bucketing on fractional panels. - The published fit-time key set is int-bucketed by the ANALYTICAL - aggregator regardless of the bootstrap prep's keying, so the - discriminating surfaces are (a) the prep's own key set and (b) the - off-grid-onset arm, where pre-fix NO raw key intersected the analytical - buckets and the fit-time rows kept analytical inference. + The container's key set is int-bucketed by the ANALYTICAL aggregator and + the bootstrap prep keys the same buckets, so the discriminating surfaces + are (a) the replay's own key set and (b) the off-grid-onset arm, where + pre-fix NO raw key intersected the analytical buckets and the rows kept + analytical inference. """ - def test_prep_keys_match_analytical_buckets_and_warns(self): - from diff_diff import EfficientDiD - + def test_replay_keys_are_analytical_buckets_and_warns(self): d = _efficient_fractional_panel() - # _fit_efficient suppresses warnings, so fit directly to pin the - # fractional-truncation warning alongside the prep-key assertion. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res = _efficient_boot_fit(d) + # _fit_efficient suppresses warnings, so aggregate directly to pin the + # fractional-truncation warning alongside the key-set assertion. with pytest.warns(UserWarning, match="bucketed by int"): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - res = EfficientDiD(n_bootstrap=_EDID_NBOOT, seed=42).fit( - d, aggregate="all", **EFFICIENT_KW - ) - assert set(res.bootstrap_results.event_study_ses) >= set(res.event_study_effects) - assert all(isinstance(e, int) for e in res.bootstrap_results.event_study_ses) + es = res.aggregate("event_study") + labels = np.asarray(es.event_time, dtype=float) + assert np.all(labels == np.round(labels)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ana = _fit_efficient(d).aggregate("event_study") + assert list(es.event_time) == list(ana.event_time) def test_offgrid_rows_carry_percentile_inference(self): # EVERY treated cohort off-grid: raw t-g is never an integer, so - # pre-fix no override would land and the fit-time rows would keep - # ANALYTICAL inference - the end-to-end discriminator. + # pre-fix no override would land and the rows would keep ANALYTICAL + # inference - the end-to-end discriminator. d = _efficient_fractional_panel(cohorts=(2.25, 3.75)) with warnings.catch_warnings(): warnings.simplefilter("ignore") - boot = _efficient_boot_fit_time(d) - ana = _fit_efficient(d, aggregate="all") - for e, row in boot.event_study_effects.items(): - assert row["se"] != ana.event_study_effects[e]["se"] - - def test_replay_parity_on_fractional_panel(self): + boot = _efficient_boot_fit(d).aggregate("event_study") + ana = _fit_efficient(d).aggregate("event_study") + assert list(boot.event_time) == list(ana.event_time) + non_ref = ~np.asarray(boot.is_reference, dtype=bool) + for b_se, a_se in zip(np.asarray(boot.se)[non_ref], np.asarray(ana.se)[non_ref]): + assert b_se != a_se + + def test_replay_contract_on_fractional_panel(self): d = _efficient_fractional_panel() with warnings.catch_warnings(): warnings.simplefilter("ignore") res = _efficient_boot_fit(d) - ftime = _efficient_boot_fit_time(d) + again = _efficient_boot_fit(d) + es = _assert_edid_replay_contract(res, levels=("event_study",))["event_study"] + np.testing.assert_array_equal( + np.asarray(es.se), np.asarray(again.aggregate("event_study").se) + ) + + def test_fractional_balance_e1_matches_oracles(self): + d = _efficient_fractional_panel() with warnings.catch_warnings(): warnings.simplefilter("ignore") - es = res.aggregate("event_study") - _assert_edid_es_replay_parity(es, ftime) + ana = _fit_efficient(d).aggregate("event_study", balance_e=1) + boot = _efficient_boot_fit(d).aggregate("event_study", balance_e=1) + _assert_es_matches_oracle( + ana, _ed_oracle("fractional_balance_e1"), **ANALYTIC_TOL, name="edid/frac/be1" + ) + _assert_edid_es_replay_parity(boot, _ed_boot_oracle("fractional_balance_e1_es")) def test_n_groups_counts_distinct_cohorts(self): # Fractional buckets pool multiple cells per cohort; n_groups must @@ -2731,22 +3408,24 @@ def test_n_groups_counts_distinct_cohorts(self): d = _efficient_fractional_panel() with warnings.catch_warnings(): warnings.simplefilter("ignore") - res = _fit_efficient(d, aggregate="event_study") - for e, row in res.event_study_effects.items(): - assert row["n_groups"] <= 2, (e, row["n_groups"]) + es = _fit_efficient(d).aggregate("event_study") + for e, n in zip(es.event_time, es.n): + if np.isfinite(n): + assert n <= 2, (e, n) - def test_integer_panel_n_groups_regression(self, efficient_fit_time): + def test_integer_panel_n_groups_regression(self, efficient_fitted): # Identity claim: on integer panels one cell per cohort per bucket, # so distinct-cohort counting equals the old cell count. Pin the # exact values on the standard 2-cohort fixture (cohorts 4 and 6 on # an 8-period panel: both cohorts share buckets -1..1 given cohort # 6's horizons span -5..1, cohort 4's -3..3). + es = efficient_fitted.aggregate("event_study") + gte = efficient_fitted.group_time_effects expected = { - e: len({g for (g, t) in efficient_fit_time.group_time_effects if int(t - g) == e}) - for e in efficient_fit_time.event_study_effects + int(e): len({g for (g, t) in gte if int(t - g) == int(e)}) for e in es.event_time } - for e, row in efficient_fit_time.event_study_effects.items(): - assert row["n_groups"] == expected[e] + for e, n in zip(es.event_time, es.n): + assert n == expected[int(e)] assert max(expected.values()) == 2 # both cohorts pool somewhere def test_integer_panel_never_warns(self, efficient_panel): @@ -2757,18 +3436,29 @@ def test_integer_panel_never_warns(self, efficient_panel): assert not any("bucketed by int" in str(w.message) for w in caught) def test_fractional_balance_e_offgrid_anchor(self): - # Cohort 2.25 reaches the integer anchor bucket 1 only via raw - # horizons 1.25/1.75 - the :403 anchor-filter pin: the bootstrap - # balanced cohort set must match the analytical one. + # (v) Cohort 2.25 reaches the integer anchor bucket 1 only via raw + # horizons 1.25/1.75 - the bucket rule keeps it (the exact-anchor CS + # rule would not), both cohorts are anchored, and the replay's + # balanced cohort set matches the analytical one. d = _efficient_fractional_panel() with warnings.catch_warnings(): warnings.simplefilter("ignore") - boot = _efficient_boot_fit_time(d, aggregate="event_study", balance_e=1) - ana = _fit_efficient(d, aggregate="event_study", balance_e=1) - assert set(boot.bootstrap_results.event_study_ses) >= set(ana.event_study_effects) - assert ana.event_study_effects[1]["n_groups"] == 2 # both cohorts anchored - for e, row in boot.event_study_effects.items(): - assert row["se"] != ana.event_study_effects[e]["se"] or np.isnan(row["se"]) + plain = _fit_efficient(d) + ana = plain.aggregate("event_study", balance_e=1) + boot = _efficient_boot_fit(d).aggregate("event_study", balance_e=1) + gte = plain.group_time_effects + retained = {g for (g, t), c in gte.items() if int(t - g) == 1 and np.isfinite(c["effect"])} + assert 2.25 in retained and 3.0 in retained + assert not any(t - g == 1 for (g, t) in gte if g == 2.25) # never at the EXACT anchor + i = list(ana.event_time).index(1) + assert ana.n[i] == 2 # both cohorts anchored + assert sorted(int(e) for e in ana.event_time) == sorted( + {int(t - g) for (g, t) in gte if g in retained} + ) + assert list(boot.event_time) == list(ana.event_time) + non_ref = ~np.asarray(boot.is_reference, dtype=bool) + for b_se, a_se in zip(np.asarray(boot.se)[non_ref], np.asarray(ana.se)[non_ref]): + assert b_se != a_se or np.isnan(b_se) def test_bucket_pooled_att_cell_mass_weighting(self): # Hand-computed oracle for the REGISTRY Note's weighting clause: a @@ -2778,46 +3468,49 @@ def test_bucket_pooled_att_cell_mass_weighting(self): d = _efficient_fractional_panel() with warnings.catch_warnings(): warnings.simplefilter("ignore") - res = _fit_efficient(d, aggregate="event_study") + res = _fit_efficient(d) + es = res.aggregate("event_study") kit = res._aggregation_kit fracs = kit.bookkeeping["cohort_fractions"] gt = kit.bookkeeping["group_time_effects"] - for e, row in res.event_study_effects.items(): + for e, att, is_ref in zip(es.event_time, es.att, es.is_reference): + if is_ref: + continue cells = [ (d_["effect"], fracs.get(g, 0.0)) for (g, t), d_ in gt.items() - if int(t - g) == e and np.isfinite(d_["effect"]) + if int(t - g) == int(e) and np.isfinite(d_["effect"]) ] w = np.array([c[1] for c in cells]) effs = np.array([c[0] for c in cells]) w = w / w.sum() if w.sum() > 0 else np.ones(len(w)) / len(w) - np.testing.assert_allclose(row["effect"], float(np.sum(w * effs)), rtol=1e-12) + np.testing.assert_allclose(att, float(np.sum(w * effs)), rtol=1e-12) def test_fractional_pt_post_reference_collision(self): - # Note clause (iii), PER-ROUTE oracles: reference normalization is - # CONTAINER-only. The bucket at -1-anticipation pools a genuine - # fractional pre-treatment estimate with the mechanical zero anchor; - # the container publishes it as the reference (att=0, NaN - # inference), while the fit-time DICT keeps the pooled effect with - # percentile inference. Parity asserted on NON-reference rows only. + # Note clause (iii): reference normalization is CONTAINER-only. The + # bucket at -1-anticipation pools a genuine fractional pre-treatment + # estimate with the mechanical zero anchor; the container publishes + # it as the reference (att=0, NaN inference) although the PUBLIC + # cells pool to a non-zero estimate there. d = _efficient_fractional_panel() with warnings.catch_warnings(): warnings.simplefilter("ignore") res = _efficient_boot_fit(d, est_kw={"pt_assumption": "post"}) - ftime = _efficient_boot_fit_time(d, est_kw={"pt_assumption": "post"}) es = res.aggregate("event_study") df = es.to_dataframe() anchor = df[df["is_reference"]] assert len(anchor) == 1 and int(anchor["event_time"].iloc[0]) == -1 assert float(anchor["att"].iloc[0]) == 0.0 assert np.isnan(float(anchor["se"].iloc[0])) - # The fit-time DICT entry for the same bucket is NOT rewritten. - dict_row = ftime.event_study_effects[-1] - assert dict_row["effect"] != 0.0 or not np.isnan(dict_row["se"]) - for _, row in df[~df["is_reference"]].iterrows(): - ref = ftime.event_study_effects[int(row["event_time"])] - assert row["att"] == ref["effect"] - np.testing.assert_allclose(row["se"], ref["se"], rtol=1e-13, atol=1e-13) + pooled = [ + c["effect"] + for (g, t), c in res.group_time_effects.items() + if int(t - g) == -1 and np.isfinite(c["effect"]) + ] + assert len(pooled) > 1 and any(v != 0.0 for v in pooled) + non_ref = df[~df["is_reference"]] + assert np.all(np.isfinite(non_ref["se"].to_numpy(dtype=float))) + _assert_bootstrap_p_values_in_unit_interval(es) def test_hausman_pretest_fractional_warns(self): d = _efficient_fractional_panel() @@ -2853,8 +3546,9 @@ def test_hausman_pretest_emits_no_future_warning(self, efficient_panel): # --------------------------------------------------------------------------- # -# ImputationDiD (rows M-021/M-118): fit(aggregate=/balance_e=) shim + the -# PANEL-BACKED recompute aggregate() (kit refs = the _fit_data objects) +# ImputationDiD (rows M-021/M-118): fit(aggregate=/balance_e=) shim pins + the +# PANEL-BACKED recompute aggregate() (kit refs = the _fit_data objects) on +# analytic fits; bootstrapped fits RELAY the fit-time percentile surfaces # --------------------------------------------------------------------------- # IMPUTATION_KW = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") @@ -2924,51 +3618,6 @@ def _fit_imputation(data, *, est_kw=None, **fit_kw): return ImputationDiD(**(est_kw or {})).fit(data, **IMPUTATION_KW, **fit_kw) -def _assert_es_container_matches_fit_time(container, fit_time_results, name=""): - """Post-fit EventStudyResults must equal the fit-time-built surface - column-for-column at 1e-14 (incl. reference marking and df provenance).""" - from diff_diff.results_base import build_event_study_surface - - ref = build_event_study_surface(fit_time_results) - da, db = container.to_dataframe(), ref.to_dataframe() - assert list(da.columns) == list(db.columns) - for c in da.columns: - x, y = da[c].to_numpy(), db[c].to_numpy() - if x.dtype.kind in "fc": - np.testing.assert_allclose( - x, y, rtol=0, atol=1e-14, equal_nan=True, err_msg=f"{name}/{c}" - ) - else: - assert (x == y).all(), (name, c) - for attr in ("df_survey", "anticipation", "alpha"): - va, vb = getattr(container, attr), getattr(ref, attr) - same = va == vb or (va is None and vb is None) - try: - same = same or (np.isnan(va) and np.isnan(vb)) - except TypeError: - pass - assert same, (name, attr, va, vb) - - -def _assert_group_matches_fit_time(agg, fit_time_results, name=""): - for i, g in enumerate(agg.label): - row = fit_time_results.group_effects[g] - for field_, key in ( - ("att", "effect"), - ("se", "se"), - ("t_stat", "t_stat"), - ("p_value", "p_value"), - ): - np.testing.assert_allclose( - getattr(agg, field_)[i], - row[key], - rtol=0, - atol=1e-14, - equal_nan=True, - err_msg=f"{name}/{g}/{field_}", - ) - - @pytest.fixture(scope="module") def imputation_panel(): return _imputation_panel() @@ -2986,6 +3635,18 @@ def imputation_fit_time(imputation_panel): return _fit_imputation(imputation_panel, aggregate="all") +def _im_oracle(arm): + return load_oracles()["analytic"]["ImputationDiD"][arm] + + +def _im_boot_oracle(arm): + return boot_oracles()["ImputationDiD"][arm] + + +_IM_BOOT_KW = {"n_bootstrap": 19, "seed": 1} +_IM_P_ATOL = 2.0 / _IM_BOOT_KW["n_bootstrap"] + + class TestImputationShim: def test_plain_fit_does_not_warn(self, imputation_panel): from diff_diff import ImputationDiD @@ -3037,124 +3698,172 @@ def test_warn_and_still_work(self, imputation_fitted, imputation_fit_time): class TestImputationAggregate: - def test_event_study_inert(self, imputation_fitted, imputation_fit_time): + def test_event_study_matches_oracle(self, imputation_fitted): es = imputation_fitted.aggregate("event_study") - _assert_es_container_matches_fit_time(es, imputation_fit_time, "imp/es") + _assert_es_matches_oracle(es, _im_oracle("default_all"), **ANALYTIC_TOL, name="imp/es") - @pytest.mark.parametrize("balance_e", [0, 1, 2]) - def test_balance_e_inert(self, imputation_panel, imputation_fitted, balance_e): - ref = _fit_imputation(imputation_panel, aggregate="event_study", balance_e=balance_e) - es = imputation_fitted.aggregate("event_study", balance_e=balance_e) - _assert_es_container_matches_fit_time(es, ref, f"imp/es/be{balance_e}") - - def test_group_inert(self, imputation_fitted, imputation_fit_time): - _assert_group_matches_fit_time( - imputation_fitted.aggregate("group"), imputation_fit_time, "imp/gr" + def test_balance_e1_matches_oracle(self, imputation_fitted): + es = imputation_fitted.aggregate("event_study", balance_e=1) + _assert_es_matches_oracle( + es, _im_oracle("default_balance_e1"), **ANALYTIC_TOL, name="imp/es/be1" + ) + + def test_group_matches_oracle(self, imputation_fitted): + _assert_group_matches_oracle( + imputation_fitted.aggregate("group"), + _im_oracle("default_all"), + **ANALYTIC_TOL, + name="imp/gr", ) def test_simple_relay_bit_exact(self, imputation_fitted): - sm = imputation_fitted.aggregate("simple") - assert sm.att[0] == imputation_fitted.overall_att - assert sm.se[0] == imputation_fitted.overall_se - assert sm.t_stat[0] == imputation_fitted.overall_t_stat - assert sm.p_value[0] == imputation_fitted.overall_p_value + sm = _assert_simple_relays_stored_overall(imputation_fitted) + _assert_overall_matches_oracle( + imputation_fitted, _im_oracle("default_all")["overall"], **ANALYTIC_TOL, name="imp" + ) assert sm.n_kind == "obs" assert sm.n[0] == imputation_fitted.n_treated_obs assert np.isnan(sm.to_dataframe()["df"].to_numpy()).all() # non-survey fit - def test_cluster_arm_inert(self): - d = _imputation_clustered_panel() - ref = _fit_imputation(d, est_kw={"cluster": "cl"}, aggregate="all") - plain = _fit_imputation(d, est_kw={"cluster": "cl"}) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "imp/cluster") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "imp/cluster") + def test_post_point_estimates_from_public_tau(self, imputation_fitted): + """(iii) att[e >= 0] is the plain mean of treatment_effects.tau_hat.""" + _assert_post_att_is_tau_mean( + imputation_fitted, imputation_fitted.aggregate("event_study"), name="imp" + ) + + def test_analytic_inference_contract(self, imputation_fitted): + """(iv) NaN-consistency everywhere (the zero-count reference row is + normalized by the container; the plain panel carries no df).""" + for level in ("event_study", "group", "simple"): + _assert_analytic_inference_contract( + imputation_fitted.aggregate(level), alpha=imputation_fitted.alpha + ) + + def test_cluster_arm_matches_oracle(self): + plain = _fit_imputation(_imputation_clustered_panel(), est_kw={"cluster": "cl"}) + orow = _im_oracle("clustered_all") + _assert_es_matches_oracle( + plain.aggregate("event_study"), orow, **ANALYTIC_TOL, name="imp/cluster" + ) + _assert_group_matches_oracle( + plain.aggregate("group"), orow, **ANALYTIC_TOL, name="imp/cluster" + ) - def test_covariate_arm_inert(self): + def test_covariate_arm_point_path(self): # Time-varying covariate: the kit must carry delta_hat / - # kept_cov_mask / the covariate columns; an omission escapes every - # covariate-free arm (local-review P2). + # kept_cov_mask / the covariate columns; the post-fit point path must + # reproduce the PUBLIC imputed tau_hat means, which an omission of + # the covariate adjustment could not (local-review P2). d = _imputation_panel(seed=23).copy() rng = np.random.default_rng(23) d["x1"] = rng.normal(size=len(d)) + 0.1 * d["period"] - ref = _fit_imputation(d, aggregate="all", covariates=["x1"]) plain = _fit_imputation(d, covariates=["x1"]) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "imp/cov") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "imp/cov") + es = plain.aggregate("event_study") + _assert_post_att_is_tau_mean(plain, es, name="imp/cov") + _assert_analytic_inference_contract(es, alpha=plain.alpha) + grp = plain.aggregate("group") + orow = _im_oracle("covariates_all") + _assert_es_matches_oracle(es, orow, **ANALYTIC_TOL, name="imp/cov") + _assert_group_matches_oracle(grp, orow, **ANALYTIC_TOL, name="imp/cov") + te = plain.treatment_effects.merge( + d[["unit", "first_treat"]].drop_duplicates("unit"), on="unit", how="left" + ) + for i, g in enumerate(grp.label): + sel = te.loc[(te["first_treat"] == g) & (te["rel_time"] >= 0), "tau_hat"] + np.testing.assert_allclose(grp.att[i], float(sel[np.isfinite(sel)].mean()), rtol=1e-12) - def test_survey_tsl_arm_inert(self): + def test_survey_tsl_arm_matches_oracle(self): d, _ = _imputation_survey_panel() sd = _imputation_survey_design() - ref = _fit_imputation(d, aggregate="all", survey_design=sd) plain = _fit_imputation(d, survey_design=sd) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "imp/tsl") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "imp/tsl") + orow = _im_oracle("survey_tsl_all") + es = _assert_es_matches_oracle( + plain.aggregate("event_study"), orow, **ANALYTIC_TOL, name="imp/tsl" + ) + _assert_group_matches_oracle(plain.aggregate("group"), orow, **ANALYTIC_TOL, name="imp/tsl") sm = plain.aggregate("simple") df_col = sm.to_dataframe()["df"].to_numpy() assert np.isfinite(df_col).all() # survey df threads the simple row + # (iv) identity leg: finite survey df on every estimable row. + assert _assert_analytic_inference_contract(es, alpha=plain.alpha) > 0 + assert _assert_analytic_inference_contract(plain.aggregate("group"), alpha=plain.alpha) > 0 - @pytest.mark.parametrize("degenerate", [None, "dropped", "undefined"]) + @pytest.mark.parametrize("degenerate", [None, "dropped", "undefined", "cohort_zero"]) def test_replicate_arms_level_matched(self, degenerate): - # LEVEL-MATCHED references: aggregate(L) reproduces fit(aggregate=L), - # not fit(aggregate='all') (the joint replicate stack couples rows). + # LEVEL-MATCHED oracles: aggregate(L) reproduces the pre-removal + # fit(aggregate=L), not fit(aggregate='all') (the joint replicate + # stack couples rows). d, rep_cols = _imputation_survey_panel(replicate=True, degenerate=degenerate) sd = _imputation_survey_design(rep_cols) - ref_es = _fit_imputation(d, aggregate="event_study", survey_design=sd) - ref_gr = _fit_imputation(d, aggregate="group", survey_design=sd) plain = _fit_imputation(d, survey_design=sd) - _assert_es_container_matches_fit_time( - plain.aggregate("event_study"), ref_es, f"imp/rep/{degenerate}" + tag = degenerate or "none" + _assert_es_matches_oracle( + plain.aggregate("event_study"), + _im_oracle(f"replicate_{tag}_event_study"), + **ANALYTIC_TOL, + name=f"imp/rep/{tag}", + ) + _assert_group_matches_oracle( + plain.aggregate("group"), + _im_oracle(f"replicate_{tag}_group"), + **ANALYTIC_TOL, + name=f"imp/rep/{tag}", ) - _assert_group_matches_fit_time(plain.aggregate("group"), ref_gr, f"imp/rep/{degenerate}") sm = plain.aggregate("simple") np.testing.assert_allclose(sm.se[0], plain.overall_se, rtol=0, atol=0, equal_nan=True) + # (iv) the degenerate="undefined" arm's df=0 sentinel NaNs t/p beside + # finite effects - a valid NaN-consistent state. + _assert_analytic_inference_contract(plain.aggregate("event_study"), alpha=plain.alpha) + _assert_analytic_inference_contract(plain.aggregate("group"), alpha=plain.alpha) def test_replicate_undefined_df_sentinel(self): # n_valid <= 1: the working df degenerates; the ES container's - # df_survey resolves to the 0.0 replicate-undefined sentinel on - # BOTH routes (the metadata-copy discriminator). + # df_survey resolves to the 0.0 replicate-undefined sentinel (the + # metadata-copy discriminator) and the per-row df column is NaN. d, rep_cols = _imputation_survey_panel(replicate=True, degenerate="undefined") sd = _imputation_survey_design(rep_cols) plain = _fit_imputation(d, survey_design=sd) es = plain.aggregate("event_study") - ref = _fit_imputation(d, aggregate="event_study", survey_design=sd) - from diff_diff.results_base import build_event_study_surface - - assert es.df_survey == build_event_study_surface(ref).df_survey == 0.0 + assert es.df_survey == 0.0 + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) + assert _im_oracle("replicate_undefined_event_study")["event_study_df"] is None def test_replicate_overall_row_migration_delta(self): # The documented migration consequence: the deprecated fit(aggregate=) # coupled the OVERALL row to the joint replicate stack. On the # cohort-zero design (one replicate NaNs one cohort's group target - # while overall stays finite) the joint [overall, groups] stack drops - # a replicate the [overall]-only stack keeps -> plain-fit overall_se - # differs from fit(aggregate='group') overall_se; each surface is - # self-consistent, and post-fit aggregate('group') level-matches the - # deprecated fit(aggregate='group') rows exactly. + # while overall stays finite) the joint [overall, groups] stack + # dropped a replicate the [overall]-only stack keeps -> the plain + # fit's overall_se differs from the pre-removal fit(aggregate='group') + # overall_se (oracle); post-fit aggregate('group') level-matches the + # oracle rows exactly. d, rep_cols = _imputation_survey_panel(replicate=True, degenerate="cohort_zero") sd = _imputation_survey_design(rep_cols) plain = _fit_imputation(d, survey_design=sd) - ref_gr = _fit_imputation(d, aggregate="group", survey_design=sd) - assert plain.overall_se != ref_gr.overall_se - _assert_group_matches_fit_time(plain.aggregate("group"), ref_gr, "imp/delta") - # healthy design: no coupling -> equality - dh, rep_h = _imputation_survey_panel(replicate=True) - sdh = _imputation_survey_design(rep_h) + orow = _im_oracle("replicate_cohort_zero_group") + assert plain.overall_se != oracle_value(orow["overall"]["se"]) + _assert_group_matches_oracle( + plain.aggregate("group"), orow, **ANALYTIC_TOL, name="imp/delta" + ) # Healthy design: no replicate-drop coupling, so the two stacks # agree - but NOT bit-identically: the [overall] vs # [overall, groups] layouts route the replicate-variance matmul - # through different BLAS kernel shapes (~1 ULP reassociation; - # bit-identical on Accelerate, not on OpenBLAS-ARM/Windows). + # through different BLAS kernel shapes (~1 ULP reassociation). # A REAL coupling delta is O(se) itself, far above this band. + dh, rep_h = _imputation_survey_panel(replicate=True) + healthy = _fit_imputation(dh, survey_design=_imputation_survey_design(rep_h)) np.testing.assert_allclose( - _fit_imputation(dh, survey_design=sdh).overall_se, - _fit_imputation(dh, aggregate="group", survey_design=sdh).overall_se, + healthy.overall_se, + oracle_value(_im_oracle("replicate_none_group")["overall"]["se"]), rtol=1e-12, ) - def test_pretrends_arm_inert(self, imputation_panel): - ref = _fit_imputation(imputation_panel, est_kw={"pretrends": True}, aggregate="event_study") + def test_pretrends_arm_matches_oracle(self, imputation_panel): plain = _fit_imputation(imputation_panel, est_kw={"pretrends": True}) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "imp/pretrends") + es = plain.aggregate("event_study") + _assert_es_matches_oracle(es, _im_oracle("pretrends_all"), **ANALYTIC_TOL, name="imp/pre") + assert (np.asarray(es.event_time) < -1).any() # leads present + _assert_post_att_is_tau_mean(plain, es, name="imp/pre") # leads are oracle-only def test_isolation_under_public_field_mutation(self, imputation_panel): res = _fit_imputation(imputation_panel) @@ -3192,6 +3901,8 @@ def test_repeated_and_order_independent(self, imputation_fitted): b = imputation_fitted.aggregate("group").to_dataframe() pd.testing.assert_frame_equal(a, b) + # ---------------- bootstrapped fits (fail closed until the PR 3c relay) ---------------- + def test_bootstrap_recompute_levels_fail_closed(self, imputation_panel): res = _fit_imputation(imputation_panel, est_kw={"n_bootstrap": 19, "seed": 1}) for level in ("event_study", "group"): @@ -3289,8 +4000,9 @@ def test_empty_balance_window(self, imputation_fitted): # --------------------------------------------------------------------------- # -# TwoStageDiD (rows M-022/M-119): fit(aggregate=/balance_e=) shim + the -# PANEL-BACKED recompute aggregate() (column-subset working-frame kit) +# TwoStageDiD (rows M-022/M-119): fit(aggregate=/balance_e=) shim pins + the +# PANEL-BACKED recompute aggregate() (column-subset working-frame kit) on +# analytic fits; bootstrapped fits RELAY the fit-time percentile surfaces # --------------------------------------------------------------------------- # TWOSTAGE_KW = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") @@ -3304,6 +4016,12 @@ def _twostage_panel(seed=42, n_units=120, n_periods=8): ) +def _twostage_clustered_panel(seed=11): + d = _twostage_panel(seed=seed).copy() + d["cl"] = (d["unit"] // 3).astype(int) + return d + + def _twostage_prop5_panel(seed=13): """No never-treated units, multiple cohorts -> Proposition-5 NaN rows.""" d = _twostage_panel(seed=seed) @@ -3383,6 +4101,18 @@ def twostage_fit_time(twostage_panel): return _fit_twostage(twostage_panel, aggregate="all") +def _ts_oracle(arm): + return load_oracles()["analytic"]["TwoStageDiD"][arm] + + +def _ts_boot_oracle(arm): + return boot_oracles()["TwoStageDiD"][arm] + + +_TS_BOOT_KW = {"n_bootstrap": 19, "seed": 1} +_TS_P_ATOL = 2.0 / _TS_BOOT_KW["n_bootstrap"] + + class TestTwoStageShim: def test_plain_fit_does_not_warn(self, twostage_panel): from diff_diff import TwoStageDiD @@ -3432,39 +4162,57 @@ def test_warn_and_still_work(self, twostage_fitted, twostage_fit_time): class TestTwoStageAggregate: - def test_event_study_inert(self, twostage_fitted, twostage_fit_time): + def test_event_study_matches_oracle(self, twostage_fitted): es = twostage_fitted.aggregate("event_study") - _assert_es_container_matches_fit_time(es, twostage_fit_time, "ts/es") + _assert_es_matches_oracle(es, _ts_oracle("default_all"), **ANALYTIC_TOL, name="ts/es") - @pytest.mark.parametrize("balance_e", [0, 1, 2]) - def test_balance_e_inert(self, twostage_panel, twostage_fitted, balance_e): - ref = _fit_twostage(twostage_panel, aggregate="event_study", balance_e=balance_e) - es = twostage_fitted.aggregate("event_study", balance_e=balance_e) - _assert_es_container_matches_fit_time(es, ref, f"ts/es/be{balance_e}") - - def test_group_inert(self, twostage_fitted, twostage_fit_time): - _assert_group_matches_fit_time( - twostage_fitted.aggregate("group"), twostage_fit_time, "ts/gr" + def test_balance_e1_matches_oracle(self, twostage_fitted): + es = twostage_fitted.aggregate("event_study", balance_e=1) + _assert_es_matches_oracle( + es, _ts_oracle("default_balance_e1"), **ANALYTIC_TOL, name="ts/es/be1" + ) + + def test_group_matches_oracle(self, twostage_fitted): + _assert_group_matches_oracle( + twostage_fitted.aggregate("group"), _ts_oracle("default_all"), **ANALYTIC_TOL, name="ts" ) def test_simple_relay_bit_exact(self, twostage_fitted): - sm = twostage_fitted.aggregate("simple") - assert sm.att[0] == twostage_fitted.overall_att - assert sm.se[0] == twostage_fitted.overall_se + sm = _assert_simple_relays_stored_overall(twostage_fitted) + _assert_overall_matches_oracle( + twostage_fitted, _ts_oracle("default_all")["overall"], **ANALYTIC_TOL, name="ts" + ) assert sm.n_kind == "obs" assert sm.n[0] == twostage_fitted.n_treated_obs assert np.isnan(sm.to_dataframe()["df"].to_numpy()).all() - def test_m092_vcov_parity_analytical(self, twostage_fitted, twostage_panel): - # The post-fit container threads the recomputed joint GMM vcov + - # index + df exactly as the level-matched fit-time container does. - from diff_diff.results_base import build_event_study_surface + def test_post_point_estimates_from_public_tau(self, twostage_fitted): + """(iii) att[e >= 0] is the plain mean of treatment_effects.tau_hat + (the Stage-2 OLS on horizon dummies IS that mean).""" + _assert_post_att_is_tau_mean( + twostage_fitted, twostage_fitted.aggregate("event_study"), name="ts" + ) - ref = build_event_study_surface(_fit_twostage(twostage_panel, aggregate="event_study")) + def test_analytic_inference_contract(self, twostage_fitted): + for level in ("event_study", "group", "simple"): + _assert_analytic_inference_contract( + twostage_fitted.aggregate(level), alpha=twostage_fitted.alpha + ) + + def test_m092_vcov_matches_oracle(self, twostage_fitted): + # The post-fit container threads the recomputed joint GMM vcov + + # index exactly as the pre-removal fit-time container did. es = twostage_fitted.aggregate("event_study") - assert es.vcov is not None and ref.vcov is not None - np.testing.assert_allclose(es.vcov, ref.vcov, rtol=0, atol=1e-14, equal_nan=True) - assert list(es.vcov_index) == list(ref.vcov_index) + orow = _ts_oracle("default_all") + assert es.vcov is not None and orow["event_study_vcov"] is not None + assert [float(x) for x in es.vcov_index] == orow["event_study_vcov_index"] + want = np.array([[oracle_value(x) for x in r] for r in orow["event_study_vcov"]]) + np.testing.assert_allclose(np.asarray(es.vcov), want, **ANALYTIC_TOL, equal_nan=True) + # The ES diagonal is the per-horizon variance. + non_ref = ~np.asarray(es.is_reference, dtype=bool) + np.testing.assert_allclose( + np.diag(np.asarray(es.vcov)), np.asarray(es.se)[non_ref] ** 2, rtol=1e-12 + ) def test_m092_vcov_cleared_on_replicate(self): d, rep_cols = _twostage_survey_panel(replicate=True) @@ -3472,97 +4220,154 @@ def test_m092_vcov_cleared_on_replicate(self): plain = _fit_twostage(d, survey_design=sd) es = plain.aggregate("event_study") assert es.vcov is None and es.vcov_index is None - # df provenance still threads (level-matched replayed value) - ref = _fit_twostage(d, aggregate="event_study", survey_design=sd) - from diff_diff.results_base import build_event_study_surface + # df provenance still threads (the level-matched replayed value). + orow = _ts_oracle("replicate_none_event_study") + assert orow["event_study_vcov"] is None + assert es.df_survey == float(orow["event_study_df"]) - ref_surface = build_event_study_surface(ref) - assert (es.df_survey == ref_surface.df_survey) or ( - es.df_survey is None and ref_surface.df_survey is None - ) - - def test_cluster_arm_inert(self): - d = _twostage_panel(seed=11).copy() - d["cl"] = (d["unit"] // 3).astype(int) - ref = _fit_twostage(d, est_kw={"cluster": "cl"}, aggregate="all") + def test_cluster_arm_matches_oracle(self): + d = _twostage_clustered_panel() plain = _fit_twostage(d, est_kw={"cluster": "cl"}) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "ts/cl") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "ts/cl") - - def test_cluster_naming_unit_column_inert(self, twostage_panel): + es = plain.aggregate("event_study") + orow = _ts_oracle("clustered_all") + _assert_es_matches_oracle(es, orow, **ANALYTIC_TOL, name="ts/cl") + _assert_group_matches_oracle(plain.aggregate("group"), orow, **ANALYTIC_TOL, name="ts/cl") + _assert_post_att_is_tau_mean(plain, es, name="ts/cl") + _assert_analytic_inference_contract(es, alpha=plain.alpha) + # Clustering changes the variance, never the point path. Compare the + # estimated rows only: the reference row's SE is NaN on both surfaces, + # which would make a whole-array (in)equality check vacuous. + unclustered = _fit_twostage(d).aggregate("event_study") + np.testing.assert_array_equal(np.asarray(es.att), np.asarray(unclustered.att)) + est_rows = ~np.asarray(es.is_reference, dtype=bool) + assert np.array_equal(est_rows, ~np.asarray(unclustered.is_reference, dtype=bool)) + se_cl = np.asarray(es.se, dtype=float)[est_rows] + se_un = np.asarray(unclustered.se, dtype=float)[est_rows] + assert np.isfinite(se_cl).all() and np.isfinite(se_un).all() + assert not np.allclose(se_cl, se_un, rtol=1e-8, atol=0) + + def test_cluster_naming_unit_column_inert(self, twostage_panel, twostage_fitted): # cluster= legally names the unit column - the kit's column-subset # dedup must keep the frame single-labeled (a duplicated column - # would break df[unit].map in the moved Stage-1 helpers). - ref = _fit_twostage(twostage_panel, est_kw={"cluster": "unit"}, aggregate="all") + # would break df[unit].map in the moved Stage-1 helpers). The + # default fit clusters by unit too, so the surfaces are identical. plain = _fit_twostage(twostage_panel, est_kw={"cluster": "unit"}) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "ts/cl-unit") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "ts/cl-unit") + _frame_equal( + plain.aggregate("event_study").to_dataframe(), + twostage_fitted.aggregate("event_study").to_dataframe(), + ) + _frame_equal( + plain.aggregate("group").to_dataframe(), + twostage_fitted.aggregate("group").to_dataframe(), + ) - def test_covariate_arm_inert(self): + def test_covariate_arm_point_path(self): d = _twostage_panel(seed=23).copy() rng = np.random.default_rng(23) d["x1"] = rng.normal(size=len(d)) + 0.1 * d["period"] - ref = _fit_twostage(d, aggregate="all", covariates=["x1"]) plain = _fit_twostage(d, covariates=["x1"]) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "ts/cov") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "ts/cov") - - def test_prop5_arm_inert(self): + es = plain.aggregate("event_study") + _assert_post_att_is_tau_mean(plain, es, name="ts/cov") + _assert_analytic_inference_contract(es, alpha=plain.alpha) + orow = _ts_oracle("covariates_all") + _assert_es_matches_oracle(es, orow, **ANALYTIC_TOL, name="ts/cov") + _assert_group_matches_oracle(plain.aggregate("group"), orow, **ANALYTIC_TOL, name="ts/cov") + # The covariate adjustment moves the imputed tau_hat (and so the att). + bare = _fit_twostage(d).aggregate("event_study") + assert not np.array_equal(np.asarray(es.att), np.asarray(bare.att)) + + def test_prop5_arm_matches_oracle(self): d = _twostage_prop5_panel() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - ref = _fit_twostage(d, aggregate="event_study") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") plain = _fit_twostage(d) es = plain.aggregate("event_study") - _assert_es_container_matches_fit_time(es, ref, "ts/prop5") + assert any("Proposition 5" in str(w.message) for w in caught) + _assert_es_matches_oracle(es, _ts_oracle("prop5_all"), **ANALYTIC_TOL, name="ts/prop5") + # (iv) a NaN effect with n > 0 is valid (Prop-5 unidentified horizons). + frame = es.to_dataframe() + nan_rows = frame[~np.isfinite(frame["att"]) & ~frame["is_reference"]] + assert len(nan_rows) > 0 and (nan_rows["n"] > 0).all() + _assert_analytic_inference_contract(es, alpha=plain.alpha) - def test_survey_tsl_arm_inert(self): + def test_survey_tsl_arm_matches_oracle(self): + d, _ = _twostage_survey_panel() + plain = _fit_twostage(d, survey_design=_twostage_survey_design()) + orow = _ts_oracle("survey_tsl_all") + _assert_es_matches_oracle( + plain.aggregate("event_study"), orow, **ANALYTIC_TOL, name="ts/tsl" + ) + _assert_group_matches_oracle(plain.aggregate("group"), orow, **ANALYTIC_TOL, name="ts/tsl") + + def test_survey_psu_arm_matches_oracle_and_df(self): d, _ = _twostage_survey_panel() sd = _twostage_survey_design(psu=True) - ref = _fit_twostage(d, aggregate="all", survey_design=sd) plain = _fit_twostage(d, survey_design=sd) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "ts/tsl") - _assert_group_matches_fit_time(plain.aggregate("group"), ref, "ts/tsl") + orow = _ts_oracle("survey_psu_all") + es = _assert_es_matches_oracle( + plain.aggregate("event_study"), orow, **ANALYTIC_TOL, name="ts/psu" + ) gr = plain.aggregate("group") + _assert_group_matches_oracle(gr, orow, **ANALYTIC_TOL, name="ts/psu") df_col = gr.to_dataframe()["df"].to_numpy() finite_p = np.isfinite(gr.p_value) assert np.isfinite(df_col[finite_p]).all() # scalar survey-df broadcast + assert _assert_analytic_inference_contract(es, alpha=plain.alpha) > 0 + assert _assert_analytic_inference_contract(gr, alpha=plain.alpha) > 0 - def test_always_treated_pad_arm_inert(self): + def test_always_treated_pad_arm_matches_oracle(self): # The only arm where score_pad_mask/cluster_ids_full are non-None: # an implementation storing None unconditionally passes every other - # arm but breaks inertness here. + # arm but breaks parity here. d = _twostage_always_treated_panel() - sd = _twostage_survey_design(psu=True) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ref = _fit_twostage(d, aggregate="all", survey_design=sd) - plain = _fit_twostage(d, survey_design=sd) + plain = _fit_twostage(d, survey_design=_twostage_survey_design()) es = plain.aggregate("event_study") gr = plain.aggregate("group") - _assert_es_container_matches_fit_time(es, ref, "ts/pad") - _assert_group_matches_fit_time(gr, ref, "ts/pad") - - @pytest.mark.parametrize("degenerate", [None, "dropped", "undefined"]) + orow = _ts_oracle("always_treated_survey_all") + _assert_es_matches_oracle(es, orow, **ANALYTIC_TOL, name="ts/pad") + _assert_group_matches_oracle(gr, orow, **ANALYTIC_TOL, name="ts/pad") + # The PSU design is the shape the 3.x test used (score_pad_mask + + # cluster_ids_full both non-None at once). + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + plain_psu = _fit_twostage(d, survey_design=_twostage_survey_design(psu=True)) + es_psu = plain_psu.aggregate("event_study") + gr_psu = plain_psu.aggregate("group") + orow_psu = _ts_oracle("always_treated_survey_psu_all") + _assert_es_matches_oracle(es_psu, orow_psu, **ANALYTIC_TOL, name="ts/pad-psu") + _assert_group_matches_oracle(gr_psu, orow_psu, **ANALYTIC_TOL, name="ts/pad-psu") + + @pytest.mark.parametrize("degenerate", [None, "dropped", "undefined", "cohort_zero"]) def test_replicate_arms_level_matched(self, degenerate): d, rep_cols = _twostage_survey_panel(replicate=True, degenerate=degenerate) sd = _twostage_survey_design(rep_cols) - ref_es = _fit_twostage(d, aggregate="event_study", survey_design=sd) - ref_gr = _fit_twostage(d, aggregate="group", survey_design=sd) plain = _fit_twostage(d, survey_design=sd) - _assert_es_container_matches_fit_time( - plain.aggregate("event_study"), ref_es, f"ts/rep/{degenerate}" + tag = degenerate or "none" + _assert_es_matches_oracle( + plain.aggregate("event_study"), + _ts_oracle(f"replicate_{tag}_event_study"), + **ANALYTIC_TOL, + name=f"ts/rep/{tag}", ) - _assert_group_matches_fit_time(plain.aggregate("group"), ref_gr, f"ts/rep/{degenerate}") + _assert_group_matches_oracle( + plain.aggregate("group"), + _ts_oracle(f"replicate_{tag}_group"), + **ANALYTIC_TOL, + name=f"ts/rep/{tag}", + ) + _assert_analytic_inference_contract(plain.aggregate("event_study"), alpha=plain.alpha) + _assert_analytic_inference_contract(plain.aggregate("group"), alpha=plain.alpha) def test_all_prop5_horizons_survive_early_return(self): # Local-review P1 (pre-existing corner in the verbatim-moved code, # fixed in this PR): when EVERY non-reference horizon is # Proposition-5-unidentified, est_horizons empties and the early # return used to DROP the built prop5 rows - real treated horizons - # reported as absent instead of unidentified. Both the fit-time - # and post-fit surfaces must keep them as all-NaN rows with - # n_obs > 0, plus the consolidated Prop-5 warning. + # reported as absent instead of unidentified. The surface must keep + # them as all-NaN rows with n_obs > 0, plus the consolidated Prop-5 + # warning. rng = np.random.default_rng(5) rows = [] for u in range(12): # cohort 2: pre at t=1, post only at h >= h_bar @@ -3574,43 +4379,40 @@ def test_all_prop5_horizons_survive_early_return(self): d = pd.DataFrame(rows, columns=["unit", "period", "outcome", "first_treat"]) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - ref = _fit_twostage(d.copy(), aggregate="event_study") plain = _fit_twostage(d.copy()) es = plain.aggregate("event_study") assert any("Proposition 5" in str(w.message) for w in caught) - for surface_dict in (ref.event_study_effects,): - prop5 = {h: r for h, r in surface_dict.items() if h != -1} - assert prop5, "Prop-5 rows must survive the early return" - for h, r in prop5.items(): - assert h >= 6 and r["n_obs"] > 0 - assert np.isnan(r["effect"]) and np.isnan(r["se"]) df_ = es.to_dataframe() - prop5_rows = df_[(df_["event_time"] >= 6)] - assert len(prop5_rows) > 0 - assert np.isnan(prop5_rows["att"]).all() + prop5_rows = df_[~df_["is_reference"]] + assert len(prop5_rows) > 0, "Prop-5 rows must survive the early return" + assert (prop5_rows["event_time"] >= 6).all() + assert np.isnan(prop5_rows["att"]).all() and np.isnan(prop5_rows["se"]).all() assert (prop5_rows["n"] > 0).all() - _assert_es_container_matches_fit_time(es, ref, "ts/all-prop5") + _assert_analytic_inference_contract(es, alpha=plain.alpha) def test_replicate_overall_row_migration_delta(self): d, rep_cols = _twostage_survey_panel(replicate=True, degenerate="cohort_zero") sd = _twostage_survey_design(rep_cols) plain = _fit_twostage(d, survey_design=sd) - ref_gr = _fit_twostage(d, aggregate="group", survey_design=sd) - assert plain.overall_se != ref_gr.overall_se - _assert_group_matches_fit_time(plain.aggregate("group"), ref_gr, "ts/delta") + orow = _ts_oracle("replicate_cohort_zero_group") + assert plain.overall_se != oracle_value(orow["overall"]["se"]) + _assert_group_matches_oracle( + plain.aggregate("group"), orow, **ANALYTIC_TOL, name="ts/delta" + ) dh, rep_h = _twostage_survey_panel(replicate=True) - sdh = _twostage_survey_design(rep_h) + healthy = _fit_twostage(dh, survey_design=_twostage_survey_design(rep_h)) # Same BLAS-shape caveat as the Imputation twin above. np.testing.assert_allclose( - _fit_twostage(dh, survey_design=sdh).overall_se, - _fit_twostage(dh, aggregate="group", survey_design=sdh).overall_se, + healthy.overall_se, + oracle_value(_ts_oracle("replicate_none_group")["overall"]["se"]), rtol=1e-12, ) - def test_pretrends_arm_inert(self, twostage_panel): - ref = _fit_twostage(twostage_panel, est_kw={"pretrends": True}, aggregate="event_study") + def test_pretrends_arm_matches_oracle(self, twostage_panel): plain = _fit_twostage(twostage_panel, est_kw={"pretrends": True}) - _assert_es_container_matches_fit_time(plain.aggregate("event_study"), ref, "ts/pretrends") + es = plain.aggregate("event_study") + _assert_es_matches_oracle(es, _ts_oracle("pretrends_all"), **ANALYTIC_TOL, name="ts/pre") + _assert_post_att_is_tau_mean(plain, es, name="ts/pre") def test_isolation_under_public_field_mutation(self, twostage_panel): res = _fit_twostage(twostage_panel) @@ -3645,6 +4447,8 @@ def test_repeated_and_order_independent(self, twostage_fitted): b = twostage_fitted.aggregate("group").to_dataframe() pd.testing.assert_frame_equal(a, b) + # ---------------- bootstrapped fits (fail closed until the PR 3c relay) ---------------- + def test_bootstrap_recompute_levels_fail_closed(self, twostage_panel): res = _fit_twostage(twostage_panel, est_kw={"n_bootstrap": 19, "seed": 1}) for level in ("event_study", "group"): @@ -3704,10 +4508,11 @@ def test_empty_balance_window(self, twostage_fitted): # --------------------------------------------------------------------------- # -# ContinuousDiD (row M-025): fit(aggregate=) shim + the MIXED view/recompute -# aggregate() - 'simple'/'dose' are pure VIEWS over stored fields (permitted -# on bootstrap fits), 'event_study' is a pruned-IF-payload kit recompute -# (bootstrap fails closed; replicate designs supported, no refit replay) +# ContinuousDiD (row M-025): fit(aggregate=) shim pin + the MIXED +# view/relay/recompute aggregate() - 'simple'/'dose' are pure VIEWS over stored +# fields (permitted on bootstrap fits), 'event_study' is a pruned-IF-payload kit +# recompute on analytic fits and a RELAY of the fit-time percentile surface on +# bootstrapped fits (replicate designs supported analytically, no refit replay) # --------------------------------------------------------------------------- # CONT_KW = dict( @@ -3858,6 +4663,31 @@ def cont_bootstrap(cont_panel): return _fit_cont(cont_panel, est_kw=dict(n_bootstrap=30, seed=3)) +_CO_BOOT_KW = dict(n_bootstrap=30, seed=3) +_CO_P_ATOL = 2.0 / _CO_BOOT_KW["n_bootstrap"] + + +def _co_oracle(arm): + return load_oracles()["analytic"]["ContinuousDiD"][arm] + + +def _co_boot_oracle(arm): + return boot_oracles()["ContinuousDiD"][arm] + + +def _assert_cont_overall_matches_oracle(res, orow, *, p_atol=_CO_P_ATOL): + """ContinuousDiD overall ATT quintet + the ACRT scalars the capture kept.""" + o = orow["overall"] + _close(res.overall_att, o["att"], **BOOT_TOL, what="cont.overall_att") + _close(res.overall_att_se, o["se"], **BOOT_TOL, what="cont.overall_att_se") + _close(res.overall_att_t_stat, o["t_stat"], **BOOT_TOL, what="cont.overall_att_t") + _close(res.overall_att_p_value, o["p_value"], rtol=1e-13, atol=p_atol, what="cont.overall_p") + _close(res.overall_att_conf_int[0], o["ci_lo"], **BOOT_TOL, what="cont.ci_lo") + _close(res.overall_att_conf_int[1], o["ci_hi"], **BOOT_TOL, what="cont.ci_hi") + _close(res.overall_acrt, orow["overall_acrt"], **BOOT_TOL, what="cont.acrt") + _close(res.overall_acrt_se, orow["overall_acrt_se"], **BOOT_TOL, what="cont.acrt_se") + + class TestContinuousShim: def test_plain_fit_does_not_warn(self, cont_panel): from diff_diff import ContinuousDiD @@ -3905,14 +4735,31 @@ def test_invalid_value_warns_then_raises(self, cont_panel): class TestContinuousAggregate: - # ---------------- event_study: kit-recompute inertness ---------------- + # ---------------- event_study: kit-recompute oracle parity ---------------- - def test_event_study_inert(self, cont_fitted, cont_fit_time): + def test_event_study_matches_oracle(self, cont_fitted): es = cont_fitted.aggregate("event_study") - _assert_es_container_matches_fit_time(es, cont_fit_time, "cont/es") + _assert_es_matches_oracle(es, _co_oracle("default_eventstudy"), **ANALYTIC_TOL, name="cont") # n_kind is a container ATTRIBUTE (not an EVENT_STUDY_SCHEMA # column): ContinuousDiD's ES dict carries no count key. assert es.n_kind is None + _assert_analytic_inference_contract(es, alpha=cont_fitted.alpha) + assert cont_fitted.event_study_effects is None # analytic: nothing stored + + _ORACLE_ARMS = { + "survey_tsl": "survey_tsl_eventstudy", + "covariates": "covariate_eventstudy", + "anticipation": "anticipation1_eventstudy", + "multi_cohort": "multi_cohort_eventstudy", + "survey_zero_dose_drop": "survey_zero_dose_drop_eventstudy", + "replicate_healthy": "replicate_healthy_eventstudy", + "replicate_dropped": "replicate_dropped_eventstudy", + "replicate_undefined": "replicate_undefined_eventstudy", + "discrete": "discrete_eventstudy", + "lowest_dose": "lowest_dose_eventstudy", + "not_yet_treated": "not_yet_treated_eventstudy", + "nondefault_config": "nondefault_config_eventstudy", + } @pytest.mark.parametrize( "arm", @@ -3931,7 +4778,10 @@ def test_event_study_inert(self, cont_fitted, cont_fit_time): "nondefault_config", ], ) - def test_event_study_inert_across_designs(self, arm): + def test_event_study_across_designs(self, arm): + """Every arm pins its numbers against its captured oracle AND the + structural contract ((iv) NaN-consistency / safe_inference identity + where se and df are finite) and same-data idempotency.""" est_kw, fit_kw = {}, {} if arm == "multi_cohort": data = _cont_panel(seed=43, cohort_periods=[3, 4, 5]) @@ -3969,9 +4819,20 @@ def test_event_study_inert_across_designs(self, arm): data = _cont_panel(seed=59) est_kw = dict(alpha=0.10, base_period="universal") plain = _fit_cont(data, est_kw=est_kw, **fit_kw) - ref = _fit_cont(data, est_kw=est_kw, aggregate="eventstudy", **fit_kw) es = plain.aggregate("event_study") - _assert_es_container_matches_fit_time(es, ref, f"cont/{arm}") + _assert_es_matches_oracle( + es, _co_oracle(self._ORACLE_ARMS[arm]), **ANALYTIC_TOL, name=f"cont/{arm}" + ) + checked = _assert_analytic_inference_contract(es, alpha=plain.alpha) + if arm in ("survey_tsl", "survey_zero_dose_drop", "replicate_healthy", "replicate_dropped"): + assert checked > 0, arm # finite survey df -> the identity leg ran + _frame_equal(es.to_dataframe(), plain.aggregate("event_study").to_dataframe()) + assert plain.event_study_effects is None + if arm == "covariates": + # The covariate adjustment must move the point path off the + # plain fit's. + bare = _fit_cont(data).aggregate("event_study") + assert not np.array_equal(np.asarray(es.att), np.asarray(bare.att)) def test_nondefault_provenance_discriminates_defaults(self): """alpha=0.10 / base_period='universal' must SURFACE - a kit that @@ -3985,15 +4846,18 @@ def test_nondefault_provenance_discriminates_defaults(self): assert s.alpha == 0.10 def test_empty_post_gt_all_levels(self): - """Fit-faithful empty-post_gt quirk: ES rows keep NaN inference on - BOTH routes; the views relay the stored all-NaN fields.""" + """Fit-faithful empty-post_gt quirk: ES rows keep NaN inference (the + oracle pins the pre-period point estimates); the views relay the + stored all-NaN fields.""" data = _cont_empty_post_panel() plain = _fit_cont(data) - ref = _fit_cont(data, aggregate="eventstudy") es = plain.aggregate("event_study") - _assert_es_container_matches_fit_time(es, ref, "cont/empty_post") + _assert_es_matches_oracle( + es, _co_oracle("empty_post_eventstudy"), **ANALYTIC_TOL, name="cont/empty_post" + ) frame = es.to_dataframe() assert not np.isfinite(frame["se"]).any() + _assert_analytic_inference_contract(es, alpha=plain.alpha) s = plain.aggregate("simple") assert np.isnan(s.att).all() and np.isnan(s.se).all() d = plain.aggregate("dose") @@ -4149,12 +5013,18 @@ def test_mixed_type_labels_preserve_producer_order(self): assert list(frame["target"]) == ["att", "att", "acrt", "acrt"] assert "target" in agg.summary() - # ---------------- bootstrap gating + kit shape ---------------- + # ---------------- bootstrap: fail-closed gate (relay lands in PR 3c) + kit shape ---------------- def test_bootstrap_event_study_fails_closed(self, cont_bootstrap): with pytest.raises(NotImplementedError, match="bootstrap"): cont_bootstrap.aggregate("event_study") + def test_bootstrap_plain_overall_matches_oracle(self, cont_bootstrap): + """The headline bootstrap (overall ATT + ACRT) is untouched by the + always-on surface materialization.""" + _assert_cont_overall_matches_oracle(cont_bootstrap, _co_boot_oracle("default_plain")) + _assert_cont_overall_matches_oracle(cont_bootstrap, _co_boot_oracle("default_eventstudy")) + def test_bootstrap_views_still_work(self, cont_bootstrap): assert cont_bootstrap.aggregate("simple") is not None assert cont_bootstrap.aggregate("dose") is not None @@ -4267,7 +5137,7 @@ def _walk(obj): # --------------------------------------------------------------------------- # # HeterogeneousAdoptionDiD (rows M-027/M-139): the fit + workflow MODE-SELECTOR -# shims (panel-shape inference) and the two PURE-VIEW results classes. +# removals (panel-shape inference only) and the two PURE-VIEW results classes. # --------------------------------------------------------------------------- # HAD_KW = dict(outcome="outcome", dose="dose", time="period", unit="unit") @@ -5380,31 +6250,27 @@ def test_degenerate_inference_passes_through(self): # --------------------------------------------------------------------------- -from diff_diff.results_base import build_event_study_surface # noqa: E402 - - class TestPerRowDfProvenanceCompletion: """Per-row ES df threading for the four former M-092 holes (EfficientDiD, ImputationDiD, ContinuousDiD, HAD) and the documented two-channel contract: the scalar ``df_survey`` (the fit's resolved scalar inference df) persists on bootstrapped surfaces, while the per-row ``df`` column is the inference-provenance channel and is cleared under percentile - bootstrap (M-027).""" + bootstrap (M-027). The container is the canonical surface on + analytic fits and bootstrapped Imputation/ContinuousDiD fits RELAY their + stored percentile surface.""" # ---- EfficientDiD ------------------------------------------------- def test_edid_es_per_row_df_survey_tsl_oracle(self): - from diff_diff.utils import safe_inference - d, _ = _efficient_survey_panel() sd = _efficient_survey_design() plain = _fit_efficient(d, survey_design=sd) expected = plain._aggregation_kit.bookkeeping["df_survey"] - # Modern fits build no fit-time ES surface -> attr None (absent- - # surface gate); the deprecated fit-time route carries the value. + # Modern fits build no fit-time ES surface -> attr None; the oracle + # recorded the same scalar on the pre-removal fit-time route. assert plain.event_study_df is None - fit_time = _fit_efficient(d, survey_design=sd, aggregate="event_study") - assert fit_time.event_study_df == expected + assert expected == _ed_oracle("survey_weights_all")["event_study_df"] es = plain.aggregate("event_study") finite = np.isfinite(np.asarray(es.df, dtype=float)) assert finite.any() @@ -5419,49 +6285,46 @@ def test_edid_es_per_row_df_survey_tsl_oracle(self): np.testing.assert_allclose(p, es.p_value[i], rtol=1e-12) def test_edid_es_per_row_df_replicate_arms(self): - from diff_diff.utils import safe_inference - for degenerate, expected_df in ((None, 7.0), ("dropped", 5.0)): d, rep_cols = _efficient_survey_panel(replicate=True, degenerate=degenerate) sd = _efficient_survey_design(rep_cols) plain = _fit_efficient(d, survey_design=sd) - fit_time = _fit_efficient(d, survey_design=sd, aggregate="event_study") - assert fit_time.event_study_df == expected_df + assert plain.event_study_df is None es = plain.aggregate("event_study") df_col = np.asarray(es.df, dtype=float) finite = np.isfinite(df_col) + assert finite.any() assert set(np.unique(df_col[finite])) <= {expected_df} + assert es.df_survey == expected_df for i in np.flatnonzero(finite): t, p, _ = safe_inference( float(es.att[i]), float(es.se[i]), alpha=plain.alpha, df=float(es.df[i]) ) np.testing.assert_allclose(p, es.p_value[i], rtol=1e-12) - def test_edid_replicate_undefined_zero_sentinel_is_none(self): + def test_edid_replicate_undefined_zero_sentinel(self): # 0-sentinel normalization: the replicate-undefined df publishes the - # ATTRIBUTE as None, never 0 (the sentinel stays representable only - # via survey_metadata). + # per-row column as NaN and the scalar df_survey as the 0.0 sentinel + # (representable only via survey_metadata; the attribute stays None). d, rep_cols = _efficient_survey_panel(replicate=True, degenerate="undefined") sd = _efficient_survey_design(rep_cols) - fit_time = _fit_efficient(d, survey_design=sd, aggregate="event_study") - assert fit_time.event_study_df is None + plain = _fit_efficient(d, survey_design=sd) + assert plain.event_study_df is None + es = plain.aggregate("event_study") + assert es.df_survey == 0.0 + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) def test_edid_bootstrap_clears_per_row_df_scalar_persists(self): d, _ = _efficient_survey_panel() sd = _efficient_survey_design() boot = _efficient_boot_fit(d, survey_design=sd) - # Fit-time clearing: percentile inference used no df. - assert boot.event_study_df is None + # Nothing is stored at fit; percentile inference used no df. + assert boot.event_study_df is None and boot.event_study_effects is None es = boot.aggregate("event_study") assert np.all(np.isnan(np.asarray(es.df, dtype=float))) # Row B: the scalar df_survey persists beside percentile inference. assert es.df_survey is not None and np.isfinite(es.df_survey) assert es.df_survey == boot.survey_metadata.df_survey - # Fit-time (deprecated route) surface behaves identically. - fit_time = _efficient_boot_fit_time(d, aggregate="event_study", survey_design=sd) - surface = build_event_study_surface(fit_time) - assert np.all(np.isnan(np.asarray(surface.df, dtype=float))) - assert surface.df_survey == fit_time.survey_metadata.df_survey def test_edid_absent_es_surface_gate(self, efficient_panel): plain = _fit_efficient(efficient_panel) @@ -5471,16 +6334,11 @@ def test_edid_absent_es_surface_gate(self, efficient_panel): # ---- ImputationDiD ------------------------------------------------ def test_imputation_es_per_row_df_covers_leads(self): - from diff_diff.utils import safe_inference - d, _ = _imputation_survey_panel() sd = _imputation_survey_design() plain = _fit_imputation(d, survey_design=sd, est_kw={"pretrends": True}) expected = plain.survey_metadata.df_survey - fit_time = _fit_imputation( - d, survey_design=sd, aggregate="event_study", est_kw={"pretrends": True} - ) - assert fit_time.event_study_df == expected + assert plain.event_study_df is None # analytic fits store no surface es = plain.aggregate("event_study") df_col = np.asarray(es.df, dtype=float) ev = np.asarray(es.event_time) @@ -5488,6 +6346,7 @@ def test_imputation_es_per_row_df_covers_leads(self): # Leads included: at least one pre-period row carries the finite df. assert finite.any() and (ev[finite] < 0).any() assert set(np.unique(df_col[finite])) == {float(expected)} + assert es.df_survey == expected for i in np.flatnonzero(finite): t, p, _ = safe_inference( float(es.att[i]), float(es.se[i]), alpha=plain.alpha, df=float(es.df[i]) @@ -5506,12 +6365,16 @@ def test_imputation_replicate_replay_df_is_level_matched(self): finite = np.isfinite(df_col) assert finite.any() assert set(np.unique(df_col[finite])) == {float(es.df_survey)} + assert es.df_survey == _im_oracle("replicate_cohort_zero_event_study")["event_study_df"] - def test_imputation_replicate_undefined_zero_sentinel_is_none(self): + def test_imputation_replicate_undefined_zero_sentinel(self): d, rep_cols = _imputation_survey_panel(replicate=True, degenerate="undefined") sd = _imputation_survey_design(rep_cols) - fit_time = _fit_imputation(d, survey_design=sd, aggregate="event_study") - assert fit_time.event_study_df is None + plain = _fit_imputation(d, survey_design=sd) + assert plain.event_study_df is None + es = plain.aggregate("event_study") + assert es.df_survey == 0.0 + assert np.all(np.isnan(np.asarray(es.df, dtype=float))) def test_imputation_bootstrap_clears_at_fit_and_replay_fails_closed(self): d, _ = _imputation_survey_panel() @@ -5534,12 +6397,8 @@ def test_cont_es_per_row_df_threads_survey_df(self): finite = np.isfinite(df_col) assert finite.any() assert set(np.unique(df_col[finite])) == {float(expected)} - # Route parity: the fit-time surface carries the same column. - fit_time = _fit_cont(d, survey_design=sd, aggregate="eventstudy") - surface = build_event_study_surface(fit_time) - np.testing.assert_array_equal( - np.asarray(surface.df, dtype=float), np.asarray(es.df, dtype=float) - ) + # The pre-removal fit-time surface carried the same scalar. + assert float(expected) == _co_oracle("survey_tsl_eventstudy")["event_study_df"] def test_cont_bootstrap_fit_clears_df_scalar_is_unit_level(self): # Bootstrapped fit: _survey_df stays None -> attr None; the fit-time @@ -5572,7 +6431,6 @@ def test_cont_absent_es_surface_gate(self, cont_panel): def test_had_es_per_row_df_survey(self): from diff_diff import SurveyDesign - from diff_diff.utils import safe_inference d = _had_panel_multi() rng = np.random.default_rng(7) @@ -5606,11 +6464,8 @@ def test_cs_bootstrapped_scalar_persists_per_row_cleared(self): d = _survey_panel() sd = _survey_design() boot = _boot_fit(d, fit_kwargs={"survey_design": sd}) + assert boot.event_study_effects is None # nothing stored; the replay serves es = boot.aggregate("event_study") assert np.all(np.isnan(np.asarray(es.df, dtype=float))) assert es.df_survey is not None and np.isfinite(es.df_survey) assert es.df_survey == boot.survey_metadata.df_survey - fit_time = _boot_fit_time(d, aggregate="event_study", fit_kwargs={"survey_design": sd}) - surface = build_event_study_surface(fit_time) - assert np.all(np.isnan(np.asarray(surface.df, dtype=float))) - assert surface.df_survey == fit_time.survey_metadata.df_survey diff --git a/tests/test_bootstrap_chunking.py b/tests/test_bootstrap_chunking.py index 634ec0f26..07ee692a8 100644 --- a/tests/test_bootstrap_chunking.py +++ b/tests/test_bootstrap_chunking.py @@ -329,7 +329,9 @@ def _panel(): return pd.DataFrame({"unit": units, "period": periods, "y": y, "first_treat": ft}) def _fit(self): - return CallawaySantAnna( + # A plain fit stores no fit-time surface; the post-fit aggregate() REPLAYS the + # bootstrap, so it runs here - inside whatever monkeypatch the caller set. + res = CallawaySantAnna( control_group="never_treated", estimation_method="dr", cluster="unit", @@ -341,20 +343,22 @@ def _fit(self): unit="unit", time="period", first_treat="first_treat", - aggregate="all", ) + return _with_aggregates(res) def test_tiny_chunks_match_single_chunk(self, monkeypatch): # Default path: this small panel fits in a single block. - base = self._fit() + base, base_es, base_grp = self._fit() # Force many tiny blocks on every weight path (unit + survey). monkeypatch.setattr("diff_diff.bootstrap_chunking.compute_block_size", lambda *a, **k: 7) monkeypatch.setattr("diff_diff.staggered_bootstrap.compute_block_size", lambda *a, **k: 7) - tiny = self._fit() + tiny, tiny_es, tiny_grp = self._fit() # Continuous bootstrap statistics match to within BLAS reassociation. assert tiny.overall_se == pytest.approx(base.overall_se, rel=1e-10, abs=1e-12) - assert tiny.cband_crit_value == pytest.approx(base.cband_crit_value, rel=1e-10, abs=1e-12) + assert tiny_es.cband_crit_value == pytest.approx( + base_es.cband_crit_value, rel=1e-10, abs=1e-12 + ) # p-values are discrete proportions over draws; a borderline draw can # flip under reassociation, shifting a p-value by O(1/n_bootstrap). assert tiny.overall_p_value == pytest.approx(base.overall_p_value, abs=0.02) @@ -366,9 +370,9 @@ def test_tiny_chunks_match_single_chunk(self, monkeypatch): np.testing.assert_allclose(t["p_value"].to_numpy(), b["p_value"].to_numpy(), atol=0.02) # Event-study and group aggregate effects/SEs/CIs also match under chunking. - for level in ("event_study", "group"): - bl = base.to_dataframe(level=level).reset_index(drop=True) - tl = tiny.to_dataframe(level=level).reset_index(drop=True) + for level, bc, tc in (("event_study", base_es, tiny_es), ("group", base_grp, tiny_grp)): + bl = bc.to_dataframe().reset_index(drop=True) + tl = tc.to_dataframe().reset_index(drop=True) num_cols = [c for c in bl.columns if bl[c].dtype.kind in "fi" and c != "p_value"] assert num_cols, f"no numeric columns to compare for level={level}" for col in num_cols: @@ -392,7 +396,6 @@ def fit(): unit="unit", time="period", first_treat="first_treat", - aggregate="simple", ) base = fit() @@ -423,7 +426,7 @@ def _clustered_panel(): ) def _fit_clustered(self): - return CallawaySantAnna( + res = CallawaySantAnna( control_group="never_treated", estimation_method="dr", cluster="state", @@ -435,15 +438,15 @@ def _fit_clustered(self): unit="unit", time="period", first_treat="first_treat", - aggregate="all", ) + return _with_aggregates(res) def test_nonidentity_cluster_chunks_match_single(self, monkeypatch): # Exercises the non-identity PSU fan-out expansion under tiny chunks. - base = self._fit_clustered() + base, _, _ = self._fit_clustered() monkeypatch.setattr("diff_diff.bootstrap_chunking.compute_block_size", lambda *a, **k: 7) monkeypatch.setattr("diff_diff.staggered_bootstrap.compute_block_size", lambda *a, **k: 7) - tiny = self._fit_clustered() + tiny, _, _ = self._fit_clustered() assert tiny.overall_se == pytest.approx(base.overall_se, rel=1e-10, abs=1e-12) b = base.to_dataframe().sort_values(["group", "time"]).reset_index(drop=True) @@ -472,20 +475,22 @@ def _force_one_column_tiles(monkeypatch): def test_unit_path_tiles_match_single_tile(self, monkeypatch): fit = TestCSBootstrapChunkInvariance()._fit - base = fit() + base, base_es, base_grp = fit() self._force_one_column_tiles(monkeypatch) - tiled = fit() + tiled, tiled_es, tiled_grp = fit() assert tiled.overall_se == pytest.approx(base.overall_se, rel=1e-10, abs=1e-12) - assert tiled.cband_crit_value == pytest.approx(base.cband_crit_value, rel=1e-10, abs=1e-12) + assert tiled_es.cband_crit_value == pytest.approx( + base_es.cband_crit_value, rel=1e-10, abs=1e-12 + ) assert tiled.overall_p_value == pytest.approx(base.overall_p_value, abs=0.02) b = base.to_dataframe().sort_values(["group", "time"]).reset_index(drop=True) t = tiled.to_dataframe().sort_values(["group", "time"]).reset_index(drop=True) for col in ["se", "conf_int_lower", "conf_int_upper"]: np.testing.assert_allclose(t[col].to_numpy(), b[col].to_numpy(), rtol=1e-10, atol=1e-12) - for level in ("event_study", "group"): - bl = base.to_dataframe(level=level).reset_index(drop=True) - tl = tiled.to_dataframe(level=level).reset_index(drop=True) + for bc, tc in ((base_es, tiled_es), (base_grp, tiled_grp)): + bl = bc.to_dataframe().reset_index(drop=True) + tl = tc.to_dataframe().reset_index(drop=True) num_cols = [c for c in bl.columns if bl[c].dtype.kind in "fi" and c != "p_value"] for col in num_cols: np.testing.assert_allclose( @@ -495,9 +500,9 @@ def test_unit_path_tiles_match_single_tile(self, monkeypatch): def test_cluster_path_tiles_match_single_tile(self, monkeypatch): # Non-identity PSU fan-out: the per-pass expansion copy must replay too. fit = TestCSBootstrapChunkInvariance()._fit_clustered - base = fit() + base, _, _ = fit() self._force_one_column_tiles(monkeypatch) - tiled = fit() + tiled, _, _ = fit() assert tiled.overall_se == pytest.approx(base.overall_se, rel=1e-10, abs=1e-12) b = base.to_dataframe().sort_values(["group", "time"]).reset_index(drop=True) @@ -539,7 +544,7 @@ def _fit_survey(self, stratified): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - return CallawaySantAnna( + res = CallawaySantAnna( control_group="never_treated", estimation_method="dr", n_bootstrap=200, @@ -550,9 +555,12 @@ def _fit_survey(self, stratified): unit="unit", time="period", first_treat="first_treat", - aggregate="all", survey_design=design, ) + # Replays the bootstrap under the caller's chunking monkeypatch. + res.aggregate("event_study") + res.aggregate("group") + return res @pytest.mark.parametrize("stratified", [False, True], ids=["psu", "stratified"]) def test_survey_path_tiles_match_single_tile(self, monkeypatch, stratified): @@ -586,7 +594,7 @@ def test_rcs_path_tiles_match_single_tile(self, monkeypatch): def fit(): with warnings.catch_warnings(): warnings.simplefilter("ignore") - return CallawaySantAnna( + res = CallawaySantAnna( estimation_method="reg", panel=False, n_bootstrap=200, seed=42 ).fit( self._rcs_frame(), @@ -594,8 +602,10 @@ def fit(): "unit", "period", "first_treat", - aggregate="all", ) + res.aggregate("event_study") + res.aggregate("group") + return res base = fit() self._force_one_column_tiles(monkeypatch) @@ -608,6 +618,16 @@ def fit(): np.testing.assert_allclose(t[col].to_numpy(), b[col].to_numpy(), rtol=1e-10, atol=1e-12) +def _with_aggregates(res): + """``(results, aggregate("event_study"), aggregate("group"))`` for a bootstrapped fit. + + A plain fit stores no fit-time surface, so each post-fit ``aggregate()`` call + REPLAYS the multiplier bootstrap, so the containers are materialized right + after the fit - inside the caller's chunk/tile monkeypatch context. + """ + return res, res.aggregate("event_study"), res.aggregate("group") + + def _design(psu=None, strata=None, fpc=None, weights=None, lonely_psu="adjust"): """Minimal duck-typed ResolvedSurveyDesign for the survey weight generators.""" return SimpleNamespace(psu=psu, strata=strata, fpc=fpc, weights=weights, lonely_psu=lonely_psu) @@ -697,27 +717,27 @@ def _panel(): def _fit(self, cluster=None, survey_design=None): with warnings.catch_warnings(): warnings.simplefilter("ignore") - return EfficientDiD(n_bootstrap=200, seed=42, cluster=cluster).fit( + res = EfficientDiD(n_bootstrap=200, seed=42, cluster=cluster).fit( self._panel(), "y", "unit", "period", "first_treat", - aggregate="all", survey_design=survey_design, ) + # A plain EfficientDiD fit stores no fit-time surface; the post-fit + # containers replay the bootstrap under the caller's monkeypatch. + assert res.event_study_effects is None and res.group_effects is None + return _with_aggregates(res) @staticmethod - def _ses(r): + def _ses(fitted): # Flatten every bootstrap SE (overall + group_time + event_study + group) # into one vector, ordered by sorted keys, for an nan-safe comparison. + r, es_c, grp_c = fitted gt = [r.group_time_effects[k]["se"] for k in sorted(r.group_time_effects)] - es = ( - [r.event_study_effects[k]["se"] for k in sorted(r.event_study_effects)] - if r.event_study_effects - else [] - ) - gp = [r.group_effects[k]["se"] for k in sorted(r.group_effects)] if r.group_effects else [] + es = [float(x) for x in es_c.se] + gp = [float(x) for x in grp_c.se] return np.array([r.overall_se, *gt, *es, *gp], dtype=float) def _run(self, monkeypatch, **fit_kwargs): @@ -837,7 +857,6 @@ def _fit(self): "dose", "period", "unit", - aggregate="event_study", survey_design=SurveyDesign(weights="w"), ) diff --git a/tests/test_br_dr_canonical_datasets.py b/tests/test_br_dr_canonical_datasets.py index 74bd05c2f..bcabdd35a 100644 --- a/tests/test_br_dr_canonical_datasets.py +++ b/tests/test_br_dr_canonical_datasets.py @@ -149,7 +149,6 @@ def test_negative_att_robust_sensitivity_clean_pretrends(self, mpdta_panel): unit="countyreal", time="year", first_treat="first_treat", - aggregate="event_study", ) br = BusinessReport( cs, @@ -202,7 +201,6 @@ def test_cs_positive_att_clear_violation_fragile_sensitivity(self, castle_panel) unit="state", time="year", first_treat="first_treat", - aggregate="event_study", ) br = BusinessReport( cs, @@ -248,7 +246,6 @@ def test_treatment_label_proper_noun_preserved(self, castle_panel): unit="state", time="year", first_treat="first_treat", - aggregate="event_study", ) br = BusinessReport( cs, @@ -274,7 +271,6 @@ def test_breakdown_m_zero_uses_smallest_m_evaluated_wording(self, castle_panel): unit="state", time="year", first_treat="first_treat", - aggregate="event_study", ) br = BusinessReport( cs, @@ -318,7 +314,6 @@ def test_sa_agrees_with_cs_on_direction_and_pt(self, castle_panel): unit="state", time="year", first_treat="first_treat", - aggregate="event_study", ) sa = SunAbraham().fit( castle_panel, diff --git a/tests/test_business_report.py b/tests/test_business_report.py index b899a84a4..84195c820 100644 --- a/tests/test_business_report.py +++ b/tests/test_business_report.py @@ -26,6 +26,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import legacy_standin import diff_diff as dd from diff_diff import ( @@ -95,13 +96,14 @@ def cs_fit(): # base_period='universal' so DR's sensitivity check can run without # hitting the round-5 methodology-critical skip (Rambachan-Roth bounds # are not interpretable on consecutive-comparison pre-periods). + # Plain fit, no fit-time surface - the report layer derives the event study + # via results.aggregate("event_study"). cs = CallawaySantAnna(base_period="universal").fit( sdf, outcome="outcome", unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) return cs, sdf @@ -1455,7 +1457,6 @@ def test_cs_fit_persists_anticipation(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) assert getattr(cs, "anticipation", None) == 1 br = BusinessReport(cs, auto_diagnostics=False) @@ -2283,7 +2284,6 @@ def test_cs_not_yet_treated_still_reinterprets(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) br = BusinessReport(cs, auto_diagnostics=False) sample = br.to_dict()["sample"] @@ -2609,8 +2609,8 @@ def test_full_vcov_path_no_downgrade_on_real_cs_fit(self, cs_fit): # Step 4 linear MDV is in Roth's γ units, a slope; the level-scale # comparable is mdv * max(|violation_weights|)). assert block["covariance_source"] == "full_pre_period_vcov", ( - "cs_fit is analytical CS with event_study_vcov populated — " - "PR-B routing must report full_pre_period_vcov" + "cs_fit is analytical CS whose derived event-study container carries " + "the full vcov — PR-B routing must report full_pre_period_vcov" ) # max_abs_pre_violation = mdv * max(|t - t_ref|) = 0.1337 * 3 ≈ 0.401 assert block.get("max_abs_pre_violation") is not None @@ -2633,7 +2633,7 @@ def test_full_vcov_path_no_downgrade_on_real_cs_fit(self, cs_fit): # compute_pretrends_power output's persisted field, locking that # provenance is recorded at fit time and consumed at the report # layer (not re-inferred from the source-fit type). - pp = compute_pretrends_power(fit, alpha=0.05, target_power=0.80) + pp = compute_pretrends_power(fit.aggregate("event_study"), alpha=0.05, target_power=0.80) assert pp.covariance_source == "full_pre_period_vcov" # Positive prose contract on the rendered surfaces. @@ -2690,7 +2690,6 @@ def test_not_yet_treated_fit_does_not_render_misleading_control_count(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) br = BusinessReport(cs, auto_diagnostics=False) sample = br.to_dict()["sample"] @@ -5062,19 +5061,12 @@ def test_plain_cs_fit_business_report_pt_verdict(self, cs_plain_fit_br): # The provenance key crosses the _lift_pre_trends whitelist. assert pt["pre_period_source"] == "aggregate_event_study" - def test_raw_route_business_report_pre_period_source_is_none(self): - # Raw-field (fit-time kwarg) route: BR still emits the key, as None. - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - sdf = generate_staggered_data(n_units=100, n_periods=6, treatment_effect=1.5, seed=7) - cs = CallawaySantAnna(base_period="universal").fit( - sdf, - outcome="outcome", - unit="unit", - time="period", - first_treat="first_treat", - aggregate="event_study", - ) + def test_raw_route_business_report_pre_period_source_is_none(self, cs_plain_fit_br): + # Raw-field route (the 3.x fit-time shape, hand-populated from the + # container - a legacy pickle today): BR still emits the key, as None. + cs, _ = cs_plain_fit_br + cs = legacy_standin(cs) + assert cs.event_study_effects is not None d = BusinessReport(cs, outcome_label="revenue").to_dict() pt = d["pre_trends"] assert pt["status"] == "computed" diff --git a/tests/test_continuous_did.py b/tests/test_continuous_did.py index f4d177864..206c116db 100644 --- a/tests/test_continuous_did.py +++ b/tests/test_continuous_did.py @@ -608,18 +608,15 @@ def test_event_study_to_dataframe(self): noise_sd=0.5, ) est = ContinuousDiD() - results = est.fit( - data, - "outcome", - "unit", - "period", - "first_treat", - "dose", - aggregate="eventstudy", - ) - df = results.to_dataframe(level="event_study") - assert "relative_period" in df.columns - assert "att_glob" in df.columns + results = est.fit(data, "outcome", "unit", "period", "first_treat", "dose") + # analytic fits leave the legacy fit-time field None; the surface is + # the post-fit container + assert results.event_study_effects is None + with pytest.raises(ValueError, match=r"aggregate\('event_study'\)"): + results.to_dataframe(level="event_study") + df = results.aggregate("event_study").to_dataframe() + assert "event_time" in df.columns + assert "att" in df.columns def test_event_study_not_yet_treated(self): """Event study with control_group='not_yet_treated' and analytic SE.""" @@ -1766,8 +1763,9 @@ def test_covariate_metadata_on_results(self): assert "Covariates" in res.summary() def test_covariate_eventstudy_and_bootstrap(self): - """Covariate paths compose with aggregate='eventstudy' and n_bootstrap>0 - (reg + dr), with finite inference and bootstrap SE near analytical.""" + """Covariate paths compose with the post-fit event study and + n_bootstrap>0 (reg + dr), with finite inference and bootstrap SE near + analytical.""" data = _cov_data(n_units=200) for method in ("reg", "dr"): es = ContinuousDiD(estimation_method=method).fit( @@ -1777,10 +1775,10 @@ def test_covariate_eventstudy_and_bootstrap(self): "period", "first_treat", "dose", - aggregate="eventstudy", covariates=["x1"], ) assert np.isfinite(es.overall_att) and np.isfinite(es.overall_att_se) + assert len(es.aggregate("event_study").event_time) > 0 ana = ContinuousDiD(estimation_method=method).fit( data, "outcome", diff --git a/tests/test_csdid_ported.py b/tests/test_csdid_ported.py index 43baa853e..8f3c70030 100644 --- a/tests/test_csdid_ported.py +++ b/tests/test_csdid_ported.py @@ -18,6 +18,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, group_dict from diff_diff import CallawaySantAnna from tests.helpers.csdid_dgp import build_csdid_sim_data, build_two_group_sim_data @@ -188,16 +189,20 @@ def test_two_period_all_aggregations(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="all", ) assert ( abs(results.overall_att - 1.0) < 0.5 ), f"Simple ATT = {results.overall_att:.3f}, expected ≈ 1.0" + # A plain fit stores no aggregation surface; both levels are post-fit + assert results.event_study_effects is None and results.group_effects is None # Event study: post-treatment effect ≈ 1 - if results.event_study_effects: - post_effects = {e: v for e, v in results.event_study_effects.items() if e >= 0} - for e, eff in post_effects.items(): - assert abs(eff["effect"] - 1.0) < 0.5, f"Event study e={e}: {eff['effect']:.3f}" + post_effects = {e: v for e, v in es_dict(results).items() if e >= 0} + assert post_effects + for e, eff in post_effects.items(): + assert abs(eff["effect"] - 1.0) < 0.5, f"Event study e={e}: {eff['effect']:.3f}" + # Group: the single cohort's effect ≈ 1 + for g, eff in group_dict(results).items(): + assert abs(eff["effect"] - 1.0) < 0.5, f"Group g={g}: {eff['effect']:.3f}" def test_dynamic_effects_dose_response(self, ci_params): """Ported from R did::test-att_gt.R: 'aggregations' (dynamic) @@ -221,14 +226,14 @@ def test_dynamic_effects_dose_response(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) - assert results.event_study_effects is not None + es_all = es_dict(results) + assert es_all # Post-treatment event study effects should be positive and increasing. # R: panel=FALSE gives att(e=2) ≈ 2. Panel=TRUE may give ≈ 3 # (different aggregation weights). Check monotonic increase. post_effects = sorted( - [(e, v["effect"]) for e, v in results.event_study_effects.items() if e >= 0], + [(e, v["effect"]) for e, v in es_all.items() if e >= 0], key=lambda x: x[0], ) if len(post_effects) >= 2: @@ -262,11 +267,11 @@ def test_aggregation_group(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="group", ) - assert results.group_effects is not None + grp = group_dict(results) + assert grp # At least one group should have a sizeable effect - any_large = any(abs(v["effect"]) > 1.0 for v in results.group_effects.values()) + any_large = any(abs(v["effect"]) > 1.0 for v in grp.values()) assert any_large, "Expected group-specific heterogeneity" def test_aggregation_balance_e(self, ci_params): @@ -293,11 +298,9 @@ def test_aggregation_balance_e(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", - balance_e=1, ) - assert results.event_study_effects is not None - es = results.event_study_effects + es = es_dict(results, balance_e=1) + assert es if 0 in es and 1 in es: diff = es[1]["effect"] - es[0]["effect"] assert abs(diff - 1.0) < 0.5, f"balance_e consecutive diff = {diff:.3f}, expected ≈ 1.0" @@ -333,12 +336,12 @@ def test_non_consecutive_time_periods(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) - assert results.event_study_effects is not None + es_all = es_dict(results) + assert es_all # R: agg_dynamic$att.egt[egt==2] ≈ 3 - if 2 in results.event_study_effects: - att_e2 = results.event_study_effects[2]["effect"] + if 2 in es_all: + att_e2 = es_all[2]["effect"] assert abs(att_e2 - 3.0) < 1.0, f"Non-consecutive e=2: {att_e2:.3f}, expected ≈ 3" def test_anticipation_with_and_without(self, ci_params): @@ -369,10 +372,10 @@ def test_anticipation_with_and_without(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) - if results.event_study_effects and 2 in results.event_study_effects: - att_e2 = results.event_study_effects[2]["effect"] + es_all = es_dict(results) + if 2 in es_all: + att_e2 = es_all[2]["effect"] assert abs(att_e2 - 2.0) < 0.5, f"Anticipation=1 e=2: {att_e2:.3f}, expected ≈ 2" # Without anticipation (incorrectly ignoring it) @@ -384,10 +387,10 @@ def test_anticipation_with_and_without(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) - if results_no.event_study_effects and 2 in results_no.event_study_effects: - att_e2_no = results_no.event_study_effects[2]["effect"] + es_no = es_dict(results_no) + if 2 in es_no: + att_e2_no = es_no[2]["effect"] assert abs(att_e2_no - 3.0) < 0.5, f"No anticipation e=2: {att_e2_no:.3f}, expected ≈ 3" def test_varying_vs_universal_pretreatment(self, ci_params): @@ -417,8 +420,8 @@ def test_varying_vs_universal_pretreatment(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) + es_v = es_dict(res_v) # Universal base period cs_u = CallawaySantAnna(estimation_method="dr", base_period="universal") @@ -429,15 +432,15 @@ def test_varying_vs_universal_pretreatment(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) + es_u = es_dict(res_u) # R: varying at e=-3 ≈ 1, universal at e=-3 ≈ -2 - if res_v.event_study_effects and -3 in res_v.event_study_effects: - att_v = res_v.event_study_effects[-3]["effect"] + if -3 in es_v: + att_v = es_v[-3]["effect"] assert abs(att_v - 1.0) < 0.5, f"Varying e=-3: {att_v:.3f}, expected ≈ 1" - if res_u.event_study_effects and -3 in res_u.event_study_effects: - att_u = res_u.event_study_effects[-3]["effect"] + if -3 in es_u: + att_u = es_u[-3]["effect"] assert abs(att_u - (-2.0)) < 0.5, f"Universal e=-3: {att_u:.3f}, expected ≈ -2" def test_small_groups_warning(self): @@ -581,8 +584,8 @@ def test_significance_level_and_cbands(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) + es05 = res05.aggregate("event_study") cs01 = CallawaySantAnna( estimation_method="dr", @@ -598,17 +601,12 @@ def test_significance_level_and_cbands(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) + es01 = res01.aggregate("event_study") # 5% CI should be narrower than (or equal to) 1% CI - if ( - res05.event_study_effects - and res01.event_study_effects - and res05.cband_crit_value - and res01.cband_crit_value - ): - assert res05.cband_crit_value <= res01.cband_crit_value + 0.01 + assert es05.cband_crit_value is not None and es01.cband_crit_value is not None + assert es05.cband_crit_value <= es01.cband_crit_value + 0.01 def test_some_units_treated_first_period(self): """Ported from R did::test-att_gt.R: 'some units treated in first period' @@ -712,18 +710,19 @@ def test_fewer_time_periods_than_groups(self, ci_params): time="period", first_treat="first_treat", covariates=["X"], - aggregate="all", ) # Dynamic aggregation should have no NaN ATTs - if results.event_study_effects: - for e, eff in results.event_study_effects.items(): - assert np.isfinite(eff["effect"]), f"NaN in event_study at e={e}" + es_all = es_dict(results) + assert es_all + for e, eff in es_all.items(): + assert np.isfinite(eff["effect"]), f"NaN in event_study at e={e}" # Group aggregation should have no NaN ATTs - if results.group_effects: - for g, eff in results.group_effects.items(): - assert np.isfinite(eff["effect"]), f"NaN in group_effects at g={g}" + grp = group_dict(results) + assert grp + for g, eff in grp.items(): + assert np.isfinite(eff["effect"]), f"NaN in group_effects at g={g}" def test_zero_pretreatment_outcomes(self): """Ported from R did::test-user_bug_fixes.R: '0 pre-treatment estimates...' @@ -1088,18 +1087,16 @@ def test_golden_dynamic_effects(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) r_dyn = scenario["results"]["dynamic"] - if results.event_study_effects: - for i, e in enumerate(r_dyn["egt"]): - e = int(e) - if e in results.event_study_effects: - py_att = results.event_study_effects[e]["effect"] - r_att = r_dyn["att"][i] - assert ( - abs(py_att - r_att) < 0.05 - ), f"Dynamic e={e}: Py={py_att:.4f}, R={r_att:.4f}" + es_all = es_dict(results) + assert es_all + for i, e in enumerate(r_dyn["egt"]): + e = int(e) + if e in es_all: + py_att = es_all[e]["effect"] + r_att = r_dyn["att"][i] + assert abs(py_att - r_att) < 0.05, f"Dynamic e={e}: Py={py_att:.4f}, R={r_att:.4f}" def test_golden_non_consecutive_periods(self, golden_values): """Non-consecutive periods {1,2,5,7} match R at the per-cell level. @@ -1125,7 +1122,6 @@ def test_golden_non_consecutive_periods(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) r_gt = scenario["results"]["group_time"] for i, (g, t) in enumerate(zip(r_gt["group"], r_gt["time"])): @@ -1161,18 +1157,18 @@ def test_golden_anticipation(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) r_dyn = scenario["results"]["dynamic"] - if results.event_study_effects: - for i, e in enumerate(r_dyn["egt"]): - e = int(e) - if e in results.event_study_effects: - py_att = results.event_study_effects[e]["effect"] - r_att = r_dyn["att"][i] - assert ( - abs(py_att - r_att) < 0.1 - ), f"Anticipation e={e}: Py={py_att:.4f}, R={r_att:.4f}" + es_all = es_dict(results) + assert es_all + for i, e in enumerate(r_dyn["egt"]): + e = int(e) + if e in es_all: + py_att = es_all[e]["effect"] + r_att = r_dyn["att"][i] + assert ( + abs(py_att - r_att) < 0.1 + ), f"Anticipation e={e}: Py={py_att:.4f}, R={r_att:.4f}" def test_golden_varying_vs_universal(self, golden_values): """Both base period modes match R.""" @@ -1265,7 +1261,6 @@ def test_golden_fewer_periods(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="all", ) r_gt = scenario["results"]["group_time"] for i, (g, t) in enumerate(zip(r_gt["group"], r_gt["time"])): @@ -1335,7 +1330,6 @@ def test_golden_simple_aggregation_se(self, golden_values): unit="unit", time="period", first_treat="first_treat", - aggregate="simple", ) r_simple = scenario["results"]["simple"] assert ( @@ -1375,14 +1369,14 @@ def test_golden_event_study_aggregation_se(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) + es_all = es_dict(results) blk = scenario["results"][f"{bp}_dynamic"] n_checked = 0 for e, r_att, r_se in zip(blk["egt"], blk["att"], blk["se"]): if r_att == "NA" or r_se == "NA": continue - es = results.event_study_effects.get(int(e)) + es = es_all.get(int(e)) assert es is not None, f"{bp}: missing event time e={e}" assert ( abs(es["effect"] - float(r_att)) < 2e-3 @@ -1457,19 +1451,18 @@ def test_golden_reg_aggregation_se(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="all", ) r = scenario["results"] assert abs(results.overall_att - r["simple"]["att"]) < 1e-8 assert abs(results.overall_se - r["simple"]["se"]) < 1e-8 g_att = np.atleast_1d(r["group"]["att"])[0] g_se = np.atleast_1d(r["group"]["se"])[0] - grp = results.group_effects[2] + grp = group_dict(results)[2] assert abs(grp["effect"] - g_att) < 1e-8 assert abs(grp["se"] - g_se) < 1e-8 d_att = np.atleast_1d(r["dynamic"]["att"])[0] d_se = np.atleast_1d(r["dynamic"]["se"])[0] - es0 = results.event_study_effects[0] + es0 = es_dict(results)[0] assert abs(es0["effect"] - d_att) < 1e-8 assert abs(es0["se"] - d_se) < 1e-8 @@ -1484,11 +1477,11 @@ def test_golden_reg_aggregation_se(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="event_study", ) r_dyn = scenario["results"]["dynamic"] + es_all = es_dict(results) for i, e in enumerate(r_dyn["egt"]): - es = results.event_study_effects.get(int(e)) + es = es_all.get(int(e)) assert es is not None, f"missing event time e={e}" assert ( abs(es["effect"] - r_dyn["att"][i]) < 1e-8 @@ -1525,7 +1518,6 @@ def test_golden_ipw_aggregation_se_vs_r_did_251(self, golden_values): time="period", first_treat="first_treat", covariates=["X"], - aggregate="all", ) # R did 2.5.1: aggte(type="simple", bstrap=FALSE, cband=FALSE) assert abs(results.overall_att - r_results["simple"]["att"]) < 1e-6 @@ -1533,15 +1525,17 @@ def test_golden_ipw_aggregation_se_vs_r_did_251(self, golden_values): # aggte(type="dynamic"): (att.egt, se.egt) per event time r_dyn = r_results["dynamic"] assert len(r_dyn["egt"]) > 0 + es_all = es_dict(results) for e, r_att, r_se in zip(r_dyn["egt"], r_dyn["att"], r_dyn["se"]): - es = results.event_study_effects[int(e)] + es = es_all[int(e)] assert abs(es["effect"] - r_att) < 1e-6, f"ipw dyn ATT(e={e})" assert abs(es["se"] - r_se) < 1e-6, f"ipw dyn SE(e={e})" # aggte(type="group"): (att.egt, se.egt) per cohort r_grp = r_results["group"] assert len(r_grp["egt"]) > 0 + grp_all = group_dict(results) for g, r_att, r_se in zip(r_grp["egt"], r_grp["att"], r_grp["se"]): - grp = results.group_effects[int(g)] + grp = grp_all[int(g)] assert abs(grp["effect"] - r_att) < 1e-6, f"ipw group ATT(g={g})" assert abs(grp["se"] - r_se) < 1e-6, f"ipw group SE(g={g})" @@ -1807,9 +1801,8 @@ def test_universal_gapped_event_study_reference_dilution_matches_r(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - es = res.event_study_effects + es = es_dict(res) gt = res.group_time_effects real = gt[(5, 3)]["effect"] # cohort 5 estimated cell at e=-2 real_se = gt[(5, 3)]["se"] @@ -1865,7 +1858,6 @@ def test_universal_reference_weight_ignores_skipped_cells(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) gt = res.group_time_effects # (7,1) is skipped; (7,5) reference still carries a positive weight. @@ -1874,9 +1866,7 @@ def test_universal_reference_weight_ignores_skipped_cells(self): # e=-2 is still diluted to half the real cell (weight not zeroed by the # skipped cell). Bootstrap SE differs from analytical but the POINT is # the same diluted estimand. - assert res.event_study_effects[-2]["effect"] == pytest.approx( - 0.5 * gt[(5, 3)]["effect"], rel=1e-6 - ) + assert es_dict(res)[-2]["effect"] == pytest.approx(0.5 * gt[(5, 3)]["effect"], rel=1e-6) def test_universal_reference_materialized_for_all_skipped_cohort(self): """A cohort whose non-reference cells are ALL non-estimable still gets its @@ -1911,8 +1901,10 @@ def test_universal_reference_materialized_for_all_skipped_cohort(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + # the post-fit event study (bootstrap replay on the n_boot=199 arm) + # must also be computable on this all-skipped-cohort panel + assert es_dict(res) gt = res.group_time_effects # Every non-reference cohort-7 cell is skipped, yet its base (7,5) # reference is materialized with the fixed cohort mass (40), not @@ -1983,7 +1975,6 @@ def _fit(self, golden): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) def test_cells_match_r(self, unbalanced_golden): @@ -2009,11 +2000,12 @@ def test_event_study_matches_r(self, unbalanced_golden): """ with pytest.warns(UserWarning): res = self._fit(unbalanced_golden) + py_es = es_dict(res) es = unbalanced_golden["event_study"] n_units = unbalanced_golden["meta"]["n_units"] fac = np.sqrt(n_units / (n_units - 1)) for i, e in enumerate(es["egt"]): - v = res.event_study_effects.get(int(e)) + v = py_es.get(int(e)) assert v is not None, f"missing event time {e}" assert v["effect"] == pytest.approx(es["att"][i], abs=1e-9) assert v["se"] == pytest.approx(es["se"][i] * fac, abs=1e-8) @@ -2041,17 +2033,15 @@ def test_flag_inert_on_balanced_panel(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) r0 = CallawaySantAnna(estimation_method="reg").fit(bal, **kw) r1 = CallawaySantAnna(estimation_method="reg", allow_unbalanced_panel=True).fit(bal, **kw) - for e in r0.event_study_effects: - assert r1.event_study_effects[e]["effect"] == pytest.approx( - r0.event_study_effects[e]["effect"], abs=1e-13 - ) - assert r1.event_study_effects[e]["se"] == pytest.approx( - r0.event_study_effects[e]["se"], abs=1e-13 - ) + es0 = es_dict(r0) + es1 = es_dict(r1) + assert es0 and set(es0) == set(es1) + for e in es0: + assert es1[e]["effect"] == pytest.approx(es0[e]["effect"], abs=1e-13) + assert es1[e]["se"] == pytest.approx(es0[e]["se"], abs=1e-13) def test_unbalanced_default_path_warns(self, unbalanced_golden): """Without the flag, an unbalanced panel warns (no-silent-failures).""" @@ -2062,7 +2052,6 @@ def test_unbalanced_default_path_warns(self, unbalanced_golden): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) def test_user_cluster_honored(self, unbalanced_golden): @@ -2082,11 +2071,11 @@ def test_user_cluster_honored(self, unbalanced_golden): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + py_es = es_dict(res) es = unbalanced_golden["event_study"] for i, e in enumerate(es["egt"]): - v = res.event_study_effects.get(int(e)) + v = py_es.get(int(e)) if v is not None: assert v["effect"] == pytest.approx(es["att"][i], abs=1e-9) @@ -2146,14 +2135,14 @@ def test_group_effects_match_r(self, unbalanced_golden): unit="unit", time="period", first_treat="first_treat", - aggregate="group", ) + grp = group_dict(res) g = unbalanced_golden["group"] n_units = unbalanced_golden["meta"]["n_units"] fac = np.sqrt(n_units / (n_units - 1)) - assert res.group_effects is not None + assert grp for i, gval in enumerate(g["egt"]): - eff = res.group_effects.get(gval) or res.group_effects.get(float(gval)) + eff = grp.get(gval) or grp.get(float(gval)) assert eff is not None, f"missing group {gval}" assert eff["effect"] == pytest.approx(g["att"][i], abs=1e-9) assert eff["se"] == pytest.approx(g["se"][i] * fac, abs=1e-8) @@ -2168,7 +2157,6 @@ def test_bootstrap_event_study_uses_unit_weights(self, unbalanced_golden): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) df = _unbal_df(unbalanced_golden) with pytest.warns(UserWarning): @@ -2183,10 +2171,11 @@ def test_bootstrap_event_study_uses_unit_weights(self, unbalanced_golden): n_bootstrap=200, seed=11, ).fit(df, **kw) - for e in a.event_study_effects: - assert b.event_study_effects[e]["effect"] == pytest.approx( - a.event_study_effects[e]["effect"], abs=1e-12 - ) + es_a = es_dict(a) + es_b = es_dict(b) # bootstrap replay + assert es_a and set(es_a) == set(es_b) + for e in es_a: + assert es_b[e]["effect"] == pytest.approx(es_a[e]["effect"], abs=1e-12) def test_result_metadata_reflects_rc_routing(self, unbalanced_golden): """The result records the flag AND whether RC-on-panel actually engaged, @@ -2296,7 +2285,6 @@ def test_nan_outcome_triggers_rc_routing(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) def mk(): @@ -2317,8 +2305,11 @@ def mk(): if v.get("is_reference"): continue assert v["effect"] == pytest.approx(ref.group_time_effects[k]["effect"], abs=1e-12) - for e, v in res.event_study_effects.items(): - assert v["effect"] == pytest.approx(ref.event_study_effects[e]["effect"], abs=1e-12) + es_res = es_dict(res) + es_ref = es_dict(ref) + assert es_res and set(es_res) == set(es_ref) + for e, v in es_res.items(): + assert v["effect"] == pytest.approx(es_ref[e]["effect"], abs=1e-12) assert res.overall_att == pytest.approx(ref.overall_att, abs=1e-12) def test_complete_case_rectangular_stays_inert(self): diff --git a/tests/test_diagnostic_report.py b/tests/test_diagnostic_report.py index 9698eb78e..b65bc9b5e 100644 --- a/tests/test_diagnostic_report.py +++ b/tests/test_diagnostic_report.py @@ -22,6 +22,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import legacy_standin import diff_diff as dd from diff_diff import ( @@ -108,17 +109,29 @@ def cs_fit(): # methodology-critical reason (Rambachan-Roth bounds are not valid for # interpretation on consecutive-comparison pre-periods). See the # round-5 CI review on PR #318. + # Plain fit, no fit-time surface - DR derives the event study via + # results.aggregate("event_study") (the default route). cs = CallawaySantAnna(base_period="universal").fit( sdf, outcome="outcome", unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) return cs, sdf +@pytest.fixture(scope="module") +def cs_raw_fit(cs_fit): + """The RAW-FIELD route: ``cs_fit`` with the 3.x fit-time fields populated by + hand from the container (the legacy-pickle shape). DR reads the raw field + FIRST, so this is the precedence-route fixture.""" + cs, sdf = cs_fit + raw = legacy_standin(cs) + assert raw.event_study_effects is not None + return raw, sdf + + @pytest.fixture(scope="module") def edid_fit(): warnings.filterwarnings("ignore") @@ -387,11 +400,14 @@ def test_precomputed_pretrends_power_parity_with_default_path(self, cs_fit): fit, data = cs_fit - # Precompute the power result from the same fit. The compute function - # populates ``original_results`` on the output so DR's precomputed - # adapter can inspect the source fit's event_study_vcov. - pp = compute_pretrends_power(fit, alpha=0.05, target_power=0.80, violation_type="linear") - assert getattr(pp, "original_results", None) is fit + # Precompute the power result from the same fit's event-study + # container (the consumer takes the post-fit container). The + # compute function records ``covariance_source`` and populates + # ``original_results`` on the output for DR's precomputed adapter. + es = fit.aggregate("event_study") + pp = compute_pretrends_power(es, alpha=0.05, target_power=0.80, violation_type="linear") + assert getattr(pp, "original_results", None) is es + assert pp.covariance_source == "full_pre_period_vcov" dr_default = DiagnosticReport(fit, data=data).to_dict() dr_precomputed = DiagnosticReport( @@ -3083,14 +3099,16 @@ def test_two_stage_nan_vcov_row_excluded_from_pt_family(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - balance_e=4, ) + # The balance_e-restricted surface reaches DR through the RAW + # field (DR derives without balance_e): a legacy stand-in carrying + # aggregate("event_study", balance_e=4) in the 3.x fields. + res = legacy_standin(res, balance_e=4) # Path-B precondition: -6 is an estimated column with NaN inference # and a NaN-filled VCV row; finite pre-periods {-4, -3, -2} remain. assert res.event_study_effects[-6]["n_obs"] == 0 assert not np.isfinite(res.event_study_effects[-6]["se"]) - idx = res.event_study_vcov_index + idx = [int(h) for h in res.event_study_vcov_index] assert -6 in idx and -4 in idx pos = idx.index(-6) assert np.isnan(res.event_study_vcov[pos, :]).all() @@ -3290,10 +3308,10 @@ def test_two_stage_singular_vcov_downgrades_to_bonferroni(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - V = res.event_study_vcov - idx = res.event_study_vcov_index + es = res.aggregate("event_study") + V = es.vcov + idx = es.vcov_index.tolist() pre_idx = [i for i, h in enumerate(idx) if h < -1 and np.isfinite(V[i, i])] sub = V[np.ix_(pre_idx, pre_idx)] sub = 0.5 * (sub + sub.T) @@ -3485,11 +3503,13 @@ def cs_bootstrap_plain_fit(): @pytest.fixture(scope="module") def imp_pretrends_fits(): - """(raw-kwarg fit, plain fit) ImputationDiD pair, both pretrends=True. + """(raw-field stand-in, plain fit) ImputationDiD pair, both pretrends=True. ``pretrends=True`` is REQUIRED for estimated pre-period horizons to exist at all — under the default ``pretrends=False`` the derived - surface carries only the reference lead. + surface carries only the reference lead. A plain analytic fit + stores no surface, so the raw-field route is a legacy stand-in with the + container's surface hand-set in the 3.x fields. """ from diff_diff import ImputationDiD @@ -3497,22 +3517,24 @@ def imp_pretrends_fits(): kw = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") with warnings.catch_warnings(): warnings.simplefilter("ignore") - raw = ImputationDiD(pretrends=True).fit(sdf, aggregate="event_study", **kw) plain = ImputationDiD(pretrends=True).fit(sdf, **kw) + assert plain.event_study_effects is None + raw = legacy_standin(plain) return raw, plain, sdf @pytest.fixture(scope="module") def ts_pretrends_fits(): - """(raw-kwarg fit, plain fit) TwoStageDiD pair, both pretrends=True.""" + """(raw-field stand-in, plain fit) TwoStageDiD pair, both pretrends=True.""" from diff_diff import TwoStageDiD sdf = generate_staggered_data(n_units=100, n_periods=6, treatment_effect=1.5, seed=7) kw = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") with warnings.catch_warnings(): warnings.simplefilter("ignore") - raw = TwoStageDiD(pretrends=True).fit(sdf, aggregate="event_study", **kw) plain = TwoStageDiD(pretrends=True).fit(sdf, **kw) + assert plain.event_study_effects is None + raw = legacy_standin(plain) return raw, plain, sdf @@ -3530,9 +3552,9 @@ def test_plain_cs_fit_runs_event_study_gated_checks(self, cs_plain_fit): for check in ("parallel_trends", "pretrends_power", "sensitivity", "heterogeneity"): assert d[check]["status"] == "ran", (check, d[check]) - def test_plain_cs_pt_parity_with_kwarg_route(self, cs_plain_fit, cs_fit): + def test_plain_cs_pt_parity_with_raw_field_route(self, cs_plain_fit, cs_raw_fit): plain, _ = cs_plain_fit - raw, _ = cs_fit + raw, _ = cs_raw_fit d_plain = DiagnosticReport(plain).run_all().to_dict()["parallel_trends"] d_raw = DiagnosticReport(raw).run_all().to_dict()["parallel_trends"] assert d_plain["method"] == "joint_wald_event_study" @@ -3552,9 +3574,9 @@ def test_plain_cs_pt_parity_with_kwarg_route(self, cs_plain_fit, cs_fit): np.testing.assert_allclose(a["coef"], b["coef"], rtol=1e-12, atol=1e-12) np.testing.assert_allclose(a["se"], b["se"], rtol=1e-12, atol=1e-12) - def test_plain_cs_pretrends_power_parity(self, cs_plain_fit, cs_fit): + def test_plain_cs_pretrends_power_parity(self, cs_plain_fit, cs_raw_fit): plain, _ = cs_plain_fit - raw, _ = cs_fit + raw, _ = cs_raw_fit d_plain = DiagnosticReport(plain).run_all().to_dict()["pretrends_power"] d_raw = DiagnosticReport(raw).run_all().to_dict()["pretrends_power"] assert d_plain["status"] == d_raw["status"] == "ran" @@ -3571,9 +3593,9 @@ def test_plain_cs_pretrends_power_parity(self, cs_plain_fit, cs_fit): atol=1e-3, ) - def test_plain_cs_sensitivity_parity(self, cs_plain_fit, cs_fit): + def test_plain_cs_sensitivity_parity(self, cs_plain_fit, cs_raw_fit): plain, _ = cs_plain_fit - raw, _ = cs_fit + raw, _ = cs_raw_fit d_plain = DiagnosticReport(plain).run_all().to_dict()["sensitivity"] d_raw = DiagnosticReport(raw).run_all().to_dict()["sensitivity"] assert d_plain["status"] == d_raw["status"] == "ran" @@ -3586,9 +3608,9 @@ def test_plain_cs_sensitivity_parity(self, cs_plain_fit, cs_fit): np.testing.assert_allclose(a["ci_lower"], b["ci_lower"], rtol=1e-12, atol=1e-12) np.testing.assert_allclose(a["ci_upper"], b["ci_upper"], rtol=1e-12, atol=1e-12) - def test_derived_route_emits_pre_period_source(self, cs_plain_fit, cs_fit): + def test_derived_route_emits_pre_period_source(self, cs_plain_fit, cs_raw_fit): plain, _ = cs_plain_fit - raw, _ = cs_fit + raw, _ = cs_raw_fit d_plain = DiagnosticReport(plain).run_all().to_dict() d_raw = DiagnosticReport(raw).run_all().to_dict() assert d_plain["parallel_trends"]["pre_period_source"] == "aggregate_event_study" @@ -3901,9 +3923,9 @@ def test_continuous_heterogeneity_numeric_parity(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - # ContinuousDiD's fit-time spelling has no underscore. - raw = ContinuousDiD().fit(cdf, aggregate="eventstudy", **kw) plain = ContinuousDiD().fit(cdf, **kw) + assert plain.event_study_effects is None + raw = legacy_standin(plain) self._assert_heterogeneity_parity(raw, plain) def test_pre_period_source_on_inconclusive_derived_return(self, cs_plain_fit): @@ -3932,23 +3954,28 @@ def test_anticipation_boundary_respected_on_derived_route(self): assert all(p["period"] < -1 for p in d["per_period"]) def test_requested_but_empty_raw_field_is_authoritative(self): - # Round-4 dual-review P0 pin: a fit whose fit-time aggregate=/ - # balance_e= produced the REQUESTED-but-empty {} sentinel must - # behave exactly as today — no silent re-derivation (which would - # drop the fit-time balance_e and report effects over cohorts the - # fit deliberately excluded). + # Round-4 dual-review P0 pin: a results object whose raw field holds + # the REQUESTED-but-empty {} sentinel (what a 3.x fit-time + # balance_e=4 stored on this panel - a legacy pickle today) must + # behave exactly as before — no silent re-derivation (which would + # drop the restriction and report effects over cohorts the surface + # deliberately excluded). + import dataclasses + with warnings.catch_warnings(): warnings.simplefilter("ignore") sdf = generate_staggered_data(n_units=100, n_periods=6, treatment_effect=1.5, seed=7) - e = EfficientDiD().fit( + plain = EfficientDiD().fit( sdf, outcome="outcome", unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", - balance_e=4, ) + # balance_e=4 retains no cohort on this 6-period panel. + assert len(plain.aggregate("event_study", balance_e=4).event_time) == 0 + assert plain.event_study_effects is None + e = dataclasses.replace(plain, event_study_effects={}) assert e.event_study_effects == {} calls = {"n": 0} orig = type(e).aggregate @@ -3983,8 +4010,8 @@ def test_no_mutation_after_run_all(self, cs_plain_fit): DiagnosticReport(cs).run_all() assert cs.event_study_effects is None - def test_raw_precedence_never_calls_aggregate(self, cs_fit): - cs, _ = cs_fit + def test_raw_precedence_never_calls_aggregate(self, cs_raw_fit): + cs, _ = cs_raw_fit def boom(self, *args, **kwargs): raise AssertionError("aggregate() must not be called when the raw field is present") @@ -4030,12 +4057,10 @@ def test_survey_backed_raw_vs_derived_pt_parity(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - cs_raw = CallawaySantAnna(base_period="universal").fit( - sdf, aggregate="event_study", **kw - ) cs_plain = CallawaySantAnna(base_period="universal").fit(sdf, **kw) - ts_raw = TwoStageDiD(pretrends=True).fit(sdf, aggregate="event_study", **kw) + cs_raw = legacy_standin(cs_plain) ts_plain = TwoStageDiD(pretrends=True).fit(sdf, **kw) + ts_raw = legacy_standin(ts_plain) for raw, plain in ((cs_raw, cs_plain), (ts_raw, ts_plain)): d_raw = DiagnosticReport(raw).run_all().to_dict()["parallel_trends"] d_plain = DiagnosticReport(plain).run_all().to_dict()["parallel_trends"] diff --git a/tests/test_did_had_parity.py b/tests/test_did_had_parity.py index caeb216d0..6bf69bef5 100644 --- a/tests/test_did_had_parity.py +++ b/tests/test_did_had_parity.py @@ -257,7 +257,7 @@ def test_point_se_ci_parity( py = _python_fit(panels[dgp_name], effects, placebo, trends_lin) # `overall_e1` returns a scalar HeterogeneousAdoptionDiDResults - # (aggregate="overall" path); all others return the array + # (two-period overall path); all others return the array # HeterogeneousAdoptionDiDEventStudyResults. Build a uniform # (rowname, py_att, py_se, py_ci_lo, py_ci_hi) iterator that # works on both. diff --git a/tests/test_efficient_did.py b/tests/test_efficient_did.py index 09bc61b74..43ccac526 100644 --- a/tests/test_efficient_did.py +++ b/tests/test_efficient_did.py @@ -14,6 +14,7 @@ import pandas as pd import pytest from edid_dgp import make_compustat_dgp +from legacy_surface import es_dict, group_dict from diff_diff import CallawaySantAnna, EDiD, EfficientDiD from diff_diff._backend import HAS_RUST_BACKEND @@ -645,11 +646,8 @@ def _cov_df(n_units=200, n_periods=9, seed=42): @staticmethod def _surfaces(res): gt = {k: (v["effect"], v["se"]) for k, v in res.group_time_effects.items()} - es = ( - {k: (v["effect"], v["se"]) for k, v in res.event_study_effects.items()} - if res.event_study_effects - else {} - ) + # post-fit container (a plain fit stores no fit-time surface) + es = {k: (v["effect"], v["se"]) for k, v in es_dict(res).items()} return gt, es, (res.overall_att, res.overall_se) @classmethod @@ -737,7 +735,7 @@ def dense_reference( return out df = self._cov_df() - fit_kwargs = dict(covariates=["x1", "x2"], aggregate="all") + fit_kwargs = dict(covariates=["x1", "x2"]) with warnings.catch_warnings(): warnings.simplefilter("ignore") r_fused = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", **fit_kwargs) @@ -757,7 +755,7 @@ def test_tile_forced_twin(self, monkeypatch): import diff_diff.efficient_did_covariates as cov_mod df = self._cov_df(n_units=150) - fit_kwargs = dict(covariates=["x1", "x2"], aggregate="all") + fit_kwargs = dict(covariates=["x1", "x2"]) with warnings.catch_warnings(): warnings.simplefilter("ignore") r_single = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", **fit_kwargs) @@ -789,9 +787,7 @@ def test_tile_forced_twin_survey_weights(self, monkeypatch): n_units = df["unit"].nunique() pw = np.exp(rng.normal(0, 0.4, n_units)) df = df.merge(pd.DataFrame({"unit": sorted(df["unit"].unique()), "pw": pw}), on="unit") - fit_kwargs = dict( - covariates=["x1", "x2"], aggregate="all", survey_design=SurveyDesign(weights="pw") - ) + fit_kwargs = dict(covariates=["x1", "x2"], survey_design=SurveyDesign(weights="pw")) with warnings.catch_warnings(): warnings.simplefilter("ignore") r_single = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", **fit_kwargs) @@ -809,7 +805,7 @@ def test_table_build_count_proportional_to_tiles(self, monkeypatch): n_units = 40 df = self._cov_df(n_units=n_units) - fit_kwargs = dict(covariates=["x1", "x2"], aggregate="all") + fit_kwargs = dict(covariates=["x1", "x2"]) calls = {"n": 0} orig_build = cov_mod._build_group_kcov_table @@ -864,7 +860,6 @@ def counting_kwm(*args, **kwargs): "time", "first_treat", covariates=["x1", "x2"], - aggregate="all", ) assert calls["tables"] == 0 assert calls["kernels"] == 0 @@ -877,7 +872,7 @@ def test_mid_size_tile_twin_with_chunked_products(self, monkeypatch): import diff_diff.efficient_did_covariates as cov_mod df = self._cov_df(n_units=150) - fit_kwargs = dict(covariates=["x1", "x2"], aggregate="all") + fit_kwargs = dict(covariates=["x1", "x2"]) with warnings.catch_warnings(): warnings.simplefilter("ignore") r_single = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", **fit_kwargs) @@ -893,7 +888,7 @@ def test_one_ulp_stability_conditional(self): per-cell and event-study ATTs and SEs by <= 1e-6 relative under the default ridge (the legacy pseudoinverse path moves ~1e-4).""" df = self._cov_df() - fit_kwargs = dict(covariates=["x1", "x2"], aggregate="all") + fit_kwargs = dict(covariates=["x1", "x2"]) with warnings.catch_warnings(): warnings.simplefilter("ignore") r_base = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", **fit_kwargs) @@ -909,12 +904,12 @@ def test_one_ulp_stability_nocov(self): df = _make_staggered_panel(n_per_group=80, n_control=100) with warnings.catch_warnings(): warnings.simplefilter("ignore") - r_base = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", aggregate="all") + r_base = EfficientDiD().fit(df, "y", "unit", "time", "first_treat") df_ulp = df.copy() df_ulp["y"] = np.nextafter(df_ulp["y"].to_numpy(), np.inf) with warnings.catch_warnings(): warnings.simplefilter("ignore") - r_ulp = EfficientDiD().fit(df_ulp, "y", "unit", "time", "first_treat", aggregate="all") + r_ulp = EfficientDiD().fit(df_ulp, "y", "unit", "time", "first_treat") self._assert_close_surfaces(r_ulp, r_base, rtol=1e-6, atol=1e-9) @@ -979,7 +974,6 @@ def spy(a_stack, ridge): "time", "first_treat", covariates=["x1", "x2"], - aggregate="all", ) assert calls["n"] > 0, "rust kernel never dispatched on the conditional fit" @@ -993,7 +987,7 @@ def test_backend_ab_conditional_fit(self, monkeypatch): import diff_diff.efficient_did_covariates as cov_mod df = _make_covariate_panel(n_units=150, n_periods=9, seed=42) - fit_kwargs = dict(covariates=["x1", "x2"], aggregate="all") + fit_kwargs = dict(covariates=["x1", "x2"]) with warnings.catch_warnings(): warnings.simplefilter("ignore") r_rust = EfficientDiD().fit(df, "y", "unit", "time", "first_treat", **fit_kwargs) @@ -1463,7 +1457,7 @@ def test_clustered_se_at_least_as_large(self): assert result_unclustered.overall_se > 0 def test_clustered_aggregate_event_study(self): - """Clustered SE with aggregate='event_study' should produce finite results.""" + """Clustered SE: the post-fit event-study aggregation should produce finite results.""" df = self._make_clustered_panel(n_clusters=60, units_per_cluster=3) result = EfficientDiD(cluster="cluster_id").fit(df, "y", "unit", "time", "first_treat") es = result.aggregate("event_study") @@ -1471,7 +1465,7 @@ def test_clustered_aggregate_event_study(self): assert np.isfinite(es.se[~es.is_reference]).all() def test_clustered_aggregate_all(self): - """Clustered SE with aggregate='all' should produce finite results.""" + """Clustered SE: both post-fit aggregations should produce finite results.""" df = self._make_clustered_panel(n_clusters=60, units_per_cluster=3) result = EfficientDiD(cluster="cluster_id").fit(df, "y", "unit", "time", "first_treat") es = result.aggregate("event_study") @@ -1560,21 +1554,19 @@ def test_cluster_plus_survey_raises(self): ) def test_clustered_bootstrap_aggregate_all(self, ci_params): - """Clustered bootstrap with aggregate='all' should produce finite results.""" + """Clustered bootstrap: the post-fit aggregations (which REPLAY the + fit-time bootstrap on n_bootstrap > 0) produce finite SEs.""" n_boot = ci_params.bootstrap(99) df = self._make_clustered_panel(n_clusters=60, units_per_cluster=3) - # Deprecated fit-time kwarg kept as the parity REFERENCE: post-fit - # aggregate() now REPLAYS the fit-time bootstrap on n_bootstrap > 0. - with pytest.warns(FutureWarning): - result = EfficientDiD(cluster="cluster_id", n_bootstrap=n_boot, seed=42).fit( - df, "y", "unit", "time", "first_treat", aggregate="all" - ) - assert result.event_study_effects is not None - assert result.group_effects is not None - for e, d in result.event_study_effects.items(): - assert np.isfinite(d["se"]) - for g, d in result.group_effects.items(): - assert np.isfinite(d["se"]) + result = EfficientDiD(cluster="cluster_id", n_bootstrap=n_boot, seed=42).fit( + df, "y", "unit", "time", "first_treat" + ) + es = result.aggregate("event_study") + grp = result.aggregate("group") + assert len(es.event_time) > 0 + assert len(grp.label) > 0 + assert np.isfinite(es.se).all() + assert np.isfinite(grp.se).all() class TestSmallCohortWarning: @@ -1770,7 +1762,7 @@ def test_last_cohort_with_anticipation_trims_at_last_g_minus_anticipation(self): assert 7 not in result_a1.groups def test_last_cohort_aggregate_event_study(self): - """last_cohort with aggregate='event_study' should produce finite results.""" + """last_cohort: the post-fit event-study aggregation should produce finite results.""" df = _make_staggered_panel( n_per_group=60, n_control=0, @@ -1786,7 +1778,7 @@ def test_last_cohort_aggregate_event_study(self): assert np.isfinite(es.att).all() def test_last_cohort_aggregate_all(self): - """last_cohort with aggregate='all' should produce finite results.""" + """last_cohort: both post-fit aggregations should produce finite results.""" df = _make_staggered_panel( n_per_group=60, n_control=0, @@ -1837,22 +1829,14 @@ def test_balance_e_with_bootstrap(self, ci_params): """Bootstrap balance_e should produce finite SEs.""" n_boot = ci_params.bootstrap(99) df = _make_staggered_panel(n_per_group=80, n_control=80, groups=(3, 5)) - # Deprecated fit-time kwargs kept as the parity REFERENCE: post-fit - # aggregate() now REPLAYS the fit-time bootstrap on n_bootstrap > 0. - with pytest.warns(FutureWarning): - result = EfficientDiD(n_bootstrap=n_boot, seed=42).fit( - df, - "y", - "unit", - "time", - "first_treat", - aggregate="event_study", - balance_e=0, - ) - assert result.event_study_effects is not None - for e, d in result.event_study_effects.items(): - if np.isfinite(d["effect"]): - assert np.isfinite(d["se"]) + result = EfficientDiD(n_bootstrap=n_boot, seed=42).fit( + df, "y", "unit", "time", "first_treat" + ) + # post-fit aggregate() REPLAYS the fit-time bootstrap on n_bootstrap > 0 + es = result.aggregate("event_study", balance_e=0) + assert len(es.event_time) > 0 + finite = np.isfinite(es.att) + assert np.isfinite(es.se[finite]).all() def test_balance_e_nan_anchor_filters_group(self): """When a group has NaN at the anchor horizon, bootstrap should @@ -1916,17 +1900,18 @@ def test_bootstrap_se_finite(self, ci_params): def test_bootstrap_with_aggregation(self, ci_params): n_boot = ci_params.bootstrap(99) df = _make_simple_panel() - # Deprecated fit-time kwarg kept as the parity REFERENCE: post-fit - # aggregate() now REPLAYS the fit-time bootstrap on n_bootstrap > 0. - with pytest.warns(FutureWarning): - result = EfficientDiD(n_bootstrap=n_boot, seed=42).fit( - df, "y", "unit", "time", "first_treat", aggregate="all" - ) + result = EfficientDiD(n_bootstrap=n_boot, seed=42).fit( + df, "y", "unit", "time", "first_treat" + ) assert result.bootstrap_results is not None - if result.event_study_effects: - for e, d in result.event_study_effects.items(): - if np.isfinite(d["effect"]): - assert np.isfinite(d["se"]) + # the legacy fit-time fields are None on a plain EDiD fit; the + # post-fit aggregate() REPLAYS the fit-time bootstrap + assert result.event_study_effects is None + assert result.group_effects is None + es = result.aggregate("event_study") + assert len(es.event_time) > 0 + finite = np.isfinite(es.att) + assert np.isfinite(es.se[finite]).all() def test_bootstrap_override_t_is_effect_over_se(self, ci_params): # Committed pin for the t-recompute semantics of the shared @@ -1937,11 +1922,10 @@ def test_bootstrap_override_t_is_effect_over_se(self, ci_params): # draw values - and catches a misaligned or altered t routine. n_boot = ci_params.bootstrap(99) df = _make_simple_panel() - with pytest.warns(FutureWarning): - result = EfficientDiD(n_bootstrap=n_boot, seed=42).fit( - df, "y", "unit", "time", "first_treat", aggregate="all" - ) - rows = list(result.event_study_effects.values()) + list(result.group_effects.values()) + result = EfficientDiD(n_bootstrap=n_boot, seed=42).fit( + df, "y", "unit", "time", "first_treat" + ) + rows = list(es_dict(result).values()) + list(group_dict(result).values()) assert rows for d in rows: se = float(d["se"]) @@ -2664,21 +2648,17 @@ def test_bootstrap_with_covariates_smoke(self): def test_covariates_pt_all_bootstrap(self): """PT-All + bootstrap + covariates end-to-end.""" df = _make_covariate_panel(n_units=300) - # Deprecated fit-time kwarg kept as the parity REFERENCE: post-fit - # aggregate() now REPLAYS the fit-time bootstrap on n_bootstrap > 0. - with pytest.warns(FutureWarning): - result = EfficientDiD(pt_assumption="all", n_bootstrap=99, seed=42).fit( - df, - "y", - "unit", - "time", - "first_treat", - covariates=["x1"], - aggregate="all", - ) + result = EfficientDiD(pt_assumption="all", n_bootstrap=99, seed=42).fit( + df, + "y", + "unit", + "time", + "first_treat", + covariates=["x1"], + ) assert result.bootstrap_results is not None - assert result.event_study_effects is not None - assert result.group_effects is not None + assert result.aggregate("event_study") is not None + assert result.aggregate("group") is not None assert np.isfinite(result.overall_att) assert result.overall_se > 0 @@ -2974,10 +2954,11 @@ class TestEfficientDiDVcovType: stay None under unclustered fits. 7-surface matrix: - 1. Default preserved bit-equally across all 4 aggregate modes - 2. Cluster path preserved bit-equally across the same aggregate grid - 3. TSL-survey path preserved bit-equally across the same aggregate grid - 4. Replicate-survey path preserved bit-equally across the same aggregate grid + 1. Default preserved bit-equally + 2. Cluster path preserved bit-equally + 3. TSL-survey path preserved bit-equally + 4. Replicate-survey path preserved bit-equally (incl. the post-fit + event-study / group containers) 5. Bootstrap × cluster + bootstrap × survey bit-equal 6. set_params(vcov_type=bad) eager revalidation 7. Bootstrap n_psu<2 NaN propagation (defensive fix regression) @@ -2986,10 +2967,9 @@ class TestEfficientDiDVcovType: ``cluster + replicate_weights`` rejection, and a DR-path bit-equality test. """ - # ---- Surface 1: default bit-equal across aggregation modes ------------ + # ---- Surface 1: default bit-equal ------------------------------------ - @pytest.mark.parametrize("aggregate", [None, "event_study", "group", "all"]) - def test_default_hc1_bit_equal_baseline(self, aggregate): + def test_default_hc1_bit_equal_baseline(self): data = _efficient_clustered_panel() common = dict( data=data, @@ -2997,21 +2977,15 @@ def test_default_hc1_bit_equal_baseline(self, aggregate): unit="unit", time="time", first_treat="first_treat", - aggregate=aggregate, ) - # Subject is bit equality, not the deprecated fit-time shim - keep - # all four aggregate arms and silence the FutureWarning. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - r_default = EfficientDiD().fit(**common) - r_explicit = EfficientDiD(vcov_type="hc1").fit(**common) + r_default = EfficientDiD().fit(**common) + r_explicit = EfficientDiD(vcov_type="hc1").fit(**common) assert r_default.overall_att == r_explicit.overall_att assert r_default.overall_se == r_explicit.overall_se # ---- Surface 2: cluster path bit-equal -------------------------------- - @pytest.mark.parametrize("aggregate", [None, "event_study", "group", "all"]) - def test_cluster_hc1_bit_equal_baseline(self, aggregate): + def test_cluster_hc1_bit_equal_baseline(self): data = _efficient_clustered_panel() common = dict( data=data, @@ -3019,19 +2993,15 @@ def test_cluster_hc1_bit_equal_baseline(self, aggregate): unit="unit", time="time", first_treat="first_treat", - aggregate=aggregate, ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - r_default = EfficientDiD(cluster="state").fit(**common) - r_explicit = EfficientDiD(cluster="state", vcov_type="hc1").fit(**common) + r_default = EfficientDiD(cluster="state").fit(**common) + r_explicit = EfficientDiD(cluster="state", vcov_type="hc1").fit(**common) assert r_default.overall_att == r_explicit.overall_att assert r_default.overall_se == r_explicit.overall_se # ---- Surface 3: TSL-survey path bit-equal ----------------------------- - @pytest.mark.parametrize("aggregate", [None, "event_study", "group", "all"]) - def test_survey_tsl_hc1_bit_equal_baseline(self, aggregate): + def test_survey_tsl_hc1_bit_equal_baseline(self): data = _efficient_survey_panel() design = SurveyDesign(weights="weight", psu="psu", strata="stratum", weight_type="pweight") common = dict( @@ -3040,20 +3010,16 @@ def test_survey_tsl_hc1_bit_equal_baseline(self, aggregate): unit="unit", time="time", first_treat="first_treat", - aggregate=aggregate, survey_design=design, ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - r_default = EfficientDiD().fit(**common) - r_explicit = EfficientDiD(vcov_type="hc1").fit(**common) + r_default = EfficientDiD().fit(**common) + r_explicit = EfficientDiD(vcov_type="hc1").fit(**common) assert r_default.overall_att == r_explicit.overall_att assert r_default.overall_se == r_explicit.overall_se # ---- Surface 4: replicate-survey path bit-equal ----------------------- - @pytest.mark.parametrize("aggregate", [None, "event_study", "group", "all"]) - def test_survey_replicate_hc1_bit_equal_baseline(self, aggregate): + def test_survey_replicate_hc1_bit_equal_baseline(self): data, rep_cols = _efficient_replicate_panel() design = SurveyDesign( weights="weight", @@ -3068,29 +3034,21 @@ def test_survey_replicate_hc1_bit_equal_baseline(self, aggregate): time="time", first_treat="first_treat", survey_design=design, - aggregate=aggregate, ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - r_default = EfficientDiD().fit(**common) - r_explicit = EfficientDiD(vcov_type="hc1").fit(**common) + r_default = EfficientDiD().fit(**common) + r_explicit = EfficientDiD(vcov_type="hc1").fit(**common) assert r_default.overall_att == r_explicit.overall_att assert r_default.overall_se == r_explicit.overall_se # Per-horizon / per-group SE override branches must also agree under - # the replicate-weight variance path. - if aggregate in ("event_study", "all"): - assert r_default.event_study_effects is not None - assert r_explicit.event_study_effects is not None - for h in r_default.event_study_effects: - assert ( - r_default.event_study_effects[h]["se"] - == r_explicit.event_study_effects[h]["se"] - ) - if aggregate in ("group", "all"): - assert r_default.group_effects is not None - assert r_explicit.group_effects is not None - for g in r_default.group_effects: - assert r_default.group_effects[g]["se"] == r_explicit.group_effects[g]["se"] + # the replicate-weight variance path (post-fit containers). + es_d, es_e = r_default.aggregate("event_study"), r_explicit.aggregate("event_study") + assert len(es_d.event_time) > 0 + np.testing.assert_array_equal(es_d.event_time, es_e.event_time) + np.testing.assert_array_equal(es_d.se, es_e.se) + g_d, g_e = r_default.aggregate("group"), r_explicit.aggregate("group") + assert len(g_d.label) > 0 + np.testing.assert_array_equal(g_d.label, g_e.label) + np.testing.assert_array_equal(g_d.se, g_e.se) # ---- Surface 5: bootstrap × cluster / × survey bit-equal -------------- @@ -3103,30 +3061,31 @@ def test_bootstrap_cluster_hc1_bit_equal(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - r_default = EfficientDiD(cluster="state", n_bootstrap=n_boot, seed=11).fit(**common) - r_explicit = EfficientDiD( - cluster="state", n_bootstrap=n_boot, seed=11, vcov_type="hc1" - ).fit(**common) + r_default = EfficientDiD(cluster="state", n_bootstrap=n_boot, seed=11).fit(**common) + r_explicit = EfficientDiD( + cluster="state", n_bootstrap=n_boot, seed=11, vcov_type="hc1" + ).fit(**common) assert r_default.bootstrap_results is not None assert r_explicit.bootstrap_results is not None assert ( r_default.bootstrap_results.overall_att_se == r_explicit.bootstrap_results.overall_att_se ) - # Per-horizon / per-group bootstrap SE override branches at - # efficient_did.py:1090-1115 must also agree. - assert r_default.bootstrap_results.event_study_ses is not None - assert r_explicit.bootstrap_results.event_study_ses is not None - for h, se in r_default.bootstrap_results.event_study_ses.items(): - assert se == r_explicit.bootstrap_results.event_study_ses[h] - assert r_default.bootstrap_results.group_effect_ses is not None - assert r_explicit.bootstrap_results.group_effect_ses is not None - for g, se in r_default.bootstrap_results.group_effect_ses.items(): - assert se == r_explicit.bootstrap_results.group_effect_ses[g] + # A plain EDiD fit computes no aggregated bootstrap SEs; the + # post-fit aggregate() REPLAYS the fit-time bootstrap, so the + # per-horizon / per-group override branches are compared on the + # replayed containers. + assert r_default.bootstrap_results.event_study_ses is None + assert r_default.bootstrap_results.group_effect_ses is None + es_d, es_e = r_default.aggregate("event_study"), r_explicit.aggregate("event_study") + assert len(es_d.event_time) > 0 + np.testing.assert_array_equal(es_d.event_time, es_e.event_time) + np.testing.assert_array_equal(es_d.se, es_e.se) + g_d, g_e = r_default.aggregate("group"), r_explicit.aggregate("group") + assert len(g_d.label) > 0 + np.testing.assert_array_equal(g_d.label, g_e.label) + np.testing.assert_array_equal(g_d.se, g_e.se) def test_bootstrap_survey_hc1_bit_equal(self, ci_params): data = _efficient_survey_panel() @@ -3139,12 +3098,9 @@ def test_bootstrap_survey_hc1_bit_equal(self, ci_params): time="time", first_treat="first_treat", survey_design=design, - aggregate="all", ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - r_default = EfficientDiD(n_bootstrap=n_boot, seed=23).fit(**common) - r_explicit = EfficientDiD(n_bootstrap=n_boot, seed=23, vcov_type="hc1").fit(**common) + r_default = EfficientDiD(n_bootstrap=n_boot, seed=23).fit(**common) + r_explicit = EfficientDiD(n_bootstrap=n_boot, seed=23, vcov_type="hc1").fit(**common) assert r_default.bootstrap_results is not None assert r_explicit.bootstrap_results is not None assert ( diff --git a/tests/test_efficient_did_validation.py b/tests/test_efficient_did_validation.py index 07909bed2..c1410f298 100644 --- a/tests/test_efficient_did_validation.py +++ b/tests/test_efficient_did_validation.py @@ -8,13 +8,13 @@ "Efficient Difference-in-Differences and Event Study Estimators." """ -import warnings from pathlib import Path import numpy as np import pandas as pd import pytest from edid_dgp import make_compustat_dgp, true_es_avg, true_overall_att +from legacy_surface import es_dict from diff_diff import CallawaySantAnna, EfficientDiD @@ -90,25 +90,9 @@ def _assert_close(actual, expected, label, se=None, se_frac=0.05): def _compute_es_avg(result): """Compute ES_avg (Eq 2.3): uniform average over post-treatment horizons.""" - if result.event_study_effects is None: - raise ValueError("No event study effects; use aggregate='all'") - es = {int(e): d["effect"] for e, d in result.event_study_effects.items() if int(e) >= 0} - return np.mean(list(es.values())) - - -def _attach_post_fit_event_study(res): - """Recompute the event-study surface post-fit and hand it to the - mapping-consuming helpers/assertions in this module (``_compute_es_avg`` - and the dict-iterating tests). Post-fit ``aggregate()`` recomputes from - the retained kit and relays the same numbers as the deprecated fit-time - ``aggregate=`` surface. - """ - es = res.aggregate("event_study") - res.event_study_effects = { - int(e): {"effect": float(a), "se": float(s)} - for e, a, s in zip(es.event_time, es.att, es.se) - } - return res + es = result.aggregate("event_study") + post = np.asarray(es.event_time, dtype=int) >= 0 + return float(np.mean(np.asarray(es.att, dtype=float)[post])) _TRUE_ES_AVG_COMPUSTAT = true_es_avg() @@ -133,7 +117,6 @@ def _run_mc_simulation(n_sims, rho, seed=1000, also_cs=False): time="time", first_treat="first_treat", ) - _attach_post_fit_event_study(res) edid_estimates.append(_compute_es_avg(res)) edid_overall_att.append(res.overall_att) edid_overall_se.append(res.overall_se) @@ -141,18 +124,13 @@ def _run_mc_simulation(n_sims, rho, seed=1000, also_cs=False): if also_cs: cs = CallawaySantAnna(control_group="never_treated") - # CS fit-time aggregate= is deprecated too; migrating CS is out - # of scope here, so silence its shim warning. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - cs_res = cs.fit( - data, - outcome="y", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - ) + cs_res = cs.fit( + data, + outcome="y", + unit="unit", + time="time", + first_treat="first_treat", + ) cs_estimates_list.append(_compute_es_avg(cs_res)) return { @@ -188,7 +166,7 @@ def edid_hrs_result(hrs_data): time="time", first_treat="first_treat", ) - return _attach_post_fit_event_study(res) + return res class TestHRSReplication: @@ -225,9 +203,13 @@ def test_group_time_effects_match_table6(self, edid_hrs_result): _assert_close(info["effect"], expected_effect, f"ATT({g},{t})", se=se) def test_event_study_effects_match_table6(self, edid_hrs_result): + # EDiD fits never populate the fit-time field; the post-fit container + # (via the legacy dict shape) carries the event study. + assert edid_hrs_result.event_study_effects is None + es = es_dict(edid_hrs_result) for e, (expected_effect, se) in TABLE6_ES.items(): found = False - for rel_time, info in edid_hrs_result.event_study_effects.items(): + for rel_time, info in es.items(): if int(rel_time) == e: _assert_close(info["effect"], expected_effect, f"ES({e})", se=se) found = True diff --git a/tests/test_event_study_consumers.py b/tests/test_event_study_consumers.py index 15f9fcb83..f05568e8a 100644 --- a/tests/test_event_study_consumers.py +++ b/tests/test_event_study_consumers.py @@ -28,6 +28,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import legacy_standin from diff_diff import CallawaySantAnna, compute_honest_did, compute_pretrends_power from diff_diff.pretrends import PreTrendsPower @@ -60,12 +61,14 @@ def _panel(seed=11, n_units=80, n_periods=8): def _fit_cs(data, **cs_kw): - # The deprecated fit-time aggregate= populates the NATIVE surface the - # route-parity tests compare against; the container side re-aggregates - # from the kit either way. - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return CallawaySantAnna(**cs_kw).fit(data, aggregate="event_study", **FIT_KW) + # A plain CS fit stores no event-study surface. The route-parity tests + # compare the consumers' NATIVE (raw-field, legacy-pickle) branch against + # the container route on the same fit, so the native side is a stand-in + # carrying the container's surface in the 3.x fields (dict + vcov + + # vcov_index + provenance); the container side re-aggregates from the kit. + res = CallawaySantAnna(**cs_kw).fit(data, **FIT_KW) + assert res.event_study_effects is None + return legacy_standin(res) @pytest.fixture(scope="module") @@ -194,9 +197,7 @@ def test_survey_df_threads(self, panel): sd = SurveyDesign(weights="survey_weights", strata="strata", psu="psu") with warnings.catch_warnings(): warnings.simplefilter("ignore") - res = CallawaySantAnna().fit( - _panel(), survey_design=sd, aggregate="event_study", **FIT_KW - ) + res = legacy_standin(CallawaySantAnna().fit(_panel(), survey_design=sd, **FIT_KW)) h_native, h_container = _assert_honest_parity(res) assert h_container.df_survey is not None assert np.isfinite(float(h_container.df_survey)) @@ -220,8 +221,17 @@ def test_zero_se_rows_dropped_on_both_routes(self, panel): # and assert the same drop (the row disappears from the retained # pre set rather than entering beta with sigma 0). res = _fit_cs(panel, base_period="universal") + # The 3.x field type is ``list`` (CallawaySantAnnaResults annotates + # ``event_study_vcov_index: Optional[list]``) and the native branch's + # subsetting path calls ``.index()`` on it; the stand-in stores an + # ndarray, so restore the native type here. + import dataclasses + from diff_diff.honest_did import _extract_event_study_params + res = dataclasses.replace( + res, event_study_vcov_index=[int(t) for t in res.event_study_vcov_index] + ) pre_key = min(k for k in res.event_study_effects if k < -1) res.event_study_effects[pre_key] = dict( res.event_study_effects[pre_key], se=0.0, p_value=np.nan @@ -393,11 +403,16 @@ def _fit_gapped(periods, cohorts, **fit_kw): "y": ufe + 0.3 * t + 2.0 * post + rng.randn() * 0.5, } ) + # The native (raw-field) side is a legacy stand-in carrying the + # container's surface; ``balance_e=`` restricts that surface the way + # the 3.x fit-time ``balance_e`` did. + balance_e = fit_kw.pop("balance_e", None) with warnings.catch_warnings(): warnings.simplefilter("ignore") - return CallawaySantAnna(n_bootstrap=0, base_period="universal").fit( - pd.DataFrame(rows), aggregate="event_study", **fit_kw, **FIT_KW + res = CallawaySantAnna(n_bootstrap=0, base_period="universal").fit( + pd.DataFrame(rows), **fit_kw, **FIT_KW ) + return legacy_standin(res, balance_e=balance_e) def test_overlap_layout_provenance(self): res = self._fit_gapped((1, 2, 3, 5), (2, 3, 5)) @@ -512,13 +527,16 @@ def test_missing_container_provenance_warns(self): with pytest.warns(UserWarning, match="no reference_event_times provenance"): compute_pretrends_power(surface, M=0.1) - def test_fit_time_balance_e_provenance_matches_surface(self): - # The deprecated fit-time aggregate="event_study" + balance_e - # stores a RESTRICTED surface: its provenance must describe the - # retained cohorts too, or the native route would reject a fit - # whose equivalent post-fit container is accepted (route parity). + def test_native_balance_e_provenance_matches_surface(self): + # A balance_e-RESTRICTED native surface (the 3.x fit-time shape, here a + # legacy stand-in) carries provenance describing the retained cohorts + # only, so the native route accepts a surface whose equivalent post-fit + # container is accepted (route parity). The unrestricted FIT keeps the + # fit-wide tuple. res = self._fit_gapped((1, 2, 3, 4, 6), (3, 6), balance_e=1) assert tuple(int(e) for e in res.reference_event_times) == (-1,) + unrestricted = self._fit_gapped((1, 2, 3, 4, 6), (3, 6)) + assert tuple(int(e) for e in unrestricted.reference_event_times) == (-2, -1) with warnings.catch_warnings(): warnings.simplefilter("ignore") p = compute_pretrends_power(res, M=0.1) diff --git a/tests/test_event_study_surface.py b/tests/test_event_study_surface.py index fc816f5eb..02348dc36 100644 --- a/tests/test_event_study_surface.py +++ b/tests/test_event_study_surface.py @@ -19,6 +19,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import legacy_standin from diff_diff.results_base import ( EVENT_STUDY_SCHEMA, @@ -263,7 +264,18 @@ class _Fake: @pytest.fixture(scope="module") def surfaces(): - """Build every producer's surface once; return {label: (native, surface)}.""" + """Build every producer's surface once; return {label: (native, surface)}. + + Producer routes as they stand in this (tests-first) migration step: the + CallawaySantAnna "native" side is a ``legacy_standin`` carrying + ``aggregate("event_study")`` in the 3.x fields (a plain CS fit stores no + fit-time dict; the stand-in is the legacy-pickle shape + ``build_event_study_surface`` still accepts), while the ImputationDiD, + TwoStageDiD, ContinuousDiD and EfficientDiD producers still use the + deprecated fit-time ``aggregate=`` route, which populates the dict directly. + Those sites move to stand-ins / the post-fit relay when the fit-time + keywords are removed (4.0 PRs 3b/3c). + """ out = {} from diff_diff import ( @@ -303,8 +315,9 @@ def surfaces(): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + assert cs.event_study_effects is None # plain fit: not populated at fit + cs = legacy_standin(cs) out["CallawaySantAnna"] = (cs, build_event_study_surface(cs)) # 2. SunAbraham (always-on) @@ -789,13 +802,14 @@ def test_cs_universal_gapped_multiple_references(): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - cs = CallawaySantAnna(n_bootstrap=0, base_period="universal").fit( - pd.DataFrame(rows), - outcome="outcome", - unit="unit", - time="period", - first_treat="first_treat", - aggregate="event_study", + cs = legacy_standin( + CallawaySantAnna(n_bootstrap=0, base_period="universal").fit( + pd.DataFrame(rows), + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + ) ) native_refs = sorted(e for e, r in cs.event_study_effects.items() if r.get("n_groups") == 0) assert len(native_refs) >= 2 # the case only matters when >1 @@ -1030,7 +1044,8 @@ def test_absent_surface_raises(): from diff_diff.prep import generate_staggered_data as prep_staggered data = prep_staggered(n_units=60, n_periods=6, cohort_periods=[3], treatment_effect=2.0, seed=1) - # aggregate defaults to "simple": no event_study_effects populated. + # A plain CallawaySantAnna fit does not populate event_study_effects; the builder + # distinguishes that ABSENT surface (raise) from an empty one. cs = CallawaySantAnna(n_bootstrap=0).fit( data, outcome="outcome", unit="unit", time="period", first_treat="first_treat" ) @@ -1298,7 +1313,7 @@ def test_two_stage_vcov_alignment(surfaces): idx = surface.vcov_index.tolist() assert set(idx).issubset(set(surface.event_time.tolist())) assert -1 not in idx # ref_period never a Stage-2 column - assert idx == native.event_study_vcov_index + assert idx == np.asarray(native.event_study_vcov_index).tolist() ses = {h: d["se"] for h, d in native.event_study_effects.items()} diag = np.sqrt(np.maximum(np.diag(surface.vcov), 0.0)) for i, h in enumerate(idx): @@ -1330,18 +1345,17 @@ def test_two_stage_prop5_horizons_excluded_from_vcov_index(): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + surface = res.aggregate("event_study") prop5 = [ - h - for h, d in res.event_study_effects.items() - if d["n_obs"] > 0 and not np.isfinite(d["effect"]) + int(h) + for h, n, att in zip(surface.event_time, surface.n, surface.att) + if n > 0 and not np.isfinite(att) ] assert prop5, "fixture must produce Proposition-5 horizons" - assert res.event_study_vcov_index is not None - assert not set(prop5) & set(res.event_study_vcov_index) - surface = build_event_study_surface(res) assert surface.vcov is not None + assert surface.vcov_index is not None + assert not set(prop5) & set(surface.vcov_index.tolist()) assert set(surface.vcov_index.tolist()).issubset(set(surface.event_time.tolist())) @@ -1434,14 +1448,14 @@ def test_two_stage_survey_df_threaded(): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", survey_design=design, ) - assert res.event_study_df is not None and res.event_study_df > 0 - surface = build_event_study_surface(res) + surface = res.aggregate("event_study") + assert surface.df_survey is not None and surface.df_survey > 0 + assert surface.df_survey == float(res.survey_metadata.df_survey) finite_p = np.isfinite(surface.p_value) assert finite_p.any() - assert set(surface.df[finite_p].tolist()) == {float(res.event_study_df)} + assert set(surface.df[finite_p].tolist()) == {float(surface.df_survey)} def test_stacked_replicate_weight_vcov_diag_matches_ses(): @@ -1511,13 +1525,12 @@ def test_cs_bare_cluster_df_fallback(): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + surface = res.aggregate("event_study") # Bare-cluster synthesize path: the ES rows' safe_inference used G-1, - # recorded on BOTH channels (primary event_study_df; df_inference keeps - # its narrow HonestDiD contract). - assert res.event_study_df == float(res.df_inference) == 7.0 - surface = build_event_study_surface(res) + # recorded on BOTH channels (the container's per-row df + df_survey; + # df_inference keeps its narrow HonestDiD contract). + assert surface.df_survey == float(res.df_inference) == 7.0 finite_p = np.isfinite(surface.p_value) assert set(surface.df[finite_p].tolist()) == {7.0} @@ -1533,16 +1546,14 @@ def test_cs_bare_cluster_bootstrap_df_is_nan(): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - # Bootstrap overrode the stored ES p/CIs with percentile values that + surface = res.aggregate("event_study") + # The bootstrap replay overrides the ES p/CIs with percentile values that # never used the analytical df: the surface must show NO df even though # df_inference (the HonestDiD contract field) still carries G-1 on the - # container. Bootstrap p-values are finite, so only the gated + # results object. Bootstrap p-values are finite, so only the gated # df_inference fallback - not the NaN-p mask - prevents the leak. assert res.df_inference == 7 - assert res.event_study_df is None - surface = build_event_study_surface(res) assert np.isfinite(surface.p_value).any() assert np.isnan(surface.df).all() @@ -1562,18 +1573,19 @@ def test_cs_survey_min_df_on_surface(): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", survey_design=design, ) + surface = res.aggregate("event_study") # Explicit-survey fits: df_inference stays None (PR #487 narrow # HonestDiD contract - channel separation), while the surface carries # the ONE df every ES row's safe_inference_batch actually applied. assert res.df_inference is None - assert res.event_study_df is not None and res.event_study_df > 0 - surface = build_event_study_surface(res) finite_p = np.isfinite(surface.p_value) assert finite_p.any() - assert set(surface.df[finite_p].tolist()) == {float(res.event_study_df)} + applied = set(surface.df[finite_p].tolist()) + assert len(applied) == 1 + (df_applied,) = applied + assert np.isfinite(df_applied) and df_applied > 0 def test_mpd_hc2_bm_per_period_df(): diff --git a/tests/test_had.py b/tests/test_had.py index d85464745..abaa26ce5 100644 --- a/tests/test_had.py +++ b/tests/test_had.py @@ -2196,11 +2196,10 @@ def _fit_es(est, *args, **kwargs) -> HeterogeneousAdoptionDiDEventStudyResults: The public ``fit()`` is annotated to return a union over the single-period ``HeterogeneousAdoptionDiDResults`` and the multi- period ``HeterogeneousAdoptionDiDEventStudyResults``, matching its - runtime polymorphism on ``aggregate``. When ``aggregate="event_study"`` - is requested, this helper narrows the union to the event-study - branch for the test body via ``typing.cast``. + runtime polymorphism on the panel shape (T > 2 selects the event + study). This helper narrows the union to the event-study branch for + the test body via ``typing.cast``. """ - kwargs.setdefault("aggregate", "event_study") result = est.fit(*args, **kwargs) return cast(HeterogeneousAdoptionDiDEventStudyResults, result) @@ -2210,8 +2209,8 @@ class TestFitReturnAnnotation: The annotation MUST be a union over the single-period and event-study result classes — narrowing one branch out silently would make the - static type contract diverge from the runtime polymorphism on - ``aggregate``. + static type contract diverge from the runtime polymorphism on the + panel shape. """ def test_fit_return_annotation_is_union_of_result_classes(self): @@ -5431,8 +5430,8 @@ def test_mass_point_default_vcov_event_study_cband_rejected(self): def test_survey_event_study_continuous_end_to_end(self): """Review R6 P3: estimator-level - ``fit(aggregate='event_study', survey_design=SurveyDesign(...))`` - integration lock for the continuous path. Verifies + multi-period ``fit(survey_design=SurveyDesign(...))`` (event-study + mode) integration lock for the continuous path. Verifies variance_formula, survey_metadata.df_survey (t-inference path), cband_* population, and stratified PSU dispatch through _aggregate_unit_resolved_survey.""" @@ -5480,8 +5479,8 @@ def test_survey_event_study_continuous_end_to_end(self): def test_survey_event_study_mass_point_end_to_end(self): """Review R6 P3: estimator-level - ``fit(design='mass_point', aggregate='event_study', - survey_design=...)`` integration lock. Verifies + multi-period ``fit(design='mass_point', survey_design=...)`` + (event-study mode) integration lock. Verifies variance_formula='survey_binder_tsl_2sls' and that the weighted 2SLS IF flows correctly through per-horizon Binder-TSL + sup-t bootstrap.""" diff --git a/tests/test_had_pretests.py b/tests/test_had_pretests.py index 90f04671e..0f278a6f2 100644 --- a/tests/test_had_pretests.py +++ b/tests/test_had_pretests.py @@ -2789,7 +2789,8 @@ def test_joint_homogeneity_test_uses_chronological_rank(self): assert np.isfinite(result.p_value) def test_workflow_event_study_ordered_categorical(self): - """did_had_pretest_workflow(aggregate="event_study") must pick + """did_had_pretest_workflow on a multi-period panel (event-study + battery) must pick up BOTH earlier pre-periods ("q1", "q2") from an ordered- categorical panel where lexical order would silently drop one of them. Regression against the `earlier_pre` raw-< fix.""" diff --git a/tests/test_honest_did.py b/tests/test_honest_did.py index 9e5e4442c..cee1e4428 100644 --- a/tests/test_honest_did.py +++ b/tests/test_honest_did.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import legacy_standin from diff_diff import MultiPeriodDiD from diff_diff.honest_did import ( @@ -862,12 +863,13 @@ def test_callaway_santanna_universal_base_period(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", - ) + ).aggregate("event_study") # Verify reference period exists with NaN SE - assert -1 in results.event_study_effects - assert np.isnan(results.event_study_effects[-1]["se"]) + times = results.event_time.tolist() + assert -1 in times + assert bool(results.is_reference[times.index(-1)]) + assert np.isnan(results.se[times.index(-1)]) # HonestDiD should work without errors (reference period filtered out) honest = HonestDiD(method="relative_magnitude", M=1.0) @@ -890,33 +892,41 @@ def test_max_pre_violation_excludes_reference_period(self): # Generate data with universal base period data = generate_staggered_data(n_units=200, n_periods=10, seed=42) cs = CallawaySantAnna(base_period="universal") - results = cs.fit( + fit = cs.fit( data, outcome="outcome", unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + es = fit.aggregate("event_study") - # Verify reference period exists with n_groups=0 - assert -1 in results.event_study_effects - assert results.event_study_effects[-1]["n_groups"] == 0 + # Verify the reference period exists and is marked on the container + times = es.event_time.tolist() + assert -1 in times + assert bool(es.is_reference[times.index(-1)]) + assert np.isnan(es.n[times.index(-1)]) # The max pre-violation calculation should exclude the reference period honest = HonestDiD(method="relative_magnitude", M=1.0) - # Get pre_periods excluding reference (n_groups=0) + # Get pre_periods excluding the reference row real_pre_periods = [ - t - for t in results.event_study_effects - if t < 0 and results.event_study_effects[t].get("n_groups", 1) > 0 + int(t) for t, ref in zip(es.event_time, es.is_reference) if t < 0 and not ref ] + # ``_estimate_max_pre_violation`` reads the CS raw ``event_study_effects`` + # dict (the 3.x fit-time shape, now the legacy-pickle route): feed it a + # stand-in carrying the container's surface. Its reference row is the + # zero-count sentinel the method excludes. + results = legacy_standin(fit) + assert results.event_study_effects[-1]["n_groups"] == 0 + # If there are real pre-periods, max_violation should be > 0 # (based on actual pre-period effects, not the reference period's effect=0) if real_pre_periods: max_violation = honest._estimate_max_pre_violation(results, real_pre_periods) + assert max_violation == max(abs(es.att[times.index(t)]) for t in real_pre_periods) # Max violation should reflect actual pre-period coefficients, not 0 # The actual effects are non-zero due to sampling variation assert max_violation > 0, "max_pre_violation should be > 0 when real pre-periods exist" @@ -1246,25 +1256,29 @@ def test_df_survey_extracted_from_cs_results(self): "period", "first_treat", survey_design=sd, - aggregate="event_study", ) + es = cs_result.aggregate("event_study") honest = HonestDiD(method="smoothness", M=0.0) - h_result = honest.fit(cs_result) + h_result = honest.fit(es) - # df_survey should be propagated + # df_survey should be propagated through the container's df_survey assert h_result.df_survey is not None assert h_result.df_survey > 0 - assert h_result.survey_metadata is not None - # Single-source oracle: the CS branch resolves via the shared helper - # (aggregation.resolve_inference_df) - value-equal, float-typed. + assert h_result.df_survey == es.df_survey + # The container carries no survey-metadata OBJECT (documented + # divergence from the raw-field route); df_survey is its replacement. + assert h_result.survey_metadata is None + # Single-source oracle: the container's df_survey resolves via the + # shared helper (aggregation.resolve_inference_df) - value-equal, + # float-typed. from diff_diff.aggregation import resolve_inference_df assert h_result.df_survey == resolve_inference_df(cs_result) assert isinstance(h_result.df_survey, float) def test_event_study_vcov_computed(self): - """CallawaySantAnna event_study_vcov is computed and used by HonestDiD.""" + """The CallawaySantAnna event-study container carries the full vcov HonestDiD uses.""" from diff_diff import CallawaySantAnna, generate_staggered_data data = generate_staggered_data(n_units=100, n_periods=6, seed=42) @@ -1274,23 +1288,32 @@ def test_event_study_vcov_computed(self): "unit", "period", "first_treat", - aggregate="event_study", ) + # Plain-fit invariant: no fit-time surface or vcov is stored. + assert cs_result.event_study_effects is None + assert cs_result.event_study_vcov is None + assert cs_result.event_study_vcov_index is None - # VCV should be computed - assert cs_result.event_study_vcov is not None - n_effects = len( - [ - e - for e, d in cs_result.event_study_effects.items() - if d.get("n_groups", 1) > 0 and np.isfinite(d.get("se", np.nan)) - ] - ) - assert cs_result.event_study_vcov.shape == (n_effects, n_effects) + es = cs_result.aggregate("event_study") + # VCV should be computed on the container + assert es.vcov is not None + estimated = ~es.is_reference & np.isfinite(es.se) + n_effects = int(estimated.sum()) + assert es.vcov.shape == (n_effects, n_effects) + np.testing.assert_array_equal(es.vcov_index, es.event_time[estimated]) - # Diagonal should be positive + # Diagonal should be positive and equal the stored SEs squared for i in range(n_effects): - assert cs_result.event_study_vcov[i, i] > 0 + assert es.vcov[i, i] > 0 + np.testing.assert_allclose(np.diag(es.vcov), es.se[estimated] ** 2, rtol=1e-12) + + # HonestDiD consumes it without the diagonal fallback + import warnings as _w + + with _w.catch_warnings(record=True) as caught: + _w.simplefilter("always") + HonestDiD(method="smoothness", M=0.0).fit(es) + assert not [w for w in caught if "covariance" in str(w.message)] def test_survey_df_widens_bounds(self): """Survey df (t-distribution) should give wider CIs than normal.""" @@ -1319,13 +1342,12 @@ def test_survey_df_widens_bounds(self): "period", "first_treat", survey_design=sd, - aggregate="event_study", ) # Use RM method (naive FLCI) which honors df via _get_critical_value. # The optimal FLCI (smoothness) uses folded normal which doesn't use df. honest = HonestDiD(method="relative_magnitude", M=1.0) - h_result = honest.fit(cs_result) + h_result = honest.fit(cs_result.aggregate("event_study")) # With df=8, t critical value (~2.31) > z critical value (1.96) # So CI width should be wider than 2*1.96*SE @@ -1346,17 +1368,16 @@ def test_no_survey_gives_none_df(self): "unit", "period", "first_treat", - aggregate="event_study", ) honest = HonestDiD(method="smoothness", M=0.0) - h_result = honest.fit(cs_result) + h_result = honest.fit(cs_result.aggregate("event_study")) assert h_result.df_survey is None assert h_result.survey_metadata is None def test_replicate_weight_uses_diagonal_fallback(self): - """Replicate-weight designs should NOT produce full event_study_vcov.""" + """Replicate-weight designs should NOT produce a full event-study vcov.""" from diff_diff import CallawaySantAnna, SurveyDesign, generate_staggered_data data = generate_staggered_data(n_units=100, n_periods=5, seed=42) @@ -1386,15 +1407,16 @@ def test_replicate_weight_uses_diagonal_fallback(self): "period", "first_treat", survey_design=sd, - aggregate="event_study", ) - # event_study_vcov should be None (diagonal fallback for replicate designs) - assert cs_result.event_study_vcov is None + # The container's vcov should be None (diagonal fallback for replicate designs) + es = cs_result.aggregate("event_study") + assert es.vcov is None and es.vcov_index is None + assert np.isfinite(es.se[~es.is_reference]).any() def test_bootstrap_fit_carries_bootstrap_vcov(self): - """Bootstrap CS results carry the covariance of the bootstrap draws - (never the analytical event_study_vcov): its diagonal is the + """A bootstrapped CS event-study container carries the covariance of + the bootstrap draws (never the analytical vcov): its diagonal is the bootstrap SEs squared, and HonestDiD uses it without the diagonal fallback.""" from diff_diff import CallawaySantAnna, generate_staggered_data @@ -1406,13 +1428,14 @@ def test_bootstrap_fit_carries_bootstrap_vcov(self): "unit", "period", "first_treat", - aggregate="event_study", ) + es = cs_result.aggregate("event_study") - vcov = cs_result.event_study_vcov - index = cs_result.event_study_vcov_index + vcov = es.vcov + index = es.vcov_index assert vcov is not None and vcov.shape == (len(index), len(index)) - ses = np.array([cs_result.event_study_effects[e]["se"] for e in index]) + times = es.event_time.tolist() + ses = np.array([es.se[times.index(e)] for e in index]) np.testing.assert_allclose(np.diag(vcov), ses**2, rtol=1e-12, atol=0.0) honest = HonestDiD(method="relative_magnitude", M=1.0) @@ -1420,16 +1443,22 @@ def test_bootstrap_fit_carries_bootstrap_vcov(self): with _w.catch_warnings(record=True) as caught: _w.simplefilter("always") - h_result = honest.fit(cs_result) - diag_warnings = [w for w in caught if "diagonal covariance" in str(w.message)] - assert not diag_warnings + h_result = honest.fit(es) + fallback_warnings = [ + w + for w in caught + if "diagonal covariance" in str(w.message) or "no full covariance" in str(w.message) + ] + assert not fallback_warnings assert np.isfinite(h_result.original_se) assert h_result.original_se > 0 def test_bootstrap_vcov_same_on_native_and_container_routes(self): - """On a bootstrapped fit HonestDiD reads the bootstrap covariance on - both input routes: the native results object and the replayed - aggregate('event_study') container give the same sigma and bounds.""" + """On a bootstrapped fit the replayed aggregate('event_study') + container's vcov diagonal is the stored bootstrap SEs squared, and + HonestDiD reads that covariance identically on the container route + and on the native (raw-field, legacy-pickle) route - here a stand-in + carrying the same surface in the 3.x fields.""" import warnings as _w from diff_diff import CallawaySantAnna, generate_staggered_data @@ -1443,15 +1472,20 @@ def test_bootstrap_vcov_same_on_native_and_container_routes(self): "unit", "period", "first_treat", - aggregate="event_study", ) es = cs_result.aggregate("event_study") - native = _extract_event_study_params(cs_result) + standin = legacy_standin(cs_result) + native = _extract_event_study_params(standin) container = _extract_event_study_params(es) - h_native = HonestDiD(method="smoothness", M=0.0).fit(cs_result) + h_native = HonestDiD(method="smoothness", M=0.0).fit(standin) h_container = HonestDiD(method="smoothness", M=0.0).fit(es) - np.testing.assert_array_equal(native[1], cs_result.event_study_vcov) + # The fit itself stores no surface; the container's vcov is the + # bootstrap-draw covariance whose diagonal is se**2. + assert cs_result.event_study_vcov is None + in_vcov = np.isin(es.event_time, es.vcov_index) + np.testing.assert_allclose(np.diag(es.vcov), es.se[in_vcov] ** 2, rtol=1e-12, atol=0.0) + np.testing.assert_array_equal(native[1], standin.event_study_vcov) np.testing.assert_array_equal(container[0], native[0]) np.testing.assert_allclose(container[1], native[1], rtol=1e-12, atol=0.0) np.testing.assert_allclose(h_container.original_se, h_native.original_se, rtol=1e-12) @@ -1472,17 +1506,18 @@ def test_bootstrap_fit_without_vcov_falls_back_to_diagonal(self): def fit(n_bootstrap): with _w.catch_warnings(): _w.simplefilter("ignore") - return CallawaySantAnna( + res = CallawaySantAnna( n_bootstrap=n_bootstrap, seed=42, base_period="universal" - ).fit(data, "outcome", "unit", "period", "first_treat", aggregate="event_study") - - cs_result = fit(len(fit(49).event_study_vcov_index)) - with _w.catch_warnings(): - _w.simplefilter("ignore") - es = cs_result.aggregate("event_study") - assert cs_result.event_study_vcov is None and es.vcov is None + ).fit(data, "outcome", "unit", "period", "first_treat") + return res, res.aggregate("event_study") + + n_horizons = len(fit(49)[1].vcov_index) + cs_result, es = fit(n_horizons) + assert es.vcov is None and es.vcov_index is None + # Native (raw-field) route on the same surface: a stand-in with the + # dict but no vcov takes the diagonal fallback too. with pytest.warns(UserWarning, match="diagonal covariance"): - HonestDiD(method="relative_magnitude", M=1.0).fit(cs_result) + HonestDiD(method="relative_magnitude", M=1.0).fit(legacy_standin(cs_result)) with pytest.warns(UserWarning, match="no full covariance"): HonestDiD(method="relative_magnitude", M=1.0).fit(es) @@ -1503,15 +1538,16 @@ def test_singular_vcov_few_psus_raises_on_both_routes(self, n_bootstrap): _w.simplefilter("ignore") cs_result = CallawaySantAnna( n_bootstrap=n_bootstrap, seed=1, base_period="universal", cluster="psu" - ).fit(data, "outcome", "unit", "period", "first_treat", aggregate="event_study") + ).fit(data, "outcome", "unit", "period", "first_treat") es = cs_result.aggregate("event_study") - vcov = cs_result.event_study_vcov + standin = legacy_standin(cs_result) + vcov = standin.event_study_vcov assert vcov is not None and vcov.shape[0] > 7 # Precondition: both PSU-built matrices are genuinely rank-deficient. assert np.linalg.matrix_rank(vcov) < vcov.shape[0] assert np.linalg.matrix_rank(es.vcov) < es.vcov.shape[0] with pytest.raises(ValueError, match="singular"): - HonestDiD(method="relative_magnitude", M=1.0).fit(cs_result) + HonestDiD(method="relative_magnitude", M=1.0).fit(standin) with pytest.raises(ValueError, match="singular"): HonestDiD(method="relative_magnitude", M=1.0).fit(es) diff --git a/tests/test_imputation.py b/tests/test_imputation.py index be907ad59..4377daf21 100644 --- a/tests/test_imputation.py +++ b/tests/test_imputation.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, group_dict from diff_diff.imputation import ( ImputationBootstrapResults, @@ -172,7 +173,7 @@ def test_aggregate_simple(self): assert results.group_effects is None def test_aggregate_event_study(self): - """Test event study aggregation.""" + """Post-fit event-study aggregation on an analytical fit.""" data = generate_test_data() est = ImputationDiD() results = est.fit( @@ -181,20 +182,27 @@ def test_aggregate_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + # Analytical fits store no fit-time surface; the route is post-fit. + assert results.event_study_effects is None assert results.group_effects is None - - for h, eff in results.event_study_effects.items(): - assert "effect" in eff - assert "se" in eff - assert "t_stat" in eff - assert "p_value" in eff - assert "conf_int" in eff - assert "n_obs" in eff + es = results.aggregate("event_study") + assert len(es.event_time) > 0 + assert es.n_kind == "obs" + frame = es.to_dataframe() + for col in ( + "event_time", + "att", + "se", + "t_stat", + "p_value", + "conf_int_lower", + "conf_int_upper", + "n", + "is_reference", + ): + assert col in frame.columns def test_aggregate_group(self): """Test group aggregation.""" @@ -206,20 +214,17 @@ def test_aggregate_group(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - assert results.group_effects is not None - assert len(results.group_effects) == 3 # 3 cohorts + grp = results.aggregate("group") + assert len(grp.label) == 3 # 3 cohorts assert results.event_study_effects is None - - for g, eff in results.group_effects.items(): - assert "effect" in eff - assert "se" in eff - assert eff["se"] > 0 + assert results.group_effects is None + assert np.all(np.isfinite(grp.att)) + assert np.all(grp.se > 0) def test_aggregate_all(self): - """Test 'all' aggregation computes both event study and group.""" + """Both post-fit levels are available from one analytical fit.""" data = generate_test_data() est = ImputationDiD() results = est.fit( @@ -228,11 +233,10 @@ def test_aggregate_all(self): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) - assert results.event_study_effects is not None - assert results.group_effects is not None + assert len(results.aggregate("event_study").event_time) > 0 + assert len(results.aggregate("group").label) > 0 def test_covariates(self): """Test estimation with covariates.""" @@ -265,7 +269,6 @@ def test_anticipation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) est1 = ImputationDiD(anticipation=1) @@ -275,15 +278,16 @@ def test_anticipation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # With anticipation=1, more obs are treated, fewer untreated assert results1.n_treated_obs > results0.n_treated_obs # Reference period changes - ref0 = [h for h, e in results0.event_study_effects.items() if e.get("n_obs", 1) == 0] - ref1 = [h for h, e in results1.event_study_effects.items() if e.get("n_obs", 1) == 0] + es0 = results0.aggregate("event_study") + es1 = results1.aggregate("event_study") + ref0 = es0.event_time[es0.is_reference].tolist() + ref1 = es1.event_time[es1.is_reference].tolist() assert -1 in ref0 assert -2 in ref1 @@ -292,26 +296,18 @@ def test_balance_e(self): data = generate_test_data() est = ImputationDiD() - results_unbal = est.fit( - data, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - ) - results_bal = est.fit( + results = est.fit( data, outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - balance_e=2, ) + es_unbal = results.aggregate("event_study") + es_bal = results.aggregate("event_study", balance_e=2) # Balanced should have same or fewer horizons - assert len(results_bal.event_study_effects) <= len(results_unbal.event_study_effects) + 5 + assert len(es_bal.event_time) <= len(es_unbal.event_time) + 5 def test_horizon_max(self): """Test horizon_max caps event study horizons.""" @@ -324,11 +320,10 @@ def test_horizon_max(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - for h in results.event_study_effects: - if results.event_study_effects[h].get("n_obs", 0) > 0: + for h, eff in es_dict(results).items(): + if eff["n_obs"] > 0: assert abs(h) <= 3 def test_summary(self): @@ -341,14 +336,16 @@ def test_summary(self): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) summary = results.summary() assert "Imputation DiD" in summary assert "ATT" in summary - assert "Event Study" in summary - assert "Group" in summary + # Analytical fits carry no fit-time surface: the per-level tables + # live on the post-fit containers' own summaries. + assert "Event Study" not in summary + assert "Event-Study Effects" in results.aggregate("event_study").summary() + assert "aggregate(type='group')" in results.aggregate("group").summary() def test_to_dataframe_observation(self): """Test to_dataframe at observation level.""" @@ -377,12 +374,11 @@ def test_to_dataframe_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - df = results.to_dataframe("event_study") - assert "relative_period" in df.columns - assert "effect" in df.columns + df = results.aggregate("event_study").to_dataframe() + assert "event_time" in df.columns + assert "att" in df.columns assert "se" in df.columns def test_to_dataframe_group(self): @@ -395,11 +391,10 @@ def test_to_dataframe_group(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - df = results.to_dataframe("group") - assert "group" in df.columns + df = results.aggregate("group").to_dataframe() + assert "label" in df.columns assert len(df) == 3 def test_to_dataframe_errors(self): @@ -592,33 +587,25 @@ def test_balance_e_checks_pre_treatment_periods(self): data = pd.DataFrame(rows) est = ImputationDiD(horizon_max=1) - # balance_e=2, horizon_max=1: required = {-2,-1,0,1} - # Cohort B missing -2 => should be dropped - results_bal2 = est.fit( + results = est.fit( data, outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - balance_e=2, ) + # balance_e=2, horizon_max=1: required = {-2,-1,0,1} + # Cohort B missing -2 => should be dropped + es_bal2 = es_dict(results, balance_e=2) + # balance_e=1, horizon_max=1: required = {-1,0,1} # Both cohorts have -1 => both kept - results_bal1 = est.fit( - data, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - balance_e=1, - ) + es_bal1 = es_dict(results, balance_e=1) # Cohort B dropped at balance_e=2 => fewer obs at horizon 0 - n_obs_bal2_h0 = results_bal2.event_study_effects[0]["n_obs"] - n_obs_bal1_h0 = results_bal1.event_study_effects[0]["n_obs"] + n_obs_bal2_h0 = es_bal2[0]["n_obs"] + n_obs_bal1_h0 = es_bal1[0]["n_obs"] assert n_obs_bal2_h0 < n_obs_bal1_h0, ( f"balance_e=2 should drop cohort B (missing rel_time=-2), " f"got n_obs={n_obs_bal2_h0} vs {n_obs_bal1_h0}" @@ -748,11 +735,10 @@ def test_se_positive_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - for h, eff in results.event_study_effects.items(): - if eff.get("n_obs", 0) > 0 and np.isfinite(eff["se"]): + for h, eff in es_dict(results).items(): + if eff["n_obs"] > 0 and np.isfinite(eff["se"]): assert eff["se"] > 0 def test_aux_partition_cohort_horizon(self): @@ -1424,8 +1410,8 @@ def test_no_never_treated(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + effects = es_dict(results) # Should still estimate assert results.overall_att is not None @@ -1433,15 +1419,13 @@ def test_no_never_treated(self): # Proposition 5: long-run horizons should be NaN prop5_nans = [ - h - for h, eff in results.event_study_effects.items() - if np.isnan(eff["effect"]) and eff.get("n_obs", 0) > 0 + h for h, eff in effects.items() if np.isnan(eff["effect"]) and eff["n_obs"] > 0 ] assert len(prop5_nans) > 0, "Should have Prop 5 NaN horizons" # Check all inference fields are NaN for Prop 5 horizons for h in prop5_nans: - eff = results.event_study_effects[h] + eff = effects[h] assert np.isnan(eff["se"]) assert np.isnan(eff["t_stat"]) assert np.isnan(eff["p_value"]) @@ -1602,12 +1586,12 @@ def test_nan_propagation_all_nan_horizon(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + effects = es_dict(results) # Check that NaN horizons have all-NaN inference - for h, eff in results.event_study_effects.items(): - if eff.get("n_obs", 0) > 0 and np.isnan(eff["effect"]): + for h, eff in effects.items(): + if eff["n_obs"] > 0 and np.isnan(eff["effect"]): assert np.isnan(eff["se"]) assert np.isnan(eff["t_stat"]) assert np.isnan(eff["p_value"]) @@ -1652,19 +1636,14 @@ def test_rank_condition_missing_untreated_period(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) rank_warnings = [x for x in w if "Rank condition" in str(x.message)] assert len(rank_warnings) > 0, "Should warn about rank condition violation" + effects = es_dict(results) # Affected horizons should have NaN effects (periods with no untreated units) - if results.event_study_effects: - nan_effects = [ - h - for h, d in results.event_study_effects.items() - if np.isnan(d["effect"]) and d.get("n_obs", 1) > 0 - ] - assert len(nan_effects) > 0, "Some horizons should have NaN effects" + nan_effects = [h for h, d in effects.items() if np.isnan(d["effect"]) and d["n_obs"] > 0] + assert len(nan_effects) > 0, "Some horizons should have NaN effects" def test_rank_condition_error_mode(self): """Test error raised when rank condition fails with action='error'.""" @@ -1754,17 +1733,17 @@ def test_plot_reference_with_anticipation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Reference should be at -2 (= -1 - anticipation) - assert -2 in results.event_study_effects - assert results.event_study_effects[-2]["n_obs"] == 0 # reference marker + es = results.aggregate("event_study") + assert es.event_time[es.is_reference].tolist() == [-2] + assert es_dict(results)[-2]["n_obs"] == 0 # legacy zero-count marker - # Test that plot_event_study auto-detects it + # Test that plot_event_study auto-detects it on the container pytest.importorskip("matplotlib") from diff_diff import plot_event_study - fig = plot_event_study(results) + fig = plot_event_study(es) assert fig is not None def test_overall_se_with_partial_nan_tau_hat(self): @@ -2085,28 +2064,23 @@ def test_event_study_empty_after_filtering(self): # All cohorts fail the balanced check, so all horizons have n_h=0. est = ImputationDiD() + results = est.fit( + data, + outcome="outcome", + unit="unit", + time="time", + first_treat="first_treat", + ) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - results = est.fit( - data, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - balance_e=100, - ) + effects = es_dict(results, balance_e=100) empty_warnings = [x for x in w if "no horizons with observations" in str(x.message)] assert len(empty_warnings) >= 1, "Expected warning about empty event study" # Only reference period should remain ref_period = -1 - assert ref_period in results.event_study_effects - real_effects = { - h: v - for h, v in results.event_study_effects.items() - if h != ref_period and v.get("n_obs", 0) > 0 - } + assert ref_period in effects + real_effects = {h: v for h, v in effects.items() if h != ref_period and v["n_obs"] > 0} assert len(real_effects) == 0 def test_balanced_cohort_mask_requires_negative_horizons(self): @@ -2687,7 +2661,6 @@ def test_pretrends_hc1_bit_equal_with_cluster(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) r_default = ImputationDiD(cluster="state", pretrends=True).fit(**common) r_explicit = ImputationDiD(cluster="state", pretrends=True, vcov_type="hc1").fit(**common) @@ -2702,18 +2675,19 @@ def test_pretrends_hc1_bit_equal_with_cluster(self): pt_explicit = r_explicit.pretrend_test() assert np.isclose(pt_default["f_stat"], pt_explicit["f_stat"], rtol=0, atol=1e-12) assert np.isclose(pt_default["p_value"], pt_explicit["p_value"], rtol=0, atol=1e-12) - # Event-study SE (computed during fit() via Theorem 3 machinery on + # Event-study SE (post-fit recompute via Theorem 3 machinery on # within-transformed residuals; pretrends=True path includes the # pre-period horizons). Sub-ULP differences come from BLAS # non-associativity in the shared MAP demean engine # (`demean_by_groups`) across distinct estimator instances and are # not methodological divergence under the narrow vcov_type contract. - assert r_default.event_study_effects is not None - assert r_explicit.event_study_effects is not None - for h in r_default.event_study_effects: + es_default = es_dict(r_default) + es_explicit = es_dict(r_explicit) + assert set(es_default) == set(es_explicit) + for h in es_default: assert np.isclose( - r_default.event_study_effects[h]["se"], - r_explicit.event_study_effects[h]["se"], + es_default[h]["se"], + es_explicit[h]["se"], rtol=0, atol=1e-12, equal_nan=True, @@ -2983,17 +2957,16 @@ def test_main_fit_zero_weight_treated_unit_covariates(self): first_treat="first_treat", covariates=["x1", "x2"], survey_design=SurveyDesign(weights="w"), - aggregate="group", ) + groups = group_dict(r) from tests.conftest import assert_nan_inference assert np.isfinite(r.overall_att) - assert r.group_effects is not None # Cohort 2 contains only the zero-weight unit: NaN across ALL fields. - assert_nan_inference(r.group_effects[2]) - assert np.isnan(r.group_effects[2]["effect"]) + assert_nan_inference(groups[2]) + assert np.isnan(groups[2]["effect"]) # Cohort 3 is unaffected. - assert np.isfinite(r.group_effects[3]["effect"]) + assert np.isfinite(groups[3]["effect"]) class TestLeadSnapAbsorbed: @@ -3034,10 +3007,10 @@ def test_spanned_lead_snaps_to_nan_with_cause_warning(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None - eff = res.event_study_effects + # The lead regression runs on the post-fit route; its warnings + # are recorded in the same block. + eff = es_dict(res) # The spanned lead is deterministically NaN — the FULL inference # tuple (review P3: assert every field, not just effect/se). assert np.isnan(eff[-2]["effect"]) and np.isnan(eff[-2]["se"]) @@ -3074,12 +3047,9 @@ def test_identified_leads_unchanged_no_snap_warning(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None - finite_leads = [ - h for h, e in res.event_study_effects.items() if h < -1 and np.isfinite(e["effect"]) - ] + effects = es_dict(res) + finite_leads = [h for h, e in effects.items() if h < -1 and np.isfinite(e["effect"])] assert len(finite_leads) >= 4 assert not any("collinear with the absorbed fixed effects" in str(x.message) for x in w) @@ -3261,7 +3231,6 @@ def _panel(seed=13): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) @staticmethod @@ -3277,8 +3246,9 @@ def _z_p(t_stat): return 2 * stats.norm.sf(abs(t_stat)) def _lead(self, res): - h = min(k for k in res.event_study_effects if k < -1) - return h, res.event_study_effects[h] + effects = es_dict(res) + h = min(k for k in effects if k < -1) + return h, effects[h] def test_leads_are_t_residual_not_z(self): res = ImputationDiD(pretrends=True).fit(self._panel(), **self._kw) @@ -3292,20 +3262,21 @@ def test_cluster_matches_g_minus_1_on_leads_only(self): r0 = ImputationDiD(pretrends=True).fit(data, **self._kw) rc = ImputationDiD(pretrends=True, df_convention="cluster").fit(data, **self._kw) h, e0 = self._lead(r0) - ec = rc.event_study_effects[h] + es0, esc = es_dict(r0), es_dict(rc) + ec = esc[h] G = data["unit"].nunique() assert ec["effect"] == e0["effect"] and ec["se"] == e0["se"] assert ec["p_value"] == pytest.approx(self._t_p(ec["t_stat"], G - 1), rel=1e-12) # post rows are BJS (knob-independent) - hp = min(k for k in r0.event_study_effects if k >= 0) - assert r0.event_study_effects[hp]["p_value"] == rc.event_study_effects[hp]["p_value"] + hp = min(k for k in es0 if k >= 0) + assert es0[hp]["p_value"] == esc[hp]["p_value"] def test_normal_reproduces_pre39_z_on_leads(self): data = self._panel() r0 = ImputationDiD(pretrends=True).fit(data, **self._kw) rn = ImputationDiD(pretrends=True, df_convention="normal").fit(data, **self._kw) h, e0 = self._lead(r0) - en = rn.event_study_effects[h] + en = es_dict(rn)[h] assert en["effect"] == e0["effect"] and en["se"] == e0["se"] assert en["p_value"] == pytest.approx(self._z_p(en["t_stat"]), rel=1e-14) @@ -3336,7 +3307,7 @@ def test_survey_df_precedence_on_leads(self): } h, e0 = self._lead(fits["residual"]) for conv in ("cluster", "normal"): - e = fits[conv].event_study_effects[h] + e = es_dict(fits[conv])[h] assert e["p_value"] == e0["p_value"] assert e["conf_int"] == e0["conf_int"] diff --git a/tests/test_imputation_loo_stata_parity.py b/tests/test_imputation_loo_stata_parity.py index 20abebb27..3026a2558 100644 --- a/tests/test_imputation_loo_stata_parity.py +++ b/tests/test_imputation_loo_stata_parity.py @@ -74,25 +74,11 @@ import pandas as pd import pytest +from legacy_surface import es_dict from numpy.testing import assert_allclose from diff_diff import ImputationDiD -# --------------------------------------------------------------------------- -# Rows M-021/M-022 (+ M-118/M-119): ImputationDiD / TwoStageDiD -# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed 4.0) and warns on -# ANY supplied value. The deprecated fit-time route is kept DELIBERATELY here: -# these tests pin FIT-TIME surface behaviour (bit-equality grids, bootstrap -# aggregation, R/Stata parity, replicate overrides, native effect dicts) that -# the post-fit ``results.aggregate(...)`` container route does not reproduce -# shape-for-shape. The shim warning is therefore filtered BY MESSAGE, scoped to -# these two estimators only - every other FutureWarning (including the other -# estimators' aggregate() shims) still surfaces. -# --------------------------------------------------------------------------- -pytestmark = pytest.mark.filterwarnings( - r"ignore:(ImputationDiD|TwoStageDiD)\.fit\((aggregate=|balance_e=|aggregate= / balance_e=)\):FutureWarning" -) - _DATA = Path(__file__).parent.parent / "benchmarks" / "data" STATA_GOLDEN_PATH = _DATA / "didimputation_loo_stata_golden.json" R_GOLDEN_PATH = _DATA / "didimputation_golden.json" @@ -144,10 +130,10 @@ def r_golden() -> dict: def library_fit() -> dict: """Fit the LOO and non-LOO paths once each; return per-horizon + overall + warnings. - ImputationDiD exposes ``res.event_study_effects[h]["effect"]`` / ``["se"]`` (a - dict-of-dicts, NOT a ``.event_study`` DataFrame) and scalar ``res.overall_att`` / - ``res.overall_se``. ``aggregate="event_study"`` also computes the overall - unconditionally, so one fit per variance mode suffices. + The fit stores scalar ``res.overall_att`` / ``res.overall_se``; the per-horizon + surface is the post-fit ``res.aggregate("event_study")`` container, read here + through the legacy dict shape (``es_dict``: ``{h: {"effect", "se", ...}}``). + The LOO warnings of both stages (fit + event-study recompute) are recorded. """ _skip_if_missing() panel = pd.read_csv(PANEL_PATH) @@ -159,8 +145,8 @@ def library_fit() -> dict: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + loo_es = es_dict(loo) loo_msgs = [str(w.message) for w in caught] nonloo = ImputationDiD(leave_one_out=False).fit( panel, @@ -168,17 +154,17 @@ def library_fit() -> dict: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + nonloo_es = es_dict(nonloo) - def by_h(res): - return {h: res.event_study_effects[h] for h in HORIZONS} + def by_h(es): + return {h: es[h] for h in HORIZONS} return { "loo_overall": (float(loo.overall_att), float(loo.overall_se)), - "loo_es": {h: (float(v["effect"]), float(v["se"])) for h, v in by_h(loo).items()}, + "loo_es": {h: (float(v["effect"]), float(v["se"])) for h, v in by_h(loo_es).items()}, "nonloo_overall_se": float(nonloo.overall_se), - "nonloo_es_se": {h: float(v["se"]) for h, v in by_h(nonloo).items()}, + "nonloo_es_se": {h: float(v["se"]) for h, v in by_h(nonloo_es).items()}, "loo_warnings": loo_msgs, } @@ -194,11 +180,9 @@ def _require_block(golden: dict, key: str) -> dict: def _fit_partition(panel, aux_partition: str, leave_one_out: bool): - """One event-study fit; returns (results, [warning messages]). The event_study - aggregate populates the per-horizon surfaces gates 6/8 read. The DEPRECATED - fit-time ``aggregate=`` route is deliberate, matching this module's convention - (see the M-021/M-022 pytestmark comment above: parity tests pin the fit-time - surface shape-for-shape; the shim warning is filtered by message).""" + """One fit plus its post-fit event-study recompute; returns (results, + per-horizon legacy dict, [warning messages]). The event-study surface is the + per-horizon surface gates 6/8 read; warnings from both stages are recorded.""" with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") res = ImputationDiD(leave_one_out=leave_one_out, aux_partition=aux_partition).fit( @@ -207,10 +191,10 @@ def _fit_partition(panel, aux_partition: str, leave_one_out: bool): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None - return res, [str(w.message) for w in caught] + es = es_dict(res) + assert len(es) > 0 + return res, es, [str(w.message) for w in caught] @pytest.fixture(scope="module") @@ -223,11 +207,9 @@ def library_fit_variants() -> dict: out: dict = {"loo_warnings": []} for part in ("cohort", "horizon", "cohort_horizon"): for loo in (True, False): - res, msgs = _fit_partition(panel, part, loo) + res, es, msgs = _fit_partition(panel, part, loo) if loo: out["loo_warnings"] += msgs - es = res.event_study_effects - assert es is not None mode = "loo" if loo else "nonloo" out[f"{part}_{mode}"] = { "att": float(res.overall_att), diff --git a/tests/test_lpdid.py b/tests/test_lpdid.py index d69daa10c..47f1a1aed 100644 --- a/tests/test_lpdid.py +++ b/tests/test_lpdid.py @@ -25,6 +25,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from _dgp_utils import make_lpdid_panel # noqa: E402 +from legacy_surface import es_dict # noqa: E402 from diff_diff import ( # noqa: E402 CallawaySantAnna, @@ -722,19 +723,11 @@ def test_ce2_reweighted_equals_callaway_santanna(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + cs_es = es_dict(cs) for h in range(0, 4): - assert _event_coef(lp, h) == pytest.approx( - cs.event_study_effects[h]["effect"], abs=1e-9 - ) + assert _event_coef(lp, h) == pytest.approx(cs_es[h]["effect"], abs=1e-9) - # The BJS cross-check reads `bjs.event_study_effects` (the native fit-time - # dict) so it keeps the deprecated `fit(aggregate=)` route (rows M-021); - # scoped by message to the ImputationDiD shim only. - @pytest.mark.filterwarnings( - r"ignore:ImputationDiD\.fit\((aggregate=|balance_e=|aggregate= / balance_e=)\):FutureWarning" - ) def test_ce3_pmd_single_cohort_equals_bjs_imputation(self): # PMD LP-DiD (k=t-1, "max") with a SINGLE treated cohort == BJS # imputation [paper Section 3.4, footnotes 10-11; only single-cohort @@ -750,12 +743,10 @@ def test_ce3_pmd_single_cohort_equals_bjs_imputation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + bjs_es = es_dict(bjs) for h in range(0, 4): - assert _event_coef(lp, h) == pytest.approx( - bjs.event_study_effects[h]["effect"], abs=1e-6 - ) + assert _event_coef(lp, h) == pytest.approx(bjs_es[h]["effect"], abs=1e-6) # CE-4 (variance-weighted LP-DiD == stacked regression) is intentionally # omitted: the paper's equivalence is to the Cengiz et al. (2019) stacking @@ -850,12 +841,10 @@ def test_reweight_matches_cs_on_unbalanced(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + cs_es = es_dict(cs) for h in range(0, 4): - assert _event_coef(lp, h) == pytest.approx( - cs.event_study_effects[h]["effect"], abs=1e-9 - ) + assert _event_coef(lp, h) == pytest.approx(cs_es[h]["effect"], abs=1e-9) def test_ra_nan_consistent_on_uncontrolled_event_time(self): # An event time with treated units but no clean control is unidentified: diff --git a/tests/test_methodology_callaway.py b/tests/test_methodology_callaway.py index 34655bbf9..8f08cc569 100644 --- a/tests/test_methodology_callaway.py +++ b/tests/test_methodology_callaway.py @@ -20,6 +20,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, group_dict from diff_diff import CallawaySantAnna, SurveyDesign from diff_diff.bootstrap_utils import ( @@ -258,16 +259,16 @@ def test_base_period_universal_includes_reference_period(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + # A plain fit stores no event study; it is computed post-fit + assert results.event_study_effects is None + es = es_dict(results) + # Reference period e=-1 should exist with effect=0 - assert results.event_study_effects is not None, "Event study effects should be computed" - assert ( - -1 in results.event_study_effects - ), "Universal base period should include e=-1 in event study" + assert -1 in es, "Universal base period should include e=-1 in event study" - ref = results.event_study_effects[-1] + ref = es[-1] assert ref["effect"] == 0.0, "Reference period effect should be 0" assert np.isnan(ref["se"]), "Reference period SE should be NaN" assert ref["n_groups"] == 0, "Reference period n_groups should be 0" @@ -339,7 +340,7 @@ class TestDRNoCovariateSEUniformity: fed aggregation.""" @staticmethod - def _fit(data, method, aggregate=None): + def _fit(data, method): cs = CallawaySantAnna(estimation_method=method, n_bootstrap=0) return cs.fit( data, @@ -347,7 +348,6 @@ def _fit(data, method, aggregate=None): unit="unit", time="period", first_treat="first_treat", - aggregate=aggregate, ) def test_dr_reg_per_cell_se_identical(self): @@ -406,16 +406,18 @@ def test_dr_no_cov_aggregated_se_matches_reg(self): """Overall and event-study SEs are identical between dr and reg (both consume the same per-cell IF) — guards against an accidental IF change.""" data = generate_staggered_data(n_units=120, n_periods=6, never_treated_frac=0.3, seed=11) - reg = self._fit(data, "reg", aggregate="event_study") - dr = self._fit(data, "dr", aggregate="event_study") + reg = self._fit(data, "reg") + dr = self._fit(data, "dr") np.testing.assert_allclose(dr.overall_att, reg.overall_att, rtol=0, atol=1e-12) np.testing.assert_allclose(dr.overall_se, reg.overall_se, rtol=0, atol=1e-12) - assert reg.event_study_effects is not None and dr.event_study_effects is not None - assert set(reg.event_study_effects) == set(dr.event_study_effects) - for e, reg_es in reg.event_study_effects.items(): - dr_es = dr.event_study_effects[e] + reg_es_all = es_dict(reg) + dr_es_all = es_dict(dr) + assert reg_es_all and dr_es_all + assert set(reg_es_all) == set(dr_es_all) + for e, reg_es in reg_es_all.items(): + dr_es = dr_es_all[e] if np.isnan(reg_es["se"]): assert np.isnan(dr_es["se"]) else: @@ -434,8 +436,8 @@ def test_ipw_no_cov_per_cell_identical_to_reg(self): branch has diverged from the diff-in-means contract and the REGISTRY note must be revisited.""" data = generate_staggered_data(n_units=120, n_periods=6, never_treated_frac=0.3, seed=7) - reg = self._fit(data, "reg", aggregate="event_study") - ipw = self._fit(data, "ipw", aggregate="event_study") + reg = self._fit(data, "reg") + ipw = self._fit(data, "ipw") assert set(reg.group_time_effects) == set(ipw.group_time_effects) compared = 0 @@ -1063,14 +1065,13 @@ def test_event_study_aggregation_by_relative_time(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + es = results.aggregate("event_study") + assert len(es.event_time) > 0 # Should have both pre and post treatment periods - rel_times = list(results.event_study_effects.keys()) + rel_times = list(es.event_time) assert any(e < 0 for e in rel_times), "Should have pre-treatment periods" assert any(e >= 0 for e in rel_times), "Should have post-treatment periods" @@ -1091,17 +1092,16 @@ def test_group_aggregation_by_cohort(self): unit="unit", time="period", first_treat="first_treat", - aggregate="group", ) - assert results.group_effects is not None - assert len(results.group_effects) > 0 + grp = group_dict(results) + assert len(grp) > 0 # Should have effects for each cohort - assert 4 in results.group_effects or 6 in results.group_effects + assert 4 in grp or 6 in grp def test_all_aggregation_computes_everything(self): - """Test aggregate='all' computes event_study and group effects.""" + """Both post-fit aggregation levels are available on one fit.""" data = generate_staggered_data( n_units=100, n_periods=8, cohort_periods=[4], treatment_effect=2.0, seed=42 ) @@ -1113,11 +1113,10 @@ def test_all_aggregation_computes_everything(self): unit="unit", time="period", first_treat="first_treat", - aggregate="all", ) - assert results.event_study_effects is not None - assert results.group_effects is not None + assert len(results.aggregate("event_study").event_time) > 0 + assert len(results.aggregate("group").label) > 0 class TestGetSetParams: @@ -1211,7 +1210,7 @@ def test_results_to_dataframe_group_time(self): assert len(df) == len(results.group_time_effects) def test_results_to_dataframe_event_study(self): - """Test to_dataframe with level='event_study'.""" + """The post-fit event study container converts to a DataFrame.""" data = generate_staggered_data( n_units=50, n_periods=6, cohort_periods=[3], treatment_effect=2.0, seed=42 ) @@ -1223,13 +1222,12 @@ def test_results_to_dataframe_event_study(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - df = results.to_dataframe(level="event_study") + df = results.aggregate("event_study").to_dataframe() - assert "relative_period" in df.columns - assert "effect" in df.columns + assert "event_time" in df.columns + assert "att" in df.columns def test_results_significance_properties(self): """Test is_significant and significance_stars properties.""" @@ -1282,12 +1280,12 @@ def test_event_study_analytical_se_includes_wif(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None + es = es_dict(results) + assert es - for e, eff_data in results.event_study_effects.items(): + for e, eff_data in es.items(): se_with_wif = eff_data["se"] if np.isfinite(se_with_wif) and se_with_wif > 0: # WIF-adjusted SE should be positive @@ -1310,10 +1308,10 @@ def test_event_study_bootstrap_consistent_with_analytical(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + es_analytical = es_dict(results_analytical) - # Bootstrap SEs + # Bootstrap SEs (post-fit aggregation replays the bootstrap) cs_boot = CallawaySantAnna(n_bootstrap=n_boot, seed=42, cband=False) results_boot = cs_boot.fit( data, @@ -1321,19 +1319,19 @@ def test_event_study_bootstrap_consistent_with_analytical(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + es_boot = es_dict(results_boot) - assert results_analytical.event_study_effects is not None - assert results_boot.event_study_effects is not None + assert es_analytical + assert es_boot threshold = 0.40 if n_boot < 100 else 0.20 n_compared = 0 - for e in results_analytical.event_study_effects: - se_a = results_analytical.event_study_effects[e]["se"] - if e not in results_boot.event_study_effects: + for e in es_analytical: + se_a = es_analytical[e]["se"] + if e not in es_boot: continue - se_b = results_boot.event_study_effects[e]["se"] + se_b = es_boot[e]["se"] if np.isfinite(se_a) and se_a > 0 and np.isfinite(se_b) and se_b > 0: rel_diff = abs(se_a - se_b) / se_a assert rel_diff < threshold, ( @@ -1390,12 +1388,12 @@ def test_single_group_event_time_wif_effect(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None + es = es_dict(results) + assert es # With a single cohort, all event times have only one group - for e, eff_data in results.event_study_effects.items(): + for e, eff_data in es.items(): if eff_data["n_groups"] == 1: # SE should still be finite and positive assert np.isfinite(eff_data["se"]), f"Single-group SE for e={e} should be finite" @@ -1449,10 +1447,10 @@ def test_unbalanced_panel_bootstrap_uses_global_n(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + es_a = es_dict(results_a) - # Bootstrap SEs (the fix under test) + # Bootstrap SEs (the fix under test; post-fit aggregation replays it) cs_boot = CallawaySantAnna(n_bootstrap=n_boot, seed=42, cband=False) results_b = cs_boot.fit( data_unbalanced, @@ -1460,19 +1458,19 @@ def test_unbalanced_panel_bootstrap_uses_global_n(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + es_b = es_dict(results_b) - assert results_a.event_study_effects is not None - assert results_b.event_study_effects is not None + assert es_a + assert es_b threshold = 0.40 if n_boot < 100 else 0.25 n_compared = 0 - for e in results_a.event_study_effects: - se_a = results_a.event_study_effects[e]["se"] - if e not in results_b.event_study_effects: + for e in es_a: + se_a = es_a[e]["se"] + if e not in es_b: continue - se_b = results_b.event_study_effects[e]["se"] + se_b = es_b[e]["se"] if np.isfinite(se_a) and se_a > 0 and np.isfinite(se_b) and se_b > 0: rel_diff = abs(se_a - se_b) / se_a assert rel_diff < threshold, ( @@ -1504,14 +1502,16 @@ def test_cband_crit_value_exceeds_pointwise(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) + # A plain fit stores no band; the post-fit event study carries it + assert results.cband_crit_value is None + es = results.aggregate("event_study") assert ( - results.cband_crit_value is not None + es.cband_crit_value is not None ), "cband_crit_value should be set when cband=True and n_bootstrap > 0" - assert results.cband_crit_value > 1.96, ( - f"Simultaneous critical value ({results.cband_crit_value:.3f}) " + assert es.cband_crit_value > 1.96, ( + f"Simultaneous critical value ({es.cband_crit_value:.3f}) " "should exceed pointwise z_0.025=1.96" ) @@ -1529,11 +1529,12 @@ def test_cband_confidence_intervals_wider(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - for e, eff_data in results.event_study_effects.items(): + es = es_dict(results) + assert es + assert any("cband_conf_int" in eff_data for eff_data in es.values()) + for e, eff_data in es.items(): if "cband_conf_int" in eff_data: pw_ci = eff_data["conf_int"] cb_ci = eff_data["cband_conf_int"] @@ -1559,10 +1560,9 @@ def test_cband_false_disables_simultaneous_ci(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - assert results.cband_crit_value is None + assert results.aggregate("event_study").cband_crit_value is None def test_cband_requires_bootstrap(self): """When n_bootstrap=0, cband has no effect (remains None).""" @@ -1577,10 +1577,9 @@ def test_cband_requires_bootstrap(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - assert results.cband_crit_value is None + assert results.aggregate("event_study").cband_crit_value is None def test_cband_validity_threshold(self): """When >50% of sup-t bootstrap draws are non-finite, cband_crit_value is None.""" @@ -1606,18 +1605,20 @@ def mock_max(a, axis=None): result[:n_nan] = np.nan return result + results = cs.fit( + data, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + ) + # The sup-t band is computed by the post-fit bootstrap replay, so the + # patch and the warning capture wrap the aggregation, not the fit. with unittest.mock.patch("diff_diff.staggered_bootstrap.np.max", side_effect=mock_max): with pytest.warns(RuntimeWarning, match="Too few valid sup-t"): - results = cs.fit( - data, - outcome="outcome", - unit="unit", - time="period", - first_treat="first_treat", - aggregate="event_study", - ) + es = results.aggregate("event_study") - assert results.cband_crit_value is None + assert es.cband_crit_value is None def test_cband_in_get_params(self): """Test that cband parameter appears in get_params.""" @@ -1640,12 +1641,12 @@ def test_cband_to_dataframe_columns(self, ci_params): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - df = results.to_dataframe(level="event_study") + df = results.aggregate("event_study").to_dataframe() assert "cband_lower" in df.columns assert "cband_upper" in df.columns + assert np.isfinite(df["cband_lower"]).any() class TestMPDTARComparison: @@ -1886,10 +1887,10 @@ def test_mpdta_event_study_ses_match_r(self, require_r, tmp_path): unit="countyreal", time="year", first_treat="first_treat", - aggregate="event_study", ) - assert py_results.event_study_effects is not None, "Python event_study_effects is None" + py_es = es_dict(py_results) + assert py_es, "Python event study is empty" r_es = r_results["event_study"] r_event_times = r_es["event_time"] @@ -1908,11 +1909,11 @@ def test_mpdta_event_study_ses_match_r(self, require_r, tmp_path): if r_se is None or np.isnan(r_se) or r_se == 0: continue - if e not in py_results.event_study_effects: + if e not in py_es: unmatched_r.append(e) continue - py_eff = py_results.event_study_effects[e] + py_eff = py_es[e] py_att = py_eff["effect"] py_se = py_eff["se"] @@ -1960,10 +1961,9 @@ def test_mpdta_cband_crit_value_vs_r(self, require_r, tmp_path): unit="countyreal", time="year", first_treat="first_treat", - aggregate="event_study", ) - py_crit = py_results.cband_crit_value + py_crit = py_results.aggregate("event_study").cband_crit_value # Both should be finite and > 1.96 (exceed pointwise z-value) assert np.isfinite(r_crit), f"R crit value not finite: {r_crit}" diff --git a/tests/test_methodology_had.py b/tests/test_methodology_had.py index b9824451a..0557eecf4 100644 --- a/tests/test_methodology_had.py +++ b/tests/test_methodology_had.py @@ -153,7 +153,7 @@ def _make_two_period_panel( def _fit_overall(panel: pd.DataFrame, **kwargs) -> HeterogeneousAdoptionDiDResults: - """Fit HAD with `aggregate="overall"` and return the result.""" + """Fit HAD on a two-period panel (overall mode) and return the result.""" est = HeterogeneousAdoptionDiD(**kwargs) with warnings.catch_warnings(): # The Design 1 family (mass_point / continuous_near_d_lower) emits diff --git a/tests/test_methodology_imputation.py b/tests/test_methodology_imputation.py index f74355d35..6193281a5 100644 --- a/tests/test_methodology_imputation.py +++ b/tests/test_methodology_imputation.py @@ -49,24 +49,10 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, group_dict from diff_diff import ImputationDiD -# --------------------------------------------------------------------------- -# Rows M-021/M-022 (+ M-118/M-119): ImputationDiD / TwoStageDiD -# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed 4.0) and warns on -# ANY supplied value. The deprecated fit-time route is kept DELIBERATELY here: -# these tests pin FIT-TIME surface behaviour (bit-equality grids, bootstrap -# aggregation, R/Stata parity, replicate overrides, native effect dicts) that -# the post-fit ``results.aggregate(...)`` container route does not reproduce -# shape-for-shape. The shim warning is therefore filtered BY MESSAGE, scoped to -# these two estimators only - every other FutureWarning (including the other -# estimators' aggregate() shims) still surfaces. -# --------------------------------------------------------------------------- -pytestmark = pytest.mark.filterwarnings( - r"ignore:(ImputationDiD|TwoStageDiD)\.fit\((aggregate=|balance_e=|aggregate= / balance_e=)\):FutureWarning" -) - # ============================================================================= # Module-level R-fixture availability + per-class seed decorrelation # ============================================================================= @@ -184,12 +170,11 @@ def test_recovers_heterogeneous_event_study(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + effects = es_dict(res) for h, expected in tau_by_h.items(): - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["effect"] + assert h in effects, f"missing horizon {h}" + got = effects[h]["effect"] assert abs(got - expected) < 0.06, f"h={h}: {got:.4f} vs {expected}" def test_step1_uses_untreated_only(self) -> None: @@ -248,10 +233,8 @@ def test_event_study_ses_finite(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None - for h, eff in res.event_study_effects.items(): + for h, eff in es_dict(res).items(): # Skip the normalized reference period (effect=se=0 by construction). if h >= 0 and np.isfinite(eff["effect"]): assert np.isfinite(eff["se"]) and eff["se"] > 0, f"h={h}" @@ -406,38 +389,45 @@ def test_cache_reuse_matches_fresh_build(self) -> None: @pytest.mark.parametrize("survey", [False, True]) def test_projection_built_once_per_fit(self, survey: bool) -> None: - """The cache collapses the (1 + #horizons + #groups) per-target projection - builds -- and, with bootstrap, the analytical + precompute builds -- to a - single ``_build_untreated_projection`` per ``fit()``. Holds for the survey - path too (survey_weights is excluded from the cache key).""" - from diff_diff.imputation import ImputationDiD as _Cls + """The call-local cache collapses the per-target projection builds to a + single ``_build_untreated_projection`` per ``fit()`` and per post-fit + ``aggregate()`` call (the event-study call spans #horizons targets, the + group call #groups targets). Holds for the survey path too + (survey_weights is excluded from the cache key).""" + from diff_diff.imputation_aggregation import _ImputationAggregationMixin as _Mixin panel = self._cov_panel() base = dict(outcome="outcome", unit="unit", time="time", first_treat="first_treat") - kw: Dict[str, Any] = dict(covariates=["x1", "x2"], aggregate="all") + kw: Dict[str, Any] = dict(covariates=["x1", "x2"]) if survey: from diff_diff import SurveyDesign kw["survey_design"] = SurveyDesign(weights="weight") - orig = _Cls._build_untreated_projection + # Patch the mixin: the fit (ImputationDiD) and the post-fit kit + # aggregator (_ImputationKitAggregator) both inherit the builder. + orig = _Mixin._build_untreated_projection calls = {"n": 0} def counting(self, *a, **k): calls["n"] += 1 return orig(self, *a, **k) - _Cls._build_untreated_projection = counting # type: ignore[method-assign] + _Mixin._build_untreated_projection = counting # type: ignore[method-assign] try: with warnings.catch_warnings(): warnings.simplefilter("ignore") res = ImputationDiD().fit(panel, **base, **kw) + assert calls["n"] == 1, f"expected 1 projection build, got {calls['n']}" + es = res.aggregate("event_study") + assert calls["n"] == 2, f"event-study aggregate: {calls['n']} builds" + grp = res.aggregate("group") + assert calls["n"] == 3, f"group aggregate: {calls['n']} builds" finally: - _Cls._build_untreated_projection = orig # type: ignore[method-assign] + _Mixin._build_untreated_projection = orig # type: ignore[method-assign] # Multiple aggregation targets exist (so the naive count would be > 1). - assert res.event_study_effects and res.group_effects - assert calls["n"] == 1, f"expected 1 projection build, got {calls['n']}" + assert len(es.event_time) > 1 and len(grp.label) > 1 def test_projection_built_once_per_fit_with_bootstrap(self) -> None: """With ``n_bootstrap > 0`` the analytical aggregation AND the bootstrap @@ -478,12 +468,13 @@ def test_fit_idempotent_cache_no_leak(self) -> None: panel = self._cov_panel() base = dict(outcome="outcome", unit="unit", time="time", first_treat="first_treat") est = ImputationDiD() - r1 = est.fit(panel, **base, covariates=["x1", "x2"], aggregate="all") - r2 = est.fit(panel, **base, covariates=["x1", "x2"], aggregate="all") + r1 = est.fit(panel, **base, covariates=["x1", "x2"]) + r2 = est.fit(panel, **base, covariates=["x1", "x2"]) assert r1.overall_se == r2.overall_se - assert r1.event_study_effects is not None and r2.event_study_effects is not None - for h in r1.event_study_effects: - assert r1.event_study_effects[h]["se"] == r2.event_study_effects[h]["se"] + es1, es2 = r1.aggregate("event_study"), r2.aggregate("event_study") + assert len(es1.se) > 0 + assert np.array_equal(es1.event_time, es2.event_time) + assert np.array_equal(es1.se, es2.se, equal_nan=True) # ============================================================================= @@ -902,12 +893,11 @@ def test_loo_aggregate_all_analytical_survey(self): design = SurveyDesign(weights="weight", psu="psu") with warnings.catch_warnings(): warnings.simplefilter("ignore") - r = ImputationDiD(leave_one_out=True).fit( - panel, aggregate="all", survey_design=design, **self._COMMON - ) + r = ImputationDiD(leave_one_out=True).fit(panel, survey_design=design, **self._COMMON) + effects, groups = es_dict(r), group_dict(r) assert np.isfinite(r.overall_se) - self._assert_inference_consistent(r.event_study_effects) - self._assert_inference_consistent(r.group_effects) + self._assert_inference_consistent(effects) + self._assert_inference_consistent(groups) @pytest.mark.slow def test_loo_coverage_geq_nominal(self): @@ -971,11 +961,10 @@ def test_horizons_at_or_above_hbar_are_nan_with_warning(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + effects = es_dict(res) h_bar = 2 - for h, eff in res.event_study_effects.items(): + for h, eff in effects.items(): if h >= h_bar: assert np.isnan(eff["effect"]), f"h={h} >= H_bar should be NaN" @@ -996,13 +985,9 @@ def test_never_treated_present_identifies_all_horizons(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None # With never-treated controls, post horizons are identified (finite). - assert any( - np.isfinite(eff["effect"]) and h >= 2 for h, eff in res.event_study_effects.items() - ) + assert any(np.isfinite(eff["effect"]) and h >= 2 for h, eff in es_dict(res).items()) # ============================================================================= @@ -1151,15 +1136,14 @@ def test_event_study_atts_match_r(self, golden: dict, panel: pd.DataFrame) -> No unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + effects = es_dict(res) es = golden["event_study"] assert len(es["horizons"]) > 0 for h, att in zip(es["horizons"], es["att"]): # Every golden horizon must be present and finite -- no silent skips. - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["effect"] + assert h in effects, f"missing horizon {h}" + got = effects[h]["effect"] assert np.isfinite(got), f"non-finite ATT at h={h}" assert got == pytest.approx(att, abs=1e-6), f"h={h}" @@ -1173,15 +1157,14 @@ def test_event_study_ses_match_r(self, golden: dict, panel: pd.DataFrame) -> Non unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + effects = es_dict(res) es = golden["event_study"] assert len(es["horizons"]) > 0 for h, se in zip(es["horizons"], es["se"]): # Every golden horizon must be present and finite -- no silent skips. - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["se"] + assert h in effects, f"missing horizon {h}" + got = effects[h]["se"] assert np.isfinite(got), f"non-finite SE at h={h}" assert got == pytest.approx(se, abs=1e-7), f"h={h}" @@ -1239,13 +1222,12 @@ def test_overall_se_matches_r(self, golden: dict, covariate_panel: pd.DataFrame) assert res.overall_se == pytest.approx(golden["covariate"]["overall"]["se"], abs=1e-7) def test_event_study_atts_match_r(self, golden: dict, covariate_panel: pd.DataFrame) -> None: - res = self._fit(covariate_panel, aggregate="event_study") - assert res.event_study_effects is not None + effects = es_dict(self._fit(covariate_panel)) es = golden["covariate"]["event_study"] assert len(es["horizons"]) > 0 for h, att in zip(es["horizons"], es["att"]): - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["effect"] + assert h in effects, f"missing horizon {h}" + got = effects[h]["effect"] assert np.isfinite(got), f"non-finite ATT at h={h}" assert got == pytest.approx(att, abs=1e-6), f"h={h}" @@ -1253,11 +1235,10 @@ def test_event_study_ses_match_r(self, golden: dict, covariate_panel: pd.DataFra """Per-horizon covariate-path SEs match R (the variance machinery under a covariate-augmented first stage) — ~2e-10 observed on the reference platform, asserted at abs=1e-7 for cross-platform robustness.""" - res = self._fit(covariate_panel, aggregate="event_study") - assert res.event_study_effects is not None + effects = es_dict(self._fit(covariate_panel)) es = golden["covariate"]["event_study"] for h, se in zip(es["horizons"], es["se"]): - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["se"] + assert h in effects, f"missing horizon {h}" + got = effects[h]["se"] assert np.isfinite(got), f"non-finite SE at h={h}" assert got == pytest.approx(se, abs=1e-7), f"h={h}" diff --git a/tests/test_methodology_two_stage.py b/tests/test_methodology_two_stage.py index 4c9c1a4f6..7d09d198d 100644 --- a/tests/test_methodology_two_stage.py +++ b/tests/test_methodology_two_stage.py @@ -53,24 +53,10 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, group_dict from diff_diff import ImputationDiD, TwoStageDiD -# --------------------------------------------------------------------------- -# Rows M-021/M-022 (+ M-118/M-119): ImputationDiD / TwoStageDiD -# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed 4.0) and warns on -# ANY supplied value. The deprecated fit-time route is kept DELIBERATELY here: -# these tests pin FIT-TIME surface behaviour (bit-equality grids, bootstrap -# aggregation, R/Stata parity, replicate overrides, native effect dicts) that -# the post-fit ``results.aggregate(...)`` container route does not reproduce -# shape-for-shape. The shim warning is therefore filtered BY MESSAGE, scoped to -# these two estimators only - every other FutureWarning (including the other -# estimators' aggregate() shims) still surfaces. -# --------------------------------------------------------------------------- -pytestmark = pytest.mark.filterwarnings( - r"ignore:(ImputationDiD|TwoStageDiD)\.fit\((aggregate=|balance_e=|aggregate= / balance_e=)\):FutureWarning" -) - # ============================================================================= # Module-level R-fixture availability + per-class seed decorrelation # ============================================================================= @@ -181,12 +167,11 @@ def test_recovers_heterogeneous_event_study_eq6(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + es = es_dict(res) for h, expected in tau_by_h.items(): - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["effect"] + assert h in es, f"missing horizon {h}" + got = es[h]["effect"] assert abs(got - expected) < 0.06, f"h={h}: {got:.4f} vs {expected}" def test_stage1_uses_untreated_only(self) -> None: @@ -318,11 +303,11 @@ def test_event_study_ses_finite(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None - for h, eff in res.event_study_effects.items(): - # Skip the normalized reference period (effect=se=0 by construction). + es = es_dict(res) + assert len(es) > 0 + for h, eff in es.items(): + # Skip the normalized reference period (att=0, inference NaN). if h >= 0 and np.isfinite(eff["effect"]): assert np.isfinite(eff["se"]) and eff["se"] > 0, f"h={h}" @@ -395,16 +380,15 @@ def test_pretrends_event_study_ses_finite(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - ) - assert res.event_study_effects is not None - # At least one pre-period lead is present and has a finite SE. - leads = [h for h in res.event_study_effects if h < 0] - assert leads, "pretrends=True should expose pre-period leads" - for h in leads: - eff = res.event_study_effects[h] - if np.isfinite(eff["effect"]): - assert np.isfinite(eff["se"]), f"lead h={h} SE non-finite" + ) + es = res.aggregate("event_study") + # At least one pre-period lead is present and has a finite SE + # (the reference period's inference is NaN by container contract). + leads = (es.event_time < 0) & ~es.is_reference + assert leads.any(), "pretrends=True should expose pre-period leads" + for h, att, se in zip(es.event_time[leads], es.att[leads], es.se[leads]): + if np.isfinite(att): + assert np.isfinite(se), f"lead h={h} SE non-finite" # Overall ATT is unchanged by requesting pre-period leads. assert res.overall_att == pytest.approx(base.overall_att, abs=1e-8) @@ -616,25 +600,20 @@ def test_no_never_treated_horizons_nan(self) -> None: sigma=0.1, include_never_treated=False, ) + res = TwoStageDiD().fit( + panel, + outcome="outcome", + unit="unit", + time="time", + first_treat="first_treat", + ) with pytest.warns(UserWarning, match="identified"): - res = TwoStageDiD().fit( - panel, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - ) - assert res.event_study_effects is not None + es = es_dict(res) h_bar = 2 - flagged = [ - h - for h, eff in res.event_study_effects.items() - if h >= h_bar and np.isnan(eff["effect"]) - ] + flagged = [h for h, eff in es.items() if h >= h_bar and np.isnan(eff["effect"])] assert flagged, "expected NaN horizons at or above h_bar" for h in flagged: - assert res.event_study_effects[h]["n_obs"] > 0, f"h={h} should keep n_obs>0" + assert es[h]["n_obs"] > 0, f"h={h} should keep n_obs>0" def test_balance_e_no_qualifying_cohorts_warns(self) -> None: """balance_e larger than any cohort's available pre-window -> no cohort @@ -643,24 +622,18 @@ def test_balance_e_no_qualifying_cohorts_warns(self) -> None: panel = _make_staggered_panel( rng, cohorts=[2, 3], n_per_cohort=60, n_periods=5, tau_constant=1.0, sigma=0.1 ) + res = TwoStageDiD().fit( + panel, + outcome="outcome", + unit="unit", + time="time", + first_treat="first_treat", + ) with pytest.warns(UserWarning, match="balance_e"): - res = TwoStageDiD().fit( - panel, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - balance_e=10, # impossible pre-window - ) - assert res.event_study_effects is not None + es = es_dict(res, balance_e=10) # impossible pre-window # Only the reference period survives. - finite_effects = [ - h for h, eff in res.event_study_effects.items() if eff.get("n_obs", 0) > 0 - ] - assert finite_effects == [] or all( - res.event_study_effects[h]["effect"] == 0.0 for h in res.event_study_effects - ) + finite_effects = [h for h, eff in es.items() if eff.get("n_obs", 0) > 0] + assert finite_effects == [] or all(es[h]["effect"] == 0.0 for h in es) def test_zero_obs_cohort_nan(self) -> None: """A cohort whose treated obs ALL fall at an unidentified period -> its @@ -705,17 +678,16 @@ def test_zero_obs_cohort_nan(self) -> None: unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - assert res.group_effects is not None + grp = group_dict(res) # Cohort 4's only treated obs is the unidentified t=4 -> NaN, n_obs=0. - assert 4 in res.group_effects - c4 = res.group_effects[4] + assert 4 in grp + c4 = grp[4] assert c4["n_obs"] == 0, f"cohort 4 should have n_obs=0, got {c4['n_obs']}" assert np.isnan(c4["effect"]), "cohort 4 effect must be NaN" assert np.isnan(c4["se"]) and np.isnan(c4["t_stat"]) and np.isnan(c4["p_value"]) # Cohort 3 keeps an identified t=3 effect. - assert 3 in res.group_effects and res.group_effects[3]["n_obs"] > 0 + assert 3 in grp and grp[3]["n_obs"] > 0 # ============================================================================= @@ -805,14 +777,13 @@ def test_event_study_atts_match_r(self, golden: dict, panel: pd.DataFrame) -> No unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + got_es = es_dict(res) es = golden["event_study"] assert len(es["horizons"]) > 0 for h, att in zip(es["horizons"], es["att"]): - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["effect"] + assert h in got_es, f"missing horizon {h}" + got = got_es[h]["effect"] assert np.isfinite(got), f"non-finite ATT at h={h}" assert got == pytest.approx(att, abs=1e-6), f"h={h}" @@ -823,13 +794,12 @@ def test_event_study_ses_match_r(self, golden: dict, panel: pd.DataFrame) -> Non unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert res.event_study_effects is not None + got_es = es_dict(res) es = golden["event_study"] assert len(es["horizons"]) > 0 for h, se in zip(es["horizons"], es["se"]): - assert h in res.event_study_effects, f"missing horizon {h}" - got = res.event_study_effects[h]["se"] + assert h in got_es, f"missing horizon {h}" + got = got_es[h]["se"] assert np.isfinite(got), f"non-finite SE at h={h}" assert got == pytest.approx(se, abs=1e-7), f"h={h}" diff --git a/tests/test_pretrends.py b/tests/test_pretrends.py index f7cf74df2..ebeff0d26 100644 --- a/tests/test_pretrends.py +++ b/tests/test_pretrends.py @@ -885,16 +885,17 @@ def test_callaway_santanna_universal_base_period(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", ) - # Verify reference period exists with NaN SE - assert -1 in results.event_study_effects - assert np.isnan(results.event_study_effects[-1]["se"]) + # Verify the reference period exists with NaN SE on the post-fit container + es = results.aggregate("event_study") + es_times = list(es.event_time) + assert -1 in es_times + assert np.isnan(es.se[es_times.index(-1)]) # PreTrendsPower should work without errors (reference period filtered out) pt = PreTrendsPower() - power_results = pt.fit(results) + power_results = pt.fit(es) # Should have valid (non-NaN) results assert np.isfinite(power_results.power) diff --git a/tests/test_pretrends_event_study.py b/tests/test_pretrends_event_study.py index 4f90c18b6..0ee5b3eac 100644 --- a/tests/test_pretrends_event_study.py +++ b/tests/test_pretrends_event_study.py @@ -8,18 +8,19 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict from diff_diff.imputation import ImputationDiD from diff_diff.two_stage import TwoStageDiD # --------------------------------------------------------------------------- # Rows M-021/M-022 (+ M-118/M-119): ImputationDiD / TwoStageDiD -# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed 4.0) and warns on -# ANY supplied value. The deprecated fit-time route is kept DELIBERATELY here: -# these tests pin FIT-TIME surface behaviour (bit-equality grids, bootstrap -# aggregation, R/Stata parity, replicate overrides, native effect dicts) that -# the post-fit ``results.aggregate(...)`` container route does not reproduce -# shape-for-shape. The shim warning is therefore filtered BY MESSAGE, scoped to +# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed at the 4.0 cut) and +# warns on ANY supplied value. Analytic sites read the post-fit +# ``results.aggregate(...)`` container; the BOOTSTRAPPED sites below stay on the +# fit-time route until the 4.0 relay lands (the post-fit route raises +# NotImplementedError on a bootstrapped Imputation/TwoStage fit today). The shim +# warning is therefore filtered BY MESSAGE, scoped to # these two estimators only - every other FutureWarning (including the other # estimators' aggregate() shims) still surfaces. # --------------------------------------------------------------------------- @@ -95,11 +96,9 @@ def test_pretrends_includes_negative_horizons(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - horizons = sorted(results.event_study_effects.keys()) + horizons = sorted(es_dict(results)) negative = [h for h in horizons if h < 0] positive = [h for h in horizons if h >= 0] assert len(negative) > 1, "Should have pre-period horizons" @@ -115,11 +114,10 @@ def test_pretrends_coefficients_near_zero(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) ref_period = -1 - for h, eff in results.event_study_effects.items(): + for h, eff in es_dict(results).items(): if h >= 0 or h == ref_period: continue assert np.isfinite(eff["effect"]), f"h={h}: effect not finite" @@ -137,11 +135,10 @@ def test_pretrends_se_finite_positive(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) ref_period = -1 - for h, eff in results.event_study_effects.items(): + for h, eff in es_dict(results).items(): if h == ref_period: continue if eff["n_obs"] == 0: @@ -159,11 +156,11 @@ def test_reference_period_correct(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert -1 in results.event_study_effects - ref = results.event_study_effects[-1] + effects = es_dict(results) + assert -1 in effects + ref = effects[-1] assert ref["effect"] == 0.0 assert ref["n_obs"] == 0 @@ -177,7 +174,6 @@ def test_backward_compatibility(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) results_false = ImputationDiD(pretrends=False).fit( data, @@ -185,14 +181,11 @@ def test_backward_compatibility(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) assert results_default.overall_att == results_false.overall_att assert results_default.overall_se == results_false.overall_se - assert set(results_default.event_study_effects.keys()) == set( - results_false.event_study_effects.keys() - ) + assert set(es_dict(results_default)) == set(es_dict(results_false)) def test_post_treatment_invariance(self): """Post-treatment effects identical with pretrends=True vs False.""" @@ -204,7 +197,6 @@ def test_post_treatment_invariance(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) results_on = ImputationDiD(pretrends=True).fit( data, @@ -212,7 +204,6 @@ def test_post_treatment_invariance(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Overall ATT unchanged @@ -220,10 +211,11 @@ def test_post_treatment_invariance(self): assert results_on.overall_se == results_off.overall_se # Post-treatment event study effects unchanged - for h in results_off.event_study_effects: - assert h in results_on.event_study_effects - eff_off = results_off.event_study_effects[h] - eff_on = results_on.event_study_effects[h] + es_off, es_on = es_dict(results_off), es_dict(results_on) + for h in es_off: + assert h in es_on + eff_off = es_off[h] + eff_on = es_on[h] np.testing.assert_allclose(eff_off["effect"], eff_on["effect"], rtol=1e-10) np.testing.assert_allclose(eff_off["se"], eff_on["se"], rtol=1e-10) @@ -237,10 +229,9 @@ def test_horizon_max_interaction(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - for h in results.event_study_effects: + for h in es_dict(results): assert abs(h) <= 2, f"h={h} exceeds horizon_max=2" def test_anticipation_interaction(self): @@ -253,11 +244,11 @@ def test_anticipation_interaction(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert -2 in results.event_study_effects - ref = results.event_study_effects[-2] + effects = es_dict(results) + assert -2 in effects + ref = effects[-2] assert ref["effect"] == 0.0 assert ref["n_obs"] == 0 @@ -292,9 +283,8 @@ def test_no_pretreatment_obs_graceful(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None + assert results.aggregate("event_study") is not None # ============================================================================= @@ -315,11 +305,9 @@ def test_pretrends_includes_negative_horizons(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - horizons = sorted(results.event_study_effects.keys()) + horizons = sorted(es_dict(results)) negative = [h for h in horizons if h < 0] positive = [h for h in horizons if h >= 0] assert len(negative) > 1, "Should have pre-period horizons" @@ -335,11 +323,10 @@ def test_pretrends_coefficients_near_zero(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) ref_period = -1 - for h, eff in results.event_study_effects.items(): + for h, eff in es_dict(results).items(): if h >= 0 or h == ref_period: continue assert np.isfinite(eff["effect"]), f"h={h}: effect not finite" @@ -357,11 +344,10 @@ def test_pretrends_se_finite_positive(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) ref_period = -1 - for h, eff in results.event_study_effects.items(): + for h, eff in es_dict(results).items(): if h == ref_period: continue if eff["n_obs"] == 0: @@ -379,7 +365,6 @@ def test_post_treatment_invariance(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) results_on = TwoStageDiD(pretrends=True).fit( data, @@ -387,7 +372,6 @@ def test_post_treatment_invariance(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Overall ATT unchanged @@ -395,10 +379,11 @@ def test_post_treatment_invariance(self): assert results_on.overall_se == results_off.overall_se # Post-treatment event study effects unchanged - for h in results_off.event_study_effects: - assert h in results_on.event_study_effects - eff_off = results_off.event_study_effects[h] - eff_on = results_on.event_study_effects[h] + es_off, es_on = es_dict(results_off), es_dict(results_on) + for h in es_off: + assert h in es_on + eff_off = es_off[h] + eff_on = es_on[h] np.testing.assert_allclose(eff_off["effect"], eff_on["effect"], rtol=1e-10) np.testing.assert_allclose(eff_off["se"], eff_on["se"], rtol=1e-10) @@ -419,10 +404,9 @@ def test_horizon_max_interaction(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - for h in results.event_study_effects: + for h in es_dict(results): assert abs(h) <= 2, f"h={h} exceeds horizon_max=2" def test_anticipation_interaction(self): @@ -435,11 +419,11 @@ def test_anticipation_interaction(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert -2 in results.event_study_effects - ref = results.event_study_effects[-2] + effects = es_dict(results) + assert -2 in effects + ref = effects[-2] assert ref["effect"] == 0.0 assert ref["n_obs"] == 0 @@ -463,7 +447,6 @@ def test_imputation_pretrends_match_pretrend_test(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Get pretrend_test lead coefficients (called on results object) @@ -471,9 +454,10 @@ def test_imputation_pretrends_match_pretrend_test(self): lead_coefs = pt["lead_coefficients"] # Every lead coefficient should match the event study pre-period effect + effects = es_dict(results) for h, coef in lead_coefs.items(): - assert h in results.event_study_effects, f"h={h}: lead coefficient not in event study" - es_effect = results.event_study_effects[h]["effect"] + assert h in effects, f"h={h}: lead coefficient not in event study" + es_effect = effects[h]["effect"] np.testing.assert_allclose( es_effect, coef, @@ -537,11 +521,10 @@ def test_imputation_survey_weighted_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", survey_design=sd, ) - for h, eff in results.event_study_effects.items(): + for h, eff in es_dict(results).items(): if eff["n_obs"] == 0: continue assert np.isfinite(eff["se"]), f"h={h}: SE not finite" @@ -636,7 +619,6 @@ def test_nondefault_index_analytical(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Permuted integer index @@ -649,7 +631,6 @@ def test_nondefault_index_analytical(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Gapped/nonconsecutive index @@ -661,16 +642,18 @@ def test_nondefault_index_analytical(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # All pre-period effects and SEs must match - for h in results_default.event_study_effects: - if results_default.event_study_effects[h]["n_obs"] == 0: + es_default = es_dict(results_default) + es_perm = es_dict(results_perm) + es_gap = es_dict(results_gap) + for h in es_default: + if es_default[h]["n_obs"] == 0: continue - eff_d = results_default.event_study_effects[h] - eff_p = results_perm.event_study_effects[h] - eff_g = results_gap.event_study_effects[h] + eff_d = es_default[h] + eff_p = es_perm[h] + eff_g = es_gap[h] np.testing.assert_allclose( eff_d["effect"], eff_p["effect"], @@ -759,41 +742,32 @@ def test_imputation_pretrends_survey_accepted(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", survey_design=sd, ) - assert result.event_study_effects is not None + assert result.aggregate("event_study") is not None assert np.isfinite(result.overall_att) def test_imputation_pretrends_balance_e(self): """balance_e restricts pre-period lead regression to balanced cohorts.""" data = generate_test_data(seed=42) - results_no_be = ImputationDiD(pretrends=True).fit( + results = ImputationDiD(pretrends=True).fit( data, outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - ) - results_be = ImputationDiD(pretrends=True).fit( - data, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - balance_e=1, ) + es_no_be = results.aggregate("event_study") + es_be = results.aggregate("event_study", balance_e=1) # Both should have pre-period horizons - neg_no_be = [h for h in results_no_be.event_study_effects if h < -1] + neg_no_be = [h for h in es_no_be.event_time.tolist() if h < -1] assert len(neg_no_be) > 0 # balance_e restricts cohorts; pre-period coefficients may be NaN # if the restricted sample is rank-deficient after demeaning # The key invariant: the method runs without error - assert results_be.event_study_effects is not None + assert es_be is not None def test_two_stage_pretrends_bootstrap(self): """TwoStageDiD pretrends=True with bootstrap produces finite pre-period SEs.""" diff --git a/tests/test_replicate_weight_expansion.py b/tests/test_replicate_weight_expansion.py index 4880ba71f..24adb7d9a 100644 --- a/tests/test_replicate_weight_expansion.py +++ b/tests/test_replicate_weight_expansion.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, group_dict from diff_diff import ( DifferenceInDifferences, @@ -17,21 +18,6 @@ ) from diff_diff.survey import SurveyDesign -# --------------------------------------------------------------------------- -# Rows M-021/M-022 (+ M-118/M-119): ImputationDiD / TwoStageDiD -# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed 4.0) and warns on -# ANY supplied value. The deprecated fit-time route is kept DELIBERATELY here: -# these tests pin FIT-TIME surface behaviour (bit-equality grids, bootstrap -# aggregation, R/Stata parity, replicate overrides, native effect dicts) that -# the post-fit ``results.aggregate(...)`` container route does not reproduce -# shape-for-shape. The shim warning is therefore filtered BY MESSAGE, scoped to -# these two estimators only - every other FutureWarning (including the other -# estimators' aggregate() shims) still surfaces. -# --------------------------------------------------------------------------- -pytestmark = pytest.mark.filterwarnings( - r"ignore:(ImputationDiD|TwoStageDiD)\.fit\((aggregate=|balance_e=|aggregate= / balance_e=)\):FutureWarning" -) - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -414,15 +400,14 @@ def test_imputation_event_study_replicate(self): "unit", "time", "first_treat", - aggregate="event_study", survey_design=sd, ) assert np.isfinite(result.overall_se) and result.overall_se > 0 - assert result.event_study_effects is not None + es = es_dict(result) # At least some identified periods should have finite SE finite_ses = [ e - for e, eff in result.event_study_effects.items() + for e, eff in es.items() if np.isfinite(eff["effect"]) and np.isfinite(eff["se"]) and eff["se"] > 0 ] assert len(finite_ses) > 0, "No event-study periods have finite replicate SE" @@ -438,11 +423,11 @@ def test_imputation_group_replicate(self): "unit", "time", "first_treat", - aggregate="group", survey_design=sd, ) - assert result.group_effects is not None - for g, eff in result.group_effects.items(): + grp = group_dict(result) + assert len(grp) > 0 + for g, eff in grp.items(): assert np.isfinite(eff["se"]) and eff["se"] > 0, f"group {g}: SE not finite" def test_imputation_bootstrap_rejected(self): @@ -491,11 +476,9 @@ def test_two_stage_event_study_replicate(self): "unit", "time", "first_treat", - aggregate="event_study", survey_design=sd, ) - assert result.event_study_effects is not None - non_ref = {e: eff for e, eff in result.event_study_effects.items() if eff["effect"] != 0.0} + non_ref = {e: eff for e, eff in es_dict(result).items() if eff["effect"] != 0.0} assert len(non_ref) > 0, "No non-reference event-study effects" for e, eff in non_ref.items(): assert np.isfinite(eff["se"]) and eff["se"] > 0, f"period {e}: SE not finite" @@ -608,7 +591,6 @@ def test_two_stage_always_treated_event_study_and_group_replicate(self): "unit", "time", "first_treat", - aggregate="all", survey_design=sd, ) # Event-study surface: at least one non-reference horizon must have @@ -616,10 +598,9 @@ def test_two_stage_always_treated_event_study_and_group_replicate(self): # override path at `two_stage.py` updates SE + t_stat + p_value + # conf_int separately for each non-reference horizon via the same # _refit_ts callback, so all four fields must be locked). - assert result.event_study_effects is not None non_ref_es = { e: eff - for e, eff in result.event_study_effects.items() + for e, eff in es_dict(result).items() if eff["effect"] != 0.0 and np.isfinite(eff["effect"]) } assert len(non_ref_es) > 0, "no non-reference event-study effects" @@ -636,9 +617,8 @@ def test_two_stage_always_treated_event_study_and_group_replicate(self): ), f"event-study horizon {e}: replicate conf_int bounds must be finite" # Group surface: at least one cohort with finite replicate SE / # p_value / conf_int (same override path applies to group effects). - assert result.group_effects is not None finite_groups = { - g: eff for g, eff in result.group_effects.items() if np.isfinite(eff["effect"]) + g: eff for g, eff in group_dict(result).items() if np.isfinite(eff["effect"]) } assert len(finite_groups) > 0, "no finite group effects" for g, eff in finite_groups.items(): diff --git a/tests/test_se_accuracy.py b/tests/test_se_accuracy.py index 85dc82484..3dc6a8c73 100644 --- a/tests/test_se_accuracy.py +++ b/tests/test_se_accuracy.py @@ -126,7 +126,6 @@ def cs_results(self, benchmark_data) -> Tuple: unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) elapsed = time.perf_counter() - start @@ -195,7 +194,6 @@ def test_aggregation_se_formula(self, benchmark_data): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) # Aggregated SE should be positive diff --git a/tests/test_staggered.py b/tests/test_staggered.py index a4e85d8b1..50c4cd4fd 100644 --- a/tests/test_staggered.py +++ b/tests/test_staggered.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, es_dict_from_container, group_dict from diff_diff import CallawaySantAnna, CallawaySantAnnaResults from diff_diff.prep import generate_staggered_data as _generate_staggered_data @@ -118,7 +119,7 @@ def test_never_treated_inf_encoding(self): ), f"ATT differs: inf={results_inf.overall_att}, zero={results_zero.overall_att}" def test_event_study_aggregation(self): - """Test event study aggregation.""" + """Test post-fit event study aggregation.""" data = generate_staggered_data() cs = CallawaySantAnna() @@ -128,14 +129,16 @@ def test_event_study_aggregation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + # A plain fit does not populate the fit-time surface; aggregation is post-fit + assert results.event_study_effects is None + assert results.group_effects is None + es = results.aggregate("event_study") + assert len(es.event_time) > 0 # Check that relative periods are present - rel_periods = list(results.event_study_effects.keys()) + rel_periods = list(es.event_time) assert any(p >= 0 for p in rel_periods) # Post-treatment def test_group_aggregation(self): @@ -149,14 +152,13 @@ def test_group_aggregation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - assert results.group_effects is not None - assert len(results.group_effects) > 0 + grp = results.aggregate("group") + assert len(grp.label) > 0 def test_all_aggregation(self): - """Test computing all aggregations.""" + """Test computing both post-fit aggregation levels on one fit.""" data = generate_staggered_data() cs = CallawaySantAnna() @@ -166,11 +168,10 @@ def test_all_aggregation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) - assert results.event_study_effects is not None - assert results.group_effects is not None + assert len(results.aggregate("event_study").event_time) > 0 + assert len(results.aggregate("group").label) > 0 def test_control_group_options(self): """Test different control group options.""" @@ -221,7 +222,6 @@ def test_summary_output(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) summary = results.summary() @@ -241,7 +241,6 @@ def test_to_dataframe(self): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) # Group-time DataFrame @@ -250,13 +249,15 @@ def test_to_dataframe(self): assert "time" in df_gt.columns assert "effect" in df_gt.columns - # Event study DataFrame - df_es = results.to_dataframe(level="event_study") - assert "relative_period" in df_es.columns + # Event study DataFrame (post-fit container) + df_es = results.aggregate("event_study").to_dataframe() + assert "event_time" in df_es.columns + assert "att" in df_es.columns - # Group DataFrame - df_g = results.to_dataframe(level="group") - assert "group" in df_g.columns + # Group DataFrame (post-fit container) + df_g = results.aggregate("group").to_dataframe() + assert "label" in df_g.columns + assert "att" in df_g.columns def test_get_set_params(self): """Test sklearn-compatible parameter access.""" @@ -537,11 +538,9 @@ def test_event_study_with_covariates(self): time="time", first_treat="first_treat", covariates=["x1", "x2"], - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + assert len(results.aggregate("event_study").event_time) > 0 def test_missing_covariate_error(self): """Test error when covariate column is missing.""" @@ -1118,8 +1117,10 @@ def mock_solve_ols(*args, **kwargs): time="time", first_treat="first_treat", covariates=["x1", "x1_dup"], - aggregate="all", ) + # post-fit aggregation replays the bootstrap (low-n warnings again) + es = es_dict(results) + grp = group_dict(results) # NaN cell should be preserved in group_time_effects nan_cells = [ @@ -1140,18 +1141,18 @@ def mock_solve_ols(*args, **kwargs): assert all(np.isfinite(x) for x in results.overall_conf_int), "overall CI should be finite" # Event study: valid relative times should have finite bootstrap inference - if results.event_study_effects: - for e, data_es in results.event_study_effects.items(): - if np.isfinite(data_es["effect"]): - assert np.isfinite(data_es["se"]), f"ES e={e} se should be finite" - assert np.isfinite(data_es["p_value"]), f"ES e={e} p_value should be finite" + assert es + for e, data_es in es.items(): + if np.isfinite(data_es["effect"]): + assert np.isfinite(data_es["se"]), f"ES e={e} se should be finite" + assert np.isfinite(data_es["p_value"]), f"ES e={e} p_value should be finite" # Group effects: valid groups should have finite bootstrap inference - if results.group_effects: - for g, data_ge in results.group_effects.items(): - if np.isfinite(data_ge["effect"]): - assert np.isfinite(data_ge["se"]), f"Group {g} se should be finite" - assert np.isfinite(data_ge["p_value"]), f"Group {g} p_value should be finite" + assert grp + for g, data_ge in grp.items(): + if np.isfinite(data_ge["effect"]): + assert np.isfinite(data_ge["se"]), f"Group {g} se should be finite" + assert np.isfinite(data_ge["p_value"]), f"Group {g} p_value should be finite" def test_balance_e_excludes_nan_anchor_cohort(self, ci_params): """balance_e must exclude cohorts whose anchor-horizon effect is NaN.""" @@ -1196,9 +1197,9 @@ def mock_solve_ols(*args, **kwargs): time="time", first_treat="first_treat", covariates=["x1", "x1_dup"], - aggregate="event_study", - balance_e=0, ) + # balanced post-fit aggregation (replays the bootstrap) + es = es_dict(results, balance_e=0) # Confirm the anchor cell is NaN and is specifically the anchor (t - g == 0) assert np.isnan( @@ -1209,21 +1210,16 @@ def mock_solve_ols(*args, **kwargs): # Cohort g=3 should be excluded from ALL event-study horizons # Only g=5 and g=8 should contribute (<=2 because not all balanced # cohorts have cells at extreme horizons) - for e, es_data in results.event_study_effects.items(): + for e, es_data in es.items(): assert es_data["n_groups"] <= 2, ( f"Event time e={e} has n_groups={es_data['n_groups']}, " "expected <=2 (cohort g=3 should be excluded due to NaN anchor)" ) - # Analytical effects and SEs should be finite for all horizons - for e, es_data in results.event_study_effects.items(): - assert np.isfinite(es_data["effect"]), f"e={e}: analytical effect should be finite" - assert np.isfinite(es_data["se"]), f"e={e}: analytical SE should be finite" - - # Bootstrap SEs should also be finite - if results.bootstrap_results and results.bootstrap_results.event_study_ses: - for e, se in results.bootstrap_results.event_study_ses.items(): - assert np.isfinite(se), f"e={e}: bootstrap SE should be finite" + # Effects and (bootstrap percentile) SEs should be finite for all horizons + for e, es_data in es.items(): + assert np.isfinite(es_data["effect"]), f"e={e}: effect should be finite" + assert np.isfinite(es_data["se"]), f"e={e}: bootstrap SE should be finite" @pytest.mark.parametrize("action", ["silent", "warn"]) def test_reg_underdetermined_control_cell_no_crash(self, action): @@ -1525,9 +1521,8 @@ def test_event_study_omits_all_nonestimable_relative_time(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - es = results.event_study_effects + es = es_dict(results) # Relative time e=2 contains only (g=2, t=4), which is non-estimable (no # not-yet-treated controls at t=4) -> the whole bucket must be omitted. assert 2 not in es, "all-non-estimable relative time must be omitted, not a NaN row" @@ -2335,8 +2330,10 @@ def test_bootstrap_single_unit_cohort_handles_gracefully(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) + # both post-fit aggregation levels (bootstrap replay) must not crash either + results.aggregate("event_study") + results.aggregate("group") assert results is not None assert results.overall_att is not None @@ -2395,16 +2392,15 @@ def test_bootstrap_event_study(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) assert results.bootstrap_results is not None - assert results.bootstrap_results.event_study_ses is not None - assert results.bootstrap_results.event_study_cis is not None - assert results.bootstrap_results.event_study_p_values is not None + # the post-fit aggregation replays the bootstrap: percentile inference + es = es_dict(results) + assert es # Check event study effects have bootstrap SEs - for e, effect in results.event_study_effects.items(): + for e, effect in es.items(): assert effect["se"] > 0 assert effect["conf_int"][0] < effect["conf_int"][1] @@ -2420,16 +2416,14 @@ def test_bootstrap_group_aggregation(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) assert results.bootstrap_results is not None - assert results.bootstrap_results.group_effect_ses is not None - assert results.bootstrap_results.group_effect_cis is not None - assert results.bootstrap_results.group_effect_p_values is not None + grp = group_dict(results) + assert grp # Check group effects have bootstrap SEs - for g, effect in results.group_effects.items(): + for g, effect in grp.items(): assert effect["se"] > 0 assert effect["conf_int"][0] < effect["conf_int"][1] @@ -2445,12 +2439,13 @@ def test_bootstrap_all_aggregations(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) assert results.bootstrap_results is not None - assert results.bootstrap_results.event_study_ses is not None - assert results.bootstrap_results.group_effect_ses is not None + es = results.aggregate("event_study") + grp = results.aggregate("group") + assert np.all(np.isfinite(es.se)) and np.all(es.se > 0) + assert np.all(np.isfinite(grp.se)) and np.all(grp.se > 0) def test_bootstrap_reproducibility(self, ci_params): """Test that bootstrap is reproducible with same seed.""" @@ -2630,16 +2625,14 @@ def test_bootstrap_with_balanced_event_study(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - balance_e=0, # Balance at treatment time ) assert results.bootstrap_results is not None - assert results.bootstrap_results.event_study_ses is not None - assert results.event_study_effects is not None + es = es_dict(results, balance_e=0) # Balance at treatment time + assert es # Check that event study effects have valid bootstrap SEs - for e, effect in results.event_study_effects.items(): + for e, effect in es.items(): assert effect["se"] > 0 assert effect["conf_int"][0] < effect["conf_int"][1] @@ -2670,12 +2663,12 @@ def test_bootstrap_event_study_vcov_is_draw_covariance(self, base_period, antici unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - effects = results.event_study_effects - vcov = results.event_study_vcov - index = results.event_study_vcov_index + es = results.aggregate("event_study") + effects = es_dict_from_container(es) + vcov = es.vcov + index = list(es.vcov_index) assert index == [ e for e in sorted(effects) @@ -2701,21 +2694,21 @@ def test_bootstrap_event_study_vcov_needs_more_draws_than_horizons(self): def fit(n_bootstrap): with warnings.catch_warnings(): warnings.simplefilter("ignore") - return CallawaySantAnna(n_bootstrap=n_bootstrap, seed=7).fit( + res = CallawaySantAnna(n_bootstrap=n_bootstrap, seed=7).fit( data, outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + return res.aggregate("event_study") - n_horizons = len(fit(50).event_study_vcov_index) + n_horizons = len(fit(50).vcov_index) at_limit = fit(n_horizons) - assert at_limit.event_study_vcov is None and at_limit.event_study_vcov_index is None - assert all(np.isfinite(d["se"]) for d in at_limit.event_study_effects.values()) + assert at_limit.vcov is None and at_limit.vcov_index is None + assert np.all(np.isfinite(at_limit.se)) above_limit = fit(n_horizons + 1) - assert above_limit.event_study_vcov.shape == (n_horizons, n_horizons) + assert above_limit.vcov.shape == (n_horizons, n_horizons) # ============================================================================= @@ -2827,14 +2820,13 @@ def test_single_cohort_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + es = es_dict(results) + assert len(es) > 0 # Event study should have multiple relative periods - rel_periods = sorted(results.event_study_effects.keys()) + rel_periods = sorted(es.keys()) assert len(rel_periods) >= 2, f"Expected multiple periods, got {rel_periods}" # With single cohort, all effects are for the same group @@ -2842,7 +2834,7 @@ def test_single_cohort_event_study(self): post_periods = [e for e in rel_periods if e >= 0] if post_periods: # At least some post-periods should show positive effect - post_effects = [results.event_study_effects[e]["effect"] for e in post_periods] + post_effects = [es[e]["effect"] for e in post_periods] assert any( e > 0.5 for e in post_effects ), f"Expected positive post-period effects, got {post_effects}" @@ -3097,8 +3089,8 @@ def test_event_study_analytical_se(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + es_analytical = es_dict(results_analytical) # Bootstrap cs_bootstrap = CallawaySantAnna(n_bootstrap=n_boot, seed=42) @@ -3108,20 +3100,20 @@ def test_event_study_analytical_se(self, ci_params): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + es_bootstrap = es_dict(results_bootstrap) # Event study effects should exist - assert results_analytical.event_study_effects is not None - assert results_bootstrap.event_study_effects is not None + assert es_analytical + assert es_bootstrap # Check each event time SE is similar (wider tolerance when # min_n cap reduces bootstrap iterations in pure Python mode) threshold = 0.40 if n_boot < 100 else 0.20 - for e in results_analytical.event_study_effects: - if e in results_bootstrap.event_study_effects: - se_analytical = results_analytical.event_study_effects[e]["se"] - se_bootstrap = results_bootstrap.event_study_effects[e]["se"] + for e in es_analytical: + if e in es_bootstrap: + se_analytical = es_analytical[e]["se"] + se_bootstrap = es_bootstrap[e]["se"] if se_bootstrap > 0: rel_diff = abs(se_analytical - se_bootstrap) / se_bootstrap @@ -3239,11 +3231,9 @@ def test_non_standard_names_with_event_study(self): unit="id", time="period", first_treat="cohort", - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + assert len(results.aggregate("event_study").event_time) > 0 def test_non_standard_names_with_covariates(self): """Test non-standard column names with covariate adjustment.""" @@ -3436,13 +3426,12 @@ def test_event_study_negative_periods(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None + es = results.aggregate("event_study") # Should have negative relative periods - rel_periods = list(results.event_study_effects.keys()) + rel_periods = list(es.event_time) negative_periods = [e for e in rel_periods if e < 0] assert ( len(negative_periods) > 0 @@ -3880,21 +3869,22 @@ def test_group_effects_anticipation_boundary(self): data, outcome="outcome", unit="unit", time="time", first_treat="first_treat" ) - # Check group effects exist - if results.group_effects is not None and g in results.group_effects: - # The group effect for g should aggregate periods t >= g - 2 - # Verify by checking which group-time effects exist - gt_for_group = [(gg, t) for (gg, t) in results.group_time_effects.keys() if gg == g] + # Check group effects exist (post-fit group aggregation) + grp = group_dict(results) + assert g in grp, f"expected a group effect for cohort {g}, got {sorted(grp)}" + # The group effect for g should aggregate periods t >= g - 2 + # Verify by checking which group-time effects exist + gt_for_group = [(gg, t) for (gg, t) in results.group_time_effects.keys() if gg == g] - # There should be effects at t = g - anticipation = g - 2 - # (if the data has that period) - # Note: the anticipation period t = g - 2 may or may not be - # present depending on base_period, so it is not asserted here; - # post-treatment periods (t >= g - anticipation) should exist. + # There should be effects at t = g - anticipation = g - 2 + # (if the data has that period) + # Note: the anticipation period t = g - 2 may or may not be + # present depending on base_period, so it is not asserted here; + # post-treatment periods (t >= g - anticipation) should exist. - # Verify post-treatment periods t >= g are included - post_treatment = [t for (gg, t) in gt_for_group if t >= g] - assert len(post_treatment) > 0, "Should have post-treatment effects" + # Verify post-treatment periods t >= g are included + post_treatment = [t for (gg, t) in gt_for_group if t >= g] + assert len(post_treatment) > 0, "Should have post-treatment effects" def test_not_yet_treated_with_anticipation_excludes_anticipation_window(self): """Not-yet-treated controls must exclude cohorts in the anticipation window. @@ -4000,25 +3990,21 @@ def test_per_effect_tstat_consistency(self, ci_params): f"expected {expected}, got {t_stat}" ) - # Check event study effects if present - if results.event_study_effects is not None: - for e, effect_data in results.event_study_effects.items(): - se = effect_data["se"] - t_stat = effect_data["t_stat"] + # Check event study effects (post-fit aggregation, bootstrap replay) + for e, effect_data in es_dict(results).items(): + se = effect_data["se"] + t_stat = effect_data["t_stat"] - if not np.isfinite(se) or se == 0: - assert np.isnan( - t_stat - ), f"event study t_stat for e={e} should be NaN when SE={se}" + if not np.isfinite(se) or se == 0: + assert np.isnan(t_stat), f"event study t_stat for e={e} should be NaN when SE={se}" - # Check group effects if present - if results.group_effects is not None: - for g, effect_data in results.group_effects.items(): - se = effect_data["se"] - t_stat = effect_data["t_stat"] + # Check group effects (post-fit aggregation, bootstrap replay) + for g, effect_data in group_dict(results).items(): + se = effect_data["se"] + t_stat = effect_data["t_stat"] - if not np.isfinite(se) or se == 0: - assert np.isnan(t_stat), f"group t_stat for g={g} should be NaN when SE={se}" + if not np.isfinite(se) or se == 0: + assert np.isnan(t_stat), f"group t_stat for g={g} should be NaN when SE={se}" def test_aggregated_tstat_nan_when_se_zero(self): """Aggregated t_stat (event-study and group) is NaN when SE is zero or non-finite. @@ -4043,7 +4029,6 @@ def test_aggregated_tstat_nan_when_se_zero(self): df = pd.DataFrame(data) - # Fit with event study aggregation to get event_study_effects cs = CallawaySantAnna(n_bootstrap=0) results = cs.fit( df, @@ -4051,13 +4036,16 @@ def test_aggregated_tstat_nan_when_se_zero(self): unit="unit", time="time", first_treat="first_treat", - aggregate="all", # Get both event study and group effects ) + # Both post-fit aggregation levels + es = es_dict(results) + grp = group_dict(results) + assert es and grp # Check that t_stat computation follows the correct pattern: # t_stat = effect / se if np.isfinite(se) and se > 0 else np.nan - if results.event_study_effects: - for e, data in results.event_study_effects.items(): + if es: + for e, data in es.items(): se = data["se"] t_stat = data["t_stat"] effect = data["effect"] @@ -4074,8 +4062,8 @@ def test_aggregated_tstat_nan_when_se_zero(self): f"got {t_stat}" ) - if results.group_effects: - for g, data in results.group_effects.items(): + if grp: + for g, data in grp.items(): se = data["se"] t_stat = data["t_stat"] effect = data["effect"] @@ -4102,17 +4090,15 @@ def test_event_study_universal_includes_reference_period(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None, "event_study_effects should not be None" + es = es_dict(results) # Reference period should be included - assert -1 in results.event_study_effects, ( - f"Reference period e=-1 should be in event_study_effects, " - f"got periods: {list(results.event_study_effects.keys())}" - ) - ref = results.event_study_effects[-1] + assert ( + -1 in es + ), f"Reference period e=-1 should be in the event study, got periods: {list(es)}" + ref = es[-1] # Effect is 0 by construction (normalization) assert ref["effect"] == 0.0, f"Reference period effect should be 0.0, got {ref['effect']}" @@ -4140,17 +4126,17 @@ def test_event_study_varying_excludes_reference_period(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None, "event_study_effects should not be None" + es = es_dict(results) + assert es # Varying mode: no single reference period, e=-1 computed normally or excluded # The key is we don't artificially add a 0-effect entry - if -1 in results.event_study_effects: + if -1 in es: # If it exists, it should be an actual computed effect, not 0.0 with n_groups=0 assert ( - results.event_study_effects[-1]["n_groups"] > 0 + es[-1]["n_groups"] > 0 ), "Varying mode should not artificially add e=-1 with n_groups=0" def test_event_study_universal_with_anticipation(self): @@ -4164,17 +4150,16 @@ def test_event_study_universal_with_anticipation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None, "event_study_effects should not be None" + es = es_dict(results) # With anticipation=1, reference is e=-2 - assert -2 in results.event_study_effects, ( - f"With anticipation=1, reference period e=-2 should be in event_study_effects, " - f"got periods: {list(results.event_study_effects.keys())}" + assert -2 in es, ( + f"With anticipation=1, reference period e=-2 should be in the event study, " + f"got periods: {list(es)}" ) - ref = results.event_study_effects[-2] + ref = es[-2] assert ref["effect"] == 0.0, f"Reference period effect should be 0.0, got {ref['effect']}" # Inference fields are NaN - normalization constraint assert np.isnan(ref["se"]), f"Reference period SE should be NaN, got {ref['se']}" @@ -4209,7 +4194,6 @@ def test_event_study_universal_no_effects_raises_error(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) @@ -5841,7 +5825,6 @@ def test_bare_cluster_honest_did_uses_df_inference(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # Sanity: the CS fit populated df_inference but not survey_metadata @@ -5854,7 +5837,7 @@ def test_bare_cluster_honest_did_uses_df_inference(self): # Run HonestDiD; assert it threads df_inference into the returned df_survey honest = HonestDiD(method="relative_magnitude", M=1.0) - honest_res = honest.fit(cs_res) + honest_res = honest.fit(cs_res.aggregate("event_study")) assert honest_res.df_survey is not None, ( "HonestDiD must preserve the cluster df from CS's df_inference. " @@ -6125,8 +6108,9 @@ def test_bare_cluster_bootstrap_propagates_nan_when_g_less_than_2(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + # post-fit event study (replays the degenerate bootstrap) + es = es_dict(res) # Overall bootstrap inference must be NaN-consistent assert not np.isfinite(res.overall_se), ( @@ -6154,17 +6138,17 @@ def test_bare_cluster_bootstrap_propagates_nan_when_g_less_than_2(self): assert np.isnan(eff["p_value"]) assert np.isnan(eff["conf_int"][0]) and np.isnan(eff["conf_int"][1]) - # Requested aggregate (event-study) must also be NaN-consistent + # The event-study aggregation must also be NaN-consistent # for any aggregated horizon whose underlying cells are NaN - if res.event_study_effects: - for h, ev in res.event_study_effects.items(): - if not np.isfinite(ev["se"]): - assert np.isnan(ev["t_stat"]) - assert np.isnan(ev["p_value"]) - assert np.isnan(ev["conf_int"][0]) and np.isnan(ev["conf_int"][1]) + assert es + for h, ev in es.items(): + if not np.isfinite(ev["se"]): + assert np.isnan(ev["t_stat"]) + assert np.isnan(ev["p_value"]) + assert np.isnan(ev["conf_int"][0]) and np.isnan(ev["conf_int"][1]) def test_grouped_aggregate_se_changes_with_cluster(self): - """The ``aggregate="group"`` aggregation path + """The ``aggregate("group")`` aggregation path (``_aggregate_by_group`` at ``staggered_aggregation.py:782-860``) has its own SE computation independent of overall + event-study. Asserts grouped SEs differ between cluster=None and cluster="state" @@ -6182,7 +6166,6 @@ def test_grouped_aggregate_se_changes_with_cluster(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) cs_cluster = CallawaySantAnna(cluster="state") @@ -6192,7 +6175,6 @@ def test_grouped_aggregate_se_changes_with_cluster(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) cs_explicit = CallawaySantAnna() @@ -6202,37 +6184,37 @@ def test_grouped_aggregate_se_changes_with_cluster(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", survey_design=SurveyDesign(psu="state"), ) - assert res_unit.group_effects is not None - assert res_cluster.group_effects is not None - assert res_explicit.group_effects is not None + grp_unit = group_dict(res_unit) + grp_cluster = group_dict(res_cluster) + grp_explicit = group_dict(res_explicit) + assert grp_unit and grp_cluster and grp_explicit # Grouped SEs must differ under cluster vs unit-level (at least # one group) - common_groups = set(res_unit.group_effects.keys()) & set(res_cluster.group_effects.keys()) + common_groups = set(grp_unit) & set(grp_cluster) assert common_groups, "expected overlapping groups" diffs = [] for g in common_groups: - se_unit = res_unit.group_effects[g]["se"] - se_cluster = res_cluster.group_effects[g]["se"] + se_unit = grp_unit[g]["se"] + se_cluster = grp_cluster[g]["se"] if np.isfinite(se_unit) and np.isfinite(se_cluster): diffs.append(abs(se_unit - se_cluster)) max_diff = max(diffs) if diffs else 0.0 assert max_diff > 1e-6, ( f"Grouped SEs did not change with cluster= (max diff: " - f"{max_diff:.6g}). aggregate='group' may not be routing " + f"{max_diff:.6g}). aggregate('group') may not be routing " "through the cluster-aware IF aggregation." ) # Bare cluster vs explicit SurveyDesign must agree on grouped surface - common = set(res_cluster.group_effects.keys()) & set(res_explicit.group_effects.keys()) + common = set(grp_cluster) & set(grp_explicit) for g in common: - se_bare = res_cluster.group_effects[g]["se"] - se_explicit = res_explicit.group_effects[g]["se"] + se_bare = grp_cluster[g]["se"] + se_explicit = grp_explicit[g]["se"] if np.isfinite(se_bare) and np.isfinite(se_explicit): assert se_bare == pytest.approx(se_explicit, rel=1e-10, abs=1e-12), ( f"Grouped SE divergence at g={g}: bare cluster=state " diff --git a/tests/test_staggered_aggregation.py b/tests/test_staggered_aggregation.py index a5c96230a..97377d366 100644 --- a/tests/test_staggered_aggregation.py +++ b/tests/test_staggered_aggregation.py @@ -308,7 +308,25 @@ def __init__(self, monkeypatch, atol=1e-9): def wrapper(mixin_self, *args, **kwargs): new_psi, new_units = orig(mixin_self, *args, **kwargs) - old_psi, _ = _frozen_combined_influence_function(*args, **kwargs) + frozen_args = list(args) + precomputed = frozen_args[7] if len(frozen_args) > 7 else kwargs.get("precomputed") + if ( + frozen_args[5] is None + and precomputed is not None + and precomputed.get("is_panel", True) + and precomputed.get("survey_weights") is None + ): + # Post-fit ``aggregate()`` relays through the retained kit with + # ``df=None`` (the kit keeps no DataFrame); the fast path reads + # the cohort sizes off ``precomputed["unit_cohorts"]``. The + # frozen dense copy counts them off a per-unit frame, so hand + # it the equivalent one (one row per kit unit, canonical ids). + cohorts = np.asarray(precomputed["unit_cohorts"]) + frozen_args[5] = pd.DataFrame( + {"unit": np.arange(len(cohorts)), "first_treat": cohorts} + ) + frozen_args[6] = "unit" + old_psi, _ = _frozen_combined_influence_function(*frozen_args, **kwargs) assert new_psi.shape == old_psi.shape np.testing.assert_allclose( new_psi, @@ -330,9 +348,10 @@ def wrapper(mixin_self, *args, **kwargs): class TestCombinedIFFrozenParity: """Old-vs-new psi_total parity across data families and aggregation - targets. aggregate='all' exercises the multi-cohort overall target, - every per-horizon keeper subset (incl. single-keeper horizons), and - single-cohort group targets in one fit.""" + targets. The plain fit exercises the multi-cohort overall target; the + post-fit ``aggregate("event_study")`` / ``aggregate("group")`` calls + exercise every per-horizon keeper subset (incl. single-keeper horizons) + and the single-cohort group targets.""" def _fit_cs(self, df, covs=None, survey=False, rcs=False, boot=0): cs = CallawaySantAnna( @@ -341,16 +360,17 @@ def _fit_cs(self, df, covs=None, survey=False, rcs=False, boot=0): panel=not rcs, seed=7, ) - cs.fit( + res = cs.fit( df, outcome="outcome", unit="unit", time="time", first_treat="first_treat", covariates=covs, - aggregate="all", survey_design=SurveyDesign(weights="pw") if survey else None, ) + res.aggregate("event_study") + res.aggregate("group") return cs def test_plain_panel(self, monkeypatch): @@ -530,14 +550,15 @@ def spy( monkeypatch.setattr(CallawaySantAnnaAggregationMixin, "_combined_if_fast", spy) cs = CallawaySantAnna(estimation_method="dr", seed=7) - cs.fit( + res = cs.fit( _make_panel(), outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) + res.aggregate("event_study") + res.aggregate("group") # every aggregation target routed through the fast path, none fell # back to the general (DataFrame-scanning) branch assert calls["resolved"] > 0 @@ -563,15 +584,16 @@ def test_rcs_fast_path_taken(self, monkeypatch): calls = self._spy(monkeypatch) cs = CallawaySantAnna(estimation_method="dr", panel=False, seed=7) with pytest.warns(UserWarning, match="stationary"): - cs.fit( + res = cs.fit( _make_rcs(), outcome="outcome", unit="unit", time="time", first_treat="first_treat", covariates=["x1", "x2"], - aggregate="all", ) + res.aggregate("event_study") + res.aggregate("group") assert calls["resolved"] > 0 assert calls["fallback"] == 0 @@ -581,16 +603,17 @@ def test_survey_fast_path_taken(self, monkeypatch): optimization for the whole survey family.""" calls = self._spy(monkeypatch) cs = CallawaySantAnna(estimation_method="dr", seed=7) - cs.fit( + res = cs.fit( _make_panel(n_cov=2, survey=True), outcome="outcome", unit="unit", time="time", first_treat="first_treat", covariates=["x1", "x2"], - aggregate="all", survey_design=SurveyDesign(weights="pw"), ) + res.aggregate("event_study") + res.aggregate("group") assert calls["resolved"] > 0 assert calls["fallback"] == 0 @@ -600,14 +623,16 @@ def test_bootstrap_prep_fast_path_taken(self, monkeypatch): calls = self._spy(monkeypatch) cs = CallawaySantAnna(estimation_method="dr", n_bootstrap=19, seed=7) with pytest.warns(UserWarning, match="n_bootstrap"): - cs.fit( + res = cs.fit( _make_panel(), outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) + # the post-fit aggregation replays the bootstrap + res.aggregate("event_study") + res.aggregate("group") assert calls["resolved"] > 0 assert calls["fallback"] == 0 diff --git a/tests/test_staggered_rc.py b/tests/test_staggered_rc.py index dd71e5fd5..cb67a474e 100644 --- a/tests/test_staggered_rc.py +++ b/tests/test_staggered_rc.py @@ -130,26 +130,27 @@ def test_positive_effect(self, rc_data): class TestAggregation: - """Aggregation types work with RCS.""" + """Post-fit aggregation levels work with RCS.""" - @pytest.mark.parametrize("agg", ["simple", "event_study", "group"]) - def test_aggregation(self, rc_data, agg): + def test_aggregation(self, rc_data): result = CallawaySantAnna(estimation_method="reg", panel=False).fit( rc_data, "outcome", "unit", "period", "first_treat", - aggregate=agg, ) assert np.isfinite(result.overall_att) - if agg == "event_study": - assert result.event_study_effects is not None - for e, info in result.event_study_effects.items(): - if info["n_groups"] > 0: - assert np.isfinite(info["effect"]) - if agg == "group": - assert result.group_effects is not None + # the fit never populates the fit-time surface; aggregation is post-fit + assert result.event_study_effects is None + assert result.group_effects is None + es = result.aggregate("event_study") + for att, n in zip(es.att, es.n): + if np.isfinite(n) and n > 0: + assert np.isfinite(att) + grp = result.aggregate("group") + assert len(grp.label) > 0 + assert np.all(np.isfinite(grp.att)) # ============================================================================= diff --git a/tests/test_sun_abraham.py b/tests/test_sun_abraham.py index c236a237b..c6fbb4470 100644 --- a/tests/test_sun_abraham.py +++ b/tests/test_sun_abraham.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict from scipy import stats from diff_diff.sun_abraham import SunAbraham, SunAbrahamResults @@ -558,8 +559,8 @@ def test_pre_period_difference_expected_between_cs_sa(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + cs_varying_es = es_dict(cs_varying_results) # Callaway-Sant'Anna with universal base (all compare to g-1) cs_universal = CallawaySantAnna(base_period="universal") @@ -569,13 +570,13 @@ def test_pre_period_difference_expected_between_cs_sa(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + cs_universal_es = es_dict(cs_universal_results) # Find common event times sa_times = set(sa_results.event_study_effects.keys()) - cs_varying_times = set(cs_varying_results.event_study_effects.keys()) - cs_universal_times = set(cs_universal_results.event_study_effects.keys()) + cs_varying_times = set(cs_varying_es.keys()) + cs_universal_times = set(cs_universal_es.keys()) common_times = sa_times & cs_varying_times & cs_universal_times # Separate pre and post periods @@ -585,14 +586,14 @@ def test_pre_period_difference_expected_between_cs_sa(self): # Post-treatment effects should match across all methods for t in post_times: sa_eff = sa_results.event_study_effects[t]["effect"] - cs_vary_eff = cs_varying_results.event_study_effects[t]["effect"] - cs_univ_eff = cs_universal_results.event_study_effects[t]["effect"] + cs_vary_eff = cs_varying_es[t]["effect"] + cs_univ_eff = cs_universal_es[t]["effect"] # All three should be similar for post-treatment max_se = max( sa_results.event_study_effects[t]["se"], - cs_varying_results.event_study_effects[t]["se"], - cs_universal_results.event_study_effects[t]["se"], + cs_varying_es[t]["se"], + cs_universal_es[t]["se"], ) assert abs(sa_eff - cs_vary_eff) < 3 * max_se, ( f"Post-period t={t}: SA and CS(varying) differ too much: " @@ -614,8 +615,8 @@ def test_pre_period_difference_expected_between_cs_sa(self): total_diff_universal = 0.0 for t in pre_times: sa_eff = sa_results.event_study_effects[t]["effect"] - cs_vary_eff = cs_varying_results.event_study_effects[t]["effect"] - cs_univ_eff = cs_universal_results.event_study_effects[t]["effect"] + cs_vary_eff = cs_varying_es[t]["effect"] + cs_univ_eff = cs_universal_es[t]["effect"] total_diff_varying += abs(sa_eff - cs_vary_eff) total_diff_universal += abs(sa_eff - cs_univ_eff) diff --git a/tests/test_survey_phase3.py b/tests/test_survey_phase3.py index 8aad88eab..8618febd7 100644 --- a/tests/test_survey_phase3.py +++ b/tests/test_survey_phase3.py @@ -1375,11 +1375,9 @@ def test_efficient_did_weights_only_se(self, staggered_survey_data): assert r.overall_se > 0 assert r.overall_se > 0.01 # Not artificially tiny - @pytest.mark.filterwarnings(r"ignore:ContinuousDiD\.fit\(aggregate=\):FutureWarning") def test_continuous_did_eventstudy_survey(self, continuous_survey_data): - """The deprecated fit-time aggregate=eventstudy still works with a - survey design (M-025 legacy routing; the post-fit successor's - survey arms are pinned in tests/test_aggregate_contract.py).""" + """The post-fit event-study aggregation works with a survey design + (the survey arms are pinned in tests/test_aggregate_contract.py).""" from diff_diff import ContinuousDiD sd = SurveyDesign(weights="weight", strata="stratum") @@ -1390,14 +1388,15 @@ def test_continuous_did_eventstudy_survey(self, continuous_survey_data): "time", "first_treat", "dose", - aggregate="eventstudy", survey_design=sd, ) - assert result.event_study_effects is not None + assert result.event_study_effects is None # analytic fit: post-fit route only assert result.survey_metadata is not None - for e, eff in result.event_study_effects.items(): - if np.isfinite(eff["effect"]): - assert np.isfinite(eff["se"]), f"Non-finite SE for e={e}" + es = result.aggregate("event_study") + assert len(es.event_time) > 0 + for e, att, se, ref in zip(es.event_time, es.att, es.se, es.is_reference): + if not ref and np.isfinite(att): + assert np.isfinite(se), f"Non-finite SE for e={e}" def test_within_unit_varying_weights_rejected(self): """Time-varying survey weights within units should be rejected.""" diff --git a/tests/test_survey_phase4.py b/tests/test_survey_phase4.py index d209e1513..58b63c5dc 100644 --- a/tests/test_survey_phase4.py +++ b/tests/test_survey_phase4.py @@ -18,21 +18,6 @@ ) from diff_diff.linalg import solve_logit -# --------------------------------------------------------------------------- -# Rows M-021/M-022 (+ M-118/M-119): ImputationDiD / TwoStageDiD -# ``fit(aggregate=, balance_e=)`` is deprecated (3.9, removed 4.0) and warns on -# ANY supplied value. The deprecated fit-time route is kept DELIBERATELY here: -# these tests pin FIT-TIME surface behaviour (bit-equality grids, bootstrap -# aggregation, R/Stata parity, replicate overrides, native effect dicts) that -# the post-fit ``results.aggregate(...)`` container route does not reproduce -# shape-for-shape. The shim warning is therefore filtered BY MESSAGE, scoped to -# these two estimators only - every other FutureWarning (including the other -# estimators' aggregate() shims) still surfaces. -# --------------------------------------------------------------------------- -pytestmark = pytest.mark.filterwarnings( - r"ignore:(ImputationDiD|TwoStageDiD)\.fit\((aggregate=|balance_e=|aggregate= / balance_e=)\):FutureWarning" -) - # ============================================================================= # Shared Fixtures # ============================================================================= @@ -343,14 +328,13 @@ def test_event_study_with_survey(self, staggered_survey_data, survey_design_weig "unit", "period", "first_treat", - aggregate="event_study", survey_design=survey_design_weights_only, ) - assert result.event_study_effects is not None - assert len(result.event_study_effects) > 0 - for h, eff in result.event_study_effects.items(): - assert np.isfinite(eff["effect"]) - assert np.isfinite(eff["se"]) + assert result.event_study_effects is None # analytic fit: post-fit route only + es = result.aggregate("event_study") + assert len(es.event_time) > 0 + assert np.all(np.isfinite(es.att)) + assert np.all(np.isfinite(es.se[~es.is_reference])) def test_bootstrap_survey_supported(self, staggered_survey_data, survey_design_weights_only): """Bootstrap + survey should produce finite SE via PSU-level multiplier bootstrap.""" @@ -453,35 +437,32 @@ def test_se_scale_invariance_with_covariates(self, staggered_survey_data): ), f"SE not scale-invariant (covariates): {r1.overall_se} vs {r2.overall_se}" def test_aggregate_group_with_survey(self, staggered_survey_data, survey_design_weights_only): - """aggregate='group' works with survey design.""" + """Post-fit group aggregation works with survey design.""" result = ImputationDiD().fit( staggered_survey_data, "outcome", "unit", "period", "first_treat", - aggregate="group", survey_design=survey_design_weights_only, ) - assert result.group_effects is not None - assert len(result.group_effects) > 0 - for g, eff in result.group_effects.items(): - assert np.isfinite(eff["effect"]) + grp = result.aggregate("group") + assert len(grp.label) > 0 + assert np.all(np.isfinite(grp.att)) def test_aggregate_all_with_survey(self, staggered_survey_data, survey_design_weights_only): - """aggregate='all' works with survey design.""" + """Post-fit event-study + group aggregation work with survey design.""" result = ImputationDiD().fit( staggered_survey_data, "outcome", "unit", "period", "first_treat", - aggregate="all", survey_design=survey_design_weights_only, ) assert np.isfinite(result.overall_att) - assert result.event_study_effects is not None - assert result.group_effects is not None + assert len(result.aggregate("event_study").event_time) > 0 + assert len(result.aggregate("group").label) > 0 def test_aweight_raises(self, staggered_survey_data): """aweight survey design should raise ValueError.""" @@ -643,33 +624,31 @@ def test_summary_includes_survey(self, staggered_survey_data, survey_design_weig assert "pweight" in summary def test_aggregate_group_with_survey(self, staggered_survey_data, survey_design_weights_only): - """aggregate='group' works with survey design.""" + """Post-fit group aggregation works with survey design.""" result = TwoStageDiD().fit( staggered_survey_data, "outcome", "unit", "period", "first_treat", - aggregate="group", survey_design=survey_design_weights_only, ) - assert result.group_effects is not None - assert len(result.group_effects) > 0 + grp = result.aggregate("group") + assert len(grp.label) > 0 def test_aggregate_all_with_survey(self, staggered_survey_data, survey_design_weights_only): - """aggregate='all' works with survey design.""" + """Post-fit event-study + group aggregation work with survey design.""" result = TwoStageDiD().fit( staggered_survey_data, "outcome", "unit", "period", "first_treat", - aggregate="all", survey_design=survey_design_weights_only, ) assert np.isfinite(result.overall_att) - assert result.event_study_effects is not None - assert result.group_effects is not None + assert len(result.aggregate("event_study").event_time) > 0 + assert len(result.aggregate("group").label) > 0 def test_always_treated_with_survey(self, staggered_survey_data): """TwoStageDiD with survey + always-treated units should not crash.""" @@ -838,21 +817,20 @@ def test_strata_psu_fpc_supported(self, staggered_survey_data): assert result.survey_metadata is not None def test_aggregate_group_with_survey(self, staggered_survey_data, survey_design_weights_only): - """aggregate='group' works with weights-only survey design.""" + """Post-fit group aggregation works with weights-only survey design.""" result = CallawaySantAnna(estimation_method="reg").fit( staggered_survey_data, "outcome", "unit", "period", "first_treat", - aggregate="group", survey_design=survey_design_weights_only, ) - assert result.group_effects is not None - assert len(result.group_effects) > 0 + grp = result.aggregate("group") + assert len(grp.label) > 0 def test_aggregate_group_full_design(self, staggered_survey_data): - """aggregate='group' with full design uses design-based SEs.""" + """Post-fit group aggregation with full design uses design-based SEs.""" sd_full = SurveyDesign(weights="weight", strata="stratum", psu="psu") result = CallawaySantAnna(estimation_method="reg").fit( staggered_survey_data, @@ -860,29 +838,26 @@ def test_aggregate_group_full_design(self, staggered_survey_data): "unit", "period", "first_treat", - aggregate="group", survey_design=sd_full, ) - assert result.group_effects is not None - assert len(result.group_effects) > 0 - for g, info in result.group_effects.items(): - assert np.isfinite(info["effect"]) - assert np.isfinite(info["se"]) + grp = result.aggregate("group") + assert len(grp.label) > 0 + assert np.all(np.isfinite(grp.att)) + assert np.all(np.isfinite(grp.se)) def test_aggregate_all_with_survey(self, staggered_survey_data, survey_design_weights_only): - """aggregate='all' works with weights-only survey design.""" + """Post-fit event-study + group aggregation work with weights-only survey design.""" result = CallawaySantAnna(estimation_method="reg").fit( staggered_survey_data, "outcome", "unit", "period", "first_treat", - aggregate="all", survey_design=survey_design_weights_only, ) assert np.isfinite(result.overall_att) - assert result.event_study_effects is not None - assert result.group_effects is not None + assert len(result.aggregate("event_study").event_time) > 0 + assert len(result.aggregate("group").label) > 0 def test_bootstrap_survey_supported(self, staggered_survey_data, survey_design_weights_only): """Bootstrap + survey now works via PSU-level multiplier bootstrap.""" @@ -963,7 +938,6 @@ def test_reg_covariates_rank_deficient_survey(self, staggered_survey_data): "period", "first_treat", covariates=["x1", "x2"], - aggregate="simple", survey_design=sd, ) assert np.isfinite(result.overall_att) @@ -984,7 +958,6 @@ def test_reg_covariates_survey_se_scale_invariance(self, staggered_survey_data): "period", "first_treat", covariates=["x1"], - aggregate="simple", survey_design=sd1, ) r2 = est.fit( @@ -994,7 +967,6 @@ def test_reg_covariates_survey_se_scale_invariance(self, staggered_survey_data): "period", "first_treat", covariates=["x1"], - aggregate="simple", survey_design=sd2, ) assert np.isclose( @@ -1045,7 +1017,6 @@ def test_aggregation_with_survey(self, staggered_survey_data, survey_design_weig "unit", "period", "first_treat", - aggregate="event_study", ) r_w = CallawaySantAnna(estimation_method="reg").fit( staggered_survey_data, @@ -1053,14 +1024,13 @@ def test_aggregation_with_survey(self, staggered_survey_data, survey_design_weig "unit", "period", "first_treat", - aggregate="event_study", survey_design=survey_design_weights_only, ) # Event study ATTs should differ with non-uniform weights assert r_unw.overall_att != r_w.overall_att - # Event study effects should exist - assert r_w.event_study_effects is not None - assert len(r_w.event_study_effects) > 0 + # Event study effects should exist on the post-fit route + es_w = r_w.aggregate("event_study") + assert len(es_w.event_time) > 0 def test_summary_includes_survey(self, staggered_survey_data, survey_design_weights_only): """Summary output should include survey design section.""" @@ -1302,7 +1272,6 @@ def test_unbalanced_panel_scale_invariance(self, staggered_survey_data): "unit", "period", "first_treat", - aggregate="simple", survey_design=sd1, ) r2 = CallawaySantAnna(estimation_method="reg").fit( @@ -1311,7 +1280,6 @@ def test_unbalanced_panel_scale_invariance(self, staggered_survey_data): "unit", "period", "first_treat", - aggregate="simple", survey_design=sd2, ) assert np.isclose( @@ -1337,7 +1305,6 @@ def test_se_scale_invariance_all_methods(self, staggered_survey_data): unit="unit", time="period", first_treat="first_treat", - aggregate="simple", survey_design=sd1, ) r2 = est.fit( @@ -1346,7 +1313,6 @@ def test_se_scale_invariance_all_methods(self, staggered_survey_data): unit="unit", time="period", first_treat="first_treat", - aggregate="simple", survey_design=sd2, ) assert np.isclose( @@ -1392,7 +1358,6 @@ def test_strata_psu_fpc_works_inference(self, staggered_survey_data): "unit", "period", "first_treat", - aggregate="simple", survey_design=sd_full, ) assert np.isfinite(result.overall_att) diff --git a/tests/test_survey_phase7a.py b/tests/test_survey_phase7a.py index 3c71efc8a..782e44fcf 100644 --- a/tests/test_survey_phase7a.py +++ b/tests/test_survey_phase7a.py @@ -82,13 +82,16 @@ def test_event_study(self, staggered_data_with_covariates, survey_weights_only, "first_treat", covariates=["x1"], survey_design=survey_weights_only, - aggregate="event_study", ) - assert result.event_study_effects is not None - for e, info in result.event_study_effects.items(): - if info["n_groups"] > 0: - assert np.isfinite(info["effect"]) - assert np.isfinite(info["se"]) + # A plain CS fit leaves the legacy field None + # on every fit and the post-fit container carries the event study. + assert result.event_study_effects is None + es = result.aggregate("event_study") + assert len(es.event_time) > 0 + for i in range(len(es.event_time)): + if es.n[i] > 0: + assert np.isfinite(es.att[i]) + assert np.isfinite(es.se[i]) @pytest.mark.parametrize("method", ["ipw", "dr"]) def test_full_design(self, staggered_data_with_covariates, survey_full_design, method): @@ -263,13 +266,14 @@ def test_aggregation(self, staggered_data_with_covariates, survey_weights_only, "first_treat", covariates=["x1"], survey_design=survey_weights_only, - aggregate=agg, ) assert np.isfinite(result.overall_att) - if agg == "event_study": - assert result.event_study_effects is not None - if agg == "group": - assert result.group_effects is not None + agg_result = result.aggregate(agg) + if agg == "simple": + assert float(agg_result.att[0]) == pytest.approx(result.overall_att) + else: + assert len(agg_result.att) > 0 + assert np.isfinite(agg_result.att).any() # ============================================================================= diff --git a/tests/test_survey_phase8.py b/tests/test_survey_phase8.py index a27b15f16..46923a308 100644 --- a/tests/test_survey_phase8.py +++ b/tests/test_survey_phase8.py @@ -386,7 +386,7 @@ def test_imputation_fpc_reduces_se(self, staggered_data): assert result_fpc.overall_se <= result_no_fpc.overall_se + 1e-10 def test_imputation_fpc_event_study(self, staggered_data): - """FPC works with aggregate='event_study'.""" + """FPC works with the post-fit event-study aggregation.""" from diff_diff import ImputationDiD data = staggered_data @@ -406,14 +406,14 @@ def test_imputation_fpc_event_study(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) - assert result.event_study_effects is not None - for h, eff in result.event_study_effects.items(): - assert np.isfinite(eff["se"]), f"Non-finite SE at horizon {h}" + es = result.aggregate("event_study") + for h, se, ref in zip(es.event_time, es.se, es.is_reference): + if not ref: + assert np.isfinite(se), f"Non-finite SE at horizon {h}" def test_imputation_fpc_group(self, staggered_data): - """FPC works with aggregate='group'.""" + """FPC works with the post-fit group aggregation.""" from diff_diff import ImputationDiD data = staggered_data @@ -433,11 +433,10 @@ def test_imputation_fpc_group(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="group", ) - assert result.group_effects is not None - for g, eff in result.group_effects.items(): - assert np.isfinite(eff["se"]), f"Non-finite SE for group {g}" + grp = result.aggregate("group") + for g, se in zip(grp.label, grp.se): + assert np.isfinite(se), f"Non-finite SE for group {g}" class TestTwoStageDiDFPC: @@ -504,7 +503,7 @@ def test_two_stage_fpc_reduces_se(self, staggered_data): assert result_fpc.overall_se <= result_no_fpc.overall_se + 1e-10 def test_two_stage_fpc_event_study(self, staggered_data): - """FPC works with aggregate='event_study'.""" + """FPC works with the post-fit event-study aggregation.""" from diff_diff import TwoStageDiD data = staggered_data @@ -524,14 +523,14 @@ def test_two_stage_fpc_event_study(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) - assert result.event_study_effects is not None - for h, eff in result.event_study_effects.items(): - assert np.isfinite(eff["se"]), f"Non-finite SE at horizon {h}" + es = result.aggregate("event_study") + for h, se, ref in zip(es.event_time, es.se, es.is_reference): + if not ref: + assert np.isfinite(se), f"Non-finite SE at horizon {h}" def test_two_stage_fpc_group(self, staggered_data): - """FPC works with aggregate='group'.""" + """FPC works with the post-fit group aggregation.""" from diff_diff import TwoStageDiD data = staggered_data @@ -551,11 +550,10 @@ def test_two_stage_fpc_group(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="group", ) - assert result.group_effects is not None - for g, eff in result.group_effects.items(): - assert np.isfinite(eff["se"]), f"Non-finite SE for group {g}" + grp = result.aggregate("group") + for g, se in zip(grp.label, grp.se): + assert np.isfinite(se), f"Non-finite SE for group {g}" class TestUnstratifiedFPC: @@ -619,11 +617,11 @@ def test_two_stage_unstratified_fpc_event_study(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) - assert result.event_study_effects is not None - for h, eff in result.event_study_effects.items(): - assert np.isfinite(eff["se"]), f"Non-finite SE at horizon {h}" + es = result.aggregate("event_study") + for h, se, ref in zip(es.event_time, es.se, es.is_reference): + if not ref: + assert np.isfinite(se), f"Non-finite SE at horizon {h}" class TestSinglePSUUnstratifiedFPC: @@ -1247,11 +1245,11 @@ def test_pretrends_survey_no_raise(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) - assert result.event_study_effects is not None + assert result.event_study_effects is None # analytic fit: post-fit route only + es = result.aggregate("event_study") # Should have negative-horizon (pre-period) effects - pre_horizons = [h for h in result.event_study_effects if h < 0] + pre_horizons = [h for h in es.event_time if h < 0] assert len(pre_horizons) > 0 def test_pretrends_survey_finite_se(self, staggered_data): @@ -1268,12 +1266,12 @@ def test_pretrends_survey_finite_se(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) - for h, eff in result.event_study_effects.items(): + es = result.aggregate("event_study") + for h, se in zip(es.event_time, es.se): if h < -1: # Skip reference period (h=-1) - assert np.isfinite(eff["se"]), f"Non-finite SE at pre-horizon {h}" - assert eff["se"] > 0, f"Zero SE at pre-horizon {h}" + assert np.isfinite(se), f"Non-finite SE at pre-horizon {h}" + assert se > 0, f"Zero SE at pre-horizon {h}" def test_pretrend_test_survey_no_raise(self, staggered_data): """pretrend_test() works with survey design.""" @@ -1289,7 +1287,6 @@ def test_pretrend_test_survey_no_raise(self, staggered_data): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) pt = est._pretrend_test() assert "f_stat" in pt @@ -1385,17 +1382,17 @@ def test_pretrends_survey_always_treated_psu(self): time="time", first_treat="first_treat", survey_design=sd, - aggregate="event_study", ) # Should produce finite results even though stratum 0 has # mostly treated PSUs (sparse untreated domain) assert np.isfinite(result.overall_att) - pre = [h for h in result.event_study_effects if h < -1] - for h in pre: - assert np.isfinite(result.event_study_effects[h]["se"]) + es = result.aggregate("event_study") + pre = es.event_time < -1 + assert pre.any() + assert np.all(np.isfinite(es.se[pre])) def test_pretrends_replicate_raises(self, staggered_data): - """pretrends=True + replicate-weight survey raises NotImplementedError.""" + """pretrends=True + replicate-weight survey: aggregate('event_study') raises.""" from diff_diff import ImputationDiD data = staggered_data @@ -1419,16 +1416,19 @@ def test_pretrends_replicate_raises(self, staggered_data): replicate_method="BRR", ) est = ImputationDiD(pretrends=True, horizon_max=3) + # The fit itself succeeds; the lead regression's per-replicate refits + # are rejected post-fit by the event-study aggregation. + result = est.fit( + data, + outcome="outcome", + unit="unit", + time="time", + first_treat="first_treat", + survey_design=sd, + ) + assert np.isfinite(result.overall_att) with pytest.raises(NotImplementedError, match="replicate"): - est.fit( - data, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - survey_design=sd, - aggregate="event_study", - ) + result.aggregate("event_study") def test_pretrend_test_replicate_raises(self, staggered_data): """pretrend_test() with replicate-weight survey raises NotImplementedError.""" diff --git a/tests/test_survey_r_crossvalidation.py b/tests/test_survey_r_crossvalidation.py index e735faab0..42932192e 100644 --- a/tests/test_survey_r_crossvalidation.py +++ b/tests/test_survey_r_crossvalidation.py @@ -279,7 +279,6 @@ def _fit_scenario(self, r_results, key): "period", "first_treat", covariates=covariates, - aggregate="simple", survey_design=sd, ) return result, r @@ -362,7 +361,6 @@ def test_design_se_differs_from_naive(self, r_results, key): "period", "first_treat", covariates=covariates, - aggregate="simple", survey_design=sd, ) @@ -380,7 +378,6 @@ def test_design_se_differs_from_naive(self, r_results, key): "period", "first_treat", covariates=covariates, - aggregate="simple", ) assert result_survey.overall_se != result_naive.overall_se, ( diff --git a/tests/test_two_stage.py b/tests/test_two_stage.py index 863d971fa..68fbd3564 100644 --- a/tests/test_two_stage.py +++ b/tests/test_two_stage.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict, es_dict_from_container, group_dict from diff_diff.two_stage import ( TwoStageBootstrapResults, @@ -171,19 +172,18 @@ def test_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - assert results.event_study_effects is not None - assert len(results.event_study_effects) > 0 + es = es_dict(results) + assert len(es) > 0 # Check reference period is present ref_period = -1 - assert ref_period in results.event_study_effects - assert results.event_study_effects[ref_period]["effect"] == 0.0 + assert ref_period in es + assert es[ref_period]["effect"] == 0.0 # Post-treatment effects should be positive (treatment_effect=2.0) - post_effects = {h: e for h, e in results.event_study_effects.items() if h >= 0} + post_effects = {h: e for h, e in es.items() if h >= 0} assert len(post_effects) > 0 for h, eff in post_effects.items(): assert eff["effect"] > 0, f"Post-treatment effect at h={h} should be positive" @@ -200,19 +200,19 @@ def test_group_effects(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - assert results.group_effects is not None + grp = group_dict(results) # Should have 3 groups (cohorts 3, 5, 7) - assert len(results.group_effects) == 3 - for g, eff in results.group_effects.items(): + assert len(grp) == 3 + for g, eff in grp.items(): assert eff["effect"] > 0 assert eff["se"] > 0 assert np.isfinite(eff["t_stat"]) def test_all_aggregation(self): - """Test aggregate='all' produces both event study and group effects.""" + """An analytic fit stores NO fit-time surfaces; both event-study and + group effects come from the post-fit aggregate() route.""" data = generate_test_data() results = TwoStageDiD().fit( data, @@ -220,11 +220,17 @@ def test_all_aggregation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="all", ) - assert results.event_study_effects is not None - assert results.group_effects is not None + assert results.event_study_effects is None + assert results.group_effects is None + es = results.aggregate("event_study") + grp = results.aggregate("group") + assert len(es.event_time) > 0 + assert len(grp.label) == 3 + # Post-fit aggregation does not populate the fit-time fields. + assert results.event_study_effects is None + assert results.group_effects is None def test_summary_text(self): """Test that summary produces expected header text.""" @@ -239,7 +245,8 @@ def test_summary_text(self): assert "Overall Average Treatment Effect" in text def test_to_dataframe_event_study(self): - """Test to_dataframe with event_study level.""" + """Event-study frame comes from the aggregate() container; the + results-level to_dataframe steers there on an analytic fit.""" data = generate_test_data() results = TwoStageDiD().fit( data, @@ -247,18 +254,20 @@ def test_to_dataframe_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - df = results.to_dataframe("event_study") + with pytest.raises(ValueError, match=r"aggregate\('event_study'\)"): + results.to_dataframe("event_study") + df = results.aggregate("event_study").to_dataframe() assert isinstance(df, pd.DataFrame) - assert "relative_period" in df.columns - assert "effect" in df.columns + assert "event_time" in df.columns + assert "att" in df.columns assert "se" in df.columns assert len(df) > 0 def test_to_dataframe_group(self): - """Test to_dataframe with group level.""" + """Group frame comes from the aggregate() container; the results-level + to_dataframe steers there on an analytic fit.""" data = generate_test_data() results = TwoStageDiD().fit( data, @@ -266,12 +275,13 @@ def test_to_dataframe_group(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - df = results.to_dataframe("group") + with pytest.raises(ValueError, match=r"aggregate\('group'\)"): + results.to_dataframe("group") + df = results.aggregate("group").to_dataframe() assert isinstance(df, pd.DataFrame) - assert "group" in df.columns + assert "label" in df.columns assert len(df) == 3 def test_to_dataframe_observation(self): @@ -396,7 +406,6 @@ def test_event_study_effects_match_imputation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) imp_results = ImputationDiD().fit( data, @@ -404,18 +413,19 @@ def test_event_study_effects_match_imputation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + ts_es = es_dict(ts_results) + imp_es = es_dict(imp_results) # Both should have the same horizons - ts_horizons = set(ts_results.event_study_effects.keys()) - imp_horizons = set(imp_results.event_study_effects.keys()) + ts_horizons = set(ts_es.keys()) + imp_horizons = set(imp_es.keys()) assert ts_horizons == imp_horizons # Point estimates should match for h in ts_horizons: - ts_eff = ts_results.event_study_effects[h]["effect"] - imp_eff = imp_results.event_study_effects[h]["effect"] + ts_eff = ts_es[h]["effect"] + imp_eff = imp_es[h]["effect"] if np.isfinite(ts_eff) and np.isfinite(imp_eff): assert ( abs(ts_eff - imp_eff) < 1e-8 @@ -433,7 +443,6 @@ def test_group_effects_match_imputation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) imp_results = ImputationDiD().fit( data, @@ -441,14 +450,15 @@ def test_group_effects_match_imputation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) + ts_grp = group_dict(ts_results) + imp_grp = group_dict(imp_results) - assert set(ts_results.group_effects.keys()) == set(imp_results.group_effects.keys()) + assert set(ts_grp.keys()) == set(imp_grp.keys()) - for g in ts_results.group_effects: - ts_eff = ts_results.group_effects[g]["effect"] - imp_eff = imp_results.group_effects[g]["effect"] + for g in ts_grp: + ts_eff = ts_grp[g]["effect"] + imp_eff = imp_grp[g]["effect"] assert abs(ts_eff - imp_eff) < 1e-8 def test_ses_differ_from_imputation(self): @@ -496,10 +506,9 @@ def test_event_study_se_positive(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - for h, eff in results.event_study_effects.items(): + for h, eff in es_dict(results).items(): if eff.get("n_obs", 0) > 0: assert eff["se"] > 0, f"SE at h={h} should be positive" assert np.isfinite(eff["se"]) @@ -903,13 +912,13 @@ def test_nan_propagation(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + es = es_dict(results) # Proposition 5 horizons should have NaN inference fields - assert results.event_study_effects, "Event study should be computed" + assert es, "Event study should be computed" nan_horizons_found = 0 - for h, eff in results.event_study_effects.items(): + for h, eff in es.items(): if np.isnan(eff["effect"]): nan_horizons_found += 1 assert_nan_inference( @@ -986,12 +995,12 @@ def test_horizon_max(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) # All horizons should have |h| <= 2 - for h in results.event_study_effects: - if results.event_study_effects[h].get("n_obs", 0) > 0: + es = es_dict(results) + for h in es: + if es[h].get("n_obs", 0) > 0: assert abs(h) <= 2 def test_always_treated_warning_lists_unit_ids(self): @@ -1046,24 +1055,23 @@ def test_balance_e_empty_cohorts_warns(self): """Unreasonably large balance_e should warn when no cohorts qualify.""" data = generate_test_data() + results = TwoStageDiD().fit( + data, + outcome="outcome", + unit="unit", + time="time", + first_treat="first_treat", + ) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - results = TwoStageDiD().fit( - data, - outcome="outcome", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - balance_e=100, # No cohort can satisfy this - ) + es = results.aggregate("event_study", balance_e=100) # No cohort can satisfy this balance_warns = [x for x in w if "No cohorts satisfy" in str(x.message)] assert len(balance_warns) > 0 # Event study should contain only the reference period - assert len(results.event_study_effects) == 1 - ref_key = list(results.event_study_effects.keys())[0] - assert results.event_study_effects[ref_key]["n_obs"] == 0 + assert len(es.event_time) == 1 + assert bool(es.is_reference[0]) + assert es_dict_from_container(es)[int(es.event_time[0])]["n_obs"] == 0 def test_proposition_5_nan_for_long_run_horizons(self): """Prop 5 horizons have n_obs > 0 but NaN inference (unidentified).""" @@ -1082,10 +1090,8 @@ def test_proposition_5_nan_for_long_run_horizons(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - - assert results.event_study_effects is not None + es = es_dict(results) # Check Prop 5 warning was emitted prop5_warnings = [x for x in w if "not identified without never-treated" in str(x.message)] @@ -1093,13 +1099,13 @@ def test_proposition_5_nan_for_long_run_horizons(self): # Horizons 0-3 should have observations and finite effects for h in range(0, 4): - eff = results.event_study_effects[h] + eff = es[h] assert eff["n_obs"] > 0, f"Horizon {h} should have observations" assert np.isfinite(eff["effect"]), f"Horizon {h} effect should be finite" # Horizons 4, 5, 6: Prop 5 — n_obs > 0 but NaN inference for h in [4, 5, 6]: - eff = results.event_study_effects[h] + eff = es[h] assert eff["n_obs"] > 0, f"Horizon {h} should have n_obs > 0 (Prop 5)" assert np.isnan(eff["effect"]), f"Horizon {h} effect should be NaN" assert np.isnan(eff["se"]), f"Horizon {h} SE should be NaN" @@ -1120,19 +1126,18 @@ def test_group_effects_nan_for_all_nan_cohort(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) - assert results.group_effects is not None + grp = group_dict(results) # Cohorts 3 and 5 should have valid effects for g in [3, 5]: - eff = results.group_effects[g] + eff = grp[g] assert eff["n_obs"] > 0, f"Cohort {g} should have observations" assert np.isfinite(eff["effect"]), f"Cohort {g} effect should be finite" # Cohort 7: all treated obs have NaN y_tilde -> zero obs -> NaN - eff_7 = results.group_effects[7] + eff_7 = grp[7] assert eff_7["n_obs"] == 0, "Cohort 7 should have 0 observations" assert np.isnan(eff_7["effect"]), "Cohort 7 effect should be NaN" assert np.isnan(eff_7["se"]), "Cohort 7 SE should be NaN" @@ -1225,7 +1230,6 @@ def test_horizon_max_limits_horizons(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) results_limited = TwoStageDiD(horizon_max=2).fit( data, @@ -1233,11 +1237,10 @@ def test_horizon_max_limits_horizons(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - full_horizons = set(results_full.event_study_effects.keys()) - limited_horizons = set(results_limited.event_study_effects.keys()) + full_horizons = set(results_full.aggregate("event_study").event_time.tolist()) + limited_horizons = set(results_limited.aggregate("event_study").event_time.tolist()) assert len(limited_horizons) <= len(full_horizons) @@ -1502,8 +1505,8 @@ def test_item3_always_treated_no_survey_note_without_weights(self): survey_notes = [x for x in w if "survey weights" in str(x.message)] assert len(survey_notes) == 0, f"Unexpected survey note: {survey_notes}" - def test_item2_nan_ytilde_event_study(self): - """Item 2: y_tilde warning fires for aggregate='event_study'.""" + def test_item2_nan_ytilde_fit(self): + """Item 2: y_tilde warning fires at fit (the overall Stage-2 regression).""" data = generate_test_data(n_units=50, n_periods=10, never_treated_frac=0.0, seed=42) ts = TwoStageDiD() with pytest.warns(UserWarning, match="non-finite imputed outcomes"): @@ -1513,22 +1516,24 @@ def test_item2_nan_ytilde_event_study(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - def test_item2_nan_ytilde_group(self): - """Item 2: y_tilde warning fires for aggregate='group'.""" + @pytest.mark.parametrize("level", ["event_study", "group"]) + def test_item2_nan_ytilde_aggregate(self, level): + """Item 2: the post-fit event-study / group aggregations re-run the + Stage-2 regression on the kit and warn about non-finite y_tilde too.""" data = generate_test_data(n_units=50, n_periods=10, never_treated_frac=0.0, seed=42) - ts = TwoStageDiD() - with pytest.warns(UserWarning, match="non-finite imputed outcomes"): - ts.fit( + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res = TwoStageDiD().fit( data, outcome="outcome", unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) + with pytest.warns(UserWarning, match="non-finite imputed outcomes"): + res.aggregate(level) def test_iterative_fe_warns_on_nonconvergence(self): """Silent-failure audit axis B: _iterative_fe must warn when max_iter exhausts.""" @@ -1715,11 +1720,8 @@ def drop_last(A, **kwargs): return drop_last - @pytest.mark.parametrize( - "aggregate,attr", - [("event_study", "event_study_effects"), ("group", "group_effects")], - ) - def test_dropped_coefficient_propagates_nan_inference(self, aggregate, attr): + @pytest.mark.parametrize("aggregate", ["event_study", "group"]) + def test_dropped_coefficient_propagates_nan_inference(self, aggregate): """A dropped (unidentified) Stage-2 coefficient must report NaN se / t_stat / p_value / conf_int at the ESTIMATOR level (not the zero-filled se=0) — the per-coefficient propagation the rank-guard enables (CI codex P2 test-depth). @@ -1740,9 +1742,20 @@ def test_dropped_coefficient_propagates_nan_inference(self, aggregate, attr): unit="unit", time="time", first_treat="first_treat", - aggregate=aggregate, ) - effects = getattr(res, attr) + # The analytical Stage-2 aggregation recomputes post-fit, so the + # rank-guard patch must still be live when aggregate() runs. + if aggregate == "event_study": + es = res.aggregate("event_study") + # The reference row's inference is NaN by container + # contract - exclude it so nan_k reflects dropped coordinates. + effects = { + k: e + for (k, e), ref in zip(es_dict_from_container(es).items(), es.is_reference) + if not ref + } + else: + effects = group_dict(res) nan_k = [k for k, e in effects.items() if np.isnan(e["se"])] fin_k = [k for k, e in effects.items() if np.isfinite(e["se"]) and e["se"] > 0] assert nan_k, ( @@ -2280,7 +2293,7 @@ def test_default_vcov_type(self): def test_results_carry_vcov_metadata(self): data = generate_test_data(n_units=80, seed=1) - r = _fit_ts(TwoStageDiD(), data, aggregate="all") + r = _fit_ts(TwoStageDiD(), data) assert r.vcov_type == "hc1" assert r.cluster_name == "unit" assert r.n_clusters == 80 @@ -2434,8 +2447,12 @@ def test_bootstrap_single_cluster_returns_nan(self): unit="unit", time="time", first_treat="first_treat", + # Fit-time aggregation kept on this BOOTSTRAPPED fit until the + # 4.0 relay: the NaN guard must cover the per-horizon and + # per-group payloads, not just the overall quintet. aggregate="all", ) + assert r.event_study_effects and r.group_effects _assert_full_bootstrap_nan(r) def test_bootstrap_single_psu_survey_returns_nan(self): @@ -2451,8 +2468,9 @@ def test_bootstrap_single_psu_survey_returns_nan(self): time="time", first_treat="first_treat", survey_design=design, - aggregate="event_study", + aggregate="event_study", # bootstrapped: fit-time route until the 4.0 relay ) + assert r.event_study_effects _assert_full_bootstrap_nan(r) # ---- metadata reflects the post-drop fit sample (codex P2) ---- @@ -2621,7 +2639,7 @@ def _panel(seed=7): ) return pd.DataFrame(rows) - def test_analytical_fit_persists_vcov_diag_matches_ses(self): + def test_analytical_aggregate_vcov_diag_matches_ses(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") res = TwoStageDiD().fit( @@ -2676,21 +2694,18 @@ def test_replicate_weight_survey_clears_vcov_and_threads_final_df(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", survey_design=design, ) - assert res.event_study_vcov is None - assert res.event_study_vcov_index is None - assert res.event_study_df is not None and res.event_study_df > 0 - assert res.event_study_df == float(res.survey_metadata.df_survey) - from diff_diff.results_base import build_event_study_surface - - surface = build_event_study_surface(res) + surface = res.aggregate("event_study") + assert surface.vcov is None + assert surface.vcov_index is None + assert surface.df_survey is not None and surface.df_survey > 0 + assert surface.df_survey == float(res.survey_metadata.df_survey) finite_p = np.isfinite(surface.p_value) assert finite_p.any() - assert set(surface.df[finite_p].tolist()) == {float(res.event_study_df)} + assert set(np.asarray(surface.df)[finite_p].tolist()) == {float(surface.df_survey)} - def test_group_only_aggregate_has_no_es_vcov(self): + def test_analytic_fit_leaves_fit_time_surfaces_none(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") res = TwoStageDiD().fit( @@ -2699,9 +2714,12 @@ def test_group_only_aggregate_has_no_es_vcov(self): unit="unit", time="time", first_treat="first_treat", - aggregate="group", ) + res.aggregate("group") + res.aggregate("event_study") + # Post-fit aggregation is a pure read: nothing is written back. assert res.event_study_effects is None + assert res.group_effects is None assert res.event_study_vcov is None assert res.event_study_vcov_index is None assert res.event_study_df is None diff --git a/tests/test_variance_conventions.py b/tests/test_variance_conventions.py index d4fdd79ed..1062243cd 100644 --- a/tests/test_variance_conventions.py +++ b/tests/test_variance_conventions.py @@ -385,14 +385,11 @@ def snapshot(self): ), dict( key="imputation_pretrends_event_study", - fit=lambda df: diff_diff.ImputationDiD(pretrends=True).fit( - df, - outcome="y", - unit="unit", - time="time", - first_treat="first_treat", - aggregate="event_study", - ), + # The pretrends lead regression runs on the post-fit event-study + # route (no fit-time aggregate=), so the row spies fit + aggregate. + fit=lambda df: diff_diff.ImputationDiD(pretrends=True) + .fit(df, outcome="y", unit="unit", time="time", first_treat="first_treat") + .aggregate("event_study"), cr1_k=None, # contract row: shared CR1 REACHED; exact k is config-detail # Pinned since 3.9 (M-127): the three lead calls carry the knob- # resolved residual df (157 = n_untreated - k_kept - absorbed rank diff --git a/tests/test_visualization.py b/tests/test_visualization.py index dc23fd697..c49f1e64c 100644 --- a/tests/test_visualization.py +++ b/tests/test_visualization.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd import pytest +from legacy_surface import es_dict_from_container, legacy_standin from diff_diff import ( CallawaySantAnna, @@ -97,8 +98,8 @@ def multi_period_results(self): ) @pytest.fixture - def cs_results(self): - """Fixture for CallawaySantAnna results with event study.""" + def cs_fit(self): + """A plain CallawaySantAnna fit (no fit-time event-study surface).""" data = generate_staggered_data() cs = CallawaySantAnna() return cs.fit( @@ -107,9 +108,13 @@ def cs_results(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) + @pytest.fixture + def cs_results(self, cs_fit): + """The post-fit event-study container - what plot_event_study consumes.""" + return cs_fit.aggregate("event_study") + def test_plot_from_multi_period_results(self, multi_period_results): """Test plotting from MultiPeriodDiD results.""" pytest.importorskip("matplotlib") @@ -118,12 +123,52 @@ def test_plot_from_multi_period_results(self, multi_period_results): assert ax is not None def test_plot_from_cs_results(self, cs_results): - """Test plotting from CallawaySantAnna results.""" + """Test plotting from the CallawaySantAnna event-study container.""" pytest.importorskip("matplotlib") ax = plot_event_study(cs_results, show=False) assert ax is not None + def test_cs_results_object_has_no_fit_time_surface(self, cs_fit): + """Plain-fit invariant: a CS results object carries no event-study surface. + + Every CallawaySantAnna fit leaves the legacy field None; plotting the + results object itself fails closed and points at the container. + """ + assert cs_fit.event_study_effects is None + with pytest.raises(TypeError, match=r"aggregate\(\s*'event_study'\)"): + plot_event_study(cs_fit, show=False) + + def test_plot_from_legacy_raw_field_results(self, cs_fit): + """Raw-field branch pin: a results object whose ``event_study_effects`` + dict is populated by hand (the legacy-pickle shape) still plots, and + the plotted effects equal the container's. + """ + pytest.importorskip("matplotlib") + import matplotlib.pyplot as plt + + from diff_diff.visualization import _extract_plot_data + + standin = legacy_standin(cs_fit) + assert standin.event_study_effects is not None + es = cs_fit.aggregate("event_study") + + effects, se, periods, _, _, ref, inferred, *_ = _extract_plot_data( + standin, None, None, None, None + ) + assert sorted(periods) == sorted(es.event_time.tolist()) + for i, e in enumerate(es.event_time.tolist()): + assert effects[e] == pytest.approx(float(es.att[i]), nan_ok=True) + assert se[e] == pytest.approx(float(es.se[i]), nan_ok=True) + # Varying-base CS marks no reference row on the container; the raw-field + # branch keeps its legacy ``-1`` fallback for that case. + assert ref == (es.reference_period if es.reference_period is not None else -1) + assert inferred is True + + ax = plot_event_study(standin, show=False) + assert ax is not None + plt.close() + def test_plot_from_dataframe(self): """Test plotting from DataFrame.""" pytest.importorskip("matplotlib") @@ -331,8 +376,7 @@ def test_plot_cs_universal_base_period(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", - ) + ).aggregate("event_study") # Should not raise even with NaN SE in reference period ax = plot_event_study(results, show=False) @@ -362,12 +406,13 @@ def test_plot_cs_with_anticipation(self): unit="unit", time="period", first_treat="first_treat", - aggregate="event_study", - ) + ).aggregate("event_study") # Reference period should be at e=-2 (not e=-1) with anticipation=1 - assert -2 in results.event_study_effects - assert results.event_study_effects[-2]["n_groups"] == 0 + times = results.event_time.tolist() + assert -2 in times + assert bool(results.is_reference[times.index(-2)]) + assert results.reference_period == -2 ax = plot_event_study(results, show=False) assert ax is not None @@ -538,7 +583,7 @@ def test_plot_cs_results_no_auto_normalization(self, cs_results): # Get original effects from results (before any normalization) original_effects = { period: effect_data["effect"] - for period, effect_data in results.event_study_effects.items() + for period, effect_data in es_dict_from_container(results).items() } # Plot WITHOUT explicitly specifying reference_period @@ -586,7 +631,7 @@ def test_plot_cs_results_explicit_reference_normalizes(self, cs_results): # Get original effects from results original_effects = { period: effect_data["effect"] - for period, effect_data in results.event_study_effects.items() + for period, effect_data in es_dict_from_container(results).items() } # Choose reference period (typically -1) @@ -658,8 +703,7 @@ def cs_cband_results(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", - ) + ).aggregate("event_study") def test_plot_uses_cband_cis_by_default(self, cs_cband_results): """Test that cband CIs are used by default when available.""" @@ -668,16 +712,36 @@ def test_plot_uses_cband_cis_by_default(self, cs_cband_results): from diff_diff.visualization import _extract_plot_data + # The container carries the simultaneous band. + assert cs_cband_results.cband_lower is not None + assert cs_cband_results.cband_upper is not None + assert np.isfinite(cs_cband_results.cband_crit_value) + finite = np.isfinite(cs_cband_results.cband_lower) & np.isfinite( + cs_cband_results.cband_upper + ) + assert finite.any() + # The band is at least as wide as the pointwise interval on every finite row. + assert np.all( + cs_cband_results.cband_lower[finite] <= cs_cband_results.conf_int_lower[finite] + 1e-12 + ) + assert np.all( + cs_cband_results.cband_upper[finite] >= cs_cband_results.conf_int_upper[finite] - 1e-12 + ) + # Verify plot succeeds ax = plot_event_study(cs_cband_results, show=False) assert ax is not None - # Verify cband CIs are extracted + # Verify cband CIs are extracted and equal the container's band _, _, _, _, _, _, _, ci_lower_override, ci_upper_override, _, _, _ = _extract_plot_data( cs_cband_results, None, None, None, None ) assert ci_lower_override is not None assert ci_upper_override is not None + for i, e in enumerate(cs_cband_results.event_time.tolist()): + if finite[i]: + assert ci_lower_override[e] == float(cs_cband_results.cband_lower[i]) + assert ci_upper_override[e] == float(cs_cband_results.cband_upper[i]) plt.close() @@ -737,11 +801,12 @@ def test_full_workflow_callaway_santanna(self): unit="unit", time="time", first_treat="first_treat", - aggregate="event_study", ) - # Plot - ax = plot_event_study(results, title="Staggered DiD Event Study", show=False) + # Plot the post-fit container + ax = plot_event_study( + results.aggregate("event_study"), title="Staggered DiD Event Study", show=False + ) assert ax is not None plt.close() @@ -760,24 +825,28 @@ class TestEfficientDiDPlotReference: def test_pt_post_anticipation_reference_shifts(self): pytest.importorskip("matplotlib") - import warnings from diff_diff import EfficientDiD from diff_diff.prep_dgp import generate_staggered_data from diff_diff.visualization._event_study import _extract_plot_data d = generate_staggered_data(n_units=80, n_periods=8, cohort_periods=[4, 6], seed=9) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - res = EfficientDiD(pt_assumption="post", anticipation=1).fit( - d, - outcome="outcome", - unit="unit", - time="period", - first_treat="first_treat", - aggregate="event_study", - ) - assert res.reference_period == -2 - extracted = _extract_plot_data(res, None, None, None, None) + res = EfficientDiD(pt_assumption="post", anticipation=1).fit( + d, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + ) + # A plain EfficientDiD fit stores no fit-time surface, so the MEMBERSHIP-GATED + # property is None on the results object itself ... + assert res.event_study_effects is None + assert res.reference_period is None + # ... and reads -2 on a surface-bearing stand-in (the legacy-pickle shape), + # while the container carries the anchor as its single is_reference row. + assert legacy_standin(res).reference_period == -2 + es = res.aggregate("event_study") + assert es.reference_period == -2 + extracted = _extract_plot_data(es, None, None, None, None) assert extracted[5] == -2 # reference_period slot assert extracted[6] is True # inferred, not caller-supplied