# ============================================================================
# POL 213 — Week 5: Count Models
# ============================================================================

# ---- Setup ----

library(tidyverse)
library(juanr)
library(broom)
library(modelsummary)
library(marginaleffects)
library(MASS, exclude = "select")   # for glm.nb (loaded after dplyr)


# Prep `invasions` (juanr): Brazilian municípios, 1988–2004.
#   occs      — count of MST land occupations 1988–2004 (outcome)
#   landgini  — land Gini in the município (focal predictor); rescaled to 0–100
#               so a "1-unit increase" is 1 percentage point of inequality
#   ext_pov   — extreme poverty rate (1991, percent of population)
#   logpop    — log population (1991)
#   logarea   — log municipal land area
#
# Substantive question: in municípios with more unequal land distribution,
# do we observe more peasant land occupations?

inv_clean = juanr::invasions |>
  filter(!is.na(occs), !is.na(landgini),
         !is.na(ext_pov), !is.na(logpop), !is.na(logarea)) |>
  mutate(landgini = 100 * landgini) |>
  dplyr::select(municipio, occs, landgini, ext_pov, logpop, logarea)


# ============================================================================
# Section 1: Counts in political science — OLS as a baseline
# ============================================================================

# ---- The outcome ----

# count of land occupations is non-negative, integer, with a long right tail
# and a big pile-up at zero
summary(inv_clean$occs)
mean(inv_clean$occs == 0)   # share of municípios with zero occupations

ggplot(inv_clean, aes(occs)) +
  geom_histogram(bins = 60, fill = "#022851", color = "white") +
  labs(x = "occupations 1988–2004", y = "municípios")


# ---- OLS as a baseline ----

# OLS estimates the conditional mean E[Y|X] = Xβ. Even with a count outcome,
# this gives the ATE on the count scale — same Angrist-Pischke defense as
# the LPM in Week 3
mod_ols = lm(occs ~ landgini + ext_pov + logpop + logarea,
             data = inv_clean)
tidy(mod_ols)


# ---- Anchor the OLS effect against the outcome's distribution ----

# raw coefficient on its own is hard to read — context the effect against
# the mean of Y, the SD of Y, and a realistic (IQR) shift in X
beta_ols   = coef(mod_ols)["landgini"]
mean_y     = mean(inv_clean$occs)
sd_y       = sd(inv_clean$occs)
q_lg       = quantile(inv_clean$landgini, c(0.25, 0.75))
shift_q    = q_lg[2] - q_lg[1]

# anchor 1: 10-pp shift in landgini, as a % of mean(Y)
100 * 10 * beta_ols / mean_y

# anchor 2: 10-pp shift in landgini, as a fraction of SD(Y)
10 * beta_ols / sd_y

# anchor 3: a realistic IQR shift in landgini → predicted change in counts
shift_q * beta_ols


# ============================================================================
# Section 2: Why not OLS? — predictions can fall below zero
# ============================================================================

# OLS is a line — it must eventually cross 0. With a count outcome, that's
# nonsensical: negative occupations don't exist
augment(mod_ols) |>
  ggplot(aes(.fitted)) +
  geom_histogram(bins = 80, fill = "#022851", color = "white") +
  geom_vline(xintercept = 0, linetype = 2, color = "#B8611A", linewidth = 1) +
  labs(x = "OLS-predicted expected count", y = "municípios")

# how many predictions are below zero?
augment(mod_ols) |> filter(.fitted < 0) |> nrow()


# ---- The OLS slope is constant — implausible for counts ----

# OLS forces ∂E[Y]/∂X to be the same at E[Y] = 0.1 and at E[Y] = 20
coef(mod_ols)["landgini"]

# +10pp landgini changes the predicted count by the same amount regardless
# of the baseline — almost certainly wrong substantively
delta = 10
tibble(
  baseline_EY = c(0.1, 1, 5, 20),
  effect      = delta * coef(mod_ols)["landgini"]
)


# ============================================================================
# Section 3: The Poisson model
# ============================================================================

# ---- The Poisson distribution ----

# Y ~ Poisson(λ): single parameter λ controls both mean and variance
# E[Y] = Var(Y) = λ
crossing(k = 0:20, lambda = c(1, 4, 10)) |>
  mutate(p = dpois(k, lambda)) |>
  ggplot(aes(k, p)) +
  geom_col(fill = "#022851") +
  facet_wrap(~ paste("lambda =", lambda)) +
  labs(x = "k", y = "Pr(Y = k)")

# rpois(n, lambda): n draws from Poisson(λ)
rpois(10, lambda = 4)


# ---- Fit a Poisson regression ----

# log(λ) = Xβ → exp() takes any real number into (0, ∞), respecting λ > 0
# same architecture as logit, with a different link function
mod_pois = glm(occs ~ landgini + ext_pov + logpop + logarea,
               data = inv_clean,
               family = poisson(link = "log"))
summary(mod_pois)
# coefficients are on the log-rate scale — not raw counts


# ---- Way 1: exponentiate to get a rate ratio ----

# +1pp landgini multiplies the expected count by exp(β)
exp(coef(mod_pois)["landgini"])

# a 10-pp shift in landgini
exp(10 * coef(mod_pois)["landgini"])
# rate ratios are correct but feel hollow without a baseline count


# ---- Way 2: predict expected counts for a scenario ----

# vary landgini, hold controls at sample means
scenario = crossing(
  landgini = seq(0, 100, by = 5),
  logpop   = mean(inv_clean$logpop),
  ext_pov  = mean(inv_clean$ext_pov),
  logarea  = mean(inv_clean$logarea)
)

preds = predictions(mod_pois, newdata = scenario, type = "response")

preds |>
  as_tibble() |>
  dplyr::select(estimate, conf.low, conf.high, landgini)

# visualize: the curve bends upward — multiplicative response from the log link
ggplot(preds, aes(landgini, estimate)) +
  geom_ribbon(aes(ymin = conf.low, ymax = conf.high),
              alpha = 0.2, fill = "#022851") +
  geom_line(color = "#022851", linewidth = 1.4) +
  labs(x = "landgini (Gini, 0–100)",
       y = "Expected count of occupations",
       subtitle = "Controls held at sample means; 95% CI shown")


# ---- Way 3: marginal effect via calculus ----

# from λ = exp(Xβ), chain rule gives ∂λ/∂X = β · λ
# the marginal effect SCALES WITH λ — not constant across the data

b   = coef(mod_pois)
mns = c(mean(inv_clean$ext_pov), mean(inv_clean$logpop), mean(inv_clean$logarea))

tibble(landgini = c(55, 75, 95)) |>
  mutate(
    eta           = b["(Intercept)"] + b["landgini"] * landgini +
                    b["ext_pov"] * mns[1] + b["logpop"] * mns[2] +
                    b["logarea"] * mns[3],
    lambda        = exp(eta),
    beta_landgini = b["landgini"],
    ME_landgini   = beta_landgini * lambda
  )
# same β, three different MEs — driven entirely by changes in λ


# ---- ME of landgini across the data (slopes) ----

me_lg = slopes(mod_pois, variables = "landgini",
               newdata = datagrid(landgini = seq(20, 95, by = 5),
                                  logpop  = mean(inv_clean$logpop),
                                  ext_pov = mean(inv_clean$ext_pov),
                                  logarea = mean(inv_clean$logarea)))

ggplot(me_lg, aes(landgini, estimate)) +
  geom_ribbon(aes(ymin = conf.low, ymax = conf.high),
              alpha = 0.2, fill = "#B8611A") +
  geom_line(color = "#B8611A", linewidth = 1.2) +
  labs(x = "landgini", y = "ME of landgini on expected occs")


# ---- The AME — one number for the paper ----

# averages the per-observation ME across all municípios
avg_slopes(mod_pois, variables = "landgini")


# ---- When does Poisson = OLS? Saturated models ----

# binary X, no other covariates → both models compute group means → identical
inv_demo = inv_clean |>
  mutate(high_gini = as.numeric(landgini > median(landgini)))

mod_ols_simple  = lm(occs  ~ high_gini, data = inv_demo)
mod_pois_simple = glm(occs ~ high_gini, data = inv_demo, family = poisson)

tibble(high_gini = c(0, 1)) |>
  mutate(OLS     = predict(mod_ols_simple,  newdata = pick(everything())),
         Poisson = predict(mod_pois_simple, newdata = pick(everything()),
                           type = "response"))

# add controls back in → models are no longer saturated → predictions diverge
mod_ols_full  = lm(occs  ~ high_gini + ext_pov + logpop + logarea,
                   data = inv_demo)
mod_pois_full = glm(occs ~ high_gini + ext_pov + logpop + logarea,
                    data = inv_demo, family = poisson)

crossing(high_gini = c(0, 0.5, 1)) |>
  mutate(ext_pov = mean(inv_demo$ext_pov),
         logpop  = mean(inv_demo$logpop),
         logarea = mean(inv_demo$logarea)) |>
  mutate(OLS     = predict(mod_ols_full,  newdata = pick(everything())),
         Poisson = predict(mod_pois_full, newdata = pick(everything()),
                           type = "response"),
         diff    = Poisson - OLS)


# ============================================================================
# Section 4: MLE for Poisson — the likelihood under the hood
# ============================================================================

# Poisson log-likelihood (drop the log(Y!) constant — it doesn't depend on β):
#   ℓ(β) = Σ [ Y_i · X_i'β  −  exp(X_i'β) ]
# this is the function glm() climbs

b_mle = coef(mod_pois)

loglik_pois = function(b) {
  eta = b[1] + b[2] * inv_clean$landgini + b[3] * inv_clean$ext_pov +
        b[4] * inv_clean$logpop + b[5] * inv_clean$logarea
  sum(inv_clean$occs * eta - exp(eta))
}

# evaluate at a few candidate β vectors — the MLE has the highest log-lik
tibble(
  candidate = c("All zeros",
                "Intercept only at log(mean Y)",
                "MLE (the peak)",
                "Flipped signs"),
  log_lik   = c(loglik_pois(c(0, 0, 0, 0, 0)),
                loglik_pois(c(log(mean(inv_clean$occs)), 0, 0, 0, 0)),
                loglik_pois(b_mle),
                loglik_pois(-b_mle)) |> round(0)
)


# ============================================================================
# Section 5: Overdispersion
# ============================================================================

# Poisson assumes Var(Y|X) = E[Y|X] = λ. The Pearson residual standardizes
# (Y_i - λ̂_i) by the predicted SD √λ̂_i. If Poisson is right, the dispersion
# statistic φ̂ = Σr̂_i² / (n - k) ≈ 1
sum(residuals(mod_pois, type = "pearson")^2) / df.residual(mod_pois)
# φ̂ >> 1 → massive overdispersion (the norm for political event counts)


# ---- Fix 1: quasi-Poisson — keep the β̂, inflate the SEs by √φ̂ ----

mod_qpois = glm(occs ~ landgini + ext_pov + logpop + logarea,
                data = inv_clean, family = quasipoisson)
tidy(mod_qpois)


# ============================================================================
# Section 6: Negative Binomial — change the DGP
# ============================================================================

# NB unties the variance from the mean:
#   Var(Y) = μ + μ²/θ
# θ → ∞ collapses NB back to Poisson; small θ → strongly overdispersed

mod_nb = MASS::glm.nb(occs ~ landgini + ext_pov + logpop + logarea,
                      data = inv_clean)


# coefficients usually close across all three; SEs are larger under
# quasi-Poisson and NB (both acknowledge the extra variance, in different ways)
modelsummary(list("Poisson"       = mod_pois,
                  "Quasi-Poisson" = mod_qpois,
                  "NB"            = mod_nb),
             stars = TRUE,
             gof_omit = "IC|Log|Adj|F|RMSE")

# interpretation transfers exactly — log-rate units, exponentiate for rate
# ratios, ME = β · λ, all the marginaleffects machinery still works
avg_slopes(mod_nb, variables = "landgini")


# ============================================================================
# Recap: count-model toolkit
# ============================================================================

# | Outcome features                              | Model         | R call                |
# |-----------------------------------------------|---------------|-----------------------|
# | Counts, design identifies ATE                 | OLS + HC SE   | estimatr::lm_robust() |
# | Counts, Var(Y) = E[Y] holds                   | Poisson       | glm(family = poisson) |
# | Counts, Var(Y) > E[Y], keep Poisson β̂        | Quasi-Poisson | glm(family = quasipoisson) |
# | Counts, Var(Y) > E[Y] (the usual case)        | NB            | MASS::glm.nb()        |
#
# For most political-science count outcomes, NB is the right default.
