# ============================================================================
# POL 213 — Week 4: Maximum Likelihood Estimation
# ============================================================================

# ---- Setup ----

library(tidyverse)
library(juanr)
library(broom)
library(modelsummary)
library(marginaleffects)

# same 2020 CES working dataset as Week 3: binary outcomes + Protestant indicator
bot_clean = bot |>
  filter(
    sex %in% c("Male", "Female"),
    aca  %in% c("Support", "Oppose"),
    wall %in% c("Support", "Oppose"),
    !religion %in% c("skipped", "not asked"),
    !is.na(religion),
    !is.na(age)
  ) |>
  mutate(
    protestant = factor(
      ifelse(religion == "Protestant", "Protestant", "Non-Protestant"),
      levels = c("Non-Protestant", "Protestant")
    ),
    wall = as.numeric(wall == "Support")
  ) |>
  select(protestant, wall, age)


# ============================================================================
# Section 1: the logit workflow — fit, convert, report
# ============================================================================

# ---- Fit the model ----

# wall support on Protestant + age; glm's default binomial link is logit
mod_wall = glm(wall ~ protestant + age, data = bot_clean,
               family = binomial(link = "logit"))
summary(mod_wall)
# coefficients are on the log-odds scale — not probability changes


# ---- Step 2: plug in a scenario, get probabilities ----

scenario = tibble(
  protestant = factor(c("Non-Protestant", "Protestant"),
                      levels = c("Non-Protestant", "Protestant")),
  age        = median(bot_clean$age, na.rm = TRUE)
)

# predictions() on the response scale: probabilities with CI by default
predictions(mod_wall, newdata = scenario, type = "response") |>
  as_tibble() |>
  select(protestant, age, estimate, conf.low, conf.high)


# ---- Same thing by hand: get eta first, then apply the sigmoid ----

# step 1: plug coefficients + scenario values into the linear predictor
#   eta = beta_0 + beta_1 * Protestant + beta_2 * age   (log-odds scale)
# step 2: push eta through the sigmoid
#   p = 1 / (1 + exp(-eta))                             (probability scale)
coefs   = coef(mod_wall)
age_med = median(bot_clean$age, na.rm = TRUE)

scenario |>
  mutate(
    eta = coefs["(Intercept)"] +
          coefs["protestantProtestant"] * (protestant == "Protestant") +
          coefs["age"]                  * age,
    p   = 1 / (1 + exp(-eta))
  )


# ---- Three equivalent ways to get predicted probabilities ----

# 1. predictions() on the link scale returns eta (log-odds)
predictions(mod_wall, newdata = scenario, type = "link") |>
  as_tibble() |> select(protestant, age, estimate, conf.low, conf.high)

# 2. predictions() on the response scale returns p (automatic conversion)
predictions(mod_wall, newdata = scenario, type = "response") |>
  as_tibble() |> select(protestant, age, estimate, conf.low, conf.high)

# 3. link scale + manual sigmoid — same answer as #2
predictions(mod_wall, newdata = scenario, type = "link") |>
  as_tibble() |>
  mutate(p = 1 / (1 + exp(-estimate))) |>
  select(protestant, age, eta = estimate, p)


# ---- Step 3: marginal effects differ at every age ----

# the ME of being Protestant depends on where you are on the sigmoid;
# comparisons() computes the contrast (Protestant vs Non-Protestant) with CI
comparisons(mod_wall, variables = "protestant",
            newdata = datagrid(age = c(25, 45, 65)))


# ---- Step 4: the AME is the one-number summary ----

# avg_comparisons() averages the per-observation ME across the whole sample
avg_comparisons(mod_wall, variables = "protestant")
# headline: "Protestants are ~16 percentage points more likely to support the wall"


# ---- Step 5: visualize the effect ----

preds = predictions(mod_wall, newdata = scenario, type = "response")

ggplot(preds, aes(y = protestant, x = estimate,
                  xmin = conf.low, xmax = conf.high,
                  color = protestant)) +
  geom_pointrange(linewidth = 1.2, size = 0.9) +
  scale_color_manual(values = c("Non-Protestant" = "#022851",
                                "Protestant"     = "#B8611A"),
                     guide = "none") +
  scale_x_continuous(labels = scales::percent) +
  labs(x = "Pr(support wall)", y = NULL,
       title = "The jump from Non-Protestant to Protestant is the effect")


# ============================================================================
# Section 2: the puzzle — why does the effect of X depend on Z
#                        even without an interaction term?
# ============================================================================

# ---- Chain rule by hand ----

# on the probability scale, dP/dX = p(1-p) * beta_1
# so the effect changes across ages even without Protestant:age in the model —
# the scaling factor p(1-p) varies with age
tibble(age = c(25, 45, 65), protestant = 0) |>
  mutate(
    eta     = coefs["(Intercept)"] +
              coefs["protestantProtestant"] * protestant +
              coefs["age"]                  * age,
    p       = 1 / (1 + exp(-eta)),
    scaling = p * (1 - p),
    ME      = coefs["protestantProtestant"] * scaling
  )
# same beta_1, three different MEs — driven entirely by sigmoid curvature


# ---- Fit with a product term (genuine interaction) ----

# now age moderates the Protestant effect on the log-odds scale itself —
# a "second source" of Z-dependence, stacked on top of the curvature effect
mod_wall_int = glm(wall ~ protestant * age, data = bot_clean,
                   family = binomial(link = "logit"))
tidy(mod_wall_int)


# ---- Compare ME curves across the two models ----

# additive model: curvature-only dependence on age
me_add = comparisons(mod_wall, variables = "protestant",
                     newdata = datagrid(age = seq(20, 85, by = 1))) |>
  as_tibble() |> mutate(model = "Additive")

# interaction model: curvature + structural dependence on age
me_int = comparisons(mod_wall_int, variables = "protestant",
                     newdata = datagrid(age = seq(20, 85, by = 1))) |>
  as_tibble() |> mutate(model = "Interaction")

bind_rows(me_add, me_int) |>
  ggplot(aes(age, estimate, color = model, fill = model)) +
  geom_ribbon(aes(ymin = conf.low, ymax = conf.high),
              alpha = 0.15, color = NA) +
  geom_line(linewidth = 1.2) +
  geom_hline(yintercept = 0, linetype = 3, color = "grey50") +
  scale_color_manual(values = c("Additive"    = "#022851",
                                "Interaction" = "#B8611A")) +
  scale_fill_manual(values  = c("Additive"    = "#022851",
                                "Interaction" = "#B8611A")) +
  labs(x = "Age", y = "ME of being Protestant on Pr(wall)",
       color = NULL, fill = NULL)


# ============================================================================
# Section 3: MLE under the hood
# ============================================================================

# ---- OLS has a closed-form solution: (X'X)^-1 X'Y ----

# plug in your data, get the answer — no iteration needed
X_mat    = cbind(1, bot_clean$age)
y_vec    = bot_clean$wall
beta_ols = solve(t(X_mat) %*% X_mat) %*% t(X_mat) %*% y_vec
beta_ols

# matches lm() exactly
coef(lm(wall ~ age, data = bot_clean))


# ---- Logit has no closed form — we have to search ----

# simplified setup: wall ~ protestant (no age); fix beta_0 at its MLE,
# search for beta_1 by evaluating the log-likelihood at candidate values

X = as.numeric(bot_clean$protestant == "Protestant")
Y = bot_clean$wall
b0_hat = coef(glm(Y ~ X, family = binomial))[1]

# log-likelihood as a function of a single parameter beta_1
log_lik = function(b1) {
  p = 1 / (1 + exp(-(b0_hat + b1 * X)))
  sum(Y * log(p) + (1 - Y) * log(1 - p))
}

# evaluate at a grid of beta_1 values
tibble(b1 = c(-1, -0.5, 0, 0.3, 0.7, 1.5)) |>
  mutate(log_lik = map_dbl(b1, log_lik))
# some beta_1 values make the data more probable than others —
# MLE picks the one that makes it most probable


# ---- The log-likelihood is a curve with one peak ----

b1_grid  = seq(-1.5, 2.5, length.out = 200)
ll_curve = tibble(b1 = b1_grid, ll = map_dbl(b1_grid, log_lik))
b1_mle   = coef(glm(Y ~ X, family = binomial))[2]

ggplot(ll_curve, aes(b1, ll)) +
  geom_line(color = "#022851", linewidth = 1.3) +
  geom_vline(xintercept = b1_mle, linetype = 2, color = "#B8611A") +
  labs(x = expression(beta[1]), y = expression(ell(beta[1])),
       title = "One peak, and we want to find it")


# ---- Gradient: one nudge tells us direction ----

# to find which way to step, evaluate ell twice — once at current beta_1,
# once a tiny step to the right. Slope > 0 means step right, < 0 means left
b1_0    = -0.5
epsilon = 0.01

tibble(b1 = c(b1_0, b1_0 + epsilon),
       ll = c(log_lik(b1_0), log_lik(b1_0 + epsilon))) |>
  mutate(slope = (ll - lag(ll)) / epsilon)


# ---- Hessian: how fast the slope itself is changing ----

# three evaluations of ell → two slopes → one Hessian
# big |Hessian| = curve bends sharply = take a small step
# small |Hessian| = curve is broad = take a big step
tibble(
  b1 = c(b1_0, b1_0 + epsilon, b1_0 + 2 * epsilon),
  ll = c(log_lik(b1_0),
         log_lik(b1_0 + epsilon),
         log_lik(b1_0 + 2 * epsilon))
) |>
  mutate(slope   = (ll - lag(ll))       / epsilon,
         hessian = (slope - lag(slope)) / epsilon)


# ---- Newton-Raphson: combine gradient + Hessian to take one step ----

# beta_new = beta_old - slope / hessian
# glm() does this automatically; the "Fisher Scoring iterations: 4" output
# means it took 4 of these steps before converging
coef(glm(Y ~ X, family = binomial))


# ---- MLE recovers the true beta in simulation ----

# ground-truth beta = 1.2, generate binary data, fit glm, check estimate
set.seed(213)
n         = 500
X_sim     = rnorm(n)
beta_true = 1.2
Y_sim     = rbinom(n, 1, 1 / (1 + exp(-beta_true * X_sim)))

mod_demo = glm(Y_sim ~ X_sim - 1, family = binomial)
tidy(mod_demo)
# estimate should be close to 1.2


# ---- OLS is MLE under Gaussian errors ----

# if you assume Y ~ Normal, maximizing the likelihood reduces to minimizing SSR
# so glm(..., family = gaussian) and lm() return identical coefficients
coef(glm(mpg ~ hp + wt, data = mtcars, family = gaussian()))
coef(lm (mpg ~ hp + wt, data = mtcars))


# ---- Standard errors come from the curvature at the peak ----

# Fisher information = -ell''(beta_hat); more curvature = smaller SE
# same three-evaluations trick, just applied AT the MLE instead of at a guess
tibble(
  b1 = c(b1_mle, b1_mle + epsilon, b1_mle + 2 * epsilon),
  ll = c(log_lik(b1_mle),
         log_lik(b1_mle + epsilon),
         log_lik(b1_mle + 2 * epsilon))
) |>
  mutate(slope   = (ll - lag(ll))       / epsilon,
         hessian = (slope - lag(slope)) / epsilon,
         fisher  = -hessian,           # Fisher info = -Hessian
         se      = 1 / sqrt(fisher))   # SE = 1 / sqrt(I)

# compare to glm's SE — close but not exact, because the by-hand version
# profiles over beta_0 (treats it as known) while glm inverts the full
# 2x2 Fisher information matrix
sqrt(diag(vcov(glm(Y ~ X, family = binomial))))


# ============================================================================
# Section 4: when logit breaks — perfect separation
# ============================================================================

# ---- A toy dataset where one predictor splits Y perfectly ----

# 40 post-communist countries; every EU-joiner transitioned, every
# non-joiner didn't. joined_eu is a perfect predictor of democratic
set.seed(213)
transitions = tibble(
  country    = paste0("C", 1:40),
  joined_eu  = c(rep(1, 15), rep(0, 25)),
  democratic = c(rep(1, 15), rep(0, 25))
)

# perfect separation is obvious in the crosstab
table(transitions$joined_eu, transitions$democratic)


# ---- glm returns absurd coefficients and SEs (and often DOESN'T warn) ----

mod_sep = glm(democratic ~ joined_eu, data = transitions,
              family = binomial(link = "logit"))
summary(mod_sep)

# at n = 40, the deviance flattens below the convergence threshold before
# numerical precision breaks; glm reports converged = TRUE WITHOUT firing
# the usual "fitted probabilities numerically 0 or 1" warning
mod_sep$converged
mod_sep$iter

# huge beta + absurd SE is the real signal, not the warning
coef(mod_sep)["joined_eu"]
sqrt(diag(vcov(mod_sep)))["joined_eu"]


# ---- Why: the log-likelihood has no finite maximum ----

# with perfect separation, a larger beta always fits the data better,
# so the log-likelihood climbs monotonically toward 0 as beta -> infinity
loglik_logit = function(beta, X, Y) {
  eta = X * beta
  sum(Y * eta - log1p(exp(eta)))
}

# a well-behaved case (for comparison) and a separated case
set.seed(213)
X_good = rnorm(100)
Y_good = rbinom(100, 1, 1 / (1 + exp(-0.8 * X_good)))

X_bad  = rnorm(100)
Y_bad  = as.numeric(X_bad > 0)   # perfectly separated by X = 0

beta_grid = seq(-2, 30, length.out = 400)
tibble(
  beta = rep(beta_grid, 2),
  ll   = c(map_dbl(beta_grid, \(b) loglik_logit(b, X_good, Y_good)),
           map_dbl(beta_grid, \(b) loglik_logit(b, X_bad,  Y_bad))),
  case = rep(c("Well-behaved data", "Perfectly separated"),
             each = length(beta_grid))
) |>
  ggplot(aes(beta, ll, color = case)) +
  geom_line(linewidth = 1.3) +
  geom_hline(yintercept = 0, linetype = 3, color = "grey50") +
  scale_color_manual(values = c("Well-behaved data"   = "#022851",
                                "Perfectly separated" = "#B8611A")) +
  labs(x = expression(beta), y = "Log-likelihood",
       color = NULL,
       title = "A peak vs. a ramp",
       subtitle = "Orange keeps climbing --- its maximum is at infinity")

# takeaway: if glm gives you a huge beta and an absurd SE (with or without
# a warning), check your data for separation before reaching for a
# technical fix. Separation should make you question your theory first,
# your data second, and whether quant methods fit your question third.
