# ============================================================================
# POL 213 — Week 6: Duration Models
# ============================================================================

# ---- Setup ----

library(tidyverse)
library(juanr)
library(broom)
library(modelsummary)
library(marginaleffects)
library(survival)        # Surv(), survfit(), survreg(), coxph(), basehaz()
library(flexsurv)        # flexsurvreg() — gives CIs on predicted S(t|X)


# Prep `leader` (juanr): country-leader-year panel, 1840–2015.
# We collapse to one row per LEADER SPELL:
#   duration  — total years in office (max yr_office for that leader)
#   event     — 1 if the leader exited office during the sample,
#               0 if right-censored (still in office in 2015)
#   covariates — characteristics from the leader's first year in office
#
# Substantive question: which leaders survive longer in office?
# Focal predictor: `rebel` — did the leader come to power through armed
# rebellion (guerrilla, insurgency, war of national liberation)?

leader_spells = juanr::leader |>
  group_by(country, leader) |>
  summarise(
    gwcode      = first(gwcode),
    duration    = max(yr_office),
    end_year    = max(year),
    age_start   = first(age,         order_by = year),
    edu         = first(edu,         order_by = year),
    mil_service = first(mil_service, order_by = year),
    combat      = first(combat,      order_by = year),
    rebel       = first(rebel,       order_by = year),
    .groups     = "drop"
  ) |>
  mutate(event = as.integer(end_year < 2015)) |>   # 1 = exited; 0 = censored
  drop_na(duration, age_start, mil_service, combat, rebel, edu)


# ============================================================================
# Section 1: Time-to-event in political science — leader survival
# ============================================================================

# ---- The outcome ----

# Each row is one leader-spell. duration = years in office. event = exited (1)
# vs still in office at 2015 (0, right-censored).
nrow(leader_spells)
sum(leader_spells$event)              # number of observed exits
sum(1 - leader_spells$event)          # number of right-censored spells

# Censoring: Sogavare and Taraki appear to last the same amount of time, but
# Sogavare may still be in office while Taraki actually exited
leader_spells |> filter(leader %in% c("Sogavare", "Taraki"))

# Histogram of duration, faceted by event status
ggplot(leader_spells, aes(x = duration)) +
  geom_histogram(bins = 35, fill = "#022851", color = "white") +
  facet_wrap(~ event, scales = "free_y",
             labeller = as_labeller(c(`1` = "Exited", `0` = "Still in office"))) +
  labs(x = "duration (years in office)", y = "leader-spells")


# ---- OLS as a baseline ----

# OLS treats `duration` as a continuous outcome and ignores censoring. Three
# problems: (1) predicts negative durations, (2) heavy right skew, (3) no way
# to handle censored cases without bias.
mod_ols = lm(duration ~ age_start + mil_service + combat + rebel,
             data = leader_spells)
modelsummary(list("OLS on duration" = mod_ols),
             stars = TRUE, gof_omit = "IC|Log|Adj|F|RMSE")


# ============================================================================
# Section 2: Why not OLS? — censoring biases toward zero
# ============================================================================

# Three options for handling censoring with OLS, all biased:
#   A. Treat censored spells as if they ended at the last observation (clip)
#   B. Drop censored cases entirely
#   C. Log(duration) on uncensored only (helps skew, not censoring)


# ---- Simulate to see the bias ----

# DGP: same variable names as the leaders data; one binary predictor `rebel`,
# no controls. rebel = 1 doubles expected survival (5 vs 10 years). Censor at
# C = 7 years of follow-up.
set.seed(213)
n        = 2000
rebel    = rbinom(n, size = 1, prob = 0.5)
dur_true = rexp(n, rate = ifelse(rebel == 1, 0.1, 0.2))   # rate 0.1 → mean 10
C        = 7                                              # censoring threshold
event    = as.integer(dur_true <= C)
duration = pmin(dur_true, C)
sim      = tibble(rebel, dur_true, duration, event)

# True effect of rebel on E[T] = 10 - 5 = 5 years

# Three OLS fits — none correct
m_oracle = lm(dur_true ~ rebel, data = sim)                    # sees true T (oracle)
m_naive  = lm(duration ~ rebel, data = sim)                    # Option A: clip at C
m_drop   = lm(duration ~ rebel, data = filter(sim, event == 1))# Option B: drop censored

# Compare the three estimates against the truth (= 5)
bind_rows(
  tidy(m_oracle, conf.int = TRUE) |> mutate(method = "Oracle"),
  tidy(m_naive,  conf.int = TRUE) |> mutate(method = "Option A: clip at C"),
  tidy(m_drop,   conf.int = TRUE) |> mutate(method = "Option B: drop censored")
) |> filter(term == "rebel") |>
  select(method, estimate, conf.low, conf.high)

# Both options shrink the estimate toward 0. Right-censoring clips the right
# tail of every group; the longer-lived (rebel = 1) group has more density
# beyond C, so loses more from clipping → gap between groups narrows.


# ============================================================================
# Section 3: The survivor function and Kaplan-Meier
# ============================================================================

# Reframe the question: instead of "how long until?" (which censoring blocks
# for some leaders), ask "what fraction of leaders are still in office after
# t years?". That's the survivor function:
#   S(t) = Pr(T > t)

# ---- Kaplan-Meier estimator ----

# survfit(Surv(time, event) ~ 1) computes Ŝ(t) nonparametrically, handling
# censored cases by counting them in the at-risk denominator until they're
# censored, then dropping them without an event.
km_all = survfit(Surv(duration, event) ~ 1, data = leader_spells)

# Read off Ŝ(5): fraction of leaders still in office after 5 years
summary(km_all, times = 5)$surv

# Plot the curve with CIs
broom::tidy(km_all) |>
  ggplot(aes(x = time, y = estimate)) +
  geom_step(linewidth = 1, color = "#022851") +
  geom_ribbon(aes(ymin = conf.low, ymax = conf.high), alpha = 0.2,
              fill = "#022851") +
  scale_y_continuous(labels = scales::percent, limits = c(0, 1)) +
  labs(x = "years in office", y = expression(hat(S)(t)))


# ---- KM by group (rebel background vs. not) ----

km_grp = survfit(Surv(duration, event) ~ rebel, data = leader_spells)
broom::tidy(km_grp) |>
  mutate(rebel = ifelse(strata == "rebel=1", "Rebel background",
                        "No rebel background")) |>
  ggplot(aes(x = time, y = estimate, color = rebel, fill = rebel)) +
  geom_step(linewidth = 1) +
  geom_ribbon(aes(ymin = conf.low, ymax = conf.high), alpha = 0.15,
              color = NA) +
  scale_y_continuous(labels = scales::percent, limits = c(0, 1)) +
  labs(x = "years in office", y = expression(hat(S)(t)), color = NULL,
       fill = NULL)


# ============================================================================
# Section 4: Exponential MLE — the simplest parametric duration model
# ============================================================================

# Hazard rate h(t) = "rate of failure for those who've made it to time t".
# Exponential assumption: hazard is *constant over time* — h(t) = λ.
# Equivalent to mean tenure = 1/λ. Empirical estimate:
#   λ̂ = events / person-years at risk
n_events      = sum(leader_spells$event)
person_years  = sum(leader_spells$duration)
n_events / person_years                    # ≈ 0.17 per year → mean tenure ≈ 5.9 years


# ---- Fit an exponential model with covariates ----

# survreg uses an AFT (accelerated failure time) parameterization:
#   log T = X'β + ε   →   coefficients are on the log of expected survival time
# So exp(β) is the time ratio (multiplicative factor on E[T]).
mod_exp = survreg(Surv(duration, event) ~ age_start + mil_service + combat + rebel,
                  data = leader_spells, dist = "exponential")
modelsummary(list("Exponential" = mod_exp),
             stars = TRUE, gof_omit = "IC|Log|Adj|F|RMSE")


# ---- Way 1: exponentiate β̂ → time ratio (TR) ----

# TR > 1 means the covariate is associated with longer expected survival.
# Note: tidy.survreg ignores `exponentiate = TRUE`; do it manually:
broom::tidy(mod_exp, conf.int = TRUE) |>
  filter(term != "(Intercept)") |>
  mutate(across(c(estimate, conf.low, conf.high), exp)) |>
  select(term, time_ratio = estimate, conf.low, conf.high, p.value)


# ---- Way 2: predicted survival at chosen scenarios ----

# Two scenarios: typical leader with rebel = 0 vs rebel = 1
exp_scen = tibble(
  age_start   = mean(leader_spells$age_start),
  mil_service = mean(leader_spells$mil_service),
  combat      = mean(leader_spells$combat),
  edu         = "Tertiary",
  rebel       = c(0, 1)
)

# Scalar predicted E[T] at each scenario
predictions(mod_exp, newdata = exp_scen) |>
  as_tibble() |> select(rebel, estimate, conf.low, conf.high)

# Full predicted survival curves with CIs — refit with flexsurv (same MLE,
# but its summary() returns Ŝ(t|X) with confidence intervals)
mod_exp_fs = flexsurvreg(
  Surv(duration, event) ~ age_start + mil_service + combat + rebel,
  data = leader_spells, dist = "exp"
)
summary(mod_exp_fs, newdata = exp_scen, type = "survival",
        t = seq(0, 60, length.out = 121), tidy = TRUE) |>
  mutate(Group = ifelse(rebel == 0, "Non-rebel", "Rebel")) |>
  ggplot(aes(x = time, y = est, color = Group, fill = Group)) +
  geom_ribbon(aes(ymin = lcl, ymax = ucl), alpha = 0.15, color = NA) +
  geom_line(linewidth = 1) +
  scale_y_continuous(labels = scales::percent, limits = c(0, 1)) +
  labs(x = "years in office", y = expression(hat(S)(t)),
       color = NULL, fill = NULL)


# ---- Way 3: marginal effect on E[T] ----

# For exponential AFT: ∂E[T|X]/∂X_k = β_k · E[T|X] — same multiplicative form
# as Poisson (effect scales with the level of E[T])
avg_slopes(mod_exp, variables = "rebel")     # AME — one number for the paper

# ME varies across leaders (depends on each leader's predicted E[T])
slopes(mod_exp, variables = "rebel") |>
  as_tibble() |>
  ggplot(aes(x = estimate)) +
  geom_histogram(bins = 35, fill = "#022851", color = "white") +
  labs(x = "ME of rebel on E[T] (years), per leader", y = "leaders")


# ============================================================================
# Section 5: Weibull — let the hazard rise or fall with t
# ============================================================================

# Weibull adds a shape parameter p:
#   h(t) = λ · p · (λt)^{p-1}
# p = 1 → exponential. p > 1 → hazard rises. p < 1 → hazard falls.

mod_wb = survreg(Surv(duration, event) ~ age_start + mil_service + combat + rebel,
                 data = leader_spells, dist = "weibull")
modelsummary(list("Exponential" = mod_exp, "Weibull" = mod_wb),
             stars = TRUE, gof_omit = "IC|Log|Adj|F|RMSE")


# ---- Was constant hazard OK? ----

# survreg parameterizes Weibull as log T = Xβ + σW. The "Log(scale)" row in
# the regression table is log(σ̂); the Weibull shape is p̂ = 1/σ̂
1 / mod_wb$scale          # = p̂


# ---- Interpretation: same three-step workflow as exponential ----

# Way 1: exponentiate β̂ → time ratio (still AFT on log-time scale)
broom::tidy(mod_wb, conf.int = TRUE) |>
  filter(!term %in% c("(Intercept)", "Log(scale)")) |>
  mutate(across(c(estimate, conf.low, conf.high), exp)) |>
  select(term, time_ratio = estimate, conf.low, conf.high, p.value)

# Way 2: predicted survival at scenarios — same recipe with flexsurv
mod_wb_fs = flexsurvreg(
  Surv(duration, event) ~ age_start + mil_service + combat + rebel,
  data = leader_spells, dist = "weibull"
)
summary(mod_wb_fs, newdata = exp_scen, type = "survival",
        t = seq(0, 60, length.out = 121), tidy = TRUE) |>
  mutate(Group = ifelse(rebel == 0, "Non-rebel", "Rebel")) |>
  ggplot(aes(x = time, y = est, color = Group, fill = Group)) +
  geom_ribbon(aes(ymin = lcl, ymax = ucl), alpha = 0.15, color = NA) +
  geom_line(linewidth = 1) +
  scale_y_continuous(labels = scales::percent, limits = c(0, 1)) +
  labs(x = "years in office", y = expression(hat(S)(t)),
       color = NULL, fill = NULL)

# Way 3: same multiplicative ME formula as exponential
avg_slopes(mod_wb, variables = "rebel")


# ============================================================================
# Section 6: Cox proportional hazards — don't pick a hazard shape
# ============================================================================

# Cox lets the baseline hazard h_0(t) be anything — flat, monotone, U-shaped,
# lumpy — and only models how covariates *scale* it:
#   h(t | X) = h_0(t) · exp(X'β)
# The PH assumption: covariates multiply the baseline by a constant factor
# that doesn't change with t.

mod_cox = coxph(Surv(duration, event) ~ age_start + mil_service + combat + rebel,
                data = leader_spells)


# ---- Side by side with Exp and Weibull ----

# Raw coefficients: Cox is on log-hazard scale (no intercept — h_0(t) is
# unspecified). AFT (Exp/Weibull) and PH (Cox) coefficients have OPPOSITE
# signs (positive AFT = longer survival = lower hazard = negative PH).
modelsummary(list("Exponential" = mod_exp, "Weibull" = mod_wb, "Cox" = mod_cox),
             stars = TRUE, gof_omit = "IC|Log|Adj|F|RMSE",
             coef_omit = "Log\\(scale\\)")

# Exponentiated: time ratios (Exp/Weibull) vs hazard ratios (Cox).
# Reciprocally related under PH: TR ≈ 1/HR.
modelsummary(list("Exp (TR)" = mod_exp, "Weibull (TR)" = mod_wb,
                  "Cox PH (HR)" = mod_cox),
             stars = TRUE, gof_omit = "IC|Log|Adj|F|RMSE",
             coef_omit = "Log\\(scale\\)",
             exponentiate = TRUE)


# ---- Predicted survival curves at scenarios ----

# survfit(coxph_obj, newdata = ...) returns predicted S(t|X) with CIs computed
# via delta method on the cumulative hazard. The curves are step functions
# (Cox's baseline is the nonparametric Breslow estimator).
sf_scen = survfit(mod_cox, newdata = exp_scen)

tibble(
  time  = rep(sf_scen$time, 2),
  S     = c(sf_scen$surv[, 1],  sf_scen$surv[, 2]),
  S_lo  = c(sf_scen$lower[, 1], sf_scen$lower[, 2]),
  S_hi  = c(sf_scen$upper[, 1], sf_scen$upper[, 2]),
  Group = factor(rep(c("Non-rebel", "Rebel"), each = length(sf_scen$time)))
) |>
  ggplot(aes(x = time, y = S, color = Group, fill = Group)) +
  geom_ribbon(aes(ymin = S_lo, ymax = S_hi), alpha = 0.15, color = NA) +
  geom_step(linewidth = 1) +
  scale_y_continuous(labels = scales::percent, limits = c(0, 1)) +
  labs(x = "years in office", y = expression(hat(S)(t)),
       color = NULL, fill = NULL)


# ============================================================================
# Section 7: Competing risks — not all exits are the same
# ============================================================================

# Same observed exit at year 8 might be a term-limited departure (regular)
# or a coup-driven ouster (irregular). Standard Cox averages across causes;
# cause-specific Cox decomposes by cause type.


# ---- Code exit type using a coarse democracy proxy ----

# In established post-1950 democracies, exits are *typically* regular
# (term limits, election losses); in non-democracies, *typically* irregular
# (coups, in-office death). Coarse — for teaching only. Real research uses
# Archigos exit codes.
established_democracies_gw = c(
  2,   200, 220, 255, 325, 20,  900, 920, 740, 205, 210, 211,
  225, 305, 350, 375, 380, 385, 390, 395, 230, 235
)

leader_cr = leader_spells |>
  mutate(
    is_democracy = gwcode %in% established_democracies_gw & end_year >= 1950,
    exit_type    = case_when(
      event == 0    ~ "censored",
      is_democracy  ~ "regular",     # democracy → term limit / election loss
      !is_democracy ~ "irregular"    # autocracy → coup / in-office death
    )
  )

leader_cr |> count(exit_type)


# ---- Cause-specific Cox: fit one model per cause ----

# Treat other-cause exits as CENSORED. Why? If a democratic leader exits via
# term limit, they can no longer exit by coup — for the irregular-exit model,
# that observation tells us only "they made it t years without an irregular
# exit." That's exactly censoring.

mod_cox_reg = coxph(
  Surv(duration, exit_type == "regular") ~ age_start + mil_service + combat + rebel,
  data = leader_cr
)

mod_cox_irr = coxph(
  Surv(duration, exit_type == "irregular") ~ age_start + mil_service + combat + rebel,
  data = leader_cr
)

# Refit standard Cox on the same dataset for fair side-by-side comparison
mod_cox_lumped = coxph(
  Surv(duration, event) ~ age_start + mil_service + combat + rebel,
  data = leader_cr
)


# ---- Same data, three stories ----

# The lumped HR for `rebel` is a weighted average of the cause-specific HRs.
# Cause-specific decomposition reveals *whether* rebel matters for both
# exit types or just one.
#
# Note: the slide deck renders this same table with kableExtra styling, so
# the formatting looks slightly different there — but the coefficients,
# standard errors, and gof statistics are identical. Use `output = "markdown"`
# for a console-friendly table.
modelsummary(
  list("Standard Cox (lumped)" = mod_cox_lumped,
       "Regular exits"          = mod_cox_reg,
       "Irregular exits"        = mod_cox_irr),
  stars = TRUE,
  gof_omit = "IC|Log|Adj|F|RMSE",
  exponentiate = TRUE,
  output = "markdown"
)


# ============================================================================
# Recap: duration-model toolkit
# ============================================================================

# | Question you're asking                                  | Right tool                  | R                              |
# |---------------------------------------------------------|------------------------------|--------------------------------|
# | What does survival look like in my data, by group?      | Kaplan-Meier (nonparametric) | survfit(Surv(t, e) ~ X)        |
# | What's the predicted survival time at scenario X?       | Exponential / Weibull (AFT)  | survreg(..., dist = "weibull") |
# | How does X affect the hazard, holding shape free?       | Cox PH (partial likelihood)  | coxph(...)                     |
# | Are different *causes* of exit driven by different things? | Cause-specific Cox        | coxph(Surv(t, type == "k") ~ ...) |
#
# For most political-science effect-of-X questions, Cox is the right default.
# But if your question is descriptive or predictive, reach for the right
# neighbor — these aren't a ladder, they're a tool drawer.
