#!/usr/bin/env python3
"""The mean hides the tail, DDIA 2e Chapter 2.

"Average response time" feels like the typical experience. It is not. For a
right-skewed (lognormal) latency distribution, the arithmetic mean is dragged
UP by the long tail, so:

  - it sits well ABOVE the median (the actual midpoint), and
  - it sits BELOW a large majority of requests in rank -- most requests are
    FASTER than "average" -- yet
  - it is still many times SMALLER than a high percentile like p99.

So a single "average" both overstates the typical request and understates the
tail. We measure exactly where the mean falls in the sorted request order, and
how far p99 sits above it.

Same lognormal as tail_latency.py (median ~10 ms, p99 ~100 ms). Pure standard
library, fixed seed -> reproducible.
"""
import math
import random
from bisect import bisect_left

SEED = 42
POOL = 1_000_000  # individual request latency samples

# lognormal tuned so median = exp(mu) = 10 ms, p99 = exp(mu + 2.326*sigma) = 100 ms
MU = math.log(10)
SIGMA = (math.log(100) - MU) / 2.326  # ~0.99


def pct(sorted_vals, p):
    """The p-th percentile (0..100) of an already-sorted list."""
    if p <= 0:
        return sorted_vals[0]
    if p >= 100:
        return sorted_vals[-1]
    k = (len(sorted_vals) - 1) * (p / 100)
    lo = math.floor(k)
    hi = math.ceil(k)
    if lo == hi:
        return sorted_vals[int(k)]
    return sorted_vals[lo] * (hi - k) + sorted_vals[hi] * (k - lo)


def main():
    rng = random.Random(SEED)

    pool = [rng.lognormvariate(MU, SIGMA) for _ in range(POOL)]
    pool.sort()

    mean = sum(pool) / len(pool)
    p50 = pct(pool, 50)
    p90 = pct(pool, 90)
    p99 = pct(pool, 99)
    p999 = pct(pool, 99.9)

    # Where does the mean land in the sorted order? What % of requests are
    # faster than the "average" request?
    rank = bisect_left(pool, mean)
    faster_than_mean = 100 * rank / len(pool)

    print(f"request latency (n={POOL:,}, lognormal mu={MU:.3f} sigma={SIGMA:.3f}):")
    print(f"    mean = {mean:7.1f} ms")
    print(f"    p50  = {p50:7.1f} ms   (median -- the true midpoint)")
    print(f"    p90  = {p90:7.1f} ms")
    print(f"    p99  = {p99:7.1f} ms")
    print(f"    p999 = {p999:7.1f} ms")
    print()
    print("what the single 'average' number hides:")
    print(f"    mean / median          = {mean / p50:4.2f}x   (mean is pulled above the midpoint)")
    print(f"    % of requests faster than the mean = {faster_than_mean:4.1f}%")
    print(f"    p99  / mean            = {p99 / mean:4.2f}x   (the tail the mean doesn't show)")
    print(f"    p999 / mean            = {p999 / mean:4.2f}x")
    print()

    # Analytic checks for a lognormal(mu, sigma):
    #   mean          = exp(mu + sigma^2/2)
    #   P(X < mean)   = Phi(sigma/2)      -> rank of the mean
    analytic_mean = math.exp(MU + SIGMA * SIGMA / 2)
    phi = 0.5 * (1 + math.erf((SIGMA / 2) / math.sqrt(2)))
    print(f"analytic check: mean = exp(mu + sigma^2/2) = {analytic_mean:.1f} ms")
    print(f"analytic check: P(X < mean) = Phi(sigma/2) = {100 * phi:.1f}%")


if __name__ == "__main__":
    main()