"""osrs-poll-check.py - re-count an Old School RuneScape poll the way the
polling charter counts it, and re-read the other public figures behind the
Old School RuneScape article (published 11 October 2026).

Jagex publishes every poll's results at oldschool.runescape.com/polls/<year>/<id>:
for each question, the Yes and No shares with their vote counts, and a count
of players who skipped it. The polling charter says a feature poll passes when
it "has achieved over 70% yes votes when it closes". This script reads a poll
page, recomputes each question's Yes share from the counts (Yes divided by
Yes plus No: skipped ballots are left out, which is how the published
percentages work out), marks each question passed or failed against the 70%
line, and for a failed question prints how many No voters would have had to
vote Yes instead for it to pass. It also prints the live player count from the
game's home page, the Steam membership prices in three currencies and the
Steam review summary. Nothing here needs a key or an account.

What it reads:

  Jagex polls     oldschool.runescape.com/polls/latest (the newest poll's id),
                  then oldschool.runescape.com/polls/<year>/<id> for each poll
  Jagex home      oldschool.runescape.com, the "There are currently N people
                  playing!" line
  Steam           store.steampowered.com/api/appdetails for the 1-month (app
                  1474630) and 12-month (app 1401462) memberships, cc=us/gb/de,
                  and store.steampowered.com/appreviews/1343370 (all languages)

Robots rules. Before its first call to a host the script reads that host's
robots.txt and skips the host, saying why, when the file cannot be read, when
the server refuses the script, or when any group in the file closes the page,
whichever crawler the group names. On 11 October 2026 all three hosts allowed
these pages. The script does not call api.steampowered.com, whose robots.txt
closes every path to every crawler, so it prints no Steam player count. Every
call names this script in its User-Agent header, and calls to one host are at
least two seconds apart, or as far apart as the host's Crawl-delay asks.

Standard library only (urllib, json, re, html, time, sys). Python 3.9 or newer.

    python osrs-poll-check.py                 the newest poll, plus 2026/1768 and 2026/1764
    python osrs-poll-check.py 2026/1764       one or more polls by year/id
"""
import html
import json
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

UA = "osrs-poll-check/1.0 (+https://blastproc.com/code/osrs-poll-check.py; reader re-check script)"
HEADERS = {"User-Agent": UA, "Accept": "text/html,application/json;q=0.9,*/*;q=0.5"}
THRESHOLD = 70.0            # the polling charter: "over 70% yes votes"
DEFAULT_POLLS = ["2026/1768", "2026/1764"]   # the RuneFest 2026 Summit Poll; the Bronzeman poll
APP = 1343370
DLC = {1474630: "1-month membership", 1401462: "12-month membership + soundtrack"}

# What the article printed on 11 October 2026, for comparison.
PRINTED = {
    "players": "110,335 at 14:18 UTC on 11 October 2026",
    1474630: {"us": "$14.99", "gb": "£10.99", "de": "13,49€"},
    1401462: {"us": "$131.88", "gb": "£95.88", "de": "119,88€"},
    "reviews": "20,917 reviews, 17,326 positive",
}
SKIPPED = {}   # url -> why the robots rules ruled it out

# ------------------------------------------------------------------ robots.txt, read before the first call to a host
MIN_GAP = 2.0
_ROBOTS = {}   # host -> (groups or None, why the host is skipped, seconds between calls)
_LAST = {}


def _wait(host):
    gap = _ROBOTS.get(host, (None, None, MIN_GAP))[2]
    if host in _LAST:
        time.sleep(max(0.0, _LAST[host] + gap - time.monotonic()))
    _LAST[host] = time.monotonic()


def _parse(text):
    """robots.txt -> ([(agent names, [(allow?, pattern), ...]), ...], the longest Crawl-delay in the file)."""
    groups, gap, in_agents = [], 0.0, False
    for line in text.splitlines():
        key, colon, value = line.split("#", 1)[0].partition(":")
        if not colon:
            continue
        key, value = key.strip().lower(), value.strip()
        if key == "user-agent":
            if not in_agents:
                groups.append(([], []))
            groups[-1][0].append(value)
            in_agents = True
            continue
        in_agents = False
        if key in ("allow", "disallow") and groups and value:
            groups[-1][1].append((key == "allow", value))
        elif key == "crawl-delay":
            try:
                gap = max(gap, float(value))
            except ValueError:
                pass
    return groups, gap


def _rules(host):
    """Read https://<host>/robots.txt once per run. A file that cannot be read, or a server that refuses this
    script, rules the host out; a 404 or another 4xx means the host sets no rules."""
    if host not in _ROBOTS:
        _wait(host)
        try:
            req = urllib.request.Request(f"https://{host}/robots.txt", headers=HEADERS)
            with urllib.request.urlopen(req, timeout=60) as r:
                status, body = r.status, r.read()
        except urllib.error.HTTPError as e:
            status, body = e.code, b""
        except Exception as e:  # noqa: BLE001  (no answer or a dropped connection: the file cannot be read)
            status, body = None, str(e).encode()
        webpage = bool(re.search(rb"(?i)<!doctype|<html", body[:2000])) and not re.search(rb"(?im)^\s*user-agent\s*:", body)
        if status == 200 and not webpage:
            groups, gap = _parse(body.decode("utf-8", "replace"))
            _ROBOTS[host] = (groups, None, max(MIN_GAP, gap))
        elif status in (401, 403):
            _ROBOTS[host] = (None, f"the server refuses this script (HTTP {status} for robots.txt)", MIN_GAP)
        elif status is not None and 400 <= status < 500 and status != 429:
            _ROBOTS[host] = ([], None, MIN_GAP)
        else:
            why = body.decode("utf-8", "replace")[:80] if status is None else f"HTTP {status}" + (", a web page in its place" if webpage else "")
            _ROBOTS[host] = (None, f"its robots.txt could not be read ({why})", MIN_GAP)
    return _ROBOTS[host]


def _closed(rules, path):
    """True when the longest pattern matching the path is a Disallow (an Allow of equal length wins)."""
    best = (0, True)
    for allow, pattern in rules:
        rx = re.escape(pattern.rstrip("$")).replace(r"\*", ".*") + ("$" if pattern.endswith("$") else "")
        if re.match(rx, path) and (len(pattern), allow) > best:
            best = (len(pattern), allow)
    return not best[1]


def robots_block(url):
    parts = urllib.parse.urlsplit(url)
    groups, why, _ = _rules(parts.hostname)
    if groups is None:
        return f"{parts.hostname}: {why}"
    path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
    for agents, rules in groups:
        if _closed(rules, path):
            who = "every crawler" if "*" in agents else "the crawlers it names (" + ", ".join(agents) + ")"
            return f"{parts.hostname}: robots.txt closes {parts.path} to {who}"
    return None


def fetch(url):
    """(final url, text) or None when the robots rules rule the url out (the reason goes to SKIPPED)."""
    why = robots_block(url)
    if why:
        SKIPPED[url] = why
        return None
    _wait(urllib.parse.urlsplit(url).hostname)
    req = urllib.request.Request(url, headers=HEADERS)
    with urllib.request.urlopen(req, timeout=60) as r:
        raw = r.read()
        final = r.geturl()
    if urllib.parse.urlsplit(final).hostname != urllib.parse.urlsplit(url).hostname and robots_block(final):
        SKIPPED[url] = robots_block(final)
        return None
    try:
        return final, raw.decode("utf-8")
    except UnicodeDecodeError:
        return final, raw.decode("cp1252", "replace")


def page_lines(markup):
    """A web page as its visible lines of text."""
    t = re.sub(r"(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", markup)
    t = re.sub(r"(?i)<br\s*/?>|</p>|</li>|</h\d>|</div>|</span>", "\n", t)
    t = html.unescape(re.sub(r"<[^>]+>", " ", t))
    return [" ".join(line.split()) for line in t.splitlines() if line.strip()]


# ------------------------------------------------------------------ polls
def newest_poll():
    got = fetch("https://oldschool.runescape.com/polls/latest")
    if got is None:
        return None
    ids = re.findall(r'href="(20\d\d/\d+)"', got[1])
    return ids[0] if ids else None


def read_poll(year_id):
    """-> (title, dates, total votes, [(question, yes, no, skipped, printed yes %)]) or None."""
    got = fetch(f"https://oldschool.runescape.com/polls/{year_id}")
    if got is None:
        return None
    lines = page_lines(got[1])
    title = lines[0].split(" | ")[0] if lines else year_id
    total = next((int(m.group(1).replace(",", "")) for m in (re.match(r"Total Votes: ([\d,]+)", x) for x in lines) if m), None)
    dates = next((x for x in lines if re.match(r"\d\d \w+ \d{4} - \d\d \w+ \d{4}$", x)), "")
    questions = []
    for i, line in enumerate(lines):
        if line != "Most Popular Option":
            continue
        # the question is the nearest line above that asks something (a note may follow it)
        q = next((lines[j] for j in range(i - 1, max(i - 4, 0), -1) if "?" in lines[j]), lines[i - 1])
        block = " ".join(lines[i + 1:i + 9])
        m = re.match(r"(?:Yes|No) ([\d.]+)% \( ([\d,]+) \) (?:Yes|No) ([\d.]+)% \( ([\d,]+) \) Skipped \( ([\d,]+) \)", block)
        if not m:
            continue
        first_is_yes = lines[i + 1].startswith("Yes")
        a, b = int(m.group(2).replace(",", "")), int(m.group(4).replace(",", ""))
        yes, no = (a, b) if first_is_yes else (b, a)
        printed = float(m.group(1) if first_is_yes else m.group(3))
        questions.append((q, yes, no, int(m.group(5).replace(",", "")), printed))
    return title, dates, total, questions


def votes_to_flip(yes, no):
    """How many No voters would have to vote Yes instead for Yes to be over 70% of Yes plus No."""
    n = yes + no
    need = int(n * THRESHOLD / 100) + 1          # the smallest Yes count strictly over 70%
    return max(0, need - yes)


def show_poll(year_id):
    got = read_poll(year_id)
    if got is None:
        print(f"poll {year_id}: not read ({SKIPPED.get('https://oldschool.runescape.com/polls/' + year_id, 'robots')})")
        return
    title, dates, total, questions = got
    print(f"\n{title} ({year_id}), {dates}; total votes {total:,}" if total else f"\n{title} ({year_id})")
    for q, yes, no, skipped, printed in questions:
        share = 100.0 * yes / (yes + no)
        verdict = "PASSED" if share > THRESHOLD else f"FAILED: {votes_to_flip(yes, no):,} No voters would have had to switch"
        check = "" if abs(round(share, 1) - printed) < 0.051 else f"  (page prints {printed}%)"
        print(f"  {share:5.1f}% yes  {yes:>7,} yes  {no:>7,} no  {skipped:>6,} skipped  {verdict}{check}")
        print(f"         {q[:110]}")
        if total and yes + no + skipped != total:
            print(f"         note: yes + no + skipped = {yes + no + skipped:,}, not the page's total of {total:,}")


# ------------------------------------------------------------------ the other figures
def players():
    got = fetch("https://oldschool.runescape.com/")
    if got is None:
        print("players: not read")
        return
    m = re.search(r"There are currently ([\d,]+) people playing", got[1])
    print("\nJagex player count:", m.group(1) if m else "line not found", f"  (printed {PRINTED['players']})")


def steam():
    print("\nSteam membership prices")
    for app, name in DLC.items():
        for cc in ("us", "gb", "de"):
            got = fetch(f"https://store.steampowered.com/api/appdetails?appids={app}&cc={cc}&l=en")
            if got is None:
                print(f"  {name} {cc}: not read")
                continue
            d = json.loads(got[1])[str(app)]
            po = (d.get("data") or {}).get("price_overview") or {}
            off = po.get("discount_percent", 0)
            print(f"  {name} {cc}: {po.get('final_formatted', 'no price')}" + (f" ({off}% off)" if off else "")
                  + f"   printed {PRINTED[app][cc]}")
    got = fetch(f"https://store.steampowered.com/appreviews/{APP}?json=1&language=all&purchase_type=all&num_per_page=0")
    if got is not None:
        s = json.loads(got[1])["query_summary"]
        pos, tot = s["total_positive"], s["total_reviews"]
        print(f"  reviews: {tot:,}, {pos:,} positive ({100 * pos / tot:.1f}%), \"{s['review_score_desc']}\""
              f"   printed {PRINTED['reviews']}")


if __name__ == "__main__":
    sys.stdout.reconfigure(encoding="utf-8")
    polls = sys.argv[1:]
    if not polls:
        newest = newest_poll()
        polls = ([newest] if newest and newest not in DEFAULT_POLLS else []) + DEFAULT_POLLS
    for p in polls:
        show_poll(p)
    if not sys.argv[1:]:
        players()
        steam()
    for url, why in SKIPPED.items():
        print(f"skipped {url}: {why}")