"""factorio-2-1-check.py - re-check whether Factorio 2.1 has been marked stable yet,
and re-count the numbers the article printed on 22 October 2026.

On 22 October 2026 Wube Software's own release API answered stable 2.0.77 and
experimental 2.1.23. 2.1 had been public as an opt-in experimental branch since
2.1.7 on 23 June 2026, and Wube had given no date for marking it stable. This
script re-reads the same public records and marks every figure that no longer
matches the one the article printed:

  1. Wube's release API: the current stable and experimental version numbers.
  2. The official wiki's 2.1 version history: every 2.1.x build with its date,
     the number of builds, and the days from the first public build to today.
  3. The game's Steam news feed (newest 50 posts): the newest post whose title
     says a version was released or marked "stable", and whether any post title
     names 2.1 as stable.
  4. With --mods, the mod portal's full mod list: how many mods' latest release
     declares factorio_version "2.1" against "2.0", overall and among the most
     downloaded. This is one request of about 14 MB, so it is off by default.

It then sets the 2.1 wait beside the two earlier experimental runs it is
compared with in the article (0.17 and 1.1; dates from Steam posts, Wube's
Friday Facts and the wiki, all named in the article's Sources list).

What it reads:

  factorio.com           /api/latest-releases
  wiki.factorio.com      /Version_history/2.1.0
  store.steampowered.com /events/ajaxgetpartnereventspageable/?appid=427520
  mods.factorio.com      /api/mods?page_size=max        (only with --mods)

Robots rules. Before its first call to a host the script reads that host's
robots.txt and skips the host, saying why, when the file cannot be read, when
the server refuses the script, or when any group in the file closes the page,
whichever crawler the group names. On 22 October 2026 all four hosts allowed
these paths. The script never calls api.steampowered.com, whose robots.txt
closes every path to every crawler. Every call names this script in its
User-Agent header, and calls to one host are at least two seconds apart, or as
far apart as the host's Crawl-delay asks.

Standard library only (urllib, json, re, html, statistics, time, sys, datetime). Python 3.9
or newer.

    python factorio-2-1-check.py                  read the live records
    python factorio-2-1-check.py --mods           also tally the mod portal
    python factorio-2-1-check.py --saved DIR      read saved copies instead:
                                                  DIR/latest.json, DIR/wiki.html,
                                                  DIR/events.json and, with --mods,
                                                  DIR/mods.json

Exit code is 0 when every record answered, 1 when any of them did not.
"""
import datetime
import html
import json
import os
import re
import statistics
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

UA = "factorio-2-1-check/1.0 (+https://blastproc.com/code/factorio-2-1-check.py; reader re-check script)"
HEADERS = {"User-Agent": UA, "Accept": "text/html,application/json;q=0.9,*/*;q=0.5"}
LATEST = "https://factorio.com/api/latest-releases"
WIKI = "https://wiki.factorio.com/Version_history/2.1.0"
EVENTS = ("https://store.steampowered.com/events/ajaxgetpartnereventspageable/"
          "?clan_accountid=0&appid=427520&offset=0&count=50&l=english")
MODS = "https://mods.factorio.com/api/mods?page_size=max"

PRINTED = {"stable": "2.0.77", "experimental": "2.1.23", "builds": 17, "first": "2.1.7",
           "first_date": datetime.date(2026, 6, 23), "last_stable_post": "Version 2.0.77 released as stable"}
WRITTEN = datetime.date(2026, 10, 22)
# Earlier experimental runs: first experimental build -> the day Wube marked the line stable.
PRECEDENTS = [("0.17", "0.17.0", datetime.date(2019, 2, 26), "0.17.69", datetime.date(2019, 9, 24)),
              ("1.1", "1.1.0", datetime.date(2020, 11, 23), "1.1.19", datetime.date(2021, 1, 29))]
SKIPPED = {}
FAILED = []

# ------------------------------------------------------------------ robots.txt, read before the first call to a host
MIN_GAP = 2.0
_ROBOTS = {}   # host -> (groups or None, why the host is skipped, seconds between calls)
_LAST = {}


def _wait(host):
    gap = _ROBOTS.get(host, (None, None, MIN_GAP))[2]
    if host in _LAST:
        time.sleep(max(0.0, _LAST[host] + gap - time.monotonic()))
    _LAST[host] = time.monotonic()


def _parse(text):
    """robots.txt -> ([(agent names, [(allow?, pattern), ...]), ...], the longest Crawl-delay in the file)."""
    groups, gap, in_agents = [], 0.0, False
    for line in text.splitlines():
        key, colon, value = line.split("#", 1)[0].partition(":")
        if not colon:
            continue
        key, value = key.strip().lower(), value.strip()
        if key == "user-agent":
            if not in_agents:
                groups.append(([], []))
            groups[-1][0].append(value)
            in_agents = True
            continue
        in_agents = False
        if key in ("allow", "disallow") and groups and value:
            groups[-1][1].append((key == "allow", value))
        elif key == "crawl-delay":
            try:
                gap = max(gap, float(value))
            except ValueError:
                pass
    return groups, gap


def _rules(host):
    """Read https://<host>/robots.txt once per run. A file that cannot be read, a web page served in its place,
    or a server that refuses this script rules the host out; a 404 or another 4xx means the host sets no rules."""
    if host not in _ROBOTS:
        _wait(host)
        try:
            req = urllib.request.Request(f"https://{host}/robots.txt", headers=HEADERS)
            with urllib.request.urlopen(req, timeout=60) as r:
                status, body = r.status, r.read()
        except urllib.error.HTTPError as e:
            status, body = e.code, b""
        except Exception as e:  # noqa: BLE001  (no answer or a dropped connection: the file cannot be read)
            status, body = None, str(e).encode()
        webpage = bool(re.search(rb"(?i)<!doctype|<html", body[:2000])) and not re.search(rb"(?im)^\s*user-agent\s*:", body)
        if status == 200 and not webpage:
            groups, gap = _parse(body.decode("utf-8", "replace"))
            _ROBOTS[host] = (groups, None, max(MIN_GAP, gap))
        elif status in (401, 403):
            _ROBOTS[host] = (None, f"the server refuses this script (HTTP {status} for robots.txt)", MIN_GAP)
        elif status is not None and 400 <= status < 500 and status != 429:
            _ROBOTS[host] = ([], None, MIN_GAP)
        else:
            why = body.decode("utf-8", "replace")[:80] if status is None else f"HTTP {status}" + (", a web page in its place" if webpage else "")
            _ROBOTS[host] = (None, f"its robots.txt could not be read ({why})", MIN_GAP)
    return _ROBOTS[host]


def _closed(rules, path):
    """True when the longest pattern matching the path is a Disallow (an Allow of equal length wins)."""
    best = (0, True)
    for allow, pattern in rules:
        rx = re.escape(pattern.rstrip("$")).replace(r"\*", ".*") + ("$" if pattern.endswith("$") else "")
        if re.match(rx, path) and (len(pattern), allow) > best:
            best = (len(pattern), allow)
    return not best[1]


def robots_block(url):
    parts = urllib.parse.urlsplit(url)
    groups, why, _ = _rules(parts.hostname)
    if groups is None:
        return f"{parts.hostname}: {why}"
    path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
    for agents, rules in groups:
        if _closed(rules, path):
            who = "every crawler" if "*" in agents else "the crawlers it names (" + ", ".join(agents) + ")"
            return f"{parts.hostname}: robots.txt closes {parts.path} to {who}"
    return None


def get(url):
    """The page text, or None when the robots rules rule the url out (reason in SKIPPED) or the call fails."""
    why = robots_block(url)
    if why:
        SKIPPED[url] = why
        return None
    _wait(urllib.parse.urlsplit(url).hostname)
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        with urllib.request.urlopen(req, timeout=120) as r:
            return r.read().decode("utf-8", "replace")
    except Exception as e:  # noqa: BLE001
        FAILED.append(f"{url}: {e}")
        return None


def read(url, saved, name):
    if saved:
        path = os.path.join(saved, name)
        if not os.path.exists(path):
            FAILED.append(f"{path}: no saved copy")
            return None
        return open(path, encoding="utf-8", errors="replace").read()
    return get(url)


def mark(now, printed):
    return "" if now == printed else f"   CHANGED (printed {printed})"


# ------------------------------------------------------------------ the four records
def branches(text):
    d = json.loads(text)
    return d["stable"]["alpha"], d["experimental"]["alpha"]


def wiki_builds(text):
    """[(version, date), ...] oldest first, from the wiki's 'Date: DD.MM.YYYY' lines under each 2.1.x heading."""
    plain = html.unescape(re.sub(r"<[^>]+>", "\n", text))
    found = re.findall(r"\n(2\.1\.\d+)\s*\n+\s*Date:\s*(\d\d)\.\s?(\d\d)\.\s?(\d{4})", plain)
    out = {v: datetime.date(int(y), int(m), int(d)) for v, d, m, y in found}
    return sorted(out.items(), key=lambda kv: int(kv[0].split(".")[2]))


def steam_posts(text):
    """[(date, title), ...] newest first."""
    events = json.loads(text).get("events", [])
    return [(datetime.datetime.fromtimestamp(e.get("rtime32_start_time", 0), datetime.timezone.utc).date(),
             e.get("event_name", "")) for e in events]


def mod_tally(text):
    """Counts of each mod's latest-release factorio_version: overall and in the top 100 and 1,000 by downloads."""
    mods = json.loads(text)["results"]
    ver = lambda m: ((m.get("latest_release") or {}).get("info_json") or {}).get("factorio_version")  # noqa: E731
    ranked = sorted(mods, key=lambda m: -m.get("downloads_count", 0))
    out = {"all": len(mods)}
    for label, group in (("overall", mods), ("top 100", ranked[:100]), ("top 1,000", ranked[:1000])):
        out[label] = {v: sum(1 for m in group if ver(m) == v) for v in ("2.1", "2.0", "1.1")}
    return out


def main(argv):
    saved = argv[argv.index("--saved") + 1] if "--saved" in argv else None
    today = datetime.date.today() if not saved else WRITTEN

    text = read(LATEST, saved, "latest.json")
    if text:
        stable, exp = branches(text)
        print(f"Stable branch        {stable}{mark(stable, PRINTED['stable'])}")
        print(f"Experimental branch  {exp}{mark(exp, PRINTED['experimental'])}")
        if stable.startswith("2.1"):
            print("2.1 IS NOW STABLE.")

    text = read(WIKI, saved, "wiki.html")
    if text:
        builds = wiki_builds(text)
        if builds:
            first_v, first_d = builds[0]
            print(f"2.1 builds on the wiki  {len(builds)}{mark(len(builds), PRINTED['builds'])}"
                  f"  ({first_v} on {first_d} to {builds[-1][0]} on {builds[-1][1]})")
            gaps = [(b[1] - a[1]).days for a, b in zip(builds, builds[1:])]
            print(f"Days between builds     longest {max(gaps)}, median {statistics.median(gaps)}")
            print(f"Days since {first_v}      {(today - first_d).days} on {today}")
        else:
            FAILED.append(f"{WIKI}: no 2.1.x dates found")

    text = read(EVENTS, saved, "events.json")
    if text:
        posts = steam_posts(text)
        stable_posts = [p for p in posts if re.search(r"(?i)\bstable\b", p[1])]
        if stable_posts:
            d, t = stable_posts[0]
            print(f"Newest Steam stable post  {d}  {t}{mark(t, PRINTED['last_stable_post'])}")
        named = [p for p in posts if re.search(r"(?i)2\.1", p[1]) and re.search(r"(?i)stable", p[1])]
        print("Steam post naming 2.1 as stable: " + (f"{named[0][0]} {named[0][1]}" if named else "none"))

    print("\nEarlier experimental runs, first build to stable:")
    for line, first, d0, last, d1 in PRECEDENTS:
        print(f"  {line:5s} {first} on {d0} -> {last} stable on {d1}: {(d1 - d0).days} days")
    print(f"  2.1   {PRINTED['first']} on {PRINTED['first_date']} -> not stable on {today}: "
          f"{(today - PRINTED['first_date']).days} days so far")

    if "--mods" in argv:
        text = read(MODS, saved, "mods.json")
        if text:
            t = mod_tally(text)
            print(f"\nMod portal, {t['all']:,} mods, latest release declares:")
            for label in ("overall", "top 100", "top 1,000"):
                c = t[label]
                share = 100 * c["2.1"] / (c["2.1"] + c["2.0"]) if c["2.1"] + c["2.0"] else 0
                print(f"  {label:9s} 2.1: {c['2.1']:>5,}  2.0: {c['2.0']:>5,}  1.1: {c['1.1']:>5,}"
                      f"  (2.1 share of the 2.x mods: {share:.1f}%)")

    for url, why in SKIPPED.items():
        print(f"SKIPPED {url}: {why}")
    for f in FAILED:
        print(f"FAILED {f}")
    return 1 if FAILED or SKIPPED else 0


if __name__ == "__main__":
    sys.stdout.reconfigure(encoding="utf-8")
    sys.exit(main(sys.argv[1:]))