diff --git a/README.md b/README.md index 8f412b5..c3b18f7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ share for translations. One package, `session_ops`, deployed to one self-hosted | `release-stats` | On demand: download counts of the latest releases | [release-stats](docs/jobs/release-stats.md) | | `session-ops-silence` | Discord alerts for a job that failed, or stopped running | [session-ops-silence](docs/jobs/session-ops-silence.md) | | `token-expiry` | Discord alerts 14 days, 7 days and 24 hours before a token expires | [token-expiry](docs/jobs/token-expiry.md) | +| `store-versions` | Every 5 minutes: each store's Session release against GitHub's, posted as they diverge | [store-versions](docs/jobs/store-versions.md) | Run by hand, not scheduled: [`sogs-ban`](docs/tools/sogs-ban.md) for community bans, and [`sogs-perms`](src/session_ops/sogs/perms.py) for a room's per-account permissions. diff --git a/deploy/README.md b/deploy/README.md index a1c6cf3..b7c34ff 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -58,6 +58,8 @@ Each `/etc/session-ops/.env` has a commented `.env.example` beside i installed from [`env/`](env/), saying what goes in it. `/etc/session-ops/expiry.toml` is not a secret: the expiry dates [token-expiry](../docs/jobs/token-expiry.md) cannot read from an API, from `expiry.toml.example`. +`/etc/session-ops/store-versions.toml` is not a secret either, and has to be mode 644: who +[store-versions](../docs/jobs/store-versions.md) tags, from `store-versions.toml.example`. ## Checking diff --git a/deploy/env/store-versions.env.example b/deploy/env/store-versions.env.example new file mode 100644 index 0000000..bbae6fc --- /dev/null +++ b/deploy/env/store-versions.env.example @@ -0,0 +1,2 @@ +# /etc/session-ops/store-versions.env: where store-versions posts, and its own failures. +STORE_VERSIONS_DISCORD_WEBHOOK_URL= diff --git a/deploy/env/store-versions.toml.example b/deploy/env/store-versions.toml.example new file mode 100644 index 0000000..6770270 --- /dev/null +++ b/deploy/env/store-versions.toml.example @@ -0,0 +1,11 @@ +# /etc/session-ops/store-versions.toml: who each platform's warnings tag. Not a +# secret: mode 644. Without it, warnings tag nobody. + +[tag] +#android = [111111111111111111] +#android-fdroid-merged = [111111111111111111] +#android-fdroid-live = [111111111111111111] +#ios = [222222222222222222] +#desktop = [333333333333333333] +#desktop-flathub-merged = [333333333333333333] +#desktop-flathub-live = [333333333333333333] diff --git a/deploy/install.sh b/deploy/install.sh index 74d4fde..cb5eaf7 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -71,6 +71,7 @@ move "$ETC/env" "$ETC/alerts.env" 600 root for name in zendesk github-prs crowdin publish alerts; do [ -e "$ETC/$name.env" ] || install -m 600 /dev/null "$ETC/$name.env" done +[ -e "$ETC/store-versions.env" ] || install -m 600 /dev/null "$ETC/store-versions.env" # What each file takes, commented; the env files stay empty until filled, since a job # is enabled once its env files have content. install -m 644 "$ROOT"/deploy/env/*.example "$ETC/" diff --git a/docs/jobs/store-versions.md b/docs/jobs/store-versions.md new file mode 100644 index 0000000..c4bdc8b --- /dev/null +++ b/docs/jobs/store-versions.md @@ -0,0 +1,52 @@ +# Store Versions Against GitHub Releases + +Every 5 minutes, compares each store's latest Session release with the platform's +highest GitHub release, and posts as they diverge and come back. + +| | | +| --- | --- | +| Runs | `session-ops@store-versions.timer`, every 5 minutes | +| Secrets | `/etc/session-ops/store-versions.env`: `STORE_VERSIONS_DISCORD_WEBHOOK_URL` | +| Tags | `/etc/session-ops/store-versions.toml`, from [`store-versions.toml.example`](../../deploy/env/store-versions.toml.example); without it nobody is tagged | +| Dry run | `session-ops run store-versions --dry-run` prints every version and what it would post | +| Logs | `journalctl -u session-ops@store-versions -n 50 --no-pager` | + +| Platform | GitHub | Store | Warns when the store is | +| --- | --- | --- | --- | +| `android` | `session-android` | Play listing, scraped: Play has no version API | ahead | +| `android-fdroid-merged` | `session-android` | `index-v1.json` on session-fdroid's `master` | ahead or behind | +| `android-fdroid-live` | `session-android` | `fdroid.getsession.org`'s `index-v1.json` | ahead | +| `ios` | `session-ios` | iTunes lookup API | ahead | +| `desktop` | `session-desktop` | `deb.session.foundation`, `sid`, package `session-desktop` | ahead or behind | +| `desktop-flathub-merged` | `session-desktop` | `network.loki.Session.metainfo.xml` on flathub/network.loki.Session's `master` | ahead or behind | +| `desktop-flathub-live` | `session-desktop` | Flathub's appstream API for `network.loki.Session` | ahead | + +A store ahead of GitHub is a public build whose source is not published. The channels +warned on either way are ours: each GitHub release opens a pull request on +session-fdroid and on flathub/network.loki.Session that has to be merged, and the APT +repo is published with the release. A store shipping a published prerelease above +that release is in sync with it. + +| Post | When | +| --- | --- | +| ⚠️ warning, tagging the platform's people | it diverges; on a platform's first check, if it already has | +| ⚠️ again | every 12 hours while unresolved, and at once if the gap grows | +| ✅ back in sync, or no longer ahead | after a warning only | +| ℹ️ new GitHub release, store updated | either moves without a warning | +| ⚠️ source failing, untagged | a source has failed for an hour, then every 12 hours, and ✅ once it reads again | + +A store is held at the highest version it has served: iTunes and the CDNs re-serve the +previous one for hours after a release, so a store returning to it changes nothing. +Once a store has served a lower version for 6 hours, that version counts, with nothing +to edit by hand: a bad read clears itself, a warning out is updated to the new gap, and +a rollback warns on the channels warned on either way. + +State is `/var/lib/session-ops/store-versions/state.json`. Losing it makes the next run +a first check. A message Discord refuses is posted again on the next run, as is one +still unposted 100 seconds into a run. Every request has a wall-clock limit, 60 s for a +source and 20 s for a post, so a run and its failure alert end inside the 4-minute +timeout. A version that does not start with a digit, or runs past 64 characters, fails +its source. A failed run is reported once, then every 12 hours until one succeeds. + +GitHub is read unauthenticated: one request per repository a run, 36 an hour of the 60 +allowed. The organisation refuses fine-grained tokens that live longer than 366 days. diff --git a/pyproject.toml b/pyproject.toml index b8d908c..88f3640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ session-ops = "session_ops.ops.runner:main" session-ops-alert = "session_ops.monitor.alert:main" session-ops-silence = "session_ops.monitor.silence:main" session-ops-token-expiry = "session_ops.monitor.token_expiry:main" +store-versions = "session_ops.store_versions.job:main" [dependency-groups] dev = [ diff --git a/src/session_ops/jobs.toml b/src/session_ops/jobs.toml index 9dbf23b..916a280 100644 --- a/src/session_ops/jobs.toml +++ b/src/session_ops/jobs.toml @@ -9,6 +9,8 @@ # env variables the run refuses to start without # channel_env where it posts, and where its own failures are reported; a job # without one reports to ALERT_DISCORD_WEBHOOK_URL +# repeat_alert_hours report the same failure again only this often until a run +# succeeds, for a schedule short enough to repeat it every few minutes # unit further [Service] lines, over the template's hardening # # max_age_hours for a queued job, its queue's: the weekend's 72 h (73 h across @@ -100,6 +102,21 @@ schedule = "*-*-* 09:00 Australia/Melbourne" max_age_hours = 27 timeout = "5min" +[[job]] +name = "store-versions" +description = "Store releases against GitHub releases, posted as they diverge" +entry = "session_ops.store_versions.job:main" +args = ["--state", "{state}/state.json"] +user = "sessionops" +env_files = ["/etc/session-ops/store-versions.env"] +env = ["STORE_VERSIONS_DISCORD_WEBHOOK_URL"] +channel_env = "STORE_VERSIONS_DISCORD_WEBHOOK_URL" +# Unauthenticated GitHub allows 60 requests an hour; this makes 36, one per repo a run. +schedule = "*:0/5" +max_age_hours = 1 +timeout = "4min" +repeat_alert_hours = 12 + [[job]] name = "crowdin-sync" description = "Crowdin translations into the platform repos" diff --git a/src/session_ops/ops/registry.py b/src/session_ops/ops/registry.py index 9ed9186..8348aa6 100644 --- a/src/session_ops/ops/registry.py +++ b/src/session_ops/ops/registry.py @@ -27,6 +27,7 @@ class Job: max_age_hours: float = None channel_env: str = "ALERT_DISCORD_WEBHOOK_URL" timeout: str = "30min" + repeat_alert_hours: float = None unit: tuple = field(default=()) queued: bool = False after: tuple = () diff --git a/src/session_ops/ops/runner.py b/src/session_ops/ops/runner.py index f4ab382..7724860 100644 --- a/src/session_ops/ops/runner.py +++ b/src/session_ops/ops/runner.py @@ -14,23 +14,33 @@ (session-ops-alert) knows this failure was already reported and stays quiet. The backstop is for the failures a run cannot report itself: killed, timed out, or Discord unreachable. + +A job with `repeat_alert_hours` reports the same failure again only that often, +until a run succeeds. """ import argparse import importlib +import json import os import re import shutil import socket import sys import tempfile +import time import traceback from dataclasses import dataclass, field from session_ops.ops import registry from session_ops.shared import discord, http +from session_ops.shared.state import write_json from session_ops.zendesk import claude_cli ALERTED_MARKER = "alerted" +LAST_ALERT = "last-alert.json" +# Discord answers in a second; past this the OnFailure backstop reports the run instead, +# and a job with a short timeout still has room for its alert. +ALERT_SECONDS = 45 # Present on a host rehearsing production: jobs publish beside it (env.rehearsing). REHEARSAL_MARKER = "/etc/session-ops/rehearsal" @@ -130,7 +140,8 @@ def post_alert(job, message): file=sys.stderr) return False payload = {"content": message, "allowed_mentions": {"parse": []}} - return discord.post_to_discord(http.Session(), webhook, [payload]) == 1 + session = http.Session(deadline=ALERT_SECONDS) + return discord.post_to_discord(session, webhook, [payload]) == 1 def mark_alerted(state_dir): @@ -147,6 +158,35 @@ def mark_alerted(state_dir): file=sys.stderr) +def repeated_alert(state_dir, message, hours, now): + """Whether `message` was reported less than `hours` ago; records it when not. + + Recorded whether or not the post lands: the backstop reports what it cannot.""" + path = os.path.join(state_dir, LAST_ALERT) + try: + with open(path, encoding="utf-8") as handle: + last = json.load(handle) + if last["message"] == message and now - last["at"] < hours * 3600: + return True + except (OSError, ValueError, KeyError, TypeError): + pass + try: + write_json(path, {"message": message, "at": now}) + except OSError as exc: + print(f"Could not record the alert ({exc}); the next failure alerts again.", + file=sys.stderr) + return False + + +def forget_alert(state_dir): + try: + os.remove(os.path.join(state_dir, LAST_ALERT)) + except FileNotFoundError: + pass + except OSError as exc: + print(f"Could not clear {LAST_ALERT} ({exc}).", file=sys.stderr) + + def call_target(function, argv): """Run one entry point, returning None, or its failure as one sentence. @@ -203,6 +243,8 @@ def run(job, dry_run=False, extra=()): if outcome.summary: print(outcome.summary) if not error and not outcome.failures(): + if job.repeat_alert_hours and not dry_run: + forget_alert(state_dir) return 0 message = scrub(alert_message(job, socket.gethostname(), _step, error, outcome)) @@ -210,6 +252,11 @@ def run(job, dry_run=False, extra=()): if dry_run: print(message) return code + if (code and job.repeat_alert_hours + and repeated_alert(state_dir, message, job.repeat_alert_hours, time.time())): + print(f"{message}\nAlready reported within {job.repeat_alert_hours:g}h; not posted.") + mark_alerted(state_dir) + return code if not post_alert(job, message): # Even an optional target's failure then fails the unit: OnFailure= is all # that is left to report it. diff --git a/src/session_ops/shared/github.py b/src/session_ops/shared/github.py index 84c556a..2827e50 100644 --- a/src/session_ops/shared/github.py +++ b/src/session_ops/shared/github.py @@ -1,5 +1,5 @@ -"""GitHub's REST API for the jobs that publish to the platform repos: who they -authenticate as, and the pull requests they keep open. +"""GitHub's REST API: the org's releases, and for the jobs that publish to the platform +repos, who they authenticate as and the pull requests they keep open. Publishing authenticates as a GitHub App, with an installation token minted per run and scoped to the repositories the job names: it expires in an hour, and pull @@ -23,6 +23,7 @@ APP_KEY_PATH = "/etc/session-ops/github-app.pem" ORG = "session-foundation" +RELEASES_PAGE = 20 REHEARSAL_PREFIX = "rehearsal/" BOT, DIRECT = "bot", "direct" # Every branch the App may write, and how: a BOT branch is the job's own, rebuilt and @@ -116,6 +117,16 @@ def check(resp, what): return resp.json() if resp.content else None +def releases(api, repo, count=RELEASES_PAGE): + """[(tag, page URL, prerelease)] of `repo`'s `count` most recently created + releases, drafts left out.""" + found = check(api.request("GET", f"{API}/repos/{repo}/releases", + params={"per_page": count}), + f"reading {repo}'s releases") + return [(release["tag_name"], release.get("html_url", ""), bool(release.get("prerelease"))) + for release in found if not release.get("draft")] + + def app_jwt(app_id, private_key, now=None): now = int(now or time.time()) # Backdated a minute for clock drift; GitHub refuses one living past ten minutes. diff --git a/src/session_ops/shared/http.py b/src/session_ops/shared/http.py index 8ad4e37..9e887dc 100644 --- a/src/session_ops/shared/http.py +++ b/src/session_ops/shared/http.py @@ -11,6 +11,7 @@ session = http.Session(attempts=10, timeout=60, rate=20) session.get(url) # the session's budget session.request("GET", url, attempts=2) # a shorter one for optional data + session.request("GET", url, deadline=60) # and a wall-clock limit on all of it """ import contextvars import email.utils @@ -34,6 +35,10 @@ _attempts = contextvars.ContextVar("attempts", default=None) +class DeadlineExceeded(requests.Timeout): + """No complete answer within the call's deadline.""" + + def retry_after_seconds(resp, default): """Seconds to wait per the response's rate-limit headers, else `default`. @@ -171,25 +176,59 @@ class Session(requests.Session): `limiter` may be shared between sessions, so that per-thread sessions against one API still draw on one budget. It paces each call, not each retry inside it. + + `deadline` bounds a whole call, retries, waits and the body included, in seconds. + `timeout` cannot: it applies to each socket read, so a server trickling a byte at + a time never trips it. """ def __init__(self, attempts=DEFAULT_ATTEMPTS, timeout=DEFAULT_TIMEOUT, rate=None, - limiter=None): + limiter=None, deadline=None): super().__init__() self.timeout = timeout + self.deadline = deadline self.limiter = limiter or (TokenBucket(rate) if rate else None) adapter = _Adapter(max_retries=_retry(attempts)) self.mount("https://", adapter) self.mount("http://", adapter) - def request(self, method, url, *args, attempts=None, **kwargs): + def request(self, method, url, *args, attempts=None, deadline=None, **kwargs): if attempts is not None and attempts <= 0: raise ValueError("attempts must be at least 1") kwargs.setdefault("timeout", self.timeout) if self.limiter: self.limiter.acquire() + deadline = self.deadline if deadline is None else deadline + if deadline is None: + return self._send(method, url, attempts, args, kwargs) + return within(deadline, lambda: self._send(method, url, attempts, args, kwargs)) + + def _send(self, method, url, attempts, args, kwargs): token = _attempts.set(attempts) try: return super().request(method, url, *args, **kwargs) finally: _attempts.reset(token) + + +def within(seconds, call): + """`call()`'s result, or DeadlineExceeded once `seconds` pass. + + A daemon thread runs it, since no socket option bounds a whole exchange; one left + behind dies with the process instead of holding its exit.""" + outcome = {} + + def run(): + try: + outcome["result"] = call() + except BaseException as exc: # re-raised in the caller's thread + outcome["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(seconds) + if worker.is_alive(): + raise DeadlineExceeded(f"no complete answer within {seconds:g} s") + if "error" in outcome: + raise outcome["error"] + return outcome["result"] diff --git a/src/session_ops/shared/state.py b/src/session_ops/shared/state.py index d0a801a..54c08f7 100644 --- a/src/session_ops/shared/state.py +++ b/src/session_ops/shared/state.py @@ -69,12 +69,16 @@ def save_state(path, state, records, retention_days, version, extra=None): if last >= cutoff: kept[key] = record + write_json(path, {"version": version, "updated_at": stamp, **(extra or {}), "seen": kept}) + return len(kept), len(seen) - len(kept) + + +def write_json(path, data): + """Write `data` to `path` atomically: a crash mid-write cannot corrupt the state.""" directory = os.path.dirname(path) if directory: os.makedirs(directory, exist_ok=True) temporary = f"{path}.tmp" with open(temporary, "w", encoding="utf-8") as handle: - json.dump({"version": version, "updated_at": stamp, **(extra or {}), "seen": kept}, - handle, indent=2) - os.replace(temporary, path) # atomic: a crash mid-write cannot corrupt the state - return len(kept), len(seen) - len(kept) + json.dump(data, handle, indent=2) + os.replace(temporary, path) diff --git a/src/session_ops/store_versions/__init__.py b/src/session_ops/store_versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py new file mode 100644 index 0000000..d4fc77a --- /dev/null +++ b/src/session_ops/store_versions/job.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +""" +Post when a store's Session release and the GitHub release diverge. + +Each run compares every platform's store version with its GitHub release and posts: + +- ⚠️ a warning, tagging the platform's people, when the store is ahead of GitHub (a + public build whose source is not out) or, for a channel we publish ourselves (APT, + and the merged F-Droid and Flathub manifests), when it differs either way: their + release pull requests have to be merged after each GitHub release; +- the warning again every 12 hours while unresolved, and at once if the gap grows; +- ✅ once back in sync, after a warning; +- ℹ️ when GitHub or the store moves on without a warning. + +A platform seen for the first time is recorded quietly, unless already out of sync. +A source failing for an hour is posted, without tags, then every 12 hours until it +recovers. + +Config: + STORE_VERSIONS_DISCORD_WEBHOOK_URL where it posts (not needed with --dry-run) + /etc/session-ops/store-versions.toml who each platform's warnings tag: + [tag] + android = [111111111111111111] + +Usage: + store-versions --state /var/lib/session-ops/store-versions/state.json + store-versions --dry-run # print every version and what it would post +""" +import argparse +import json +import sys +import time +import tomllib +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass + +from session_ops.shared import discord, http +from session_ops.shared.env import get_env +from session_ops.shared.state import write_json +from session_ops.store_versions import sources +from session_ops.store_versions.transitions import ( + ALL_CLEAR, INFO_GITHUB, INFO_STORE, SOURCE_RECOVERED, WARNING, + WARNING_LOWERED, WARNING_REMINDER, WARNING_UPDATE, WARNINGS, evaluate, source_events) + +TAGS_FILE = "/etc/session-ops/store-versions.toml" +STATE_VERSION = 1 +HOUR = 3600 +REMINDER_SECONDS = 12 * HOUR +FAILING_AFTER_SECONDS = HOUR +USER_AGENT = "session-ops store-versions (+https://github.com/session-foundation)" +# Inside the unit's 4-minute timeout: fetches end by 60 s, no post starts after 100 s +# or lasts past 20 s, and the runner's alert takes at most another 45 s. +HTTP_ATTEMPTS, HTTP_TIMEOUT = 2, (5, 20) +FETCH_SECONDS, POST_SECONDS = 60, 20 +POST_DEADLINE_SECONDS = 100 +EMBED_FIELD_CHARS = 1024 + +ANDROID_REPO, IOS_REPO, DESKTOP_REPO = "session-android", "session-ios", "session-desktop" +ANDROID_ID = "network.loki.messenger" +FLATHUB_ID = "network.loki.Session" + + +@dataclass(frozen=True) +class Platform: + name: str + display: str + repo: str + store: dict + strict_sync: bool = False + + +PLATFORMS = ( + Platform("android", "Session Android", ANDROID_REPO, + {"kind": "play", "app_id": ANDROID_ID}), + # Each Android release opens a "Release X.Y.Z" pull request on session-fdroid, which + # has to be merged; the merged index is what this checks. + Platform("android-fdroid-merged", "Session Android (F-Droid, merged)", ANDROID_REPO, + {"kind": "fdroid", "app_id": ANDROID_ID, + "base_url": "https://raw.githubusercontent.com/session-foundation/" + "session-fdroid/master/fdroid/repo"}, + strict_sync=True), + # What F-Droid clients pull: behind is deploy lag, so app-store rules. + Platform("android-fdroid-live", "Session Android (F-Droid, live)", ANDROID_REPO, + {"kind": "fdroid", "app_id": ANDROID_ID, + "base_url": "https://fdroid.getsession.org/fdroid/repo"}), + Platform("ios", "Session iOS", IOS_REPO, + {"kind": "appstore", "bundle_id": "com.loki-project.loki-messenger"}), + # session-desktop, not session-messenger-desktop: that one is an old transitional package. + Platform("desktop", "Session Desktop", DESKTOP_REPO, + {"kind": "deb", "base_url": "https://deb.session.foundation", + "package": "session-desktop"}, + strict_sync=True), + # flathubbot opens an update pull request on flathub/network.loki.Session per + # release, which has to be merged. + Platform("desktop-flathub-merged", "Session Desktop (Flathub, merged)", DESKTOP_REPO, + {"kind": "flathub_manifest", "app_id": FLATHUB_ID}, strict_sync=True), + # Behind is Flathub's build and publish lag after the merge. + Platform("desktop-flathub-live", "Session Desktop (Flathub, live)", DESKTOP_REPO, + {"kind": "flathub", "app_id": FLATHUB_ID}), +) + +STORE_NAMES = {"play": "Play Store", "appstore": "App Store", "deb": "APT repo", + "fdroid": "F-Droid repo", "flathub": "Flathub", + "flathub_manifest": "Flathub manifest"} + +RED, GREEN, BLUE = 0xE74C3C, 0x2ECC71, 0x3498DB + + +def store_name(platform): + return STORE_NAMES[platform.store["kind"]] + + +def load_tags(path): + """{platform: [user ids]}; nobody is tagged without the file.""" + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except FileNotFoundError: + return {} + except OSError as exc: + sys.exit(f"{path}: {exc.strerror}; it has to be readable by the job (mode 644)") + except tomllib.TOMLDecodeError as exc: + sys.exit(f"{path}: {exc}") + tags = data.get("tag", {}) + known = {platform.name for platform in PLATFORMS} + for name, ids in tags.items(): + if name not in known: + sys.exit(f"{path}: [tag] names {name}, which is not one of {', '.join(sorted(known))}") + if not (isinstance(ids, list) and all(isinstance(i, int) for i in ids)): + sys.exit(f"{path}: [tag] {name} must be a list of Discord user ids, as integers") + return {name: [str(i) for i in ids] for name, ids in tags.items()} + + +def load_state(path): + """Unreadable or another version is an empty state: every platform is then seen for + the first time, quietly unless out of sync.""" + data = {} + if path: + try: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + except (OSError, ValueError): + pass + if not isinstance(data, dict) or data.get("version") != STATE_VERSION: + data = {} + return {"version": STATE_VERSION, "platforms": data.get("platforms", {}), + "sources": data.get("sources", {})} + + +def gap(platform, event): + store = store_name(platform) + if sources.compare(event.store, event.github) > 0: + return (f"{store} is ahead of GitHub", + f"{store} has published **{event.store}**, but the latest GitHub release " + f"is **{event.github}**.") + return (f"{store} is behind GitHub", + f"GitHub has published **{event.github}**, but {store} is still at " + f"**{event.store}**.") + + +def event_message(platform, event, github_url, store_url, tags): + store = store_name(platform) + fields = [{"name": f"{store} version", "value": event.store, "inline": True}, + {"name": "GitHub version", "value": event.github, "inline": True}] + fields += [{"name": name, "value": url} for name, url in + ((f"{store} link", store_url), ("GitHub release", github_url)) + if url and len(url) <= EMBED_FIELD_CHARS] + embed = {"fields": fields} + message = {"embeds": [embed], "allowed_mentions": {"parse": []}} + + if event.kind in WARNINGS: + direction, description = gap(platform, event) + suffix = {WARNING: " (first check)" if event.initial else "", + WARNING_UPDATE: " — divergence grew", + WARNING_LOWERED: f" — {store} came down to {event.store}", + WARNING_REMINDER: " — still unresolved"}[event.kind] + embed.update(color=RED, title=f"⚠️ {platform.display}: {direction}{suffix}", + description=description) + ids = tags.get(platform.name, []) + if ids: + message["content"] = " ".join(f"<@{i}>" for i in ids) + message["allowed_mentions"] = {"users": ids} + elif event.kind == ALL_CLEAR and sources.compare(event.store, event.github) == 0: + embed.update(color=GREEN, title=f"✅ {platform.display}: back in sync", + description=f"{store} and GitHub are back in sync at **{event.github}**.") + elif event.kind == ALL_CLEAR: + embed.update(color=GREEN, title=f"✅ {platform.display}: no longer ahead of GitHub", + description=f"GitHub has published **{event.github}**; {store} is at " + f"**{event.store}**.") + elif event.kind == INFO_GITHUB: + embed.update(color=BLUE, title=f"ℹ️ {platform.display}: new GitHub release", + description=f"GitHub advanced to **{event.github}** " + f"({store} is at **{event.store}**).") + elif event.kind == INFO_STORE: + embed.update(color=BLUE, title=f"ℹ️ {platform.display}: {store} updated", + description=f"{store} advanced to **{event.store}** " + f"(GitHub is at **{event.github}**).") + return message + + +def combined(messages): + """A platform's events as one message, so they land or fail together.""" + if len(messages) < 2: + return messages + merged = {"embeds": [embed for message in messages for embed in message["embeds"]], + "allowed_mentions": {"parse": []}} + for message in messages: + if "content" in message: + merged.update(content=message["content"], + allowed_mentions=message["allowed_mentions"]) + return [merged] + + +def source_message(source, kind, failure, now): + if kind == SOURCE_RECOVERED: + embed = {"color": GREEN, "title": f"✅ {source}: readable again"} + else: + hours = max(1, round((now - failure["since"]) / HOUR)) + embed = {"color": RED, "title": f"⚠️ {source}: failing for ~{hours}h", + "description": discord.clip(failure["error"], 1000)} + return {"embeds": [embed], "allowed_mentions": {"parse": []}} + + +def http_session(): + session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT, + deadline=FETCH_SECONDS) + session.headers["User-Agent"] = USER_AGENT + return session + + +def source_keys(platform): + """(GitHub's, the store's) names for `platform`'s sources, as state["sources"] keys them.""" + return f"GitHub {platform.repo}", f"{store_name(platform)} ({platform.name})" + + +def fetch_all(wanted): + """{key: (Release, None) or (None, error)} for {key: (fetch, args)}, all at once and + each on its own session, so one host that stops answering holds up no other.""" + def one(fetch, args): + try: + return fetch(http_session(), *args), None + except Exception as exc: # one source down must not stop the others + return None, f"{type(exc).__name__}: {exc}" + + with ThreadPoolExecutor(max_workers=max(1, len(wanted))) as pool: + futures = {key: pool.submit(one, fetch, args) for key, (fetch, args) in wanted.items()} + results = {key: future.result() for key, future in futures.items()} + for key, (_, error) in results.items(): + if error: + print(f"{key}: {error}", file=sys.stderr) + return results + + +def check(state, tags, now): + """[(messages, apply)]: `apply` records a step's state once its messages are posted, + so a failed post is retried on the next run rather than lost. + + Each source is fetched once, keyed by name: three platforms share each of + session-android's and session-desktop's releases, and unauthenticated GitHub allows + 60 requests an hour.""" + keys = {platform.name: source_keys(platform) for platform in PLATFORMS} + wanted = {} + for platform in PLATFORMS: + github_key, store_key = keys[platform.name] + wanted[github_key] = (sources.fetch_github, (platform.repo,)) + wanted[store_key] = (sources.fetch_store, (platform.store,)) + results = fetch_all(wanted) + + steps = [] + for platform in PLATFORMS: + github_key, store_key = keys[platform.name] + (github, github_error), (store, store_error) = results[github_key], results[store_key] + if github_error or store_error: + print(f"{platform.name}: skipped") + continue + print(f"{platform.name}: github={github.version} {platform.store['kind']}={store.version}") + result = evaluate(state["platforms"].get(platform.name), github.version, store.version, + prereleases=[release.version for release in github.prereleases], + strict_sync=platform.strict_sync, now=now, + reminder_seconds=REMINDER_SECONDS) + github_urls = {release.version: release.url for release in github.prereleases} + messages = combined([event_message(platform, event, + github_urls.get(event.github, github.url), + store.url, tags) + for event in result.events]) + steps.append((messages, lambda name=platform.name, new=result.state: + state["platforms"].__setitem__(name, new))) + + for source in sorted(set(results) | set(state["sources"])): + error = results.get(source, (None, None))[1] + kind, failure = source_events(state["sources"].get(source), error, now=now, + after_seconds=FAILING_AFTER_SECONDS, + reminder_seconds=REMINDER_SECONDS) + messages = [source_message(source, kind, failure, now)] if kind else [] + steps.append((messages, lambda source=source, failure=failure: + state["sources"].__setitem__(source, failure) if failure + else state["sources"].pop(source, None))) + return steps + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Post when store and GitHub releases diverge.") + parser.add_argument("--state", metavar="PATH", + help="What was last seen and posted; without it every run is a first.") + parser.add_argument("--tags", default=TAGS_FILE, metavar="PATH") + parser.add_argument("--webhook", help="Discord webhook URL " + "(else STORE_VERSIONS_DISCORD_WEBHOOK_URL).") + parser.add_argument("--dry-run", action="store_true", + help="Print what it would post instead; write no state.") + args = parser.parse_args(argv) + + webhook = get_env("STORE_VERSIONS_DISCORD_WEBHOOK_URL", args.webhook, + required=not args.dry_run) + tags = load_tags(args.tags) + state = load_state(args.state) + started = time.monotonic() + steps = check(state, tags, time.time()) + messages = [message for step_messages, _ in steps for message in step_messages] + + if args.dry_run: + for message in messages: + print(json.dumps(message, ensure_ascii=False)) + print(f"Would post {len(messages)} message(s).") + return + + session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT, + deadline=POST_SECONDS) + posted, refused, deferred = 0, False, False + for step_messages, apply in steps: + if step_messages: + if time.monotonic() - started > POST_DEADLINE_SECONDS: + deferred = True + continue + landed = discord.post_to_discord(session, webhook, step_messages) + posted += landed + if landed < len(step_messages): + refused = True + continue + apply() + # Saved per step: a run killed mid-way reposts at most the step it was on. + if args.state and step_messages: + write_json(args.state, state) + if args.state: + write_json(args.state, state) + if refused: + # Without counts: the runner repeats an alert only when its text changes. + sys.exit("Discord refused a post; it is retried next run.") + if deferred: + print(f"Posted {posted} of {len(messages)} message(s); the rest go next run.") + else: + print(f"Posted {len(messages)} message(s).") + + +if __name__ == "__main__": + main() diff --git a/src/session_ops/store_versions/sources.py b/src/session_ops/store_versions/sources.py new file mode 100644 index 0000000..c122c42 --- /dev/null +++ b/src/session_ops/store_versions/sources.py @@ -0,0 +1,218 @@ +"""The latest version each release channel publishes, normalised to plain "1.2.3". + +Every fetcher takes a shared http.Session and returns a Release, or raises: the job +records a failure per source rather than per platform, since several platforms +share one. +""" +import functools +import gzip +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass + +from session_ops.shared import github + +PLAY_DETAILS = "https://play.google.com/store/apps/details" +ITUNES_LOOKUP = "https://itunes.apple.com/lookup" +FLATHUB_APPSTREAM = "https://flathub.org/api/v2/appstream/{app_id}" +FLATHUB_PAGE = "https://flathub.org/apps/{app_id}" +FLATHUB_METAINFO = "https://raw.githubusercontent.com/{repo}/{branch}/{app_id}.metainfo.xml" + +# Play has no version API: the listing bootstraps its data with the version as +# `[[["1.2.3"]]`. When Android starts failing, Google has likely changed the page. +PLAY_VERSION = re.compile(r'\[\[\["(\d+(?:\.\d+)+)"\]\]') +DEB_EPOCH = re.compile(r"^\d+:") +DEB_REVISION = re.compile(r"-[^-]+$") +RELEASE = re.compile(r"(\d+(?:\.\d+)*)(.*)", re.DOTALL) +MAX_VERSION_CHARS = 64 + + +class SourceError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Release: + version: str + url: str = "" + # GitHub only: the published prereleases above `version`. + prereleases: tuple = () + + +def normalize(version): + return re.sub(r"^[vV]", "", (version or "").strip()) + + +def version_key(version): + """Orders dotted versions numerically, so 1.10.0 sorts after 1.9.0 and 1.2 equals + 1.2.0, and ranks a suffixed one (1.2.3-beta1, 1.2.3rc1) below its bare release.""" + match = RELEASE.match(version) + if not match: + return ((), 0, ()) + parts = [int(part) for part in match.group(1).split(".")] + while parts and parts[-1] == 0: + parts.pop() + suffix = match.group(2) + return (tuple(parts), 0 if suffix else 1, + tuple(int(part) for part in re.findall(r"\d+", suffix))) + + +def compare(a, b): + """-1, 0 or 1 for a < b, a == b, a > b.""" + ka, kb = version_key(a), version_key(b) + return (ka > kb) - (ka < kb) + + +def newest(versions): + """The highest of `versions`, None if there are none. Store lists are not trusted + to be newest-first: AppStream recommends it and does not require it.""" + versions = [normalize(v) for v in versions if v] + return max(versions, key=functools.cmp_to_key(compare)) if versions else None + + +def deb_upstream(version): + """A Debian version without its epoch ("1:") or revision ("-2"); the versions + tracked here never contain a hyphen of their own.""" + version = DEB_EPOCH.sub("", version.strip()) + return normalize(DEB_REVISION.sub("", version) if "-" in version else version) + + +def checked_version(release, what): + """`release`, if its version is one an embed can carry and compare() can order.""" + if not RELEASE.match(release.version) or len(release.version) > MAX_VERSION_CHARS: + raise SourceError(f"{what} gave an unusable version: {release.version[:MAX_VERSION_CHARS]!r}") + return release + + +def checked(resp, what): + if resp.status_code >= 400: + raise SourceError(f"{what}: HTTP {resp.status_code}") + return resp + + +def fetch_github(session, repo): + """The highest release, rather than GitHub's "latest": that is the most recently + created one, which goes backwards when an older line gets a hotfix.""" + found = [(Release(normalize(tag), url), prerelease) + for tag, url, prerelease in github.releases(session, f"{github.ORG}/{repo}")] + released = [release for release, prerelease in found if not prerelease] + if not released: + raise SourceError(f"none of GitHub {repo}'s last {github.RELEASES_PAGE} releases " + "is a full release") + by_version = functools.cmp_to_key(lambda a, b: compare(a.version, b.version)) + latest = checked_version(max(released, key=by_version), f"GitHub {repo}") + above = tuple(release for release, prerelease in found + if prerelease and compare(release.version, latest.version) > 0) + return Release(latest.version, latest.url, above) + + +def fetch_play(session, app_id, country="us", lang="en"): + resp = checked(session.request("GET", PLAY_DETAILS, + params={"id": app_id, "hl": lang, "gl": country}), + f"Play listing for {app_id}") + match = PLAY_VERSION.search(resp.text) + if not match: + raise SourceError(f"no version in the Play listing for {app_id}; " + "the page layout may have changed") + return Release(match.group(1), f"{PLAY_DETAILS}?id={app_id}") + + +def fetch_appstore(session, bundle_id, country="us"): + resp = checked(session.request("GET", ITUNES_LOOKUP, + params={"bundleId": bundle_id, "country": country}), + f"App Store lookup for {bundle_id}") + results = resp.json().get("results") or [] + if not results or not results[0].get("version"): + raise SourceError(f"the App Store lists no version for {bundle_id}") + return Release(normalize(results[0]["version"]), results[0].get("trackViewUrl", "")) + + +def fetch_deb(session, base_url, package, suite="sid", component="main", arch="amd64"): + base = base_url.rstrip("/") + index = f"{base}/dists/{suite}/{component}/binary-{arch}/Packages.gz" + resp = checked(session.request("GET", index), index) + try: + text = gzip.decompress(resp.content).decode("utf-8", "replace") + except (OSError, EOFError) as exc: + raise SourceError(f"cannot decompress {index}: {exc}") from exc + version = packages_version(text, package) + if version is None: + raise SourceError(f"no {package} in {index}") + return Release(deb_upstream(version), base) + + +def packages_version(text, package): + """The highest Version: of `package` in a Packages index, or None: a repo can keep + several versions of one package.""" + versions = [] + for stanza in re.split(r"\n\s*\n", text): + fields = dict(line.split(":", 1) for line in stanza.splitlines() + if ":" in line and not line.startswith(" ")) + if fields.get("Package", "").strip() == package and fields.get("Version"): + versions.append(fields["Version"].strip()) + by_upstream = functools.cmp_to_key(lambda a, b: compare(deb_upstream(a), deb_upstream(b))) + return max(versions, key=by_upstream) if versions else None + + +def fetch_fdroid(session, base_url, app_id): + base = base_url.rstrip("/") + resp = checked(session.request("GET", f"{base}/index-v1.json"), f"{base}/index-v1.json") + try: + index = resp.json() + except ValueError as exc: + raise SourceError(f"cannot parse {base}/index-v1.json: {exc}") from exc + version = fdroid_version(index, app_id) + if version is None: + raise SourceError(f"no {app_id} in {base}/index-v1.json") + return Release(normalize(version), base) + + +def fdroid_version(index, app_id): + """versionName of the build with the highest versionCode: one entry per ABI split.""" + builds = (index.get("packages") or {}).get(app_id) + if not builds: + return None + return max(builds, key=lambda build: build.get("versionCode", 0)).get("versionName") + + +def fetch_flathub(session, app_id): + resp = checked(session.request("GET", FLATHUB_APPSTREAM.format(app_id=app_id)), + f"Flathub appstream for {app_id}") + version = newest(release.get("version") for release in resp.json().get("releases") or []) + if version is None: + raise SourceError(f"Flathub lists no release of {app_id}") + return Release(version, FLATHUB_PAGE.format(app_id=app_id)) + + +def fetch_flathub_manifest(session, app_id, repo=None, branch="master"): + repo = repo or f"flathub/{app_id}" + url = FLATHUB_METAINFO.format(repo=repo, branch=branch, app_id=app_id) + version = metainfo_version(checked(session.request("GET", url), url).content) + if version is None: + raise SourceError(f"no in {url}") + return Release(version, f"https://github.com/{repo}") + + +def metainfo_version(xml_bytes): + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + raise SourceError(f"cannot parse metainfo: {exc}") from exc + return newest(release.get("version") for release in root.iterfind("releases/release")) + + +FETCHERS = { + "play": fetch_play, + "appstore": fetch_appstore, + "deb": fetch_deb, + "fdroid": fetch_fdroid, + "flathub": fetch_flathub, + "flathub_manifest": fetch_flathub_manifest, +} + + +def fetch_store(session, store): + """`store` is a platform's store spec: its `kind`, and that fetcher's arguments.""" + arguments = dict(store) + kind = arguments.pop("kind") + return checked_version(FETCHERS[kind](session, **arguments), kind) diff --git a/src/session_ops/store_versions/transitions.py b/src/session_ops/store_versions/transitions.py new file mode 100644 index 0000000..d5cbbba --- /dev/null +++ b/src/session_ops/store_versions/transitions.py @@ -0,0 +1,127 @@ +"""What to post, given a platform's last state and this run's versions: no IO. + +A platform's state is a dict: the GitHub and store versions last seen, the store's +recent highest (`store_high`) and since when it has served lower (`low_since`), whether +a warning is out (`warning`) and when it was last posted (`warned_at`, epoch seconds). +""" +from dataclasses import dataclass, field + +from session_ops.store_versions.sources import compare + +WARNING = "warning" +WARNING_UPDATE = "warning_update" +WARNING_LOWERED = "warning_lowered" +WARNING_REMINDER = "warning_reminder" +ALL_CLEAR = "all_clear" +INFO_GITHUB = "info_github" +INFO_STORE = "info_store" +WARNINGS = (WARNING, WARNING_UPDATE, WARNING_LOWERED, WARNING_REMINDER) + +# Longer than the hours iTunes and Flathub's CDN take to stop re-serving an old version. +WATERMARK_SECONDS = 6 * 3600 + +SOURCE_FAILING = "source_failing" +SOURCE_RECOVERED = "source_recovered" + + +@dataclass(frozen=True) +class Event: + kind: str + github: str + store: str + # Raised on a platform's first check, which found it already out of sync. + initial: bool = False + + +@dataclass +class Result: + events: list = field(default_factory=list) + state: dict = field(default_factory=dict) + + +def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, + reminder_seconds=None, watermark_seconds=WATERMARK_SECONDS): + """The events for this run, and the state to keep once they are posted. + + App stores warn only when the store is ahead of GitHub: a public build whose source + is not published. A `strict_sync` channel is ours and must match GitHub, so it + warns in either direction. An unresolved warning repeats every `reminder_seconds`, + and at once if the leading side pulls further ahead or the store's watermark comes + down. `prereleases` are the + published prereleases above `github`: a store shipping one of them is in sync. + """ + # iTunes and Flathub's CDN re-serve the previous version for hours; compare against + # the highest served until a lower one has held for `watermark_seconds`. + prev_high = prev and (prev.get("store_high") or prev["store"]) + new_high = prev_high is None or compare(store, prev_high) > 0 + high, low_since = (store if new_high else prev_high), None + if not new_high and compare(store, prev_high) < 0: + low_since = now if prev.get("low_since") is None else prev["low_since"] + if now - low_since >= watermark_seconds: + high, low_since = store, None + lowered = compare(high, prev_high) < 0 if prev_high else False + if compare(high, github) > 0 and any(compare(high, p) == 0 for p in prereleases): + github = high + + order = compare(high, github) + store_ahead, store_behind = order > 0, order < 0 + out_of_sync = order != 0 if strict_sync else store_ahead + state = {"github": github, "store": store, "store_high": high, "low_since": low_since, + "warning": out_of_sync, "warned_at": None} + + if prev is None: + if out_of_sync: + state["warned_at"] = now + return Result([Event(WARNING, github, high, initial=True)], state) + return Result([], state) + + github_advanced = compare(github, prev["github"]) > 0 + warned_at = now if prev.get("warned_at") is None else prev["warned_at"] + events = [] + + if out_of_sync and not prev["warning"]: + events.append(Event(WARNING, github, high)) + warned_at = now + elif out_of_sync: + # A lowered watermark can shrink the gap or flip its direction, so it is not "grew". + if lowered: + events.append(Event(WARNING_LOWERED, github, high)) + warned_at = now + elif (store_ahead and new_high) or (store_behind and github_advanced): + events.append(Event(WARNING_UPDATE, github, high)) + warned_at = now + elif reminder_seconds is not None and now - warned_at >= reminder_seconds: + events.append(Event(WARNING_REMINDER, github, high)) + warned_at = now + elif prev["warning"]: + events.append(Event(ALL_CLEAR, github, high)) + + if not events: + if github_advanced: + events.append(Event(INFO_GITHUB, github, high)) + if new_high: + events.append(Event(INFO_STORE, github, high)) + + state["warned_at"] = warned_at if out_of_sync else None + return Result(events, state) + + +def source_events(prev, error, *, now, after_seconds, reminder_seconds): + """(event or None, state) for one source: `error` is this run's failure, or None. + + A source can fail for a run or two (Play throttles a scrape, a CDN hiccups), so it + is reported only once it has failed for `after_seconds`, then every + `reminder_seconds`, and its recovery only if its failure was reported. + """ + if error is None: + recovered = prev and prev.get("alerted_at") is not None + return (SOURCE_RECOVERED if recovered else None), None + since = prev["since"] if prev else now + alerted_at = prev.get("alerted_at") if prev else None + state = {"since": since, "alerted_at": alerted_at, "error": error} + due = (now - since >= after_seconds if alerted_at is None + else now - alerted_at >= reminder_seconds) + if due: + state["alerted_at"] = now + return SOURCE_FAILING, state + return None, state diff --git a/tests/goldens/units/dropins.txt b/tests/goldens/units/dropins.txt index 7f3cb1a..51ff8e7 100644 --- a/tests/goldens/units/dropins.txt +++ b/tests/goldens/units/dropins.txt @@ -89,6 +89,24 @@ EnvironmentFile=/etc/session-ops/publish.env TimeoutStartSec=10min LoadCredential=github-app.pem:/etc/session-ops/github-app.pem +==> session-ops@store-versions.service.d/job.conf <== +# Generated by `session-ops units` from jobs.toml. Edit the registry, not this. + +[Unit] +Description=Store releases against GitHub releases, posted as they diverge + +[Service] +User=sessionops +Group=sessionops +EnvironmentFile=/etc/session-ops/store-versions.env +TimeoutStartSec=4min + +==> session-ops@store-versions.timer.d/schedule.conf <== +# Generated by `session-ops units` from jobs.toml. Edit the registry, not this. + +[Timer] +OnCalendar=*:0/5 + ==> session-ops@token-expiry.service.d/job.conf <== # Generated by `session-ops units` from jobs.toml. Edit the registry, not this. diff --git a/tests/ops/test_runner.py b/tests/ops/test_runner.py index 25821d3..08efb14 100644 --- a/tests/ops/test_runner.py +++ b/tests/ops/test_runner.py @@ -22,7 +22,7 @@ def job(entry, **extra): class TestRun(unittest.TestCase): def setUp(self): self.state = tempfile.mkdtemp() - self.posted = [] + self.posted, self.sessions = [], [] env = {"STATE_DIRECTORY": self.state, "INVOCATION_ID": "inv-1", "ALERT_DISCORD_WEBHOOK_URL": HOOK, "DEMO_TOKEN": "s3cr3t-value"} for patcher in (mock.patch.dict(os.environ, env), @@ -34,6 +34,7 @@ def setUp(self): fake_jobs.calls.clear() def post(self, session, url, messages): + self.sessions.append(session) self.posted.append((url, messages)) return len(messages) @@ -154,6 +155,38 @@ def test_an_optional_failure_whose_alert_cannot_be_posted_fails_the_run(self): self.assertEqual(self.run_job(job("optional_failure")), 1) self.assertIsNone(self.marker()) + def test_a_repeated_failure_is_reported_once_until_its_interval_passes(self): + throttled = job("exits", repeat_alert_hours=12) + with mock.patch.object(runner.time, "time", return_value=0.0): + self.assertEqual(self.run_job(throttled), 1) + os.environ["INVOCATION_ID"] = "inv-2" + with mock.patch.object(runner.time, "time", return_value=11 * 3600.0): + self.assertEqual(self.run_job(throttled), 1) + self.assertEqual(len(self.posted), 1) + self.assertEqual(self.marker(), "inv-2") + with mock.patch.object(runner.time, "time", return_value=12 * 3600.0): + self.run_job(throttled) + self.assertEqual(len(self.posted), 2) + + def test_a_success_resets_the_repeat_interval(self): + with mock.patch.object(runner.time, "time", return_value=0.0): + self.run_job(job("exits", repeat_alert_hours=12)) + self.run_job(job("succeeds", repeat_alert_hours=12)) + self.run_job(job("exits", repeat_alert_hours=12)) + self.assertEqual(len(self.posted), 2) + + def test_a_different_failure_is_reported_at_once(self): + with mock.patch.object(runner.time, "time", return_value=0.0): + self.run_job(job("exits", repeat_alert_hours=12)) + self.run_job(job("raises", args=("x",), repeat_alert_hours=12)) + self.assertEqual(len(self.posted), 2) + + def test_without_an_interval_every_failure_is_reported(self): + self.run_job(job("exits")) + self.run_job(job("exits")) + self.assertEqual(len(self.posted), 2) + self.assertFalse(os.path.exists(os.path.join(self.state, runner.LAST_ALERT))) + def test_a_dry_run_prints_the_alert_instead(self): out = io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): @@ -178,6 +211,10 @@ def test_an_alert_mentions_nobody(self): self.assertTrue(payload["content"].startswith("❌ ")) self.assertEqual(payload["allowed_mentions"], {"parse": []}) + def test_an_alert_post_has_a_deadline(self): + self.run_job(job("exits")) + self.assertEqual(self.sessions[0].deadline, runner.ALERT_SECONDS) + class TestCommandLine(unittest.TestCase): def test_dry_run_is_the_runners_and_what_follows_dashes_is_the_jobs(self): diff --git a/tests/shared/test_http.py b/tests/shared/test_http.py index 7fced3e..40859fa 100644 --- a/tests/shared/test_http.py +++ b/tests/shared/test_http.py @@ -260,6 +260,98 @@ def test_the_limiter_paces_every_call(self): self.assertEqual(limiter.acquire.call_count, 2) +class Trickle: + """Sends a 200 whose body comes one byte every `gap` seconds: each read is quick, + so only a deadline on the whole call stops it.""" + + def __init__(self, size=20, gap=0.2): + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(5) + self.url = f"http://127.0.0.1:{self.sock.getsockname()[1]}/" + self.size, self.gap = size, gap + # Event.wait, not time.sleep: TestSession's NoSleep patches time.sleep globally, and + # a thread still trickling into a later test would land in its recorded sleeps. + self.stop = threading.Event() + self.threads = [threading.Thread(target=self.serve, daemon=True)] + self.threads[0].start() + + def serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + worker = threading.Thread(target=self.answer, args=(conn,), daemon=True) + self.threads.append(worker) + worker.start() + + def answer(self, conn): + with conn: + conn.recv(4096) + conn.sendall(f"HTTP/1.1 200 OK\r\nContent-Length: {self.size}\r\n\r\n".encode()) + for _ in range(self.size): + if self.stop.wait(self.gap): + return + try: + conn.sendall(b"x") + except OSError: + return + + def close(self): + self.stop.set() + try: + self.sock.shutdown(socket.SHUT_RDWR) # wakes the accept(); close() alone may not + except OSError: + pass + self.sock.close() + for thread in self.threads: + thread.join(timeout=5) + + +class TestDeadline(unittest.TestCase): + def trickle(self, **kwargs): + server = Trickle(**kwargs) + self.addCleanup(server.close) + return server + + def test_a_trickling_body_stops_at_the_deadline(self): + server = self.trickle() + session = http.Session(attempts=1, timeout=(1, 1), deadline=1) + started = time.monotonic() + with self.assertRaises(http.DeadlineExceeded): + session.get(server.url) + self.assertLess(time.monotonic() - started, 2) + + def test_a_deadline_is_a_requests_timeout(self): + self.assertTrue(issubclass(http.DeadlineExceeded, requests.Timeout)) + + def test_an_answer_inside_the_deadline_is_returned_whole(self): + server = self.trickle(size=3, gap=0.05) + resp = http.Session(attempts=1, deadline=5).get(server.url) + self.assertEqual((resp.status_code, resp.content), (200, b"xxx")) + + def test_a_per_call_deadline_overrides_the_sessions(self): + server = self.trickle() + with self.assertRaises(http.DeadlineExceeded): + http.Session(attempts=1, timeout=(1, 1)).request("GET", server.url, deadline=0.5) + + def test_a_failure_inside_the_deadline_raises_as_itself(self): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + url = f"http://127.0.0.1:{sock.getsockname()[1]}/" + with self.assertRaises(requests.ConnectionError) as caught: + http.Session(attempts=1, deadline=5).get(url) + self.assertNotIsInstance(caught.exception, http.DeadlineExceeded) + + def test_the_per_call_attempts_reach_the_deadlines_thread(self): + server = Server(status(500), status(500), ok()) + self.addCleanup(server.close) + with NoSleep(): + resp = http.Session(attempts=1, deadline=5).request("GET", server.url, attempts=3) + self.assertEqual((resp.status_code, len(server.seen)), (200, 3)) + + class TestTokenBucket(unittest.TestCase): def test_a_burst_up_to_the_rate_is_free_then_calls_wait(self): bucket = http.TokenBucket(4) diff --git a/tests/store_versions/__init__.py b/tests/store_versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/store_versions/test_job.py b/tests/store_versions/test_job.py new file mode 100644 index 0000000..8d07f84 --- /dev/null +++ b/tests/store_versions/test_job.py @@ -0,0 +1,212 @@ +""" + uv run python -m unittest tests.store_versions.test_job +""" +import json +import os +import tempfile +import unittest + +from session_ops.ops import runner +from session_ops.shared import discord +from session_ops.shared.testing import Patched +from session_ops.store_versions import job, sources +from session_ops.store_versions.sources import Release + +PLATFORMS = (job.Platform("ios", "Session iOS", "session-ios", + {"kind": "appstore", "bundle_id": "b"}), + job.Platform("desktop", "Session Desktop", "session-desktop", + {"kind": "deb", "base_url": "u", "package": "p"}, strict_sync=True)) +WEBHOOK = "https://discord.example/api/webhooks/1/x" + + +class Run: + """job.main against fixed versions, recording what it posts; `accept` caps how + many messages Discord takes.""" + + def __init__(self, github, stores, accept=None, tags=""): + self.github, self.stores, self.accept, self.posted = github, stores, accept, [] + self.post_deadlines = [] + self.dir = tempfile.mkdtemp() + self.state = os.path.join(self.dir, "state.json") + self.tags = os.path.join(self.dir, "tags.toml") + with open(self.tags, "w", encoding="utf-8") as handle: + handle.write(tags) + + def fetch_github(self, session, repo): + return Release(self.github[repo], f"https://github.com/{repo}") + + def fetch_store(self, session, store): + value = self.stores[store["kind"]] + if isinstance(value, Exception): + raise value + return Release(value) + + def post(self, session, url, messages): + taken = messages if self.accept is None else messages[:self.accept] + if self.accept is not None: + self.accept -= len(taken) + self.post_deadlines.append(session.deadline) + self.posted += taken + return len(taken) + + def __call__(self, *extra, now=1000.0): + with Patched(job, PLATFORMS=PLATFORMS), \ + Patched(job.time, time=lambda: now), \ + Patched(sources, fetch_github=self.fetch_github, fetch_store=self.fetch_store), \ + Patched(discord, post_to_discord=self.post): + job.main(["--state", self.state, "--tags", self.tags, "--webhook", WEBHOOK, *extra]) + + def saved(self): + with open(self.state, encoding="utf-8") as handle: + return json.load(handle) + + +def titles(messages): + return [message["embeds"][0]["title"] for message in messages] + + +class TestJob(unittest.TestCase): + def test_first_run_in_sync_posts_nothing_and_records_both(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.5", "deb": "1.18.2"}) + run() + self.assertEqual(run.posted, []) + self.assertEqual(set(run.saved()["platforms"]), {"ios", "desktop"}) + + def test_warning_tags_the_platforms_people_only(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.2"}, tags="[tag]\nios = [42]\n") + run() + [message] = run.posted + self.assertEqual(message["content"], "<@42>") + self.assertEqual(message["allowed_mentions"], {"users": ["42"]}) + self.assertIn("App Store is ahead of GitHub (first check)", titles(run.posted)[0]) + + def test_info_mentions_nobody(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.5", "deb": "1.18.2"}, tags="[tag]\nios = [42]\n") + run() + run.github["session-ios"] = "2.15.6" + run() + [message] = run.posted + self.assertNotIn("content", message) + self.assertEqual(message["allowed_mentions"], {"parse": []}) + + def test_unposted_warning_is_retried_next_run(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, + {"appstore": "2.15.6", "deb": "1.18.2"}, accept=1) + with self.assertRaises(SystemExit): + run() + self.assertEqual(set(run.saved()["platforms"]), {"ios"}) + run.accept = None + run() + self.assertEqual(len(run.posted), 2) + self.assertIn("APT repo is behind GitHub", titles(run.posted)[1]) + + def test_a_refused_platform_does_not_hold_back_the_others(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + accept = run.post + + def refuse_the_app_store(session, url, messages): + if "App Store" in messages[0]["embeds"][0]["title"]: + return 0 + return accept(session, url, messages) + + run.post = refuse_the_app_store + with self.assertRaises(SystemExit): + run() + self.assertEqual(set(run.saved()["platforms"]), {"desktop"}) + self.assertIn("APT repo is behind GitHub", titles(run.posted)[0]) + + def test_a_platforms_events_go_as_one_message(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.5", "deb": "1.18.2"}) + run() + run.github["session-ios"], run.stores["appstore"] = "2.15.6", "2.15.6" + run() + [message] = run.posted + self.assertEqual([embed["title"] for embed in message["embeds"]], + ["ℹ️ Session iOS: new GitHub release", "ℹ️ Session iOS: App Store updated"]) + + def test_an_all_clear_says_what_is_true(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + run() + run.github["session-ios"] = "2.15.7" + run() + self.assertEqual(titles(run.posted)[1], "✅ Session iOS: no longer ahead of GitHub") + + def test_a_refused_post_fails_with_the_same_text_however_many_wait(self): + failures = [] + for stores in ({"appstore": "2.15.6", "deb": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.1"}): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, stores, accept=0) + with self.assertRaises(SystemExit) as raised: + run() + failures.append(str(raised.exception)) + self.assertEqual(failures[0], failures[1]) + + def test_a_lowered_watermark_is_not_titled_as_a_growing_gap(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.7", "deb": "1.18.2"}) + run(now=0.0) + run.stores["appstore"] = "2.15.6" + run(now=3600.0) + run(now=8 * 3600.0) + title = titles(run.posted)[-1] + self.assertNotIn("grew", title) + self.assertIn("App Store is ahead of GitHub — App Store came down to 2.15.6", title) + + def test_every_request_has_a_deadline_inside_the_units_timeout(self): + run = Run({"session-ios": "2.15.5"}, {"appstore": "2.15.6"}) + run() + self.assertEqual(run.post_deadlines, [job.POST_SECONDS]) + self.assertEqual(job.http_session().deadline, job.FETCH_SECONDS) + worst = (max(job.FETCH_SECONDS, job.POST_DEADLINE_SECONDS) + job.POST_SECONDS + + runner.ALERT_SECONDS) + self.assertLess(worst, 4 * 60) + + def test_past_the_deadline_messages_wait_for_the_next_run(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + clock = iter([0.0, job.POST_DEADLINE_SECONDS + 1]) + with Patched(job.time, monotonic=lambda: next(clock, 10_000.0)): + run() + self.assertEqual(run.posted, []) + self.assertEqual(set(run.saved()["platforms"]), set()) + run() + self.assertEqual(len(run.posted), 2) + + def test_failing_source_is_posted_after_an_hour_then_recovery(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": RuntimeError("boom"), "deb": "1.18.2"}) + run(now=0.0) + run(now=3600.0) + self.assertEqual(titles(run.posted), ["⚠️ App Store (ios): failing for ~1h"]) + self.assertNotIn("ios", run.saved()["platforms"]) + run.stores["appstore"] = "2.15.5" + run(now=4000.0) + self.assertEqual(titles(run.posted)[1], "✅ App Store (ios): readable again") + self.assertEqual(run.saved()["sources"], {}) + + def test_dry_run_posts_and_writes_nothing(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + run("--dry-run") + self.assertEqual(run.posted, []) + self.assertFalse(os.path.exists(run.state)) + + def test_unreadable_tags_fail_the_run_saying_why(self): + run = Run({}, {}) + os.chmod(run.tags, 0) + if os.access(run.tags, os.R_OK): + self.skipTest("running as root") + with self.assertRaises(SystemExit) as raised: + job.load_tags(run.tags) + self.assertIn("mode 644", str(raised.exception)) + + def test_tags_must_name_a_platform(self): + run = Run({}, {}, tags="[tag]\nandroind = [1]\n") + with self.assertRaises(SystemExit): + job.load_tags(run.tags) diff --git a/tests/store_versions/test_sources.py b/tests/store_versions/test_sources.py new file mode 100644 index 0000000..8cf22f8 --- /dev/null +++ b/tests/store_versions/test_sources.py @@ -0,0 +1,147 @@ +""" + uv run python -m unittest tests.store_versions.test_sources +""" +import gzip +import unittest + +from session_ops.shared.testing import FakeResponse, FakeSession +from session_ops.store_versions import sources + + +class TestVersions(unittest.TestCase): + def test_compare_is_numeric(self): + self.assertEqual(sources.compare("1.18.1", "1.17.17"), 1) + self.assertEqual(sources.compare("1.2.0", "1.10.0"), -1) + self.assertEqual(sources.compare("2.15.3", "2.15.3"), 0) + self.assertEqual(sources.compare("1.2", "1.2.0"), 0) + + def test_a_suffixed_version_is_below_its_release(self): + self.assertEqual(sources.compare("1.2.3-beta1", "1.2.3"), -1) + self.assertEqual(sources.compare("1.2.3rc1", "1.2.3"), -1) + self.assertEqual(sources.compare("1.2.3-beta1", "1.2.3-beta2"), -1) + self.assertEqual(sources.compare("1.2.3-beta1", "1.2.2"), 1) + self.assertEqual(sources.newest(["1.34.0-rc1", "1.34.0"]), "1.34.0") + + def test_an_unusable_version_fails_the_source(self): + for version in ("", "v", "beta", "1" * 65): + with self.subTest(version=version), self.assertRaises(sources.SourceError): + sources.checked_version(sources.Release(version), "test") + self.assertEqual(sources.checked_version(sources.Release("1.2.3"), "test").version, "1.2.3") + + def test_normalize_and_deb_upstream(self): + self.assertEqual(sources.normalize(" v1.18.1 "), "1.18.1") + self.assertEqual(sources.deb_upstream("1:1.17.17-2"), "1.17.17") + self.assertEqual(sources.deb_upstream("1.18.2"), "1.18.2") + + def test_newest_ignores_list_order(self): + self.assertEqual(sources.newest(["1.9.0", "v1.10.0", None, ""]), "1.10.0") + self.assertIsNone(sources.newest([])) + + +class TestParsers(unittest.TestCase): + PACKAGES = ("Package: session-messenger-desktop\nVersion: 1.0.0\n\n" + "Package: session-desktop\nArchitecture: amd64\nVersion: 1:1.18.2-1\n" + "Description: Session\n multi-line\n") + + def test_packages_finds_the_exact_package(self): + self.assertEqual(sources.packages_version(self.PACKAGES, "session-desktop"), "1:1.18.2-1") + self.assertIsNone(sources.packages_version(self.PACKAGES, "nope")) + + def test_packages_takes_the_highest_of_several_versions(self): + text = ("Package: session-desktop\nVersion: 1.18.2\n\n" + "Package: session-desktop\nVersion: 1.18.10\n\n" + "Package: session-desktop\nVersion: 1.9.0\n") + self.assertEqual(sources.packages_version(text, "session-desktop"), "1.18.10") + + def test_fdroid_takes_the_highest_version_code(self): + index = {"packages": {"app": [{"versionCode": 9, "versionName": "9.0.0"}, + {"versionCode": 10, "versionName": "10.0.0"}, + {"versionCode": 2, "versionName": "2.0.0"}]}} + self.assertEqual(sources.fdroid_version(index, "app"), "10.0.0") + self.assertIsNone(sources.fdroid_version(index, "other")) + self.assertIsNone(sources.fdroid_version({}, "app")) + + def test_metainfo_takes_the_newest_release(self): + xml = (b'' + b'' + b'') + self.assertEqual(sources.metainfo_version(xml), "1.18.2") + self.assertIsNone(sources.metainfo_version(b"")) + with self.assertRaises(sources.SourceError): + sources.metainfo_version(b"") + + +class Raw(FakeResponse): + def __init__(self, text=None, content=None, status_code=200): + super().__init__({}, status_code) + self.text = text or "" + self._content = content + + @property + def content(self): + return self._content if self._content is not None else self.text.encode() + + +class TestFetchers(unittest.TestCase): + def test_play_extracts_the_bootstrapped_version(self): + session = FakeSession([Raw('junk [[["1.33.6"]],[[[33]]]] junk')]) + release = sources.fetch_play(session, "network.loki.messenger") + self.assertEqual(release.version, "1.33.6") + self.assertEqual(session.calls[0][2]["params"]["id"], "network.loki.messenger") + + def test_play_without_a_version_fails(self): + with self.assertRaises(sources.SourceError): + sources.fetch_play(FakeSession([Raw("")]), "x") + + def test_deb(self): + packages = gzip.compress(TestParsers.PACKAGES.encode()) + release = sources.fetch_store( + FakeSession([Raw(content=packages)]), + {"kind": "deb", "base_url": "https://deb.example/", "package": "session-desktop"}) + self.assertEqual(release.version, "1.18.2") + + def test_an_empty_fdroid_version_fails_the_source(self): + index = {"packages": {"a": [{"versionCode": 1, "versionName": ""}]}} + with self.assertRaises(sources.SourceError): + sources.fetch_store(FakeSession([FakeResponse(index)]), + {"kind": "fdroid", "base_url": "https://f.example", "app_id": "a"}) + + def test_flathub_appstream(self): + session = FakeSession([FakeResponse({"releases": [{"version": "1.18.1"}, + {"version": "1.18.2"}]})]) + release = sources.fetch_flathub(session, "network.loki.Session") + self.assertEqual(release.version, "1.18.2") + self.assertEqual(release.url, "https://flathub.org/apps/network.loki.Session") + + def test_github_strips_the_v(self): + session = FakeSession([FakeResponse([{"tag_name": "v1.18.2", "html_url": "u"}])]) + release = sources.fetch_github(session, "session-desktop") + self.assertEqual((release.version, release.url), ("1.18.2", "u")) + self.assertTrue(session.calls[0][1].endswith( + "/repos/session-foundation/session-desktop/releases")) + + def test_github_takes_the_highest_release_not_the_newest(self): + session = FakeSession([FakeResponse([ + {"tag_name": "v1.32.3", "html_url": "hotfix"}, + {"tag_name": "v1.33.1", "html_url": "latest"}, + {"tag_name": "v1.32.2", "html_url": "old"}])]) + self.assertEqual(sources.fetch_github(session, "session-android").url, "latest") + + def test_github_keeps_the_prereleases_above_the_release(self): + session = FakeSession([FakeResponse([ + {"tag_name": "v1.34.0", "html_url": "draft", "draft": True}, + {"tag_name": "v1.33.0", "html_url": "pre", "prerelease": True}, + {"tag_name": "v1.32.2", "html_url": "latest"}, + {"tag_name": "v1.32.0", "html_url": "old pre", "prerelease": True}])]) + release = sources.fetch_github(session, "session-android") + self.assertEqual(release.version, "1.32.2") + self.assertEqual(release.prereleases, (sources.Release("1.33.0", "pre"),)) + + def test_github_without_a_full_release_fails_the_source(self): + session = FakeSession([FakeResponse([{"tag_name": "v1.33.0", "prerelease": True}])]) + with self.assertRaises(sources.SourceError): + sources.fetch_github(session, "session-android") + + def test_http_error_fails_the_source(self): + with self.assertRaises(sources.SourceError): + sources.fetch_appstore(FakeSession([FakeResponse({}, status_code=503)]), "x") diff --git a/tests/store_versions/test_transitions.py b/tests/store_versions/test_transitions.py new file mode 100644 index 0000000..bbad241 --- /dev/null +++ b/tests/store_versions/test_transitions.py @@ -0,0 +1,253 @@ +""" + uv run python -m unittest tests.store_versions.test_transitions +""" +import unittest + +from session_ops.store_versions.transitions import ( + ALL_CLEAR, INFO_GITHUB, INFO_STORE, SOURCE_FAILING, SOURCE_RECOVERED, WARNING, + WARNING_LOWERED, WARNING_REMINDER, WARNING_UPDATE, evaluate, source_events) + +DAY = 12 * 3600 + + +def seen(github, store, warning=False, warned_at=None, store_high=None): + return {"github": github, "store": store, "warning": warning, "warned_at": warned_at, + "store_high": store_high} + + +def kinds(result): + return [event.kind for event in result.events] + + +class TestFirstCheck(unittest.TestCase): + def test_in_sync_or_github_ahead_is_quiet(self): + for github, store in (("1.0.0", "1.0.0"), ("1.1.0", "1.0.0")): + result = evaluate(None, github, store) + self.assertEqual(kinds(result), []) + self.assertFalse(result.state["warning"]) + + def test_store_ahead_warns_as_first_check(self): + result = evaluate(None, "1.0.0", "1.1.0", now=50.0) + self.assertEqual(kinds(result), [WARNING]) + self.assertTrue(result.events[0].initial) + self.assertEqual(result.state["warned_at"], 50.0) + + def test_strict_sync_behind_warns(self): + result = evaluate(None, "1.1.0", "1.0.0", strict_sync=True) + self.assertEqual(kinds(result), [WARNING]) + + +class TestTransitions(unittest.TestCase): + def test_no_movement_is_quiet(self): + self.assertEqual(kinds(evaluate(seen("1.0.0", "1.0.0"), "1.0.0", "1.0.0")), []) + + def test_store_jumps_ahead_warns(self): + result = evaluate(seen("1.0.0", "1.0.0"), "1.0.0", "1.1.0") + self.assertEqual(kinds(result), [WARNING]) + self.assertTrue(result.state["warning"]) + + def test_store_further_ahead_while_warning_updates(self): + result = evaluate(seen("1.0.0", "1.1.0", warning=True, warned_at=0.0), "1.0.0", "1.2.0") + self.assertEqual(kinds(result), [WARNING_UPDATE]) + + def test_github_catching_up_or_passing_clears(self): + for github in ("1.1.0", "1.2.0"): + result = evaluate(seen("1.0.0", "1.1.0", warning=True, warned_at=0.0), github, "1.1.0") + self.assertEqual(kinds(result), [ALL_CLEAR]) + self.assertFalse(result.state["warning"]) + self.assertIsNone(result.state["warned_at"]) + + def test_github_advancing_is_info(self): + result = evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.0.0") + self.assertEqual(kinds(result), [INFO_GITHUB]) + + def test_store_catching_up_is_info(self): + result = evaluate(seen("1.2.0", "1.0.0"), "1.2.0", "1.1.0") + self.assertEqual(kinds(result), [INFO_STORE]) + + def test_both_advancing_store_ahead_only_warns(self): + result = evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.2.0") + self.assertEqual(kinds(result), [WARNING]) + + def test_unchanged_warning_is_not_repeated(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + self.assertEqual(kinds(evaluate(prev, "1.0.0", "1.1.0", now=60.0)), []) + + +class TestStrictSync(unittest.TestCase): + def test_behind_warns_only_under_strict_sync(self): + self.assertEqual(kinds(evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.0.0", + strict_sync=True)), [WARNING]) + self.assertEqual(kinds(evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.0.0")), + [INFO_GITHUB]) + + def test_github_pulling_further_ahead_updates(self): + prev = seen("1.1.0", "1.0.0", warning=True, warned_at=100.0) + result = evaluate(prev, "1.2.0", "1.0.0", strict_sync=True) + self.assertEqual(kinds(result), [WARNING_UPDATE]) + + def test_catching_up_clears(self): + prev = seen("1.1.0", "1.0.0", warning=True, warned_at=100.0) + self.assertEqual(kinds(evaluate(prev, "1.1.0", "1.1.0", strict_sync=True)), [ALL_CLEAR]) + + +class TestReminder(unittest.TestCase): + def test_repeats_after_the_interval_and_resets_its_clock(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + result = evaluate(prev, "1.0.0", "1.1.0", now=DAY, reminder_seconds=DAY) + self.assertEqual(kinds(result), [WARNING_REMINDER]) + self.assertEqual(result.state["warned_at"], DAY) + + def test_quiet_before_the_interval(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + result = evaluate(prev, "1.0.0", "1.1.0", now=DAY - 1, reminder_seconds=DAY) + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["warned_at"], 0.0) + + def test_growing_gap_resets_the_clock(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + result = evaluate(prev, "1.0.0", "1.2.0", now=5000.0, reminder_seconds=DAY) + self.assertEqual(kinds(result), [WARNING_UPDATE]) + self.assertEqual(result.state["warned_at"], 5000.0) + + +class TestStoreHigh(unittest.TestCase): + def test_recovering_to_an_announced_version_is_quiet(self): + result = evaluate(seen("2.15.4", "2.15.3", store_high="2.15.4"), "2.15.4", "2.15.4") + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["store_high"], "2.15.4") + + def test_new_high_is_announced(self): + result = evaluate(seen("2.15.5", "2.15.4", store_high="2.15.4"), "2.15.5", "2.15.5") + self.assertEqual(kinds(result), [INFO_STORE]) + + def test_dip_is_quiet_and_keeps_the_true_version(self): + result = evaluate(seen("2.15.4", "2.15.4", store_high="2.15.4"), "2.15.4", "2.15.3") + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["store"], "2.15.3") + self.assertEqual(result.state["store_high"], "2.15.4") + + def test_flap_announces_once(self): + state, posted = seen("2.15.4", "2.15.4"), [] + for store in ("2.15.5", "2.15.4", "2.15.5", "2.15.4", "2.15.5"): + result = evaluate(state, "2.15.5", store) + posted += kinds(result) + state = result.state + self.assertEqual(posted.count(INFO_STORE), 1) + + def test_store_flapping_ahead_warns_once(self): + state, posted = None, [] + for store in ("2.15.6", "2.15.5", "2.15.6", "2.15.5", "2.15.6"): + result = evaluate(state, "2.15.5", store) + posted += kinds(result) + state = result.state + self.assertEqual(posted, [WARNING]) + + def test_store_flapping_two_ahead_does_not_update_the_warning(self): + state = evaluate(None, "2.15.5", "2.15.7").state + for store in ("2.15.6", "2.15.7", "2.15.6"): + result = evaluate(state, "2.15.5", store) + self.assertEqual(kinds(result), []) + state = result.state + + def test_strict_sync_dip_after_a_merge_stays_in_sync(self): + state = evaluate(seen("1.18.1", "1.18.1"), "1.18.2", "1.18.2", strict_sync=True).state + result = evaluate(state, "1.18.2", "1.18.1", strict_sync=True) + self.assertEqual(kinds(result), []) + + +class TestWatermark(unittest.TestCase): + HOURS = 3600 + + def run_sequence(self, steps, strict_sync=False): + state, posted = None, [] + for at_hour, github, store in steps: + result = evaluate(state, github, store, strict_sync=strict_sync, + now=at_hour * self.HOURS) + posted.append(kinds(result)) + state = result.state + return posted, state + + def test_a_pulled_release_warns_once_the_store_has_served_lower_for_six_hours(self): + posted, state = self.run_sequence( + [(0, "1.15", "1.15"), (1, "1.15", "1.16"), (2, "1.15", "1.15"), + (3, "1.16", "1.15"), (7.9, "1.16", "1.15"), (8, "1.16", "1.15")], + strict_sync=True) + self.assertEqual(posted, [[], [WARNING], [], [ALL_CLEAR], [], [WARNING]]) + self.assertEqual(state["store_high"], "1.15") + + def test_a_bad_read_clears_after_six_hours_back_at_the_real_version(self): + posted, state = self.run_sequence( + [(0, "1.20", "1.20"), (1, "1.20", "8.0"), (2, "1.20", "1.20"), + (8, "1.20", "1.20")]) + self.assertEqual(posted, [[], [WARNING], [], [ALL_CLEAR]]) + self.assertEqual((state["store_high"], state["low_since"]), ("1.20", None)) + + def test_a_short_dip_stays_quiet_and_resets_the_clock(self): + posted, state = self.run_sequence( + [(0, "2.15.5", "2.15.5"), (1, "2.15.5", "2.15.4"), (2, "2.15.5", "2.15.5"), + (3, "2.15.5", "2.15.4"), (8, "2.15.5", "2.15.4")]) + self.assertEqual(posted, [[], [], [], [], []]) + self.assertEqual((state["store_high"], state["low_since"]), ("2.15.5", 3 * self.HOURS)) + + def test_a_state_from_before_the_clock_loads(self): + result = evaluate(seen("2.15.4", "2.15.4", store_high="2.15.4"), "2.15.4", "2.15.3", + now=100.0) + self.assertEqual(result.state["low_since"], 100.0) + + + def test_a_lowered_watermark_updates_a_warning_that_flips_direction(self): + posted, state = [], None + for hours, store in ((0, "1.6"), (1, "1.7"), (2, "1.5"), (8, "1.5")): + result = evaluate(state, "1.6", store, strict_sync=True, now=hours * 3600, + reminder_seconds=DAY) + posted.append(kinds(result)) + state = result.state + self.assertEqual(posted, [[], [WARNING], [], [WARNING_LOWERED]]) + self.assertEqual((result.events[0].store, result.state["warned_at"]), ("1.5", 8 * 3600)) + + def test_a_lowered_watermark_updates_a_shrinking_gap(self): + state = evaluate(None, "1.6", "1.8", now=0).state + state = evaluate(state, "1.6", "1.7", now=3600).state + result = evaluate(state, "1.6", "1.7", now=7 * 3600) + self.assertEqual(kinds(result), [WARNING_LOWERED]) + self.assertEqual(result.events[0].store, "1.7") + +class TestPrereleases(unittest.TestCase): + def test_a_store_shipping_a_prerelease_is_in_sync(self): + result = evaluate(None, "1.32.2", "1.33.0", prereleases=["1.33.0"]) + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["github"], "1.33.0") + + def test_strict_sync_on_the_release_below_a_prerelease_is_in_sync(self): + result = evaluate(None, "1.17.12", "1.17.12", prereleases=["1.17.13"], strict_sync=True) + self.assertEqual(kinds(result), []) + + def test_a_store_past_every_prerelease_warns(self): + result = evaluate(None, "1.32.2", "1.33.1", prereleases=["1.33.0"]) + self.assertEqual(kinds(result), [WARNING]) + self.assertEqual(result.events[0].github, "1.32.2") + + +class TestSourceEvents(unittest.TestCase): + def call(self, prev, error, now): + return source_events(prev, error, now=now, after_seconds=3600, reminder_seconds=DAY) + + def test_a_short_failure_and_its_recovery_are_quiet(self): + kind, state = self.call(None, "boom", 0.0) + self.assertIsNone(kind) + kind, state = self.call(state, "boom", 3599.0) + self.assertIsNone(kind) + self.assertEqual(self.call(state, None, 3700.0), (None, None)) + + def test_reported_after_an_hour_then_every_reminder(self): + _, state = self.call(None, "boom", 0.0) + kind, state = self.call(state, "boom", 3600.0) + self.assertEqual(kind, SOURCE_FAILING) + self.assertEqual(self.call(state, "boom", 3600.0 + DAY - 1)[0], None) + self.assertEqual(self.call(state, "boom", 3600.0 + DAY)[0], SOURCE_FAILING) + + def test_recovery_after_a_report_is_posted(self): + _, state = self.call(None, "boom", 0.0) + _, state = self.call(state, "boom", 3600.0) + self.assertEqual(self.call(state, None, 4000.0), (SOURCE_RECOVERED, None))