From a47c4aeb958a084eb9dec3956b30bf7af78b9605 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Wed, 14 Oct 2026 15:46:50 +1100 Subject: [PATCH 01/11] Add store-versions: store releases against GitHub releases Port session-divemo, the Discord bot that watches each store's Session release against the platform's GitHub release, into a session-ops job. Its seven checks (Play, App Store, the APT repo, F-Droid merged and live, and Flathub merged and live) and its warning, reminder, all-clear and info rules carry over unchanged. It runs every 5 minutes from a timer instead of a long-running gateway bot, and reuses what session-ops already shares: - shared.http for every request, so retries, timeouts and rate limits behave like every other job's - shared.discord's post_to_discord over a webhook, with embeds; a platform's state is kept only once its messages are posted, so a refused post is retried - shared.github.latest_release, new, for the GitHub side - shared.state.write_json, split out of save_state, for its atomic state file - the runner's failure alerts and the silence checker, via jobs.toml A source that keeps failing (Play's scraped listing is the fragile one) is posted after an hour and every 12 hours until it reads again, rather than alerting on every 5-minute run. The /versions slash command is not ported: `session-ops run store-versions --dry-run` prints the same versions. --- README.md | 1 + deploy/env/store-versions.env.example | 2 + deploy/env/store-versions.toml.example | 11 + deploy/install.sh | 1 + docs/jobs/store-versions.md | 41 +++ pyproject.toml | 1 + src/session_ops/jobs.toml | 14 + src/session_ops/shared/github.py | 11 +- src/session_ops/shared/state.py | 12 +- src/session_ops/store_versions/__init__.py | 0 src/session_ops/store_versions/job.py | 288 ++++++++++++++++++ src/session_ops/store_versions/sources.py | 186 +++++++++++ src/session_ops/store_versions/transitions.py | 110 +++++++ tests/goldens/units/dropins.txt | 18 ++ tests/store_versions/__init__.py | 0 tests/store_versions/test_job.py | 123 ++++++++ tests/store_versions/test_sources.py | 100 ++++++ tests/store_versions/test_transitions.py | 160 ++++++++++ 18 files changed, 1073 insertions(+), 6 deletions(-) create mode 100644 deploy/env/store-versions.env.example create mode 100644 deploy/env/store-versions.toml.example create mode 100644 docs/jobs/store-versions.md create mode 100644 src/session_ops/store_versions/__init__.py create mode 100644 src/session_ops/store_versions/job.py create mode 100644 src/session_ops/store_versions/sources.py create mode 100644 src/session_ops/store_versions/transitions.py create mode 100644 tests/store_versions/__init__.py create mode 100644 tests/store_versions/test_job.py create mode 100644 tests/store_versions/test_sources.py create mode 100644 tests/store_versions/test_transitions.py diff --git a/README.md b/README.md index 8f412b5..c3b18f7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ share for translations. One package, `session_ops`, deployed to one self-hosted | `release-stats` | On demand: download counts of the latest releases | [release-stats](docs/jobs/release-stats.md) | | `session-ops-silence` | Discord alerts for a job that failed, or stopped running | [session-ops-silence](docs/jobs/session-ops-silence.md) | | `token-expiry` | Discord alerts 14 days, 7 days and 24 hours before a token expires | [token-expiry](docs/jobs/token-expiry.md) | +| `store-versions` | Every 5 minutes: each store's Session release against GitHub's, posted as they diverge | [store-versions](docs/jobs/store-versions.md) | Run by hand, not scheduled: [`sogs-ban`](docs/tools/sogs-ban.md) for community bans, and [`sogs-perms`](src/session_ops/sogs/perms.py) for a room's per-account permissions. diff --git a/deploy/env/store-versions.env.example b/deploy/env/store-versions.env.example new file mode 100644 index 0000000..bbae6fc --- /dev/null +++ b/deploy/env/store-versions.env.example @@ -0,0 +1,2 @@ +# /etc/session-ops/store-versions.env: where store-versions posts, and its own failures. +STORE_VERSIONS_DISCORD_WEBHOOK_URL= diff --git a/deploy/env/store-versions.toml.example b/deploy/env/store-versions.toml.example new file mode 100644 index 0000000..6770270 --- /dev/null +++ b/deploy/env/store-versions.toml.example @@ -0,0 +1,11 @@ +# /etc/session-ops/store-versions.toml: who each platform's warnings tag. Not a +# secret: mode 644. Without it, warnings tag nobody. + +[tag] +#android = [111111111111111111] +#android-fdroid-merged = [111111111111111111] +#android-fdroid-live = [111111111111111111] +#ios = [222222222222222222] +#desktop = [333333333333333333] +#desktop-flathub-merged = [333333333333333333] +#desktop-flathub-live = [333333333333333333] diff --git a/deploy/install.sh b/deploy/install.sh index 74d4fde..cb5eaf7 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -71,6 +71,7 @@ move "$ETC/env" "$ETC/alerts.env" 600 root for name in zendesk github-prs crowdin publish alerts; do [ -e "$ETC/$name.env" ] || install -m 600 /dev/null "$ETC/$name.env" done +[ -e "$ETC/store-versions.env" ] || install -m 600 /dev/null "$ETC/store-versions.env" # What each file takes, commented; the env files stay empty until filled, since a job # is enabled once its env files have content. install -m 644 "$ROOT"/deploy/env/*.example "$ETC/" diff --git a/docs/jobs/store-versions.md b/docs/jobs/store-versions.md new file mode 100644 index 0000000..68e4704 --- /dev/null +++ b/docs/jobs/store-versions.md @@ -0,0 +1,41 @@ +# Store Versions Against GitHub Releases + +Every 5 minutes, compares each store's latest Session release with the platform's latest +GitHub release, and posts as they diverge and come back. + +| | | +| --- | --- | +| Runs | `session-ops@store-versions.timer`, every 5 minutes | +| Secrets | `/etc/session-ops/store-versions.env`: `STORE_VERSIONS_DISCORD_WEBHOOK_URL` | +| Tags | `/etc/session-ops/store-versions.toml`, from [`store-versions.toml.example`](../../deploy/env/store-versions.toml.example); without it nobody is tagged | +| Dry run | `session-ops run store-versions --dry-run` prints every version and what it would post | +| Logs | `journalctl -u session-ops@store-versions -n 50 --no-pager` | + +| Platform | GitHub | Store | Warns when the store is | +| --- | --- | --- | --- | +| `android` | `session-android` | Play listing, scraped: Play has no version API | ahead | +| `android-fdroid-merged` | `session-android` | `index-v1.json` on session-fdroid's `master` | ahead or behind | +| `android-fdroid-live` | `session-android` | `fdroid.getsession.org`'s `index-v1.json` | ahead | +| `ios` | `session-ios` | iTunes lookup API | ahead | +| `desktop` | `session-desktop` | `deb.session.foundation`, `sid`, package `session-desktop` | ahead or behind | +| `desktop-flathub-merged` | `session-desktop` | `network.loki.Session.metainfo.xml` on flathub/network.loki.Session's `master` | ahead or behind | +| `desktop-flathub-live` | `session-desktop` | Flathub's appstream API for `network.loki.Session` | ahead | + +A store ahead of GitHub is a public build whose source is not published. The channels +warned on either way are ours: each GitHub release opens a pull request on +session-fdroid and on flathub/network.loki.Session that has to be merged, and the APT +repo is published with the release. + +| Post | When | +| --- | --- | +| ⚠️ warning, tagging the platform's people | it diverges; on a platform's first check, if it already has | +| ⚠️ again | every 12 hours while unresolved, and at once if the gap grows | +| ✅ back in sync | after a warning only | +| ℹ️ new GitHub release, store updated | either moves without a warning; a store returning to a version it already reached stays quiet, since iTunes and Flathub's CDN re-serve the previous one for hours | +| ⚠️ source failing, untagged | a source has failed for an hour, then every 12 hours, and ✅ once it reads again | + +State is `/var/lib/session-ops/store-versions/state.json`. Losing it makes the next run +a first check. A message Discord refuses is posted again on the next run. + +GitHub is read unauthenticated: one request per repository a run, 36 an hour of the 60 +allowed. The organisation refuses fine-grained tokens that live longer than 366 days. diff --git a/pyproject.toml b/pyproject.toml index b8d908c..88f3640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ session-ops = "session_ops.ops.runner:main" session-ops-alert = "session_ops.monitor.alert:main" session-ops-silence = "session_ops.monitor.silence:main" session-ops-token-expiry = "session_ops.monitor.token_expiry:main" +store-versions = "session_ops.store_versions.job:main" [dependency-groups] dev = [ diff --git a/src/session_ops/jobs.toml b/src/session_ops/jobs.toml index 9dbf23b..5b5b34a 100644 --- a/src/session_ops/jobs.toml +++ b/src/session_ops/jobs.toml @@ -100,6 +100,20 @@ schedule = "*-*-* 09:00 Australia/Melbourne" max_age_hours = 27 timeout = "5min" +[[job]] +name = "store-versions" +description = "Store releases against GitHub releases, posted as they diverge" +entry = "session_ops.store_versions.job:main" +args = ["--state", "{state}/state.json"] +user = "sessionops" +env_files = ["/etc/session-ops/store-versions.env"] +env = ["STORE_VERSIONS_DISCORD_WEBHOOK_URL"] +channel_env = "STORE_VERSIONS_DISCORD_WEBHOOK_URL" +# Unauthenticated GitHub allows 60 requests an hour; this makes 36, one per repo a run. +schedule = "*:0/5" +max_age_hours = 1 +timeout = "4min" + [[job]] name = "crowdin-sync" description = "Crowdin translations into the platform repos" diff --git a/src/session_ops/shared/github.py b/src/session_ops/shared/github.py index 84c556a..caf1ea4 100644 --- a/src/session_ops/shared/github.py +++ b/src/session_ops/shared/github.py @@ -1,5 +1,5 @@ -"""GitHub's REST API for the jobs that publish to the platform repos: who they -authenticate as, and the pull requests they keep open. +"""GitHub's REST API: the org's releases, and for the jobs that publish to the platform +repos, who they authenticate as and the pull requests they keep open. Publishing authenticates as a GitHub App, with an installation token minted per run and scoped to the repositories the job names: it expires in an hour, and pull @@ -116,6 +116,13 @@ def check(resp, what): return resp.json() if resp.content else None +def latest_release(api, repo): + """(tag, page URL) of `repo`'s latest release: published, neither draft nor prerelease.""" + release = check(api.request("GET", f"{API}/repos/{repo}/releases/latest"), + f"reading {repo}'s latest release") + return release["tag_name"], release.get("html_url", "") + + def app_jwt(app_id, private_key, now=None): now = int(now or time.time()) # Backdated a minute for clock drift; GitHub refuses one living past ten minutes. diff --git a/src/session_ops/shared/state.py b/src/session_ops/shared/state.py index d0a801a..54c08f7 100644 --- a/src/session_ops/shared/state.py +++ b/src/session_ops/shared/state.py @@ -69,12 +69,16 @@ def save_state(path, state, records, retention_days, version, extra=None): if last >= cutoff: kept[key] = record + write_json(path, {"version": version, "updated_at": stamp, **(extra or {}), "seen": kept}) + return len(kept), len(seen) - len(kept) + + +def write_json(path, data): + """Write `data` to `path` atomically: a crash mid-write cannot corrupt the state.""" directory = os.path.dirname(path) if directory: os.makedirs(directory, exist_ok=True) temporary = f"{path}.tmp" with open(temporary, "w", encoding="utf-8") as handle: - json.dump({"version": version, "updated_at": stamp, **(extra or {}), "seen": kept}, - handle, indent=2) - os.replace(temporary, path) # atomic: a crash mid-write cannot corrupt the state - return len(kept), len(seen) - len(kept) + json.dump(data, handle, indent=2) + os.replace(temporary, path) diff --git a/src/session_ops/store_versions/__init__.py b/src/session_ops/store_versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py new file mode 100644 index 0000000..a983e68 --- /dev/null +++ b/src/session_ops/store_versions/job.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +""" +Post when a store's Session release and the GitHub release diverge. + +Each run compares every platform's store version with its GitHub release and posts: + +- ⚠️ a warning, tagging the platform's people, when the store is ahead of GitHub (a + public build whose source is not out) or, for a channel we publish ourselves (APT, + and the merged F-Droid and Flathub manifests), when it differs either way: their + release pull requests have to be merged after each GitHub release; +- the warning again every 12 hours while unresolved, and at once if the gap grows; +- ✅ once back in sync, after a warning; +- ℹ️ when GitHub or the store moves on without a warning. + +A platform seen for the first time is recorded quietly, unless already out of sync. +A source failing for an hour is posted, without tags, then every 12 hours until it +recovers. + +Config: + STORE_VERSIONS_DISCORD_WEBHOOK_URL where it posts (not needed with --dry-run) + /etc/session-ops/store-versions.toml who each platform's warnings tag: + [tag] + android = [111111111111111111] + +Usage: + store-versions --state /var/lib/session-ops/store-versions/state.json + store-versions --dry-run # print every version and what it would post +""" +import argparse +import json +import sys +import time +import tomllib +from dataclasses import dataclass + +from session_ops.shared import discord, http +from session_ops.shared.env import get_env +from session_ops.shared.state import write_json +from session_ops.store_versions import sources +from session_ops.store_versions.transitions import ( + ALL_CLEAR, INFO_GITHUB, INFO_STORE, SOURCE_RECOVERED, WARNING, + WARNING_REMINDER, WARNING_UPDATE, WARNINGS, evaluate, source_events) + +TAGS_FILE = "/etc/session-ops/store-versions.toml" +STATE_VERSION = 1 +REMINDER_SECONDS = 12 * 3600 +FAILING_AFTER_SECONDS = 3600 +USER_AGENT = "session-ops store-versions (+https://github.com/session-foundation)" + +ANDROID_ID = "network.loki.messenger" +FLATHUB_ID = "network.loki.Session" + + +@dataclass(frozen=True) +class Platform: + name: str + display: str + repo: str + store: dict + strict_sync: bool = False + + +PLATFORMS = ( + Platform("android", "Session Android", "session-android", + {"kind": "play", "app_id": ANDROID_ID}), + # Each Android release opens a "Release X.Y.Z" pull request on session-fdroid, which + # has to be merged; the merged index is what this checks. + Platform("android-fdroid-merged", "Session Android (F-Droid, merged)", "session-android", + {"kind": "fdroid", "app_id": ANDROID_ID, + "base_url": "https://raw.githubusercontent.com/session-foundation/" + "session-fdroid/master/fdroid/repo"}, + strict_sync=True), + # What F-Droid clients pull: behind is deploy lag, so app-store rules. + Platform("android-fdroid-live", "Session Android (F-Droid, live)", "session-android", + {"kind": "fdroid", "app_id": ANDROID_ID, + "base_url": "https://fdroid.getsession.org/fdroid/repo"}), + Platform("ios", "Session iOS", "session-ios", + {"kind": "appstore", "bundle_id": "com.loki-project.loki-messenger"}), + # session-desktop, not session-messenger-desktop: that one is an old transitional package. + Platform("desktop", "Session Desktop", "session-desktop", + {"kind": "deb", "base_url": "https://deb.session.foundation", + "package": "session-desktop"}, + strict_sync=True), + # flathubbot opens an update pull request on flathub/network.loki.Session per + # release, which has to be merged. + Platform("desktop-flathub-merged", "Session Desktop (Flathub, merged)", "session-desktop", + {"kind": "flathub_manifest", "app_id": FLATHUB_ID}, strict_sync=True), + # Behind is Flathub's build and publish lag after the merge. + Platform("desktop-flathub-live", "Session Desktop (Flathub, live)", "session-desktop", + {"kind": "flathub", "app_id": FLATHUB_ID}), +) + +STORE_NAMES = {"play": "Play Store", "appstore": "App Store", "deb": "APT repo", + "fdroid": "F-Droid repo", "flathub": "Flathub", + "flathub_manifest": "Flathub manifest"} + +RED, GREEN, BLUE = 0xE74C3C, 0x2ECC71, 0x3498DB + + +def store_name(platform): + return STORE_NAMES[platform.store["kind"]] + + +def load_tags(path): + """{platform: [user ids]}; nobody is tagged without the file.""" + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except FileNotFoundError: + return {} + except tomllib.TOMLDecodeError as exc: + sys.exit(f"{path}: {exc}") + tags = data.get("tag", {}) + known = {platform.name for platform in PLATFORMS} + for name, ids in tags.items(): + if name not in known: + sys.exit(f"{path}: [tag] names {name}, which is not one of {', '.join(sorted(known))}") + if not (isinstance(ids, list) and all(isinstance(i, int) for i in ids)): + sys.exit(f"{path}: [tag] {name} must be a list of Discord user ids, as integers") + return {name: [str(i) for i in ids] for name, ids in tags.items()} + + +def load_state(path): + """Unreadable or another version is an empty state: every platform is then seen for + the first time, quietly unless out of sync.""" + data = {} + if path: + try: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + except (OSError, ValueError): + pass + if not isinstance(data, dict) or data.get("version") != STATE_VERSION: + data = {} + return {"version": STATE_VERSION, "platforms": data.get("platforms", {}), + "sources": data.get("sources", {})} + + +def gap(platform, event): + store = store_name(platform) + if sources.compare(event.store, event.github) > 0: + return (f"{store} is ahead of GitHub", + f"{store} has published **{event.store}**, but the latest GitHub release " + f"is **{event.github}**.") + return (f"{store} is behind GitHub", + f"GitHub has published **{event.github}**, but {store} is still at " + f"**{event.store}**.") + + +def event_message(platform, event, github_url, store_url, tags): + store = store_name(platform) + fields = [{"name": f"{store} version", "value": event.store, "inline": True}, + {"name": "GitHub version", "value": event.github, "inline": True}] + fields += [{"name": name, "value": url} for name, url in + ((f"{store} link", store_url), ("GitHub release", github_url)) if url] + embed = {"fields": fields} + message = {"embeds": [embed], "allowed_mentions": {"parse": []}} + + if event.kind in WARNINGS: + direction, description = gap(platform, event) + suffix = {WARNING: " (first check)" if event.initial else "", + WARNING_UPDATE: " — divergence grew", + WARNING_REMINDER: " — still unresolved"}[event.kind] + if event.kind == WARNING_REMINDER: + description += f"\nStill out of sync after ~{REMINDER_SECONDS // 3600}h." + embed.update(color=RED, title=f"⚠️ {platform.display}: {direction}{suffix}", + description=description) + ids = tags.get(platform.name, []) + if ids: + message["content"] = " ".join(f"<@{i}>" for i in ids) + message["allowed_mentions"] = {"users": ids} + elif event.kind == ALL_CLEAR: + embed.update(color=GREEN, title=f"✅ {platform.display}: back in sync", + description=f"{store} and GitHub are back in sync at **{event.github}**.") + elif event.kind == INFO_GITHUB: + embed.update(color=BLUE, title=f"ℹ️ {platform.display}: new GitHub release", + description=f"GitHub advanced to **{event.github}** " + f"({store} is at **{event.store}**).") + elif event.kind == INFO_STORE: + embed.update(color=BLUE, title=f"ℹ️ {platform.display}: {store} updated", + description=f"{store} advanced to **{event.store}** " + f"(GitHub is at **{event.github}**).") + return message + + +def source_message(source, kind, failure, now): + if kind == SOURCE_RECOVERED: + embed = {"color": GREEN, "title": f"✅ {source}: readable again"} + else: + hours = max(1, round((now - failure["since"]) / 3600)) + embed = {"color": RED, "title": f"⚠️ {source}: failing for ~{hours}h", + "description": discord.clip(failure["error"], 1000)} + return {"embeds": [embed], "allowed_mentions": {"parse": []}} + + +class Fetcher: + """Each source's Release or error, fetched once per run: three platforms share + session-android's release, and unauthenticated GitHub allows 60 requests an hour.""" + + def __init__(self, session): + self.session = session + self.results = {} + + def get(self, key, fetch, *args): + if key not in self.results: + try: + self.results[key] = (fetch(self.session, *args), None) + except Exception as exc: # one source down must not stop the others + self.results[key] = (None, f"{type(exc).__name__}: {exc}") + print(f"{key}: {self.results[key][1]}", file=sys.stderr) + return self.results[key] + + +def check(state, tags, session, now): + """[(messages, apply)]: `apply` records a step's state once its messages are posted, + so a failed post is retried on the next run rather than lost.""" + fetcher = Fetcher(session) + steps = [] + for platform in PLATFORMS: + github, github_error = fetcher.get(f"GitHub {platform.repo}", sources.fetch_github, + platform.repo) + store, store_error = fetcher.get(f"{store_name(platform)} ({platform.name})", + sources.fetch_store, platform.store) + if github_error or store_error: + print(f"{platform.name}: skipped") + continue + print(f"{platform.name}: github={github.version} {platform.store['kind']}={store.version}") + result = evaluate(state["platforms"].get(platform.name), github.version, store.version, + strict_sync=platform.strict_sync, now=now, + reminder_seconds=REMINDER_SECONDS) + messages = [event_message(platform, event, github.url, store.url, tags) + for event in result.events] + steps.append((messages, lambda name=platform.name, new=result.state: + state["platforms"].__setitem__(name, new))) + + for source in sorted(set(fetcher.results) | set(state["sources"])): + error = fetcher.results.get(source, (None, None))[1] + kind, failure = source_events(state["sources"].get(source), error, now=now, + after_seconds=FAILING_AFTER_SECONDS, + reminder_seconds=REMINDER_SECONDS) + messages = [source_message(source, kind, failure, now)] if kind else [] + steps.append((messages, lambda source=source, failure=failure: + state["sources"].__setitem__(source, failure) if failure + else state["sources"].pop(source, None))) + return steps + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Post when store and GitHub releases diverge.") + parser.add_argument("--state", metavar="PATH", + help="What was last seen and posted; without it every run is a first.") + parser.add_argument("--tags", default=TAGS_FILE, metavar="PATH") + parser.add_argument("--webhook", help="Discord webhook URL " + "(else STORE_VERSIONS_DISCORD_WEBHOOK_URL).") + parser.add_argument("--dry-run", action="store_true", + help="Print what it would post instead; write no state.") + args = parser.parse_args(argv) + + webhook = get_env("STORE_VERSIONS_DISCORD_WEBHOOK_URL", args.webhook, + required=not args.dry_run) + tags = load_tags(args.tags) + state = load_state(args.state) + session = http.Session(attempts=3) + session.headers["User-Agent"] = USER_AGENT + steps = check(state, tags, session, time.time()) + messages = [message for step_messages, _ in steps for message in step_messages] + + if args.dry_run: + for message in messages: + print(json.dumps(message, ensure_ascii=False)) + print(f"Would post {len(messages)} message(s).") + return + + posted = discord.post_to_discord(http.Session(), webhook, messages) if messages else 0 + reached = 0 + for step_messages, apply in steps: + reached += len(step_messages) + if reached <= posted: + apply() + if args.state: + write_json(args.state, state) + if posted < len(messages): + sys.exit(f"Posted {posted} of {len(messages)} messages to Discord.") + print(f"Posted {len(messages)} message(s).") + + +if __name__ == "__main__": + main() diff --git a/src/session_ops/store_versions/sources.py b/src/session_ops/store_versions/sources.py new file mode 100644 index 0000000..6477cd4 --- /dev/null +++ b/src/session_ops/store_versions/sources.py @@ -0,0 +1,186 @@ +"""The latest version each release channel publishes, normalised to plain "1.2.3". + +Every fetcher takes a shared http.Session and returns a Release, or raises: the job +records a failure per source rather than per platform, since several platforms +share one. +""" +import functools +import gzip +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass + +from session_ops.shared import github + +PLAY_DETAILS = "https://play.google.com/store/apps/details" +ITUNES_LOOKUP = "https://itunes.apple.com/lookup" +FLATHUB_APPSTREAM = "https://flathub.org/api/v2/appstream/{app_id}" +FLATHUB_PAGE = "https://flathub.org/apps/{app_id}" +FLATHUB_METAINFO = "https://raw.githubusercontent.com/{repo}/{branch}/{app_id}.metainfo.xml" + +# Play has no version API: the listing bootstraps its data with the version as +# `[[["1.2.3"]]`. When Android starts failing, Google has likely changed the page. +PLAY_VERSION = re.compile(r'\[\[\["(\d+(?:\.\d+)+)"\]\]') +DEB_EPOCH = re.compile(r"^\d+:") +DEB_REVISION = re.compile(r"-[^-]+$") + + +class SourceError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Release: + version: str + url: str = "" + + +def normalize(version): + return re.sub(r"^[vV]", "", (version or "").strip()) + + +def version_key(version): + """Orders dotted versions numerically, so 1.10.0 sorts after 1.9.0 and 1.2 equals 1.2.0.""" + parts = [int(part) for part in re.findall(r"\d+", version)] + while parts and parts[-1] == 0: + parts.pop() + return tuple(parts) + + +def compare(a, b): + """-1, 0 or 1 for a < b, a == b, a > b.""" + ka, kb = version_key(a), version_key(b) + return (ka > kb) - (ka < kb) + + +def newest(versions): + """The highest of `versions`, None if there are none. Store lists are not trusted + to be newest-first: AppStream recommends it and does not require it.""" + versions = [normalize(v) for v in versions if v] + return max(versions, key=functools.cmp_to_key(compare)) if versions else None + + +def deb_upstream(version): + """A Debian version without its epoch ("1:") or revision ("-2"); the versions + tracked here never contain a hyphen of their own.""" + version = DEB_EPOCH.sub("", version.strip()) + return normalize(DEB_REVISION.sub("", version) if "-" in version else version) + + +def checked(resp, what): + if resp.status_code >= 400: + raise SourceError(f"{what}: HTTP {resp.status_code}") + return resp + + +def fetch_github(session, repo): + tag, url = github.latest_release(session, f"{github.ORG}/{repo}") + return Release(normalize(tag), url) + + +def fetch_play(session, app_id, country="us", lang="en"): + resp = checked(session.request("GET", PLAY_DETAILS, + params={"id": app_id, "hl": lang, "gl": country}), + f"Play listing for {app_id}") + match = PLAY_VERSION.search(resp.text) + if not match: + raise SourceError(f"no version in the Play listing for {app_id}; " + "the page layout may have changed") + return Release(match.group(1), f"{PLAY_DETAILS}?id={app_id}") + + +def fetch_appstore(session, bundle_id, country="us"): + resp = checked(session.request("GET", ITUNES_LOOKUP, + params={"bundleId": bundle_id, "country": country}), + f"App Store lookup for {bundle_id}") + results = resp.json().get("results") or [] + if not results or not results[0].get("version"): + raise SourceError(f"the App Store lists no version for {bundle_id}") + return Release(normalize(results[0]["version"]), results[0].get("trackViewUrl", "")) + + +def fetch_deb(session, base_url, package, suite="sid", component="main", arch="amd64"): + base = base_url.rstrip("/") + index = f"{base}/dists/{suite}/{component}/binary-{arch}/Packages.gz" + resp = checked(session.request("GET", index), index) + try: + text = gzip.decompress(resp.content).decode("utf-8", "replace") + except (OSError, EOFError) as exc: + raise SourceError(f"cannot decompress {index}: {exc}") from exc + version = packages_version(text, package) + if version is None: + raise SourceError(f"no {package} in {index}") + return Release(deb_upstream(version), base) + + +def packages_version(text, package): + """The Version: of `package` in a Packages index, or None.""" + for stanza in re.split(r"\n\s*\n", text): + fields = dict(line.split(":", 1) for line in stanza.splitlines() + if ":" in line and not line.startswith(" ")) + if fields.get("Package", "").strip() == package and fields.get("Version"): + return fields["Version"].strip() + return None + + +def fetch_fdroid(session, base_url, app_id): + base = base_url.rstrip("/") + resp = checked(session.request("GET", f"{base}/index-v1.json"), f"{base}/index-v1.json") + try: + index = resp.json() + except ValueError as exc: + raise SourceError(f"cannot parse {base}/index-v1.json: {exc}") from exc + version = fdroid_version(index, app_id) + if version is None: + raise SourceError(f"no {app_id} in {base}/index-v1.json") + return Release(normalize(version), base) + + +def fdroid_version(index, app_id): + """versionName of the build with the highest versionCode: one entry per ABI split.""" + builds = (index.get("packages") or {}).get(app_id) + if not builds: + return None + return max(builds, key=lambda build: build.get("versionCode", 0)).get("versionName") + + +def fetch_flathub(session, app_id): + resp = checked(session.request("GET", FLATHUB_APPSTREAM.format(app_id=app_id)), + f"Flathub appstream for {app_id}") + version = newest(release.get("version") for release in resp.json().get("releases") or []) + if version is None: + raise SourceError(f"Flathub lists no release of {app_id}") + return Release(version, FLATHUB_PAGE.format(app_id=app_id)) + + +def fetch_flathub_manifest(session, app_id, repo=None, branch="master"): + repo = repo or f"flathub/{app_id}" + url = FLATHUB_METAINFO.format(repo=repo, branch=branch, app_id=app_id) + version = metainfo_version(checked(session.request("GET", url), url).content) + if version is None: + raise SourceError(f"no in {url}") + return Release(version, f"https://github.com/{repo}") + + +def metainfo_version(xml_bytes): + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + raise SourceError(f"cannot parse metainfo: {exc}") from exc + return newest(release.get("version") for release in root.iterfind("releases/release")) + + +FETCHERS = { + "play": fetch_play, + "appstore": fetch_appstore, + "deb": fetch_deb, + "fdroid": fetch_fdroid, + "flathub": fetch_flathub, + "flathub_manifest": fetch_flathub_manifest, +} + + +def fetch_store(session, store): + """`store` is a platform's store spec: its `kind`, and that fetcher's arguments.""" + arguments = dict(store) + return FETCHERS[arguments.pop("kind")](session, **arguments) diff --git a/src/session_ops/store_versions/transitions.py b/src/session_ops/store_versions/transitions.py new file mode 100644 index 0000000..a7be0ba --- /dev/null +++ b/src/session_ops/store_versions/transitions.py @@ -0,0 +1,110 @@ +"""What to post, given a platform's last state and this run's versions: no IO. + +A platform's state is a dict: the GitHub and store versions last seen, the store's +highest ever (`store_high`), whether a warning is out (`warning`) and when it was last +posted (`warned_at`, epoch seconds). +""" +from dataclasses import dataclass, field + +from session_ops.store_versions.sources import compare + +WARNING = "warning" +WARNING_UPDATE = "warning_update" +WARNING_REMINDER = "warning_reminder" +ALL_CLEAR = "all_clear" +INFO_GITHUB = "info_github" +INFO_STORE = "info_store" +WARNINGS = (WARNING, WARNING_UPDATE, WARNING_REMINDER) + +SOURCE_FAILING = "source_failing" +SOURCE_RECOVERED = "source_recovered" + + +@dataclass(frozen=True) +class Event: + kind: str + github: str + store: str + # Raised on a platform's first check, which found it already out of sync. + initial: bool = False + + +@dataclass +class Result: + events: list = field(default_factory=list) + state: dict = field(default_factory=dict) + + +def evaluate(prev, github, store, *, strict_sync=False, now=0.0, reminder_seconds=None): + """The events for this run, and the state to keep once they are posted. + + App stores warn only when the store is ahead of GitHub: a public build whose source + is not published. A `strict_sync` channel is ours and must match GitHub, so it + warns in either direction. An unresolved warning repeats every `reminder_seconds`, + and at once if the leading side pulls further ahead. + """ + order = compare(store, github) + store_ahead, store_behind = order > 0, order < 0 + out_of_sync = order != 0 if strict_sync else store_ahead + + # Store APIs are eventually consistent: iTunes and Flathub's CDN re-serve the + # previous version for hours after a release. The true version is kept, but only a + # new high is announced, so a dip and recovery stays quiet. + prev_high = prev and (prev.get("store_high") or prev["store"]) + new_high = prev_high is None or compare(store, prev_high) > 0 + state = {"github": github, "store": store, "store_high": store if new_high else prev_high, + "warning": out_of_sync, "warned_at": None} + + if prev is None: + if out_of_sync: + state["warned_at"] = now + return Result([Event(WARNING, github, store, initial=True)], state) + return Result([], state) + + github_advanced = compare(github, prev["github"]) > 0 + store_advanced = compare(store, prev["store"]) > 0 + warned_at = now if prev.get("warned_at") is None else prev["warned_at"] + events = [] + + if out_of_sync and not prev["warning"]: + events.append(Event(WARNING, github, store)) + warned_at = now + elif out_of_sync: + if (store_ahead and store_advanced) or (store_behind and github_advanced): + events.append(Event(WARNING_UPDATE, github, store)) + warned_at = now + elif reminder_seconds is not None and now - warned_at >= reminder_seconds: + events.append(Event(WARNING_REMINDER, github, store)) + warned_at = now + elif prev["warning"]: + events.append(Event(ALL_CLEAR, github, store)) + + if not events: + if github_advanced: + events.append(Event(INFO_GITHUB, github, store)) + if new_high: + events.append(Event(INFO_STORE, github, store)) + + state["warned_at"] = warned_at if out_of_sync else None + return Result(events, state) + + +def source_events(prev, error, *, now, after_seconds, reminder_seconds): + """(event or None, state) for one source: `error` is this run's failure, or None. + + A source can fail for a run or two (Play throttles a scrape, a CDN hiccups), so it + is reported only once it has failed for `after_seconds`, then every + `reminder_seconds`, and its recovery only if its failure was reported. + """ + if error is None: + recovered = prev and prev.get("alerted_at") is not None + return (SOURCE_RECOVERED if recovered else None), None + since = prev["since"] if prev else now + alerted_at = prev.get("alerted_at") if prev else None + state = {"since": since, "alerted_at": alerted_at, "error": error} + due = (now - since >= after_seconds if alerted_at is None + else now - alerted_at >= reminder_seconds) + if due: + state["alerted_at"] = now + return SOURCE_FAILING, state + return None, state diff --git a/tests/goldens/units/dropins.txt b/tests/goldens/units/dropins.txt index 7f3cb1a..51ff8e7 100644 --- a/tests/goldens/units/dropins.txt +++ b/tests/goldens/units/dropins.txt @@ -89,6 +89,24 @@ EnvironmentFile=/etc/session-ops/publish.env TimeoutStartSec=10min LoadCredential=github-app.pem:/etc/session-ops/github-app.pem +==> session-ops@store-versions.service.d/job.conf <== +# Generated by `session-ops units` from jobs.toml. Edit the registry, not this. + +[Unit] +Description=Store releases against GitHub releases, posted as they diverge + +[Service] +User=sessionops +Group=sessionops +EnvironmentFile=/etc/session-ops/store-versions.env +TimeoutStartSec=4min + +==> session-ops@store-versions.timer.d/schedule.conf <== +# Generated by `session-ops units` from jobs.toml. Edit the registry, not this. + +[Timer] +OnCalendar=*:0/5 + ==> session-ops@token-expiry.service.d/job.conf <== # Generated by `session-ops units` from jobs.toml. Edit the registry, not this. diff --git a/tests/store_versions/__init__.py b/tests/store_versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/store_versions/test_job.py b/tests/store_versions/test_job.py new file mode 100644 index 0000000..0d98384 --- /dev/null +++ b/tests/store_versions/test_job.py @@ -0,0 +1,123 @@ +""" + uv run python -m unittest tests.store_versions.test_job +""" +import json +import os +import tempfile +import unittest + +from session_ops.shared import discord +from session_ops.shared.testing import Patched +from session_ops.store_versions import job, sources +from session_ops.store_versions.sources import Release + +PLATFORMS = (job.Platform("ios", "Session iOS", "session-ios", + {"kind": "appstore", "bundle_id": "b"}), + job.Platform("desktop", "Session Desktop", "session-desktop", + {"kind": "deb", "base_url": "u", "package": "p"}, strict_sync=True)) +WEBHOOK = "https://discord.example/api/webhooks/1/x" + + +class Run: + """job.main against fixed versions, recording what it posts; `accept` caps how + many messages Discord takes.""" + + def __init__(self, github, stores, accept=None, tags=""): + self.github, self.stores, self.accept, self.posted = github, stores, accept, [] + self.dir = tempfile.mkdtemp() + self.state = os.path.join(self.dir, "state.json") + self.tags = os.path.join(self.dir, "tags.toml") + with open(self.tags, "w", encoding="utf-8") as handle: + handle.write(tags) + + def fetch_github(self, session, repo): + return Release(self.github[repo], f"https://github.com/{repo}") + + def fetch_store(self, session, store): + value = self.stores[store["kind"]] + if isinstance(value, Exception): + raise value + return Release(value) + + def post(self, session, url, messages): + taken = messages if self.accept is None else messages[:self.accept] + self.posted += taken + return len(taken) + + def __call__(self, *extra, now=1000.0): + with Patched(job, PLATFORMS=PLATFORMS), \ + Patched(job.time, time=lambda: now), \ + Patched(sources, fetch_github=self.fetch_github, fetch_store=self.fetch_store), \ + Patched(discord, post_to_discord=self.post): + job.main(["--state", self.state, "--tags", self.tags, "--webhook", WEBHOOK, *extra]) + + def saved(self): + with open(self.state, encoding="utf-8") as handle: + return json.load(handle) + + +def titles(messages): + return [message["embeds"][0]["title"] for message in messages] + + +class TestJob(unittest.TestCase): + def test_first_run_in_sync_posts_nothing_and_records_both(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.5", "deb": "1.18.2"}) + run() + self.assertEqual(run.posted, []) + self.assertEqual(set(run.saved()["platforms"]), {"ios", "desktop"}) + + def test_warning_tags_the_platforms_people_only(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.2"}, tags="[tag]\nios = [42]\n") + run() + [message] = run.posted + self.assertEqual(message["content"], "<@42>") + self.assertEqual(message["allowed_mentions"], {"users": ["42"]}) + self.assertIn("App Store is ahead of GitHub (first check)", titles(run.posted)[0]) + + def test_info_mentions_nobody(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.5", "deb": "1.18.2"}, tags="[tag]\nios = [42]\n") + run() + run.github["session-ios"] = "2.15.6" + run() + [message] = run.posted + self.assertNotIn("content", message) + self.assertEqual(message["allowed_mentions"], {"parse": []}) + + def test_unposted_warning_is_retried_next_run(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, + {"appstore": "2.15.6", "deb": "1.18.2"}, accept=1) + with self.assertRaises(SystemExit): + run() + self.assertEqual(set(run.saved()["platforms"]), {"ios"}) + run.accept = None + run() + self.assertEqual(len(run.posted), 2) + self.assertIn("APT repo is behind GitHub", titles(run.posted)[1]) + + def test_failing_source_is_posted_after_an_hour_then_recovery(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": RuntimeError("boom"), "deb": "1.18.2"}) + run(now=0.0) + run(now=3600.0) + self.assertEqual(titles(run.posted), ["⚠️ App Store (ios): failing for ~1h"]) + self.assertNotIn("ios", run.saved()["platforms"]) + run.stores["appstore"] = "2.15.5" + run(now=4000.0) + self.assertEqual(titles(run.posted)[1], "✅ App Store (ios): readable again") + self.assertEqual(run.saved()["sources"], {}) + + def test_dry_run_posts_and_writes_nothing(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + run("--dry-run") + self.assertEqual(run.posted, []) + self.assertFalse(os.path.exists(run.state)) + + def test_tags_must_name_a_platform(self): + run = Run({}, {}, tags="[tag]\nandroind = [1]\n") + with self.assertRaises(SystemExit): + job.load_tags(run.tags) diff --git a/tests/store_versions/test_sources.py b/tests/store_versions/test_sources.py new file mode 100644 index 0000000..a557f47 --- /dev/null +++ b/tests/store_versions/test_sources.py @@ -0,0 +1,100 @@ +""" + uv run python -m unittest tests.store_versions.test_sources +""" +import gzip +import unittest + +from session_ops.shared.testing import FakeResponse, FakeSession +from session_ops.store_versions import sources + + +class TestVersions(unittest.TestCase): + def test_compare_is_numeric(self): + self.assertEqual(sources.compare("1.18.1", "1.17.17"), 1) + self.assertEqual(sources.compare("1.2.0", "1.10.0"), -1) + self.assertEqual(sources.compare("2.15.3", "2.15.3"), 0) + self.assertEqual(sources.compare("1.2", "1.2.0"), 0) + + def test_normalize_and_deb_upstream(self): + self.assertEqual(sources.normalize(" v1.18.1 "), "1.18.1") + self.assertEqual(sources.deb_upstream("1:1.17.17-2"), "1.17.17") + self.assertEqual(sources.deb_upstream("1.18.2"), "1.18.2") + + def test_newest_ignores_list_order(self): + self.assertEqual(sources.newest(["1.9.0", "v1.10.0", None, ""]), "1.10.0") + self.assertIsNone(sources.newest([])) + + +class TestParsers(unittest.TestCase): + PACKAGES = ("Package: session-messenger-desktop\nVersion: 1.0.0\n\n" + "Package: session-desktop\nArchitecture: amd64\nVersion: 1:1.18.2-1\n" + "Description: Session\n multi-line\n") + + def test_packages_finds_the_exact_package(self): + self.assertEqual(sources.packages_version(self.PACKAGES, "session-desktop"), "1:1.18.2-1") + self.assertIsNone(sources.packages_version(self.PACKAGES, "nope")) + + def test_fdroid_takes_the_highest_version_code(self): + index = {"packages": {"app": [{"versionCode": 9, "versionName": "9.0.0"}, + {"versionCode": 10, "versionName": "10.0.0"}, + {"versionCode": 2, "versionName": "2.0.0"}]}} + self.assertEqual(sources.fdroid_version(index, "app"), "10.0.0") + self.assertIsNone(sources.fdroid_version(index, "other")) + self.assertIsNone(sources.fdroid_version({}, "app")) + + def test_metainfo_takes_the_newest_release(self): + xml = (b'' + b'' + b'') + self.assertEqual(sources.metainfo_version(xml), "1.18.2") + self.assertIsNone(sources.metainfo_version(b"")) + with self.assertRaises(sources.SourceError): + sources.metainfo_version(b"") + + +class Raw(FakeResponse): + def __init__(self, text=None, content=None, status_code=200): + super().__init__({}, status_code) + self.text = text or "" + self._content = content + + @property + def content(self): + return self._content if self._content is not None else self.text.encode() + + +class TestFetchers(unittest.TestCase): + def test_play_extracts_the_bootstrapped_version(self): + session = FakeSession([Raw('junk [[["1.33.6"]],[[[33]]]] junk')]) + release = sources.fetch_play(session, "network.loki.messenger") + self.assertEqual(release.version, "1.33.6") + self.assertEqual(session.calls[0][2]["params"]["id"], "network.loki.messenger") + + def test_play_without_a_version_fails(self): + with self.assertRaises(sources.SourceError): + sources.fetch_play(FakeSession([Raw("")]), "x") + + def test_deb(self): + packages = gzip.compress(TestParsers.PACKAGES.encode()) + release = sources.fetch_store( + FakeSession([Raw(content=packages)]), + {"kind": "deb", "base_url": "https://deb.example/", "package": "session-desktop"}) + self.assertEqual(release.version, "1.18.2") + + def test_flathub_appstream(self): + session = FakeSession([FakeResponse({"releases": [{"version": "1.18.1"}, + {"version": "1.18.2"}]})]) + release = sources.fetch_flathub(session, "network.loki.Session") + self.assertEqual(release.version, "1.18.2") + self.assertEqual(release.url, "https://flathub.org/apps/network.loki.Session") + + def test_github_strips_the_v(self): + session = FakeSession([FakeResponse({"tag_name": "v1.18.2", "html_url": "u"})]) + release = sources.fetch_github(session, "session-desktop") + self.assertEqual(release.version, "1.18.2") + self.assertTrue(session.calls[0][1].endswith( + "/repos/session-foundation/session-desktop/releases/latest")) + + def test_http_error_fails_the_source(self): + with self.assertRaises(sources.SourceError): + sources.fetch_appstore(FakeSession([FakeResponse({}, status_code=503)]), "x") diff --git a/tests/store_versions/test_transitions.py b/tests/store_versions/test_transitions.py new file mode 100644 index 0000000..83a3f22 --- /dev/null +++ b/tests/store_versions/test_transitions.py @@ -0,0 +1,160 @@ +""" + uv run python -m unittest tests.store_versions.test_transitions +""" +import unittest + +from session_ops.store_versions.transitions import ( + ALL_CLEAR, INFO_GITHUB, INFO_STORE, SOURCE_FAILING, SOURCE_RECOVERED, WARNING, + WARNING_REMINDER, WARNING_UPDATE, evaluate, source_events) + +DAY = 12 * 3600 + + +def seen(github, store, warning=False, warned_at=None, store_high=None): + return {"github": github, "store": store, "warning": warning, "warned_at": warned_at, + "store_high": store_high} + + +def kinds(result): + return [event.kind for event in result.events] + + +class TestFirstCheck(unittest.TestCase): + def test_in_sync_or_github_ahead_is_quiet(self): + for github, store in (("1.0.0", "1.0.0"), ("1.1.0", "1.0.0")): + result = evaluate(None, github, store) + self.assertEqual(kinds(result), []) + self.assertFalse(result.state["warning"]) + + def test_store_ahead_warns_as_first_check(self): + result = evaluate(None, "1.0.0", "1.1.0", now=50.0) + self.assertEqual(kinds(result), [WARNING]) + self.assertTrue(result.events[0].initial) + self.assertEqual(result.state["warned_at"], 50.0) + + def test_strict_sync_behind_warns(self): + result = evaluate(None, "1.1.0", "1.0.0", strict_sync=True) + self.assertEqual(kinds(result), [WARNING]) + + +class TestTransitions(unittest.TestCase): + def test_no_movement_is_quiet(self): + self.assertEqual(kinds(evaluate(seen("1.0.0", "1.0.0"), "1.0.0", "1.0.0")), []) + + def test_store_jumps_ahead_warns(self): + result = evaluate(seen("1.0.0", "1.0.0"), "1.0.0", "1.1.0") + self.assertEqual(kinds(result), [WARNING]) + self.assertTrue(result.state["warning"]) + + def test_store_further_ahead_while_warning_updates(self): + result = evaluate(seen("1.0.0", "1.1.0", warning=True, warned_at=0.0), "1.0.0", "1.2.0") + self.assertEqual(kinds(result), [WARNING_UPDATE]) + + def test_github_catching_up_or_passing_clears(self): + for github in ("1.1.0", "1.2.0"): + result = evaluate(seen("1.0.0", "1.1.0", warning=True, warned_at=0.0), github, "1.1.0") + self.assertEqual(kinds(result), [ALL_CLEAR]) + self.assertFalse(result.state["warning"]) + self.assertIsNone(result.state["warned_at"]) + + def test_github_advancing_is_info(self): + result = evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.0.0") + self.assertEqual(kinds(result), [INFO_GITHUB]) + + def test_store_catching_up_is_info(self): + result = evaluate(seen("1.2.0", "1.0.0"), "1.2.0", "1.1.0") + self.assertEqual(kinds(result), [INFO_STORE]) + + def test_both_advancing_store_ahead_only_warns(self): + result = evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.2.0") + self.assertEqual(kinds(result), [WARNING]) + + def test_unchanged_warning_is_not_repeated(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + self.assertEqual(kinds(evaluate(prev, "1.0.0", "1.1.0", now=60.0)), []) + + +class TestStrictSync(unittest.TestCase): + def test_behind_warns_only_under_strict_sync(self): + self.assertEqual(kinds(evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.0.0", + strict_sync=True)), [WARNING]) + self.assertEqual(kinds(evaluate(seen("1.0.0", "1.0.0"), "1.1.0", "1.0.0")), + [INFO_GITHUB]) + + def test_github_pulling_further_ahead_updates(self): + prev = seen("1.1.0", "1.0.0", warning=True, warned_at=100.0) + result = evaluate(prev, "1.2.0", "1.0.0", strict_sync=True) + self.assertEqual(kinds(result), [WARNING_UPDATE]) + + def test_catching_up_clears(self): + prev = seen("1.1.0", "1.0.0", warning=True, warned_at=100.0) + self.assertEqual(kinds(evaluate(prev, "1.1.0", "1.1.0", strict_sync=True)), [ALL_CLEAR]) + + +class TestReminder(unittest.TestCase): + def test_repeats_after_the_interval_and_resets_its_clock(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + result = evaluate(prev, "1.0.0", "1.1.0", now=DAY, reminder_seconds=DAY) + self.assertEqual(kinds(result), [WARNING_REMINDER]) + self.assertEqual(result.state["warned_at"], DAY) + + def test_quiet_before_the_interval(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + result = evaluate(prev, "1.0.0", "1.1.0", now=DAY - 1, reminder_seconds=DAY) + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["warned_at"], 0.0) + + def test_growing_gap_resets_the_clock(self): + prev = seen("1.0.0", "1.1.0", warning=True, warned_at=0.0) + result = evaluate(prev, "1.0.0", "1.2.0", now=5000.0, reminder_seconds=DAY) + self.assertEqual(kinds(result), [WARNING_UPDATE]) + self.assertEqual(result.state["warned_at"], 5000.0) + + +class TestStoreHigh(unittest.TestCase): + def test_recovering_to_an_announced_version_is_quiet(self): + result = evaluate(seen("2.15.4", "2.15.3", store_high="2.15.4"), "2.15.4", "2.15.4") + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["store_high"], "2.15.4") + + def test_new_high_is_announced(self): + result = evaluate(seen("2.15.5", "2.15.4", store_high="2.15.4"), "2.15.5", "2.15.5") + self.assertEqual(kinds(result), [INFO_STORE]) + + def test_dip_is_quiet_and_keeps_the_true_version(self): + result = evaluate(seen("2.15.4", "2.15.4", store_high="2.15.4"), "2.15.4", "2.15.3") + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["store"], "2.15.3") + self.assertEqual(result.state["store_high"], "2.15.4") + + def test_flap_announces_once(self): + state, posted = seen("2.15.4", "2.15.4"), [] + for store in ("2.15.5", "2.15.4", "2.15.5", "2.15.4", "2.15.5"): + result = evaluate(state, "2.15.5", store) + posted += kinds(result) + state = result.state + self.assertEqual(posted.count(INFO_STORE), 1) + + +class TestSourceEvents(unittest.TestCase): + def call(self, prev, error, now): + return source_events(prev, error, now=now, after_seconds=3600, reminder_seconds=DAY) + + def test_a_short_failure_and_its_recovery_are_quiet(self): + kind, state = self.call(None, "boom", 0.0) + self.assertIsNone(kind) + kind, state = self.call(state, "boom", 3599.0) + self.assertIsNone(kind) + self.assertEqual(self.call(state, None, 3700.0), (None, None)) + + def test_reported_after_an_hour_then_every_reminder(self): + _, state = self.call(None, "boom", 0.0) + kind, state = self.call(state, "boom", 3600.0) + self.assertEqual(kind, SOURCE_FAILING) + self.assertEqual(self.call(state, "boom", 3600.0 + DAY - 1)[0], None) + self.assertEqual(self.call(state, "boom", 3600.0 + DAY)[0], SOURCE_FAILING) + + def test_recovery_after_a_report_is_posted(self): + _, state = self.call(None, "boom", 0.0) + _, state = self.call(state, "boom", 3600.0) + self.assertEqual(self.call(state, None, 4000.0), (SOURCE_RECOVERED, None)) From eb29cd8ee3363fcbba71048b5e9cd6ce2e03c4a5 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 13:32:04 +1100 Subject: [PATCH 02/11] store-versions: bound each run, fail unusable versions, limit repeat alerts Sources are fetched concurrently on short timeouts, and each platform's events go as one message saved as soon as it lands, so a host that stops answering or a kill mid-run cannot outlast the unit or repost a step. Messages still unposted 150 s in wait for the next run, and one refused message no longer holds back the other platforms. A version that does not start with a digit, or is over 64 characters, fails its source rather than producing an embed Discord refuses. A suffixed version (1.2.3-beta1, 1.2.3rc1) ranks below its release. jobs.toml's repeat_alert_hours makes the runner report the same failure again only that often until a run succeeds; store-versions sets 12, as a 5-minute timer would otherwise alert 288 times a day. --- deploy/README.md | 2 + docs/jobs/store-versions.md | 5 +- src/session_ops/jobs.toml | 3 + src/session_ops/ops/registry.py | 1 + src/session_ops/ops/runner.py | 43 +++++++ src/session_ops/store_versions/job.py | 147 +++++++++++++++------- src/session_ops/store_versions/sources.py | 26 +++- tests/ops/test_runner.py | 32 +++++ tests/store_versions/test_job.py | 48 +++++++ tests/store_versions/test_sources.py | 19 +++ 10 files changed, 273 insertions(+), 53 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index a1c6cf3..b7c34ff 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -58,6 +58,8 @@ Each `/etc/session-ops/.env` has a commented `.env.example` beside i installed from [`env/`](env/), saying what goes in it. `/etc/session-ops/expiry.toml` is not a secret: the expiry dates [token-expiry](../docs/jobs/token-expiry.md) cannot read from an API, from `expiry.toml.example`. +`/etc/session-ops/store-versions.toml` is not a secret either, and has to be mode 644: who +[store-versions](../docs/jobs/store-versions.md) tags, from `store-versions.toml.example`. ## Checking diff --git a/docs/jobs/store-versions.md b/docs/jobs/store-versions.md index 68e4704..15160ee 100644 --- a/docs/jobs/store-versions.md +++ b/docs/jobs/store-versions.md @@ -35,7 +35,10 @@ repo is published with the release. | ⚠️ source failing, untagged | a source has failed for an hour, then every 12 hours, and ✅ once it reads again | State is `/var/lib/session-ops/store-versions/state.json`. Losing it makes the next run -a first check. A message Discord refuses is posted again on the next run. +a first check. A message Discord refuses is posted again on the next run, as is one +still unposted 150 seconds into a run. A version that does not start with a digit, or +runs past 64 characters, fails its source. A failed run is reported once, then every +12 hours until one succeeds. GitHub is read unauthenticated: one request per repository a run, 36 an hour of the 60 allowed. The organisation refuses fine-grained tokens that live longer than 366 days. diff --git a/src/session_ops/jobs.toml b/src/session_ops/jobs.toml index 5b5b34a..5edff35 100644 --- a/src/session_ops/jobs.toml +++ b/src/session_ops/jobs.toml @@ -10,6 +10,8 @@ # channel_env where it posts, and where its own failures are reported; a job # without one reports to ALERT_DISCORD_WEBHOOK_URL # unit further [Service] lines, over the template's hardening +# repeat_alert_hours report the same failure again only this often until a run +# succeeds, for a schedule short enough to repeat it every few minutes # # max_age_hours for a queued job, its queue's: the weekend's 72 h (73 h across # April's DST change), plus every job ahead of it running to its timeout @@ -113,6 +115,7 @@ channel_env = "STORE_VERSIONS_DISCORD_WEBHOOK_URL" schedule = "*:0/5" max_age_hours = 1 timeout = "4min" +repeat_alert_hours = 12 [[job]] name = "crowdin-sync" diff --git a/src/session_ops/ops/registry.py b/src/session_ops/ops/registry.py index 9ed9186..fbc4ae5 100644 --- a/src/session_ops/ops/registry.py +++ b/src/session_ops/ops/registry.py @@ -30,6 +30,7 @@ class Job: unit: tuple = field(default=()) queued: bool = False after: tuple = () + repeat_alert_hours: float = None @property def scheduled(self): diff --git a/src/session_ops/ops/runner.py b/src/session_ops/ops/runner.py index f4ab382..4ace82e 100644 --- a/src/session_ops/ops/runner.py +++ b/src/session_ops/ops/runner.py @@ -14,23 +14,30 @@ (session-ops-alert) knows this failure was already reported and stays quiet. The backstop is for the failures a run cannot report itself: killed, timed out, or Discord unreachable. + +A job with `repeat_alert_hours` reports the same failure again only that often, +until a run succeeds. """ import argparse import importlib +import json import os import re import shutil import socket import sys import tempfile +import time import traceback from dataclasses import dataclass, field from session_ops.ops import registry from session_ops.shared import discord, http +from session_ops.shared.state import write_json from session_ops.zendesk import claude_cli ALERTED_MARKER = "alerted" +LAST_ALERT = "last-alert.json" # Present on a host rehearsing production: jobs publish beside it (env.rehearsing). REHEARSAL_MARKER = "/etc/session-ops/rehearsal" @@ -147,6 +154,35 @@ def mark_alerted(state_dir): file=sys.stderr) +def repeated_alert(state_dir, message, hours, now): + """Whether `message` was reported less than `hours` ago; records it when not. + + Recorded whether or not the post lands: the backstop reports what it cannot.""" + path = os.path.join(state_dir, LAST_ALERT) + try: + with open(path, encoding="utf-8") as handle: + last = json.load(handle) + if last["message"] == message and now - last["at"] < hours * 3600: + return True + except (OSError, ValueError, KeyError, TypeError): + pass + try: + write_json(path, {"message": message, "at": now}) + except OSError as exc: + print(f"Could not record the alert ({exc}); the next failure alerts again.", + file=sys.stderr) + return False + + +def forget_alert(state_dir): + try: + os.remove(os.path.join(state_dir, LAST_ALERT)) + except FileNotFoundError: + pass + except OSError as exc: + print(f"Could not clear {LAST_ALERT} ({exc}).", file=sys.stderr) + + def call_target(function, argv): """Run one entry point, returning None, or its failure as one sentence. @@ -203,6 +239,8 @@ def run(job, dry_run=False, extra=()): if outcome.summary: print(outcome.summary) if not error and not outcome.failures(): + if job.repeat_alert_hours and not dry_run: + forget_alert(state_dir) return 0 message = scrub(alert_message(job, socket.gethostname(), _step, error, outcome)) @@ -210,6 +248,11 @@ def run(job, dry_run=False, extra=()): if dry_run: print(message) return code + if (code and job.repeat_alert_hours + and repeated_alert(state_dir, message, job.repeat_alert_hours, time.time())): + print(f"{message}\nAlready reported within {job.repeat_alert_hours:g}h; not posted.") + mark_alerted(state_dir) + return code if not post_alert(job, message): # Even an optional target's failure then fails the unit: OnFailure= is all # that is left to report it. diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py index a983e68..8fb4ceb 100644 --- a/src/session_ops/store_versions/job.py +++ b/src/session_ops/store_versions/job.py @@ -31,6 +31,7 @@ import sys import time import tomllib +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from session_ops.shared import discord, http @@ -43,10 +44,17 @@ TAGS_FILE = "/etc/session-ops/store-versions.toml" STATE_VERSION = 1 -REMINDER_SECONDS = 12 * 3600 -FAILING_AFTER_SECONDS = 3600 +HOUR = 3600 +REMINDER_SECONDS = 12 * HOUR +FAILING_AFTER_SECONDS = HOUR USER_AGENT = "session-ops store-versions (+https://github.com/session-foundation)" +# Short, so a host that stops answering costs under a minute of the unit's 4. +HTTP_ATTEMPTS, HTTP_TIMEOUT = 2, (5, 20) +# Messages still unposted by then wait for the next run rather than for systemd's kill. +POST_DEADLINE_SECONDS = 150 +EMBED_FIELD_CHARS = 1024 +ANDROID_REPO, IOS_REPO, DESKTOP_REPO = "session-android", "session-ios", "session-desktop" ANDROID_ID = "network.loki.messenger" FLATHUB_ID = "network.loki.Session" @@ -61,32 +69,32 @@ class Platform: PLATFORMS = ( - Platform("android", "Session Android", "session-android", + Platform("android", "Session Android", ANDROID_REPO, {"kind": "play", "app_id": ANDROID_ID}), # Each Android release opens a "Release X.Y.Z" pull request on session-fdroid, which # has to be merged; the merged index is what this checks. - Platform("android-fdroid-merged", "Session Android (F-Droid, merged)", "session-android", + Platform("android-fdroid-merged", "Session Android (F-Droid, merged)", ANDROID_REPO, {"kind": "fdroid", "app_id": ANDROID_ID, "base_url": "https://raw.githubusercontent.com/session-foundation/" "session-fdroid/master/fdroid/repo"}, strict_sync=True), # What F-Droid clients pull: behind is deploy lag, so app-store rules. - Platform("android-fdroid-live", "Session Android (F-Droid, live)", "session-android", + Platform("android-fdroid-live", "Session Android (F-Droid, live)", ANDROID_REPO, {"kind": "fdroid", "app_id": ANDROID_ID, "base_url": "https://fdroid.getsession.org/fdroid/repo"}), - Platform("ios", "Session iOS", "session-ios", + Platform("ios", "Session iOS", IOS_REPO, {"kind": "appstore", "bundle_id": "com.loki-project.loki-messenger"}), # session-desktop, not session-messenger-desktop: that one is an old transitional package. - Platform("desktop", "Session Desktop", "session-desktop", + Platform("desktop", "Session Desktop", DESKTOP_REPO, {"kind": "deb", "base_url": "https://deb.session.foundation", "package": "session-desktop"}, strict_sync=True), # flathubbot opens an update pull request on flathub/network.loki.Session per # release, which has to be merged. - Platform("desktop-flathub-merged", "Session Desktop (Flathub, merged)", "session-desktop", + Platform("desktop-flathub-merged", "Session Desktop (Flathub, merged)", DESKTOP_REPO, {"kind": "flathub_manifest", "app_id": FLATHUB_ID}, strict_sync=True), # Behind is Flathub's build and publish lag after the merge. - Platform("desktop-flathub-live", "Session Desktop (Flathub, live)", "session-desktop", + Platform("desktop-flathub-live", "Session Desktop (Flathub, live)", DESKTOP_REPO, {"kind": "flathub", "app_id": FLATHUB_ID}), ) @@ -108,6 +116,8 @@ def load_tags(path): data = tomllib.load(handle) except FileNotFoundError: return {} + except OSError as exc: + sys.exit(f"{path}: {exc.strerror}; it has to be readable by the job (mode 644)") except tomllib.TOMLDecodeError as exc: sys.exit(f"{path}: {exc}") tags = data.get("tag", {}) @@ -152,7 +162,8 @@ def event_message(platform, event, github_url, store_url, tags): fields = [{"name": f"{store} version", "value": event.store, "inline": True}, {"name": "GitHub version", "value": event.github, "inline": True}] fields += [{"name": name, "value": url} for name, url in - ((f"{store} link", store_url), ("GitHub release", github_url)) if url] + ((f"{store} link", store_url), ("GitHub release", github_url)) + if url and len(url) <= EMBED_FIELD_CHARS] embed = {"fields": fields} message = {"embeds": [embed], "allowed_mentions": {"parse": []}} @@ -162,7 +173,7 @@ def event_message(platform, event, github_url, store_url, tags): WARNING_UPDATE: " — divergence grew", WARNING_REMINDER: " — still unresolved"}[event.kind] if event.kind == WARNING_REMINDER: - description += f"\nStill out of sync after ~{REMINDER_SECONDS // 3600}h." + description += f"\nStill out of sync after ~{REMINDER_SECONDS // HOUR}h." embed.update(color=RED, title=f"⚠️ {platform.display}: {direction}{suffix}", description=description) ids = tags.get(platform.name, []) @@ -183,44 +194,74 @@ def event_message(platform, event, github_url, store_url, tags): return message +def combined(messages): + """A platform's events as one message, so they land or fail together.""" + if len(messages) < 2: + return messages + merged = {"embeds": [embed for message in messages for embed in message["embeds"]], + "allowed_mentions": {"parse": []}} + for message in messages: + if "content" in message: + merged.update(content=message["content"], + allowed_mentions=message["allowed_mentions"]) + return [merged] + + def source_message(source, kind, failure, now): if kind == SOURCE_RECOVERED: embed = {"color": GREEN, "title": f"✅ {source}: readable again"} else: - hours = max(1, round((now - failure["since"]) / 3600)) + hours = max(1, round((now - failure["since"]) / HOUR)) embed = {"color": RED, "title": f"⚠️ {source}: failing for ~{hours}h", "description": discord.clip(failure["error"], 1000)} return {"embeds": [embed], "allowed_mentions": {"parse": []}} -class Fetcher: - """Each source's Release or error, fetched once per run: three platforms share - session-android's release, and unauthenticated GitHub allows 60 requests an hour.""" +def http_session(): + session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT) + session.headers["User-Agent"] = USER_AGENT + return session - def __init__(self, session): - self.session = session - self.results = {} - def get(self, key, fetch, *args): - if key not in self.results: - try: - self.results[key] = (fetch(self.session, *args), None) - except Exception as exc: # one source down must not stop the others - self.results[key] = (None, f"{type(exc).__name__}: {exc}") - print(f"{key}: {self.results[key][1]}", file=sys.stderr) - return self.results[key] +def fetch_all(wanted): + """{key: (Release, None) or (None, error)} for {key: (fetch, args)}, all at once and + each on its own session, so one host that stops answering holds up no other.""" + def one(fetch, args): + try: + return fetch(http_session(), *args), None + except Exception as exc: # one source down must not stop the others + return None, f"{type(exc).__name__}: {exc}" + + with ThreadPoolExecutor(max_workers=max(1, len(wanted))) as pool: + futures = {key: pool.submit(one, fetch, args) for key, (fetch, args) in wanted.items()} + results = {key: future.result() for key, future in futures.items()} + for key, (_, error) in results.items(): + if error: + print(f"{key}: {error}", file=sys.stderr) + return results -def check(state, tags, session, now): +def check(state, tags, now): """[(messages, apply)]: `apply` records a step's state once its messages are posted, - so a failed post is retried on the next run rather than lost.""" - fetcher = Fetcher(session) + so a failed post is retried on the next run rather than lost. + + Each source is fetched once, keyed by name: three platforms share each of + session-android's and session-desktop's releases, and unauthenticated GitHub allows + 60 requests an hour.""" + keys = {platform.name: (f"GitHub {platform.repo}", + f"{store_name(platform)} ({platform.name})") + for platform in PLATFORMS} + wanted = {} + for platform in PLATFORMS: + github_key, store_key = keys[platform.name] + wanted[github_key] = (sources.fetch_github, (platform.repo,)) + wanted[store_key] = (sources.fetch_store, (platform.store,)) + results = fetch_all(wanted) + steps = [] for platform in PLATFORMS: - github, github_error = fetcher.get(f"GitHub {platform.repo}", sources.fetch_github, - platform.repo) - store, store_error = fetcher.get(f"{store_name(platform)} ({platform.name})", - sources.fetch_store, platform.store) + github_key, store_key = keys[platform.name] + (github, github_error), (store, store_error) = results[github_key], results[store_key] if github_error or store_error: print(f"{platform.name}: skipped") continue @@ -228,13 +269,13 @@ def check(state, tags, session, now): result = evaluate(state["platforms"].get(platform.name), github.version, store.version, strict_sync=platform.strict_sync, now=now, reminder_seconds=REMINDER_SECONDS) - messages = [event_message(platform, event, github.url, store.url, tags) - for event in result.events] + messages = combined([event_message(platform, event, github.url, store.url, tags) + for event in result.events]) steps.append((messages, lambda name=platform.name, new=result.state: state["platforms"].__setitem__(name, new))) - for source in sorted(set(fetcher.results) | set(state["sources"])): - error = fetcher.results.get(source, (None, None))[1] + for source in sorted(set(results) | set(state["sources"])): + error = results.get(source, (None, None))[1] kind, failure = source_events(state["sources"].get(source), error, now=now, after_seconds=FAILING_AFTER_SECONDS, reminder_seconds=REMINDER_SECONDS) @@ -260,9 +301,8 @@ def main(argv=None): required=not args.dry_run) tags = load_tags(args.tags) state = load_state(args.state) - session = http.Session(attempts=3) - session.headers["User-Agent"] = USER_AGENT - steps = check(state, tags, session, time.time()) + started = time.monotonic() + steps = check(state, tags, time.time()) messages = [message for step_messages, _ in steps for message in step_messages] if args.dry_run: @@ -271,17 +311,30 @@ def main(argv=None): print(f"Would post {len(messages)} message(s).") return - posted = discord.post_to_discord(http.Session(), webhook, messages) if messages else 0 - reached = 0 + session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT) + posted, refused, deferred = 0, False, False for step_messages, apply in steps: - reached += len(step_messages) - if reached <= posted: - apply() + if step_messages: + if time.monotonic() - started > POST_DEADLINE_SECONDS: + deferred = True + continue + landed = discord.post_to_discord(session, webhook, step_messages) + posted += landed + if landed < len(step_messages): + refused = True + continue + apply() + # Saved per step: a run killed mid-way reposts at most the step it was on. + if args.state and step_messages: + write_json(args.state, state) if args.state: write_json(args.state, state) - if posted < len(messages): + if refused: sys.exit(f"Posted {posted} of {len(messages)} messages to Discord.") - print(f"Posted {len(messages)} message(s).") + if deferred: + print(f"Posted {posted} of {len(messages)} message(s); the rest go next run.") + else: + print(f"Posted {len(messages)} message(s).") if __name__ == "__main__": diff --git a/src/session_ops/store_versions/sources.py b/src/session_ops/store_versions/sources.py index 6477cd4..d8bed17 100644 --- a/src/session_ops/store_versions/sources.py +++ b/src/session_ops/store_versions/sources.py @@ -23,6 +23,8 @@ PLAY_VERSION = re.compile(r'\[\[\["(\d+(?:\.\d+)+)"\]\]') DEB_EPOCH = re.compile(r"^\d+:") DEB_REVISION = re.compile(r"-[^-]+$") +RELEASE = re.compile(r"(\d+(?:\.\d+)*)(.*)", re.DOTALL) +MAX_VERSION_CHARS = 64 class SourceError(RuntimeError): @@ -40,11 +42,17 @@ def normalize(version): def version_key(version): - """Orders dotted versions numerically, so 1.10.0 sorts after 1.9.0 and 1.2 equals 1.2.0.""" - parts = [int(part) for part in re.findall(r"\d+", version)] + """Orders dotted versions numerically, so 1.10.0 sorts after 1.9.0 and 1.2 equals + 1.2.0, and ranks a suffixed one (1.2.3-beta1, 1.2.3rc1) below its bare release.""" + match = RELEASE.match(version) + if not match: + return ((), 0, ()) + parts = [int(part) for part in match.group(1).split(".")] while parts and parts[-1] == 0: parts.pop() - return tuple(parts) + suffix = match.group(2) + return (tuple(parts), 0 if suffix else 1, + tuple(int(part) for part in re.findall(r"\d+", suffix))) def compare(a, b): @@ -67,6 +75,13 @@ def deb_upstream(version): return normalize(DEB_REVISION.sub("", version) if "-" in version else version) +def checked_version(release, what): + """`release`, if its version is one an embed can carry and compare() can order.""" + if not RELEASE.match(release.version) or len(release.version) > MAX_VERSION_CHARS: + raise SourceError(f"{what} gave an unusable version: {release.version[:MAX_VERSION_CHARS]!r}") + return release + + def checked(resp, what): if resp.status_code >= 400: raise SourceError(f"{what}: HTTP {resp.status_code}") @@ -75,7 +90,7 @@ def checked(resp, what): def fetch_github(session, repo): tag, url = github.latest_release(session, f"{github.ORG}/{repo}") - return Release(normalize(tag), url) + return checked_version(Release(normalize(tag), url), f"GitHub {repo}") def fetch_play(session, app_id, country="us", lang="en"): @@ -183,4 +198,5 @@ def metainfo_version(xml_bytes): def fetch_store(session, store): """`store` is a platform's store spec: its `kind`, and that fetcher's arguments.""" arguments = dict(store) - return FETCHERS[arguments.pop("kind")](session, **arguments) + kind = arguments.pop("kind") + return checked_version(FETCHERS[kind](session, **arguments), kind) diff --git a/tests/ops/test_runner.py b/tests/ops/test_runner.py index 25821d3..2ab1021 100644 --- a/tests/ops/test_runner.py +++ b/tests/ops/test_runner.py @@ -154,6 +154,38 @@ def test_an_optional_failure_whose_alert_cannot_be_posted_fails_the_run(self): self.assertEqual(self.run_job(job("optional_failure")), 1) self.assertIsNone(self.marker()) + def test_a_repeated_failure_is_reported_once_until_its_interval_passes(self): + throttled = job("exits", repeat_alert_hours=12) + with mock.patch.object(runner.time, "time", return_value=0.0): + self.assertEqual(self.run_job(throttled), 1) + os.environ["INVOCATION_ID"] = "inv-2" + with mock.patch.object(runner.time, "time", return_value=11 * 3600.0): + self.assertEqual(self.run_job(throttled), 1) + self.assertEqual(len(self.posted), 1) + self.assertEqual(self.marker(), "inv-2") + with mock.patch.object(runner.time, "time", return_value=12 * 3600.0): + self.run_job(throttled) + self.assertEqual(len(self.posted), 2) + + def test_a_success_resets_the_repeat_interval(self): + with mock.patch.object(runner.time, "time", return_value=0.0): + self.run_job(job("exits", repeat_alert_hours=12)) + self.run_job(job("succeeds", repeat_alert_hours=12)) + self.run_job(job("exits", repeat_alert_hours=12)) + self.assertEqual(len(self.posted), 2) + + def test_a_different_failure_is_reported_at_once(self): + with mock.patch.object(runner.time, "time", return_value=0.0): + self.run_job(job("exits", repeat_alert_hours=12)) + self.run_job(job("raises", args=("x",), repeat_alert_hours=12)) + self.assertEqual(len(self.posted), 2) + + def test_without_an_interval_every_failure_is_reported(self): + self.run_job(job("exits")) + self.run_job(job("exits")) + self.assertEqual(len(self.posted), 2) + self.assertFalse(os.path.exists(os.path.join(self.state, runner.LAST_ALERT))) + def test_a_dry_run_prints_the_alert_instead(self): out = io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): diff --git a/tests/store_versions/test_job.py b/tests/store_versions/test_job.py index 0d98384..dd5ab56 100644 --- a/tests/store_versions/test_job.py +++ b/tests/store_versions/test_job.py @@ -41,6 +41,8 @@ def fetch_store(self, session, store): def post(self, session, url, messages): taken = messages if self.accept is None else messages[:self.accept] + if self.accept is not None: + self.accept -= len(taken) self.posted += taken return len(taken) @@ -98,6 +100,43 @@ def test_unposted_warning_is_retried_next_run(self): self.assertEqual(len(run.posted), 2) self.assertIn("APT repo is behind GitHub", titles(run.posted)[1]) + def test_a_refused_platform_does_not_hold_back_the_others(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + accept = run.post + + def refuse_the_app_store(session, url, messages): + if "App Store" in messages[0]["embeds"][0]["title"]: + return 0 + return accept(session, url, messages) + + run.post = refuse_the_app_store + with self.assertRaises(SystemExit): + run() + self.assertEqual(set(run.saved()["platforms"]), {"desktop"}) + self.assertIn("APT repo is behind GitHub", titles(run.posted)[0]) + + def test_a_platforms_events_go_as_one_message(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.5", "deb": "1.18.2"}) + run() + run.github["session-ios"], run.stores["appstore"] = "2.15.6", "2.15.6" + run() + [message] = run.posted + self.assertEqual([embed["title"] for embed in message["embeds"]], + ["ℹ️ Session iOS: new GitHub release", "ℹ️ Session iOS: App Store updated"]) + + def test_past_the_deadline_messages_wait_for_the_next_run(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + clock = iter([0.0, job.POST_DEADLINE_SECONDS + 1]) + with Patched(job.time, monotonic=lambda: next(clock, 10_000.0)): + run() + self.assertEqual(run.posted, []) + self.assertEqual(set(run.saved()["platforms"]), set()) + run() + self.assertEqual(len(run.posted), 2) + def test_failing_source_is_posted_after_an_hour_then_recovery(self): run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, {"appstore": RuntimeError("boom"), "deb": "1.18.2"}) @@ -117,6 +156,15 @@ def test_dry_run_posts_and_writes_nothing(self): self.assertEqual(run.posted, []) self.assertFalse(os.path.exists(run.state)) + def test_unreadable_tags_fail_the_run_saying_why(self): + run = Run({}, {}) + os.chmod(run.tags, 0) + if os.access(run.tags, os.R_OK): + self.skipTest("running as root") + with self.assertRaises(SystemExit) as raised: + job.load_tags(run.tags) + self.assertIn("mode 644", str(raised.exception)) + def test_tags_must_name_a_platform(self): run = Run({}, {}, tags="[tag]\nandroind = [1]\n") with self.assertRaises(SystemExit): diff --git a/tests/store_versions/test_sources.py b/tests/store_versions/test_sources.py index a557f47..d04ca6c 100644 --- a/tests/store_versions/test_sources.py +++ b/tests/store_versions/test_sources.py @@ -15,6 +15,19 @@ def test_compare_is_numeric(self): self.assertEqual(sources.compare("2.15.3", "2.15.3"), 0) self.assertEqual(sources.compare("1.2", "1.2.0"), 0) + def test_a_suffixed_version_is_below_its_release(self): + self.assertEqual(sources.compare("1.2.3-beta1", "1.2.3"), -1) + self.assertEqual(sources.compare("1.2.3rc1", "1.2.3"), -1) + self.assertEqual(sources.compare("1.2.3-beta1", "1.2.3-beta2"), -1) + self.assertEqual(sources.compare("1.2.3-beta1", "1.2.2"), 1) + self.assertEqual(sources.newest(["1.34.0-rc1", "1.34.0"]), "1.34.0") + + def test_an_unusable_version_fails_the_source(self): + for version in ("", "v", "beta", "1" * 65): + with self.subTest(version=version), self.assertRaises(sources.SourceError): + sources.checked_version(sources.Release(version), "test") + self.assertEqual(sources.checked_version(sources.Release("1.2.3"), "test").version, "1.2.3") + def test_normalize_and_deb_upstream(self): self.assertEqual(sources.normalize(" v1.18.1 "), "1.18.1") self.assertEqual(sources.deb_upstream("1:1.17.17-2"), "1.17.17") @@ -81,6 +94,12 @@ def test_deb(self): {"kind": "deb", "base_url": "https://deb.example/", "package": "session-desktop"}) self.assertEqual(release.version, "1.18.2") + def test_an_empty_fdroid_version_fails_the_source(self): + index = {"packages": {"a": [{"versionCode": 1, "versionName": ""}]}} + with self.assertRaises(sources.SourceError): + sources.fetch_store(FakeSession([FakeResponse(index)]), + {"kind": "fdroid", "base_url": "https://f.example", "app_id": "a"}) + def test_flathub_appstream(self): session = FakeSession([FakeResponse({"releases": [{"version": "1.18.1"}, {"version": "1.18.2"}]})]) From 8e4e85db82459fc8b50c16980dc1a0fb7960d3f0 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 13:32:38 +1100 Subject: [PATCH 03/11] Keep repeat_alert_hours clear of #71's registry and jobs.toml hunks --- src/session_ops/jobs.toml | 2 +- src/session_ops/ops/registry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/session_ops/jobs.toml b/src/session_ops/jobs.toml index 5edff35..916a280 100644 --- a/src/session_ops/jobs.toml +++ b/src/session_ops/jobs.toml @@ -9,9 +9,9 @@ # env variables the run refuses to start without # channel_env where it posts, and where its own failures are reported; a job # without one reports to ALERT_DISCORD_WEBHOOK_URL -# unit further [Service] lines, over the template's hardening # repeat_alert_hours report the same failure again only this often until a run # succeeds, for a schedule short enough to repeat it every few minutes +# unit further [Service] lines, over the template's hardening # # max_age_hours for a queued job, its queue's: the weekend's 72 h (73 h across # April's DST change), plus every job ahead of it running to its timeout diff --git a/src/session_ops/ops/registry.py b/src/session_ops/ops/registry.py index fbc4ae5..8348aa6 100644 --- a/src/session_ops/ops/registry.py +++ b/src/session_ops/ops/registry.py @@ -27,10 +27,10 @@ class Job: max_age_hours: float = None channel_env: str = "ALERT_DISCORD_WEBHOOK_URL" timeout: str = "30min" + repeat_alert_hours: float = None unit: tuple = field(default=()) queued: bool = False after: tuple = () - repeat_alert_hours: float = None @property def scheduled(self): From 6bda13fe51b126f46d2477e591c7d87e9412a1c0 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 14:49:20 +1100 Subject: [PATCH 04/11] store-versions: hold each store at its highest version, and count prereleases A store re-serving its previous version (iTunes, Flathub's CDN, raw.githubusercontent) flipped a warning and its all-clear every run, tagging people each time: only the info post used the store's high. Every comparison now does. GitHub's /releases/latest skips prereleases, so a prerelease already on F-Droid or Play warned as unpublished source, and it follows creation date, so a hotfix on an older line looked like GitHub going backwards. One page of /releases now gives the highest full release, and a store shipping a published prerelease above it is in sync. --- docs/jobs/store-versions.md | 12 ++++-- src/session_ops/shared/github.py | 14 ++++--- src/session_ops/store_versions/job.py | 6 ++- src/session_ops/store_versions/sources.py | 17 +++++++- src/session_ops/store_versions/transitions.py | 40 ++++++++++--------- tests/store_versions/test_sources.py | 28 +++++++++++-- tests/store_versions/test_transitions.py | 36 +++++++++++++++++ 7 files changed, 120 insertions(+), 33 deletions(-) diff --git a/docs/jobs/store-versions.md b/docs/jobs/store-versions.md index 15160ee..cd78379 100644 --- a/docs/jobs/store-versions.md +++ b/docs/jobs/store-versions.md @@ -1,7 +1,7 @@ # Store Versions Against GitHub Releases -Every 5 minutes, compares each store's latest Session release with the platform's latest -GitHub release, and posts as they diverge and come back. +Every 5 minutes, compares each store's latest Session release with the platform's +highest GitHub release, and posts as they diverge and come back. | | | | --- | --- | @@ -24,16 +24,20 @@ GitHub release, and posts as they diverge and come back. A store ahead of GitHub is a public build whose source is not published. The channels warned on either way are ours: each GitHub release opens a pull request on session-fdroid and on flathub/network.loki.Session that has to be merged, and the APT -repo is published with the release. +repo is published with the release. A store shipping a published prerelease above +that release is in sync with it. | Post | When | | --- | --- | | ⚠️ warning, tagging the platform's people | it diverges; on a platform's first check, if it already has | | ⚠️ again | every 12 hours while unresolved, and at once if the gap grows | | ✅ back in sync | after a warning only | -| ℹ️ new GitHub release, store updated | either moves without a warning; a store returning to a version it already reached stays quiet, since iTunes and Flathub's CDN re-serve the previous one for hours | +| ℹ️ new GitHub release, store updated | either moves without a warning | | ⚠️ source failing, untagged | a source has failed for an hour, then every 12 hours, and ✅ once it reads again | +A store is held at the highest version it has served: iTunes and the CDNs re-serve the +previous one for hours after a release, so a store returning to it changes nothing. + State is `/var/lib/session-ops/store-versions/state.json`. Losing it makes the next run a first check. A message Discord refuses is posted again on the next run, as is one still unposted 150 seconds into a run. A version that does not start with a digit, or diff --git a/src/session_ops/shared/github.py b/src/session_ops/shared/github.py index caf1ea4..2827e50 100644 --- a/src/session_ops/shared/github.py +++ b/src/session_ops/shared/github.py @@ -23,6 +23,7 @@ APP_KEY_PATH = "/etc/session-ops/github-app.pem" ORG = "session-foundation" +RELEASES_PAGE = 20 REHEARSAL_PREFIX = "rehearsal/" BOT, DIRECT = "bot", "direct" # Every branch the App may write, and how: a BOT branch is the job's own, rebuilt and @@ -116,11 +117,14 @@ def check(resp, what): return resp.json() if resp.content else None -def latest_release(api, repo): - """(tag, page URL) of `repo`'s latest release: published, neither draft nor prerelease.""" - release = check(api.request("GET", f"{API}/repos/{repo}/releases/latest"), - f"reading {repo}'s latest release") - return release["tag_name"], release.get("html_url", "") +def releases(api, repo, count=RELEASES_PAGE): + """[(tag, page URL, prerelease)] of `repo`'s `count` most recently created + releases, drafts left out.""" + found = check(api.request("GET", f"{API}/repos/{repo}/releases", + params={"per_page": count}), + f"reading {repo}'s releases") + return [(release["tag_name"], release.get("html_url", ""), bool(release.get("prerelease"))) + for release in found if not release.get("draft")] def app_jwt(app_id, private_key, now=None): diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py index 8fb4ceb..bd3f6be 100644 --- a/src/session_ops/store_versions/job.py +++ b/src/session_ops/store_versions/job.py @@ -267,9 +267,13 @@ def check(state, tags, now): continue print(f"{platform.name}: github={github.version} {platform.store['kind']}={store.version}") result = evaluate(state["platforms"].get(platform.name), github.version, store.version, + prereleases=[release.version for release in github.prereleases], strict_sync=platform.strict_sync, now=now, reminder_seconds=REMINDER_SECONDS) - messages = combined([event_message(platform, event, github.url, store.url, tags) + github_urls = {release.version: release.url for release in github.prereleases} + messages = combined([event_message(platform, event, + github_urls.get(event.github, github.url), + store.url, tags) for event in result.events]) steps.append((messages, lambda name=platform.name, new=result.state: state["platforms"].__setitem__(name, new))) diff --git a/src/session_ops/store_versions/sources.py b/src/session_ops/store_versions/sources.py index d8bed17..ab0cc53 100644 --- a/src/session_ops/store_versions/sources.py +++ b/src/session_ops/store_versions/sources.py @@ -35,6 +35,8 @@ class SourceError(RuntimeError): class Release: version: str url: str = "" + # GitHub only: the published prereleases above `version`. + prereleases: tuple = () def normalize(version): @@ -89,8 +91,19 @@ def checked(resp, what): def fetch_github(session, repo): - tag, url = github.latest_release(session, f"{github.ORG}/{repo}") - return checked_version(Release(normalize(tag), url), f"GitHub {repo}") + """The highest release, rather than GitHub's "latest": that is the most recently + created one, which goes backwards when an older line gets a hotfix.""" + found = [(Release(normalize(tag), url), prerelease) + for tag, url, prerelease in github.releases(session, f"{github.ORG}/{repo}")] + released = [release for release, prerelease in found if not prerelease] + if not released: + raise SourceError(f"none of GitHub {repo}'s last {github.RELEASES_PAGE} releases " + "is a full release") + by_version = functools.cmp_to_key(lambda a, b: compare(a.version, b.version)) + latest = checked_version(max(released, key=by_version), f"GitHub {repo}") + above = tuple(release for release, prerelease in found + if prerelease and compare(release.version, latest.version) > 0) + return Release(latest.version, latest.url, above) def fetch_play(session, app_id, country="us", lang="en"): diff --git a/src/session_ops/store_versions/transitions.py b/src/session_ops/store_versions/transitions.py index a7be0ba..32b2009 100644 --- a/src/session_ops/store_versions/transitions.py +++ b/src/session_ops/store_versions/transitions.py @@ -35,55 +35,59 @@ class Result: state: dict = field(default_factory=dict) -def evaluate(prev, github, store, *, strict_sync=False, now=0.0, reminder_seconds=None): +def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, + reminder_seconds=None): """The events for this run, and the state to keep once they are posted. App stores warn only when the store is ahead of GitHub: a public build whose source is not published. A `strict_sync` channel is ours and must match GitHub, so it warns in either direction. An unresolved warning repeats every `reminder_seconds`, - and at once if the leading side pulls further ahead. + and at once if the leading side pulls further ahead. `prereleases` are the + published prereleases above `github`: a store shipping one of them is in sync. """ - order = compare(store, github) - store_ahead, store_behind = order > 0, order < 0 - out_of_sync = order != 0 if strict_sync else store_ahead - # Store APIs are eventually consistent: iTunes and Flathub's CDN re-serve the - # previous version for hours after a release. The true version is kept, but only a - # new high is announced, so a dip and recovery stays quiet. + # previous version for hours after a release. Every comparison uses the highest + # version the store has served, so a dip and recovery stays quiet. prev_high = prev and (prev.get("store_high") or prev["store"]) new_high = prev_high is None or compare(store, prev_high) > 0 - state = {"github": github, "store": store, "store_high": store if new_high else prev_high, + high = store if new_high else prev_high + if compare(high, github) > 0 and any(compare(high, p) == 0 for p in prereleases): + github = high + + order = compare(high, github) + store_ahead, store_behind = order > 0, order < 0 + out_of_sync = order != 0 if strict_sync else store_ahead + state = {"github": github, "store": store, "store_high": high, "warning": out_of_sync, "warned_at": None} if prev is None: if out_of_sync: state["warned_at"] = now - return Result([Event(WARNING, github, store, initial=True)], state) + return Result([Event(WARNING, github, high, initial=True)], state) return Result([], state) github_advanced = compare(github, prev["github"]) > 0 - store_advanced = compare(store, prev["store"]) > 0 warned_at = now if prev.get("warned_at") is None else prev["warned_at"] events = [] if out_of_sync and not prev["warning"]: - events.append(Event(WARNING, github, store)) + events.append(Event(WARNING, github, high)) warned_at = now elif out_of_sync: - if (store_ahead and store_advanced) or (store_behind and github_advanced): - events.append(Event(WARNING_UPDATE, github, store)) + if (store_ahead and new_high) or (store_behind and github_advanced): + events.append(Event(WARNING_UPDATE, github, high)) warned_at = now elif reminder_seconds is not None and now - warned_at >= reminder_seconds: - events.append(Event(WARNING_REMINDER, github, store)) + events.append(Event(WARNING_REMINDER, github, high)) warned_at = now elif prev["warning"]: - events.append(Event(ALL_CLEAR, github, store)) + events.append(Event(ALL_CLEAR, github, high)) if not events: if github_advanced: - events.append(Event(INFO_GITHUB, github, store)) + events.append(Event(INFO_GITHUB, github, high)) if new_high: - events.append(Event(INFO_STORE, github, store)) + events.append(Event(INFO_STORE, github, high)) state["warned_at"] = warned_at if out_of_sync else None return Result(events, state) diff --git a/tests/store_versions/test_sources.py b/tests/store_versions/test_sources.py index d04ca6c..03db0e4 100644 --- a/tests/store_versions/test_sources.py +++ b/tests/store_versions/test_sources.py @@ -108,11 +108,33 @@ def test_flathub_appstream(self): self.assertEqual(release.url, "https://flathub.org/apps/network.loki.Session") def test_github_strips_the_v(self): - session = FakeSession([FakeResponse({"tag_name": "v1.18.2", "html_url": "u"})]) + session = FakeSession([FakeResponse([{"tag_name": "v1.18.2", "html_url": "u"}])]) release = sources.fetch_github(session, "session-desktop") - self.assertEqual(release.version, "1.18.2") + self.assertEqual((release.version, release.url), ("1.18.2", "u")) self.assertTrue(session.calls[0][1].endswith( - "/repos/session-foundation/session-desktop/releases/latest")) + "/repos/session-foundation/session-desktop/releases")) + + def test_github_takes_the_highest_release_not_the_newest(self): + session = FakeSession([FakeResponse([ + {"tag_name": "v1.32.3", "html_url": "hotfix"}, + {"tag_name": "v1.33.1", "html_url": "latest"}, + {"tag_name": "v1.32.2", "html_url": "old"}])]) + self.assertEqual(sources.fetch_github(session, "session-android").url, "latest") + + def test_github_keeps_the_prereleases_above_the_release(self): + session = FakeSession([FakeResponse([ + {"tag_name": "v1.34.0", "html_url": "draft", "draft": True}, + {"tag_name": "v1.33.0", "html_url": "pre", "prerelease": True}, + {"tag_name": "v1.32.2", "html_url": "latest"}, + {"tag_name": "v1.32.0", "html_url": "old pre", "prerelease": True}])]) + release = sources.fetch_github(session, "session-android") + self.assertEqual(release.version, "1.32.2") + self.assertEqual(release.prereleases, (sources.Release("1.33.0", "pre"),)) + + def test_github_without_a_full_release_fails_the_source(self): + session = FakeSession([FakeResponse([{"tag_name": "v1.33.0", "prerelease": True}])]) + with self.assertRaises(sources.SourceError): + sources.fetch_github(session, "session-android") def test_http_error_fails_the_source(self): with self.assertRaises(sources.SourceError): diff --git a/tests/store_versions/test_transitions.py b/tests/store_versions/test_transitions.py index 83a3f22..6c29bc4 100644 --- a/tests/store_versions/test_transitions.py +++ b/tests/store_versions/test_transitions.py @@ -135,6 +135,42 @@ def test_flap_announces_once(self): state = result.state self.assertEqual(posted.count(INFO_STORE), 1) + def test_store_flapping_ahead_warns_once(self): + state, posted = None, [] + for store in ("2.15.6", "2.15.5", "2.15.6", "2.15.5", "2.15.6"): + result = evaluate(state, "2.15.5", store) + posted += kinds(result) + state = result.state + self.assertEqual(posted, [WARNING]) + + def test_store_flapping_two_ahead_does_not_update_the_warning(self): + state = evaluate(None, "2.15.5", "2.15.7").state + for store in ("2.15.6", "2.15.7", "2.15.6"): + result = evaluate(state, "2.15.5", store) + self.assertEqual(kinds(result), []) + state = result.state + + def test_strict_sync_dip_after_a_merge_stays_in_sync(self): + state = evaluate(seen("1.18.1", "1.18.1"), "1.18.2", "1.18.2", strict_sync=True).state + result = evaluate(state, "1.18.2", "1.18.1", strict_sync=True) + self.assertEqual(kinds(result), []) + + +class TestPrereleases(unittest.TestCase): + def test_a_store_shipping_a_prerelease_is_in_sync(self): + result = evaluate(None, "1.32.2", "1.33.0", prereleases=["1.33.0"]) + self.assertEqual(kinds(result), []) + self.assertEqual(result.state["github"], "1.33.0") + + def test_strict_sync_on_the_release_below_a_prerelease_is_in_sync(self): + result = evaluate(None, "1.17.12", "1.17.12", prereleases=["1.17.13"], strict_sync=True) + self.assertEqual(kinds(result), []) + + def test_a_store_past_every_prerelease_warns(self): + result = evaluate(None, "1.32.2", "1.33.1", prereleases=["1.33.0"]) + self.assertEqual(kinds(result), [WARNING]) + self.assertEqual(result.events[0].github, "1.32.2") + class TestSourceEvents(unittest.TestCase): def call(self, prev, error, now): From 70bb6166dcf2077b59f583242a01ef112eaffe57 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 14:58:46 +1100 Subject: [PATCH 05/11] store-versions: take the highest of a package's versions in an APT index --- src/session_ops/store_versions/sources.py | 9 ++++++--- tests/store_versions/test_sources.py | 6 ++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/session_ops/store_versions/sources.py b/src/session_ops/store_versions/sources.py index ab0cc53..c122c42 100644 --- a/src/session_ops/store_versions/sources.py +++ b/src/session_ops/store_versions/sources.py @@ -142,13 +142,16 @@ def fetch_deb(session, base_url, package, suite="sid", component="main", arch="a def packages_version(text, package): - """The Version: of `package` in a Packages index, or None.""" + """The highest Version: of `package` in a Packages index, or None: a repo can keep + several versions of one package.""" + versions = [] for stanza in re.split(r"\n\s*\n", text): fields = dict(line.split(":", 1) for line in stanza.splitlines() if ":" in line and not line.startswith(" ")) if fields.get("Package", "").strip() == package and fields.get("Version"): - return fields["Version"].strip() - return None + versions.append(fields["Version"].strip()) + by_upstream = functools.cmp_to_key(lambda a, b: compare(deb_upstream(a), deb_upstream(b))) + return max(versions, key=by_upstream) if versions else None def fetch_fdroid(session, base_url, app_id): diff --git a/tests/store_versions/test_sources.py b/tests/store_versions/test_sources.py index 03db0e4..8cf22f8 100644 --- a/tests/store_versions/test_sources.py +++ b/tests/store_versions/test_sources.py @@ -47,6 +47,12 @@ def test_packages_finds_the_exact_package(self): self.assertEqual(sources.packages_version(self.PACKAGES, "session-desktop"), "1:1.18.2-1") self.assertIsNone(sources.packages_version(self.PACKAGES, "nope")) + def test_packages_takes_the_highest_of_several_versions(self): + text = ("Package: session-desktop\nVersion: 1.18.2\n\n" + "Package: session-desktop\nVersion: 1.18.10\n\n" + "Package: session-desktop\nVersion: 1.9.0\n") + self.assertEqual(sources.packages_version(text, "session-desktop"), "1.18.10") + def test_fdroid_takes_the_highest_version_code(self): index = {"packages": {"app": [{"versionCode": 9, "versionName": "9.0.0"}, {"versionCode": 10, "versionName": "10.0.0"}, From 827a6c2c5f516893af92e9c2cc856a857443fd0f Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 16:29:11 +1100 Subject: [PATCH 06/11] store-versions: let the watermark come down after six hours lower store_high only ever rose, so a release pulled from APT or a reverted F-Droid or Flathub manifest read as in sync for good, and one bad Play read warned every 12 hours until state.json was edited. A store serving a lower version for six hours now has that version compared: past the iTunes and CDN lag the watermark is there to smooth. --- src/session_ops/store_versions/transitions.py | 20 +++++++--- tests/store_versions/test_transitions.py | 40 +++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/session_ops/store_versions/transitions.py b/src/session_ops/store_versions/transitions.py index 32b2009..527914b 100644 --- a/src/session_ops/store_versions/transitions.py +++ b/src/session_ops/store_versions/transitions.py @@ -1,8 +1,8 @@ """What to post, given a platform's last state and this run's versions: no IO. A platform's state is a dict: the GitHub and store versions last seen, the store's -highest ever (`store_high`), whether a warning is out (`warning`) and when it was last -posted (`warned_at`, epoch seconds). +recent highest (`store_high`) and since when it has served lower (`low_since`), whether +a warning is out (`warning`) and when it was last posted (`warned_at`, epoch seconds). """ from dataclasses import dataclass, field @@ -16,6 +16,9 @@ INFO_STORE = "info_store" WARNINGS = (WARNING, WARNING_UPDATE, WARNING_REMINDER) +# Longer than the hours iTunes and Flathub's CDN take to stop re-serving an old version. +WATERMARK_SECONDS = 6 * 3600 + SOURCE_FAILING = "source_failing" SOURCE_RECOVERED = "source_recovered" @@ -36,7 +39,7 @@ class Result: def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, - reminder_seconds=None): + reminder_seconds=None, watermark_seconds=WATERMARK_SECONDS): """The events for this run, and the state to keep once they are posted. App stores warn only when the store is ahead of GitHub: a public build whose source @@ -47,17 +50,22 @@ def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, """ # Store APIs are eventually consistent: iTunes and Flathub's CDN re-serve the # previous version for hours after a release. Every comparison uses the highest - # version the store has served, so a dip and recovery stays quiet. + # version the store has served, so a dip and recovery stays quiet, until the store + # has served lower for `watermark_seconds`: a rollback or a bad read, not a lag. prev_high = prev and (prev.get("store_high") or prev["store"]) new_high = prev_high is None or compare(store, prev_high) > 0 - high = store if new_high else prev_high + high, low_since = (store if new_high else prev_high), None + if not new_high and compare(store, prev_high) < 0: + low_since = now if prev.get("low_since") is None else prev["low_since"] + if now - low_since >= watermark_seconds: + high, low_since = store, None if compare(high, github) > 0 and any(compare(high, p) == 0 for p in prereleases): github = high order = compare(high, github) store_ahead, store_behind = order > 0, order < 0 out_of_sync = order != 0 if strict_sync else store_ahead - state = {"github": github, "store": store, "store_high": high, + state = {"github": github, "store": store, "store_high": high, "low_since": low_since, "warning": out_of_sync, "warned_at": None} if prev is None: diff --git a/tests/store_versions/test_transitions.py b/tests/store_versions/test_transitions.py index 6c29bc4..827361b 100644 --- a/tests/store_versions/test_transitions.py +++ b/tests/store_versions/test_transitions.py @@ -156,6 +156,46 @@ def test_strict_sync_dip_after_a_merge_stays_in_sync(self): self.assertEqual(kinds(result), []) +class TestWatermark(unittest.TestCase): + HOURS = 3600 + + def run_sequence(self, steps, strict_sync=False): + state, posted = None, [] + for at_hour, github, store in steps: + result = evaluate(state, github, store, strict_sync=strict_sync, + now=at_hour * self.HOURS) + posted.append(kinds(result)) + state = result.state + return posted, state + + def test_a_pulled_release_warns_once_the_store_has_served_lower_for_six_hours(self): + posted, state = self.run_sequence( + [(0, "1.15", "1.15"), (1, "1.15", "1.16"), (2, "1.15", "1.15"), + (3, "1.16", "1.15"), (7.9, "1.16", "1.15"), (8, "1.16", "1.15")], + strict_sync=True) + self.assertEqual(posted, [[], [WARNING], [], [ALL_CLEAR], [], [WARNING]]) + self.assertEqual(state["store_high"], "1.15") + + def test_a_bad_read_clears_after_six_hours_back_at_the_real_version(self): + posted, state = self.run_sequence( + [(0, "1.20", "1.20"), (1, "1.20", "8.0"), (2, "1.20", "1.20"), + (8, "1.20", "1.20")]) + self.assertEqual(posted, [[], [WARNING], [], [ALL_CLEAR]]) + self.assertEqual((state["store_high"], state["low_since"]), ("1.20", None)) + + def test_a_short_dip_stays_quiet_and_resets_the_clock(self): + posted, state = self.run_sequence( + [(0, "2.15.5", "2.15.5"), (1, "2.15.5", "2.15.4"), (2, "2.15.5", "2.15.5"), + (3, "2.15.5", "2.15.4"), (8, "2.15.5", "2.15.4")]) + self.assertEqual(posted, [[], [], [], [], []]) + self.assertEqual((state["store_high"], state["low_since"]), ("2.15.5", 3 * self.HOURS)) + + def test_a_state_from_before_the_clock_loads(self): + result = evaluate(seen("2.15.4", "2.15.4", store_high="2.15.4"), "2.15.4", "2.15.3", + now=100.0) + self.assertEqual(result.state["low_since"], 100.0) + + class TestPrereleases(unittest.TestCase): def test_a_store_shipping_a_prerelease_is_in_sync(self): result = evaluate(None, "1.32.2", "1.33.0", prereleases=["1.33.0"]) From 042aa98ce95f6a20bc87ee8be87d0fcacd4ebf11 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 16:29:11 +1100 Subject: [PATCH 07/11] store-versions: end inside the timeout, and say what is true when clearing A post started up to 150 s in could run past the unit's 4 minutes; 100 s leaves room for a request's worst case. A refused post fails with the same text however many messages wait, so the runner's 12-hour repeat limit holds. A store that was ahead and is now behind a newer GitHub release clears as "no longer ahead", not "back in sync", and reminders no longer claim ~12h at every repeat. --- docs/jobs/store-versions.md | 6 ++++-- src/session_ops/store_versions/job.py | 18 +++++++++++------- tests/store_versions/test_job.py | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/jobs/store-versions.md b/docs/jobs/store-versions.md index cd78379..94c11a4 100644 --- a/docs/jobs/store-versions.md +++ b/docs/jobs/store-versions.md @@ -31,16 +31,18 @@ that release is in sync with it. | --- | --- | | ⚠️ warning, tagging the platform's people | it diverges; on a platform's first check, if it already has | | ⚠️ again | every 12 hours while unresolved, and at once if the gap grows | -| ✅ back in sync | after a warning only | +| ✅ back in sync, or no longer ahead | after a warning only | | ℹ️ new GitHub release, store updated | either moves without a warning | | ⚠️ source failing, untagged | a source has failed for an hour, then every 12 hours, and ✅ once it reads again | A store is held at the highest version it has served: iTunes and the CDNs re-serve the previous one for hours after a release, so a store returning to it changes nothing. +Once a store has served a lower version for 6 hours, that version counts: a rollback +warns, and a bad read clears itself, with nothing to edit by hand. State is `/var/lib/session-ops/store-versions/state.json`. Losing it makes the next run a first check. A message Discord refuses is posted again on the next run, as is one -still unposted 150 seconds into a run. A version that does not start with a digit, or +still unposted 100 seconds into a run, so a run ends inside its 4-minute timeout. A version that does not start with a digit, or runs past 64 characters, fails its source. A failed run is reported once, then every 12 hours until one succeeds. diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py index bd3f6be..f9c380f 100644 --- a/src/session_ops/store_versions/job.py +++ b/src/session_ops/store_versions/job.py @@ -48,10 +48,11 @@ REMINDER_SECONDS = 12 * HOUR FAILING_AFTER_SECONDS = HOUR USER_AGENT = "session-ops store-versions (+https://github.com/session-foundation)" -# Short, so a host that stops answering costs under a minute of the unit's 4. +# A request takes at most ~110 s: two 25 s attempts and a 60 s Retry-After between them. HTTP_ATTEMPTS, HTTP_TIMEOUT = 2, (5, 20) -# Messages still unposted by then wait for the next run rather than for systemd's kill. -POST_DEADLINE_SECONDS = 150 +# Messages still unposted by then wait for the next run: a post started by 100 s ends +# inside the unit's 4-minute timeout. +POST_DEADLINE_SECONDS = 100 EMBED_FIELD_CHARS = 1024 ANDROID_REPO, IOS_REPO, DESKTOP_REPO = "session-android", "session-ios", "session-desktop" @@ -172,17 +173,19 @@ def event_message(platform, event, github_url, store_url, tags): suffix = {WARNING: " (first check)" if event.initial else "", WARNING_UPDATE: " — divergence grew", WARNING_REMINDER: " — still unresolved"}[event.kind] - if event.kind == WARNING_REMINDER: - description += f"\nStill out of sync after ~{REMINDER_SECONDS // HOUR}h." embed.update(color=RED, title=f"⚠️ {platform.display}: {direction}{suffix}", description=description) ids = tags.get(platform.name, []) if ids: message["content"] = " ".join(f"<@{i}>" for i in ids) message["allowed_mentions"] = {"users": ids} - elif event.kind == ALL_CLEAR: + elif event.kind == ALL_CLEAR and sources.compare(event.store, event.github) == 0: embed.update(color=GREEN, title=f"✅ {platform.display}: back in sync", description=f"{store} and GitHub are back in sync at **{event.github}**.") + elif event.kind == ALL_CLEAR: + embed.update(color=GREEN, title=f"✅ {platform.display}: no longer ahead of GitHub", + description=f"GitHub has published **{event.github}**; {store} is at " + f"**{event.store}**.") elif event.kind == INFO_GITHUB: embed.update(color=BLUE, title=f"ℹ️ {platform.display}: new GitHub release", description=f"GitHub advanced to **{event.github}** " @@ -334,7 +337,8 @@ def main(argv=None): if args.state: write_json(args.state, state) if refused: - sys.exit(f"Posted {posted} of {len(messages)} messages to Discord.") + # Without counts: the runner repeats an alert only when its text changes. + sys.exit("Discord refused a post; it is retried next run.") if deferred: print(f"Posted {posted} of {len(messages)} message(s); the rest go next run.") else: diff --git a/tests/store_versions/test_job.py b/tests/store_versions/test_job.py index dd5ab56..6fcd856 100644 --- a/tests/store_versions/test_job.py +++ b/tests/store_versions/test_job.py @@ -126,6 +126,24 @@ def test_a_platforms_events_go_as_one_message(self): self.assertEqual([embed["title"] for embed in message["embeds"]], ["ℹ️ Session iOS: new GitHub release", "ℹ️ Session iOS: App Store updated"]) + def test_an_all_clear_says_what_is_true(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.2"}) + run() + run.github["session-ios"] = "2.15.7" + run() + self.assertEqual(titles(run.posted)[1], "✅ Session iOS: no longer ahead of GitHub") + + def test_a_refused_post_fails_with_the_same_text_however_many_wait(self): + failures = [] + for stores in ({"appstore": "2.15.6", "deb": "1.18.2"}, + {"appstore": "2.15.6", "deb": "1.18.1"}): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, stores, accept=0) + with self.assertRaises(SystemExit) as raised: + run() + failures.append(str(raised.exception)) + self.assertEqual(failures[0], failures[1]) + def test_past_the_deadline_messages_wait_for_the_next_run(self): run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, {"appstore": "2.15.6", "deb": "1.18.2"}) From 973788e666805232d8026d15530b5df2482eb372 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 16:30:08 +1100 Subject: [PATCH 08/11] store-versions: name a platform's source keys in one place --- src/session_ops/store_versions/job.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py index f9c380f..6ebcf1e 100644 --- a/src/session_ops/store_versions/job.py +++ b/src/session_ops/store_versions/job.py @@ -226,6 +226,11 @@ def http_session(): return session +def source_keys(platform): + """(GitHub's, the store's) names for `platform`'s sources, as state["sources"] keys them.""" + return f"GitHub {platform.repo}", f"{store_name(platform)} ({platform.name})" + + def fetch_all(wanted): """{key: (Release, None) or (None, error)} for {key: (fetch, args)}, all at once and each on its own session, so one host that stops answering holds up no other.""" @@ -251,9 +256,7 @@ def check(state, tags, now): Each source is fetched once, keyed by name: three platforms share each of session-android's and session-desktop's releases, and unauthenticated GitHub allows 60 requests an hour.""" - keys = {platform.name: (f"GitHub {platform.repo}", - f"{store_name(platform)} ({platform.name})") - for platform in PLATFORMS} + keys = {platform.name: source_keys(platform) for platform in PLATFORMS} wanted = {} for platform in PLATFORMS: github_key, store_key = keys[platform.name] From e6478d95684d6ab1a42acfd626626027c1e4f240 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 17:36:47 +1100 Subject: [PATCH 09/11] store-versions: bound every request by the clock, and update a warning the watermark changes A read timeout applies per socket read, so a server trickling its body held a run past the unit's 4-minute timeout, and a killed run reaches the OnFailure backstop on every run. shared.http takes an optional deadline on a whole call, retries and body included; store-versions gives fetches 60 s and posts 20 s, so a run and its alert end by about 165 s. The runner's alert post gets 45 s for every job, past which the backstop reports the run; other requests keep no deadline. A watermark coming down while a warning is out now posts the warning's new gap, rather than turning "ahead" into "behind" unannounced until the next reminder. --- docs/jobs/store-versions.md | 12 +-- src/session_ops/ops/runner.py | 6 +- src/session_ops/shared/http.py | 43 +++++++++- src/session_ops/store_versions/job.py | 12 +-- src/session_ops/store_versions/transitions.py | 12 +-- tests/ops/test_runner.py | 7 +- tests/shared/test_http.py | 78 +++++++++++++++++++ tests/store_versions/test_job.py | 12 +++ tests/store_versions/test_transitions.py | 17 ++++ 9 files changed, 179 insertions(+), 20 deletions(-) diff --git a/docs/jobs/store-versions.md b/docs/jobs/store-versions.md index 94c11a4..c4bdc8b 100644 --- a/docs/jobs/store-versions.md +++ b/docs/jobs/store-versions.md @@ -37,14 +37,16 @@ that release is in sync with it. A store is held at the highest version it has served: iTunes and the CDNs re-serve the previous one for hours after a release, so a store returning to it changes nothing. -Once a store has served a lower version for 6 hours, that version counts: a rollback -warns, and a bad read clears itself, with nothing to edit by hand. +Once a store has served a lower version for 6 hours, that version counts, with nothing +to edit by hand: a bad read clears itself, a warning out is updated to the new gap, and +a rollback warns on the channels warned on either way. State is `/var/lib/session-ops/store-versions/state.json`. Losing it makes the next run a first check. A message Discord refuses is posted again on the next run, as is one -still unposted 100 seconds into a run, so a run ends inside its 4-minute timeout. A version that does not start with a digit, or -runs past 64 characters, fails its source. A failed run is reported once, then every -12 hours until one succeeds. +still unposted 100 seconds into a run. Every request has a wall-clock limit, 60 s for a +source and 20 s for a post, so a run and its failure alert end inside the 4-minute +timeout. A version that does not start with a digit, or runs past 64 characters, fails +its source. A failed run is reported once, then every 12 hours until one succeeds. GitHub is read unauthenticated: one request per repository a run, 36 an hour of the 60 allowed. The organisation refuses fine-grained tokens that live longer than 366 days. diff --git a/src/session_ops/ops/runner.py b/src/session_ops/ops/runner.py index 4ace82e..7724860 100644 --- a/src/session_ops/ops/runner.py +++ b/src/session_ops/ops/runner.py @@ -38,6 +38,9 @@ ALERTED_MARKER = "alerted" LAST_ALERT = "last-alert.json" +# Discord answers in a second; past this the OnFailure backstop reports the run instead, +# and a job with a short timeout still has room for its alert. +ALERT_SECONDS = 45 # Present on a host rehearsing production: jobs publish beside it (env.rehearsing). REHEARSAL_MARKER = "/etc/session-ops/rehearsal" @@ -137,7 +140,8 @@ def post_alert(job, message): file=sys.stderr) return False payload = {"content": message, "allowed_mentions": {"parse": []}} - return discord.post_to_discord(http.Session(), webhook, [payload]) == 1 + session = http.Session(deadline=ALERT_SECONDS) + return discord.post_to_discord(session, webhook, [payload]) == 1 def mark_alerted(state_dir): diff --git a/src/session_ops/shared/http.py b/src/session_ops/shared/http.py index 8ad4e37..9e887dc 100644 --- a/src/session_ops/shared/http.py +++ b/src/session_ops/shared/http.py @@ -11,6 +11,7 @@ session = http.Session(attempts=10, timeout=60, rate=20) session.get(url) # the session's budget session.request("GET", url, attempts=2) # a shorter one for optional data + session.request("GET", url, deadline=60) # and a wall-clock limit on all of it """ import contextvars import email.utils @@ -34,6 +35,10 @@ _attempts = contextvars.ContextVar("attempts", default=None) +class DeadlineExceeded(requests.Timeout): + """No complete answer within the call's deadline.""" + + def retry_after_seconds(resp, default): """Seconds to wait per the response's rate-limit headers, else `default`. @@ -171,25 +176,59 @@ class Session(requests.Session): `limiter` may be shared between sessions, so that per-thread sessions against one API still draw on one budget. It paces each call, not each retry inside it. + + `deadline` bounds a whole call, retries, waits and the body included, in seconds. + `timeout` cannot: it applies to each socket read, so a server trickling a byte at + a time never trips it. """ def __init__(self, attempts=DEFAULT_ATTEMPTS, timeout=DEFAULT_TIMEOUT, rate=None, - limiter=None): + limiter=None, deadline=None): super().__init__() self.timeout = timeout + self.deadline = deadline self.limiter = limiter or (TokenBucket(rate) if rate else None) adapter = _Adapter(max_retries=_retry(attempts)) self.mount("https://", adapter) self.mount("http://", adapter) - def request(self, method, url, *args, attempts=None, **kwargs): + def request(self, method, url, *args, attempts=None, deadline=None, **kwargs): if attempts is not None and attempts <= 0: raise ValueError("attempts must be at least 1") kwargs.setdefault("timeout", self.timeout) if self.limiter: self.limiter.acquire() + deadline = self.deadline if deadline is None else deadline + if deadline is None: + return self._send(method, url, attempts, args, kwargs) + return within(deadline, lambda: self._send(method, url, attempts, args, kwargs)) + + def _send(self, method, url, attempts, args, kwargs): token = _attempts.set(attempts) try: return super().request(method, url, *args, **kwargs) finally: _attempts.reset(token) + + +def within(seconds, call): + """`call()`'s result, or DeadlineExceeded once `seconds` pass. + + A daemon thread runs it, since no socket option bounds a whole exchange; one left + behind dies with the process instead of holding its exit.""" + outcome = {} + + def run(): + try: + outcome["result"] = call() + except BaseException as exc: # re-raised in the caller's thread + outcome["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(seconds) + if worker.is_alive(): + raise DeadlineExceeded(f"no complete answer within {seconds:g} s") + if "error" in outcome: + raise outcome["error"] + return outcome["result"] diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py index 6ebcf1e..e3ad7dd 100644 --- a/src/session_ops/store_versions/job.py +++ b/src/session_ops/store_versions/job.py @@ -48,10 +48,10 @@ REMINDER_SECONDS = 12 * HOUR FAILING_AFTER_SECONDS = HOUR USER_AGENT = "session-ops store-versions (+https://github.com/session-foundation)" -# A request takes at most ~110 s: two 25 s attempts and a 60 s Retry-After between them. +# Inside the unit's 4-minute timeout: fetches end by 60 s, no post starts after 100 s +# or lasts past 20 s, and the runner's alert takes at most another 45 s. HTTP_ATTEMPTS, HTTP_TIMEOUT = 2, (5, 20) -# Messages still unposted by then wait for the next run: a post started by 100 s ends -# inside the unit's 4-minute timeout. +FETCH_SECONDS, POST_SECONDS = 60, 20 POST_DEADLINE_SECONDS = 100 EMBED_FIELD_CHARS = 1024 @@ -221,7 +221,8 @@ def source_message(source, kind, failure, now): def http_session(): - session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT) + session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT, + deadline=FETCH_SECONDS) session.headers["User-Agent"] = USER_AGENT return session @@ -321,7 +322,8 @@ def main(argv=None): print(f"Would post {len(messages)} message(s).") return - session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT) + session = http.Session(attempts=HTTP_ATTEMPTS, timeout=HTTP_TIMEOUT, + deadline=POST_SECONDS) posted, refused, deferred = 0, False, False for step_messages, apply in steps: if step_messages: diff --git a/src/session_ops/store_versions/transitions.py b/src/session_ops/store_versions/transitions.py index 527914b..f6c2df2 100644 --- a/src/session_ops/store_versions/transitions.py +++ b/src/session_ops/store_versions/transitions.py @@ -45,13 +45,12 @@ def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, App stores warn only when the store is ahead of GitHub: a public build whose source is not published. A `strict_sync` channel is ours and must match GitHub, so it warns in either direction. An unresolved warning repeats every `reminder_seconds`, - and at once if the leading side pulls further ahead. `prereleases` are the + and at once if the leading side pulls further ahead or the store's watermark comes + down. `prereleases` are the published prereleases above `github`: a store shipping one of them is in sync. """ - # Store APIs are eventually consistent: iTunes and Flathub's CDN re-serve the - # previous version for hours after a release. Every comparison uses the highest - # version the store has served, so a dip and recovery stays quiet, until the store - # has served lower for `watermark_seconds`: a rollback or a bad read, not a lag. + # iTunes and Flathub's CDN re-serve the previous version for hours; compare against + # the highest served until a lower one has held for `watermark_seconds`. prev_high = prev and (prev.get("store_high") or prev["store"]) new_high = prev_high is None or compare(store, prev_high) > 0 high, low_since = (store if new_high else prev_high), None @@ -59,6 +58,7 @@ def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, low_since = now if prev.get("low_since") is None else prev["low_since"] if now - low_since >= watermark_seconds: high, low_since = store, None + lowered = compare(high, prev_high) < 0 if prev_high else False if compare(high, github) > 0 and any(compare(high, p) == 0 for p in prereleases): github = high @@ -82,7 +82,7 @@ def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, events.append(Event(WARNING, github, high)) warned_at = now elif out_of_sync: - if (store_ahead and new_high) or (store_behind and github_advanced): + if (store_ahead and new_high) or (store_behind and github_advanced) or lowered: events.append(Event(WARNING_UPDATE, github, high)) warned_at = now elif reminder_seconds is not None and now - warned_at >= reminder_seconds: diff --git a/tests/ops/test_runner.py b/tests/ops/test_runner.py index 2ab1021..08efb14 100644 --- a/tests/ops/test_runner.py +++ b/tests/ops/test_runner.py @@ -22,7 +22,7 @@ def job(entry, **extra): class TestRun(unittest.TestCase): def setUp(self): self.state = tempfile.mkdtemp() - self.posted = [] + self.posted, self.sessions = [], [] env = {"STATE_DIRECTORY": self.state, "INVOCATION_ID": "inv-1", "ALERT_DISCORD_WEBHOOK_URL": HOOK, "DEMO_TOKEN": "s3cr3t-value"} for patcher in (mock.patch.dict(os.environ, env), @@ -34,6 +34,7 @@ def setUp(self): fake_jobs.calls.clear() def post(self, session, url, messages): + self.sessions.append(session) self.posted.append((url, messages)) return len(messages) @@ -210,6 +211,10 @@ def test_an_alert_mentions_nobody(self): self.assertTrue(payload["content"].startswith("❌ ")) self.assertEqual(payload["allowed_mentions"], {"parse": []}) + def test_an_alert_post_has_a_deadline(self): + self.run_job(job("exits")) + self.assertEqual(self.sessions[0].deadline, runner.ALERT_SECONDS) + class TestCommandLine(unittest.TestCase): def test_dry_run_is_the_runners_and_what_follows_dashes_is_the_jobs(self): diff --git a/tests/shared/test_http.py b/tests/shared/test_http.py index 7fced3e..927e99b 100644 --- a/tests/shared/test_http.py +++ b/tests/shared/test_http.py @@ -260,6 +260,84 @@ def test_the_limiter_paces_every_call(self): self.assertEqual(limiter.acquire.call_count, 2) +class Trickle: + """Sends a 200 whose body comes one byte every `gap` seconds: each read is quick, + so only a deadline on the whole call stops it.""" + + def __init__(self, size=20, gap=0.2): + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(5) + self.url = f"http://127.0.0.1:{self.sock.getsockname()[1]}/" + self.size, self.gap = size, gap + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + threading.Thread(target=self.answer, args=(conn,), daemon=True).start() + + def answer(self, conn): + with conn: + conn.recv(4096) + conn.sendall(f"HTTP/1.1 200 OK\r\nContent-Length: {self.size}\r\n\r\n".encode()) + for _ in range(self.size): + time.sleep(self.gap) + try: + conn.sendall(b"x") + except OSError: + return + + def close(self): + self.sock.close() + + +class TestDeadline(unittest.TestCase): + def trickle(self, **kwargs): + server = Trickle(**kwargs) + self.addCleanup(server.close) + return server + + def test_a_trickling_body_stops_at_the_deadline(self): + server = self.trickle() + session = http.Session(attempts=1, timeout=(1, 1), deadline=1) + started = time.monotonic() + with self.assertRaises(http.DeadlineExceeded): + session.get(server.url) + self.assertLess(time.monotonic() - started, 2) + + def test_a_deadline_is_a_requests_timeout(self): + self.assertTrue(issubclass(http.DeadlineExceeded, requests.Timeout)) + + def test_an_answer_inside_the_deadline_is_returned_whole(self): + server = self.trickle(size=3, gap=0.05) + resp = http.Session(attempts=1, deadline=5).get(server.url) + self.assertEqual((resp.status_code, resp.content), (200, b"xxx")) + + def test_a_per_call_deadline_overrides_the_sessions(self): + server = self.trickle() + with self.assertRaises(http.DeadlineExceeded): + http.Session(attempts=1, timeout=(1, 1)).request("GET", server.url, deadline=0.5) + + def test_a_failure_inside_the_deadline_raises_as_itself(self): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + url = f"http://127.0.0.1:{sock.getsockname()[1]}/" + with self.assertRaises(requests.ConnectionError) as caught: + http.Session(attempts=1, deadline=5).get(url) + self.assertNotIsInstance(caught.exception, http.DeadlineExceeded) + + def test_the_per_call_attempts_reach_the_deadlines_thread(self): + server = Server(status(500), status(500), ok()) + self.addCleanup(server.close) + with NoSleep(): + resp = http.Session(attempts=1, deadline=5).request("GET", server.url, attempts=3) + self.assertEqual((resp.status_code, len(server.seen)), (200, 3)) + + class TestTokenBucket(unittest.TestCase): def test_a_burst_up_to_the_rate_is_free_then_calls_wait(self): bucket = http.TokenBucket(4) diff --git a/tests/store_versions/test_job.py b/tests/store_versions/test_job.py index 6fcd856..7371151 100644 --- a/tests/store_versions/test_job.py +++ b/tests/store_versions/test_job.py @@ -6,6 +6,7 @@ import tempfile import unittest +from session_ops.ops import runner from session_ops.shared import discord from session_ops.shared.testing import Patched from session_ops.store_versions import job, sources @@ -24,6 +25,7 @@ class Run: def __init__(self, github, stores, accept=None, tags=""): self.github, self.stores, self.accept, self.posted = github, stores, accept, [] + self.post_deadlines = [] self.dir = tempfile.mkdtemp() self.state = os.path.join(self.dir, "state.json") self.tags = os.path.join(self.dir, "tags.toml") @@ -43,6 +45,7 @@ def post(self, session, url, messages): taken = messages if self.accept is None else messages[:self.accept] if self.accept is not None: self.accept -= len(taken) + self.post_deadlines.append(session.deadline) self.posted += taken return len(taken) @@ -144,6 +147,15 @@ def test_a_refused_post_fails_with_the_same_text_however_many_wait(self): failures.append(str(raised.exception)) self.assertEqual(failures[0], failures[1]) + def test_every_request_has_a_deadline_inside_the_units_timeout(self): + run = Run({"session-ios": "2.15.5"}, {"appstore": "2.15.6"}) + run() + self.assertEqual(run.post_deadlines, [job.POST_SECONDS]) + self.assertEqual(job.http_session().deadline, job.FETCH_SECONDS) + worst = (max(job.FETCH_SECONDS, job.POST_DEADLINE_SECONDS) + job.POST_SECONDS + + runner.ALERT_SECONDS) + self.assertLess(worst, 4 * 60) + def test_past_the_deadline_messages_wait_for_the_next_run(self): run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.3"}, {"appstore": "2.15.6", "deb": "1.18.2"}) diff --git a/tests/store_versions/test_transitions.py b/tests/store_versions/test_transitions.py index 827361b..77e04c7 100644 --- a/tests/store_versions/test_transitions.py +++ b/tests/store_versions/test_transitions.py @@ -196,6 +196,23 @@ def test_a_state_from_before_the_clock_loads(self): self.assertEqual(result.state["low_since"], 100.0) + def test_a_lowered_watermark_updates_a_warning_that_flips_direction(self): + posted, state = [], None + for hours, store in ((0, "1.6"), (1, "1.7"), (2, "1.5"), (8, "1.5")): + result = evaluate(state, "1.6", store, strict_sync=True, now=hours * 3600, + reminder_seconds=DAY) + posted.append(kinds(result)) + state = result.state + self.assertEqual(posted, [[], [WARNING], [], [WARNING_UPDATE]]) + self.assertEqual((result.events[0].store, result.state["warned_at"]), ("1.5", 8 * 3600)) + + def test_a_lowered_watermark_updates_a_shrinking_gap(self): + state = evaluate(None, "1.6", "1.8", now=0).state + state = evaluate(state, "1.6", "1.7", now=3600).state + result = evaluate(state, "1.6", "1.7", now=7 * 3600) + self.assertEqual(kinds(result), [WARNING_UPDATE]) + self.assertEqual(result.events[0].store, "1.7") + class TestPrereleases(unittest.TestCase): def test_a_store_shipping_a_prerelease_is_in_sync(self): result = evaluate(None, "1.32.2", "1.33.0", prereleases=["1.33.0"]) From 4a7cbaa1b01cd3dcd3e550e68b577a05055646a4 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 22:35:02 +1100 Subject: [PATCH 10/11] store-versions: title a warning the watermark lowered as such, not as a growing gap A lowered watermark can shrink the gap or flip its direction, so its update now names the version the store came down to. --- src/session_ops/store_versions/job.py | 3 ++- src/session_ops/store_versions/transitions.py | 9 +++++++-- tests/store_versions/test_job.py | 11 +++++++++++ tests/store_versions/test_transitions.py | 6 +++--- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/session_ops/store_versions/job.py b/src/session_ops/store_versions/job.py index e3ad7dd..d4fc77a 100644 --- a/src/session_ops/store_versions/job.py +++ b/src/session_ops/store_versions/job.py @@ -40,7 +40,7 @@ from session_ops.store_versions import sources from session_ops.store_versions.transitions import ( ALL_CLEAR, INFO_GITHUB, INFO_STORE, SOURCE_RECOVERED, WARNING, - WARNING_REMINDER, WARNING_UPDATE, WARNINGS, evaluate, source_events) + WARNING_LOWERED, WARNING_REMINDER, WARNING_UPDATE, WARNINGS, evaluate, source_events) TAGS_FILE = "/etc/session-ops/store-versions.toml" STATE_VERSION = 1 @@ -172,6 +172,7 @@ def event_message(platform, event, github_url, store_url, tags): direction, description = gap(platform, event) suffix = {WARNING: " (first check)" if event.initial else "", WARNING_UPDATE: " — divergence grew", + WARNING_LOWERED: f" — {store} came down to {event.store}", WARNING_REMINDER: " — still unresolved"}[event.kind] embed.update(color=RED, title=f"⚠️ {platform.display}: {direction}{suffix}", description=description) diff --git a/src/session_ops/store_versions/transitions.py b/src/session_ops/store_versions/transitions.py index f6c2df2..d5cbbba 100644 --- a/src/session_ops/store_versions/transitions.py +++ b/src/session_ops/store_versions/transitions.py @@ -10,11 +10,12 @@ WARNING = "warning" WARNING_UPDATE = "warning_update" +WARNING_LOWERED = "warning_lowered" WARNING_REMINDER = "warning_reminder" ALL_CLEAR = "all_clear" INFO_GITHUB = "info_github" INFO_STORE = "info_store" -WARNINGS = (WARNING, WARNING_UPDATE, WARNING_REMINDER) +WARNINGS = (WARNING, WARNING_UPDATE, WARNING_LOWERED, WARNING_REMINDER) # Longer than the hours iTunes and Flathub's CDN take to stop re-serving an old version. WATERMARK_SECONDS = 6 * 3600 @@ -82,7 +83,11 @@ def evaluate(prev, github, store, *, prereleases=(), strict_sync=False, now=0.0, events.append(Event(WARNING, github, high)) warned_at = now elif out_of_sync: - if (store_ahead and new_high) or (store_behind and github_advanced) or lowered: + # A lowered watermark can shrink the gap or flip its direction, so it is not "grew". + if lowered: + events.append(Event(WARNING_LOWERED, github, high)) + warned_at = now + elif (store_ahead and new_high) or (store_behind and github_advanced): events.append(Event(WARNING_UPDATE, github, high)) warned_at = now elif reminder_seconds is not None and now - warned_at >= reminder_seconds: diff --git a/tests/store_versions/test_job.py b/tests/store_versions/test_job.py index 7371151..8d07f84 100644 --- a/tests/store_versions/test_job.py +++ b/tests/store_versions/test_job.py @@ -147,6 +147,17 @@ def test_a_refused_post_fails_with_the_same_text_however_many_wait(self): failures.append(str(raised.exception)) self.assertEqual(failures[0], failures[1]) + def test_a_lowered_watermark_is_not_titled_as_a_growing_gap(self): + run = Run({"session-ios": "2.15.5", "session-desktop": "1.18.2"}, + {"appstore": "2.15.7", "deb": "1.18.2"}) + run(now=0.0) + run.stores["appstore"] = "2.15.6" + run(now=3600.0) + run(now=8 * 3600.0) + title = titles(run.posted)[-1] + self.assertNotIn("grew", title) + self.assertIn("App Store is ahead of GitHub — App Store came down to 2.15.6", title) + def test_every_request_has_a_deadline_inside_the_units_timeout(self): run = Run({"session-ios": "2.15.5"}, {"appstore": "2.15.6"}) run() diff --git a/tests/store_versions/test_transitions.py b/tests/store_versions/test_transitions.py index 77e04c7..bbad241 100644 --- a/tests/store_versions/test_transitions.py +++ b/tests/store_versions/test_transitions.py @@ -5,7 +5,7 @@ from session_ops.store_versions.transitions import ( ALL_CLEAR, INFO_GITHUB, INFO_STORE, SOURCE_FAILING, SOURCE_RECOVERED, WARNING, - WARNING_REMINDER, WARNING_UPDATE, evaluate, source_events) + WARNING_LOWERED, WARNING_REMINDER, WARNING_UPDATE, evaluate, source_events) DAY = 12 * 3600 @@ -203,14 +203,14 @@ def test_a_lowered_watermark_updates_a_warning_that_flips_direction(self): reminder_seconds=DAY) posted.append(kinds(result)) state = result.state - self.assertEqual(posted, [[], [WARNING], [], [WARNING_UPDATE]]) + self.assertEqual(posted, [[], [WARNING], [], [WARNING_LOWERED]]) self.assertEqual((result.events[0].store, result.state["warned_at"]), ("1.5", 8 * 3600)) def test_a_lowered_watermark_updates_a_shrinking_gap(self): state = evaluate(None, "1.6", "1.8", now=0).state state = evaluate(state, "1.6", "1.7", now=3600).state result = evaluate(state, "1.6", "1.7", now=7 * 3600) - self.assertEqual(kinds(result), [WARNING_UPDATE]) + self.assertEqual(kinds(result), [WARNING_LOWERED]) self.assertEqual(result.events[0].store, "1.7") class TestPrereleases(unittest.TestCase): From 1ae3d80c8c0728f32e9e9a82653769ad328d201b Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 15 Oct 2026 22:35:02 +1100 Subject: [PATCH 11/11] tests: stop Trickle's threads at cleanup, and never through time.sleep TestSession's NoSleep patches time.sleep globally; a Trickle thread still sending into a later test landed in its recorded sleeps under load. --- tests/shared/test_http.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/shared/test_http.py b/tests/shared/test_http.py index 927e99b..40859fa 100644 --- a/tests/shared/test_http.py +++ b/tests/shared/test_http.py @@ -270,7 +270,11 @@ def __init__(self, size=20, gap=0.2): self.sock.listen(5) self.url = f"http://127.0.0.1:{self.sock.getsockname()[1]}/" self.size, self.gap = size, gap - threading.Thread(target=self.serve, daemon=True).start() + # Event.wait, not time.sleep: TestSession's NoSleep patches time.sleep globally, and + # a thread still trickling into a later test would land in its recorded sleeps. + self.stop = threading.Event() + self.threads = [threading.Thread(target=self.serve, daemon=True)] + self.threads[0].start() def serve(self): while True: @@ -278,21 +282,31 @@ def serve(self): conn, _ = self.sock.accept() except OSError: return - threading.Thread(target=self.answer, args=(conn,), daemon=True).start() + worker = threading.Thread(target=self.answer, args=(conn,), daemon=True) + self.threads.append(worker) + worker.start() def answer(self, conn): with conn: conn.recv(4096) conn.sendall(f"HTTP/1.1 200 OK\r\nContent-Length: {self.size}\r\n\r\n".encode()) for _ in range(self.size): - time.sleep(self.gap) + if self.stop.wait(self.gap): + return try: conn.sendall(b"x") except OSError: return def close(self): + self.stop.set() + try: + self.sock.shutdown(socket.SHUT_RDWR) # wakes the accept(); close() alone may not + except OSError: + pass self.sock.close() + for thread in self.threads: + thread.join(timeout=5) class TestDeadline(unittest.TestCase):