TPU patches

Dockerfile.tmpl

@@ -13,6 +13,14 @@ RUN pip freeze | grep -E 'tensorflow|keras|torch|jax' > /colab_requirements.txt
 RUN cat /colab_requirements.txt >> /requirements.txt
 RUN cat /kaggle_requirements.txt >> /requirements.txt
 
+# GHSA-hq85-5cpj-mr62: uv < 0.13.4 vendors a CA bundle that trusts emSign intermediates with compromised keys.
+# Upgrade it first so the patched uv performs all subsequent installs.
+# The base image's uv may not be the pip-managed one, so point every uv on PATH at the new binary.
+# Remove once the base image ships uv >= 0.13.4 itself.
+RUN uv pip install --system --no-cache "uv>=0.13.4" && \
+    NEW_UV="$(python -c 'import uv; print(uv.find_uv_bin())')" && \
+    for old in $(which -a uv); do [ "$old" -ef "$NEW_UV" ] || ln -sf "$NEW_UV" "$old"; done
+
 # Install Kaggle packages
 RUN uv pip install --system --no-cache -r /requirements.txt
 
@@ -56,6 +64,22 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list &
     apt-get install -y graphviz && pip install graphviz && \
     /tmp/clean-layer.sh
 
+# GHSA-6q48-6c7w-9gw2: emSign ECC Root CA - G3 issued intermediates with compromised keys.
+# Ubuntu's noble-security ca-certificates 20260816ubuntu0.24.04.1 blocklists it in the system store.
+# The Java keystore only drops certificates when rebuilt from scratch.
+# Remove once the base image ships that ca-certificates and a cacerts keystore without the G3 root.
+RUN apt-get install -y --only-upgrade ca-certificates && \
+    rm -f /etc/ssl/certs/java/cacerts && \
+    update-ca-certificates --fresh && \
+    /tmp/clean-layer.sh
+
+# gcloud (including the gsutil and bq it wraps), gRPC, and Node vendor unpatched bundles, so point them at the system store.
+# No upstream fix exists for these yet. Remove each one once the base image ships a release whose bundle drops the G3 root.
+# gcloud ignores the base image's installation properties file (it can't locate its SDK root), so use the env var form.
+ENV CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE=/etc/ssl/certs/ca-certificates.crt \
+    GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt \
+    NODE_OPTIONS=--use-openssl-ca
+
 # patches/keras_internal.py is not installed right now. tensorflow_decision_forests
 # is no longer in the base image and its latest release (1.12.0) ships no cp313
 # wheel, so there is no tensorflow_decision_forests/keras/ to patch.

kaggle_requirements.txt

@@ -15,6 +15,9 @@ bayesian-optimization
 boto3
 catboost
 category-encoders
+# GHSA-6q48-6c7w-9gw2: older bundles trust emSign intermediates with compromised keys.
+# Remove once the base image ships >=2026.10.19 itself.
+certifi>=2026.10.19
 cesium
 comm
 cytoolz
@@ -108,6 +111,9 @@ pdf2image
 # The base image ships pillow 11.3; pin up to a newer release.
 # Remove once the base image ships >=12.2 itself.
 pillow>=12.2
+# pip < 26.2.2 always loads its vendored certifi, so it needs the GHSA-6q48-6c7w-9gw2 fix separately.
+# Remove once the base image ships >=26.2.2 itself.
+pip>=26.2.2
 plotly-express
 pudb
 pyLDAvis
@@ -147,6 +153,9 @@ transformers>=5.0.0
 urllib3>=2.7
 universal-pathlib
 vtk
+# wandb < 0.31.0 embeds a CA bundle with the GHSA-6q48-6c7w-9gw2 emSign root in its Rust binaries.
+# Remove once the base image ships >=0.31.0 itself.
+wandb>=0.31.0
 wavio
 xvfbwrapper
 ydata-profiling

tests/test_certifi.py

@@ -0,0 +1,74 @@
+import glob
+import os
+import ssl
+import subprocess
+import unittest
+
+import certifi
+import pip
+import wandb
+from packaging.version import Version
+
+SYSTEM_BUNDLE = "/etc/ssl/certs/ca-certificates.crt"
+
+
+class TestCertifi(unittest.TestCase):
+    # GHSA-6q48-6c7w-9gw2: emSign ECC Root CA - G3 intermediates have compromised keys.
+    def test_emsign_ecc_g3_root_removed(self):
+        self.assertGreaterEqual(Version(certifi.__version__), Version("2026.10.19"))
+        with open(certifi.where()) as f:
+            self.assertNotIn("emSign ECC Root CA - G3", f.read())
+
+    def test_pip_vendored_certifi_patched(self):
+        self.assertGreaterEqual(Version(pip.__version__), Version("26.2.2"))
+
+    # GHSA-hq85-5cpj-mr62: uv vendors its own CA bundle.
+    def test_uv_vendored_certs_patched(self):
+        version = subprocess.check_output(["uv", "--version"], text=True).split()[1]
+        self.assertGreaterEqual(Version(version), Version("0.13.4"))
+
+    # Covers both the merged bundle and the per-certificate files OpenSSL looks up by hash.
+    def test_system_store_patched(self):
+        context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        for path in [SYSTEM_BUNDLE, *glob.glob("/etc/ssl/certs/*.pem")]:
+            context.load_verify_locations(cafile=path)
+        common_names = {
+            value
+            for cert in context.get_ca_certs()
+            for rdn in cert["subject"]
+            for key, value in rdn
+            if key == "commonName"
+        }
+        # The other emSign roots are unaffected and should still be trusted.
+        self.assertIn("emSign Root CA - G1", common_names)
+        self.assertNotIn("emSign ECC Root CA - G3", common_names)
+
+    def test_java_keystore_patched(self):
+        listing = subprocess.check_output(
+            ["keytool", "-list", "-v", "-cacerts", "-storepass", "changeit"], text=True
+        )
+        self.assertIn("emSign Root CA - G1", listing)
+        self.assertNotIn("emSign ECC Root CA - G3", listing)
+
+    # gcloud, gRPC, and Node vendor unpatched bundles, so they must use the system store.
+    # Remove each of these alongside its workaround in Dockerfile.tmpl.
+    def test_gcloud_uses_system_store(self):
+        path = subprocess.check_output(
+            ["gcloud", "config", "get", "core/custom_ca_certs_file"], text=True
+        ).strip()
+        self.assertEqual(SYSTEM_BUNDLE, path)
+
+    def test_grpc_uses_system_store(self):
+        self.assertEqual(SYSTEM_BUNDLE, os.environ.get("GRPC_DEFAULT_SSL_ROOTS_FILE_PATH"))
+
+    def test_node_uses_system_store(self):
+        self.assertIn("--use-openssl-ca", os.environ.get("NODE_OPTIONS", ""))
+        # Node refuses to start if NODE_OPTIONS contains a flag it doesn't accept.
+        subprocess.check_call(["node", "-e", ""])
+
+    def test_wandb_binaries_patched(self):
+        bin_dir = os.path.join(os.path.dirname(wandb.__file__), "bin")
+        for name in os.listdir(bin_dir):
+            with open(os.path.join(bin_dir, name), "rb") as f:
+                # Not assertNotIn: its failure message would dump the whole binary.
+                self.assertFalse(b"emSign ECC Root CA - G3" in f.read(), name)

tpu/Dockerfile

@@ -33,6 +33,16 @@ RUN /tmp/clean-layer.sh && \
     apt-get install --no-install-recommends -y ffmpeg libsm6 libxext6 && \
     /tmp/clean-layer.sh
 
+# GHSA-6q48-6c7w-9gw2: emSign ECC Root CA - G3 issued intermediates with compromised keys.
+# Debian's trixie-security ca-certificates 20250419+deb13u1 blocklists it in the system store.
+# Remove once the base image ships that ca-certificates itself.
+RUN apt-get install --no-install-recommends -y --only-upgrade ca-certificates && \
+    /tmp/clean-layer.sh
+
+# grpcio and libtpu embed gRPC's unpatched CA bundle, so point them at the system store.
+# Remove once both ship a release whose bundle drops the G3 root.
+ENV GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt
+
 # Install all the packages together for maximum compatibility.
 # Additional useful packages should be added in the requirements.txt
 # Bring in the requirements.txt and replace variables in it:
@@ -61,6 +71,14 @@ RUN export PATH="${HOME}/.local/bin:${PATH}" && \
     /tmp/clean-layer.sh
 ENV PATH="~/.local/bin:${PATH}"
 
+# pip < 26.2.2 always loads its vendored certifi, so it needs the GHSA-6q48-6c7w-9gw2 fix separately.
+# The compile step above excludes pip, so upgrade it here, with the patched uv rather than the old pip.
+# Remove once the base image ships pip >= 26.2.2 itself.
+RUN export PATH="${HOME}/.local/bin:${PATH}" && \
+    uv pip install --system "pip>=26.2.2" && \
+    uv cache clean && \
+    /tmp/clean-layer.sh
+
 # We install a libtpu version compatible with both jax 0.7.2 and torch 2.8.0.
 # Why? tunix latest -> flax 0.12 -> jax 0.7.2 -> libtpu 0.0.23. However, that
 # libtpu causes pjrt api errors for torch 2.8.0. screenshot/5heUtdyaJ4MmR3D