From f6173f70ef6355b5ab39293907156758448fbda8 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 23:38:29 -1000 Subject: [PATCH 01/11] Phase-2 modernization rebuilt on merged main (tree-identical to conrad/phase2-modernization 172884d) [1/4] --- .pre-commit-config.yaml | 28 +-- Dockerfile | 4 +- README.md | 2 +- bindings/jdbc/genomicsqlite-jdbc/pom.xml | 20 +- bindings/python/genomicsqlite/__init__.py | 12 +- bindings/python/genomicsqlite/__main__.py | 1 + bindings/python/release.sh | 0 bindings/python/setup.py | 3 +- bindings/python/version.py | 6 +- build.yml | 241 ++++++++++++++++++++++ docs/guide_db.md | 6 +- 11 files changed, 275 insertions(+), 48 deletions(-) mode change 100755 => 100644 bindings/python/genomicsqlite/__init__.py mode change 100755 => 100644 bindings/python/release.sh mode change 100755 => 100644 bindings/python/setup.py create mode 100644 build.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index abf7065..3e9dcfd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,26 +17,12 @@ repos: verbose: true entry: cppcheck args: [-q,--std=c++11] - - id: black - name: black - language: system - files: \.py$ - verbose: true - entry: black - args: [-l,'100'] - - id: flake8 - name: flake8 - language: system - files: \.py$ +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.1 + hooks: + - id: ruff-check + name: ruff-check verbose: true - entry: flake8 - args: [--max-line-length, "100", "--ignore=E501,W503,E722"] - - id: pylint - name: pylint - language: system - files: \.py$ - exclude: version\.py + - id: ruff-format + name: ruff-format verbose: true - entry: env PYTHONPATH=bindings/python pylint - args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,too-many-positional-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] -# CI re-run trigger 2026-10-19 (run 39294337754 lost its macOS runner to GitHub capacity) diff --git a/Dockerfile b/Dockerfile index faec530..cd2ff47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM almalinux:8 as builder ARG CMAKE_VERSION=3.17.3 -ARG SQLITE_VERSION=3410200 +ARG SQLITE_VERSION=3540100 ARG ZSTD_VERSION=1.5.5 ARG CPU_ARCH=ivybridge ENV CFLAGS="-march=${CPU_ARCH} -O3" @@ -34,7 +34,7 @@ RUN mkdir -p /work/GenomicSQLite # SQLite WORKDIR /work -RUN wget -nv http://www.sqlite.org/2023/sqlite-amalgamation-${SQLITE_VERSION}.zip \ +RUN wget -nv http://www.sqlite.org/2026/sqlite-amalgamation-${SQLITE_VERSION}.zip \ && unzip -o sqlite-amalgamation-${SQLITE_VERSION}.zip WORKDIR /work/sqlite-amalgamation-${SQLITE_VERSION} RUN bash -c "gcc -shared -o libsqlite3.so.0 -fPIC -shared -Wl,-soname,libsqlite3.so.0 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c & pid=$? \ diff --git a/README.md b/README.md index 352593f..f31a516 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ cmake --build build -j 4 --target genomicsqlite * htslib ≥ 1.9, samtools, and tabix * pigz -* Python ≥ 3.6 and packages: pytest pytest-xdist pre-commit black pylint flake8 +* Python ≥ 3.6 and packages: pytest pytest-xdist pre-commit ruff * JDK, mvn, rust * clang-format & cppcheck diff --git a/bindings/jdbc/genomicsqlite-jdbc/pom.xml b/bindings/jdbc/genomicsqlite-jdbc/pom.xml index 63b4894..b7b3646 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/pom.xml +++ b/bindings/jdbc/genomicsqlite-jdbc/pom.xml @@ -61,7 +61,7 @@ junit junit - 4.13.1 + 4.13.2 test @@ -72,44 +72,44 @@ maven-clean-plugin - 3.1.0 + 3.5.0 maven-resources-plugin - 3.0.2 + 3.5.0 maven-compiler-plugin - 3.8.0 + 3.16.0 maven-surefire-plugin - 2.22.2 + 3.6.0 false maven-jar-plugin - 3.0.2 + 3.5.1 maven-install-plugin - 2.5.2 + 3.2.0 maven-deploy-plugin - 2.8.2 + 3.2.0 maven-site-plugin - 3.7.1 + 3.22.0 maven-project-info-reports-plugin - 3.0.0 + 3.9.0 com.diffplug.spotless diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100755 new mode 100644 index c1ec17e..12cf6ec --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -3,12 +3,12 @@ GenomicSQLite Python binding """ +import json import os -import sys import platform -import json import sqlite3 -from typing import Optional, NamedTuple, Dict, Any +import sys +from typing import Any, Dict, NamedTuple, Optional _HERE = os.path.dirname(__file__) _YES = ("1", "true", "t", "yes", "y") @@ -182,9 +182,9 @@ def get_reference_sequences_by_rid( params = (assembly,) ans = {} for row in con.execute(sql, params): - assert ( - isinstance(row[0], int) and row[0] not in ans - ), "genomicsqlite: invalid or duplicate reference sequence rid" + assert isinstance(row[0], int) and row[0] not in ans, ( + "genomicsqlite: invalid or duplicate reference sequence rid" + ) ans[row[0]] = ReferenceSequence( rid=row[0], name=row[1], diff --git a/bindings/python/genomicsqlite/__main__.py b/bindings/python/genomicsqlite/__main__.py index ba91ed0..bd19d9c 100644 --- a/bindings/python/genomicsqlite/__main__.py +++ b/bindings/python/genomicsqlite/__main__.py @@ -1,4 +1,5 @@ import sys + from . import _cli if __name__ == "__main__": diff --git a/bindings/python/release.sh b/bindings/python/release.sh old mode 100755 new mode 100644 diff --git a/bindings/python/setup.py b/bindings/python/setup.py old mode 100755 new mode 100644 index b9fee7b..cfd8a19 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from os import path -from setuptools import setup, find_packages # pylint: disable=import-error + +from setuptools import find_packages, setup # pylint: disable=import-error from version import get_version HERE = path.abspath(path.dirname(__file__)) diff --git a/bindings/python/version.py b/bindings/python/version.py index 5b7cfd8..964f594 100644 --- a/bindings/python/version.py +++ b/bindings/python/version.py @@ -76,7 +76,7 @@ def read_git_version(): return None match = re.search(_GIT_DESCRIPTION_RE, ver) if not match: - sys.stderr.write("version: git description (%s) is invalid, " "ignoring\n" % ver) + sys.stderr.write("version: git description (%s) is invalid, ignoring\n" % ver) return None commits = int(match.group("commits")) @@ -90,9 +90,7 @@ def read_release_version(): with open(RELEASE_VERSION_FILE) as infile: ver = infile.readline().strip() if not re.search(_PEP386_VERSION_RE, ver): - sys.stderr.write( - "version: release version (%s) is invalid, " "will use it anyway\n" % ver - ) + sys.stderr.write("version: release version (%s) is invalid, will use it anyway\n" % ver) return ver except: return None diff --git a/build.yml b/build.yml new file mode 100644 index 0000000..3ecda10 --- /dev/null +++ b/build.yml @@ -0,0 +1,241 @@ +name: build +on: [push, pull_request] + +env: + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 + PIP_DEPS: pytest pytest-xdist + +jobs: + + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS ruff + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" + - name: pre-commit + run: pre-commit run --all-files + - name: mvn spotless:check + run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: rust lint + run: | + cd bindings/rust + ./cargo check --no-default-features + ./cargo fmt -- --check + + test: + runs-on: ubuntu-24.04 + strategy: + matrix: + BUILD_TYPE: [Debug, Release] + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build + run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) + - name: test + run: env -C build SQLITE_WEB_LOG=99 ctest -V + + linux-x86-64-so: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: build loaders + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) + rm build/libgenomicsqlite.so + sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so + - name: build portable .so in docker + run: | + # BuildKit builds only the stages the final target depends on, so the + # two minimum-runtime smoke-test stages -- which nothing depends on -- + # must be built explicitly, or they are silently skipped. + docker build --target alma-smoke-test . + docker build --target rust-smoke-test . + docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ + - name: symbols & digests + run: | + readelf -d build/libgenomicsqlite.so + ldd -r build/libgenomicsqlite.so + echo "----" + echo "libgenomicsqlite.so GLIBC version requirements:" + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 + echo "libsqlite3.so.0 GLIBC version requirements:" + objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + echo "----" + sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + # The steps below test the portable libraries against the portable SQLite + # they ship with, scoping that environment (build/ first on PATH / + # LD_LIBRARY_PATH) to the tested processes only -- never the runner's + # global loader cache or PATH. + - name: test portable .so + run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage + env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so + - name: installed-package smoke test + run: | + sudo pip3 install --break-system-packages -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" + - name: prepare artifacts + run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite.so + path: libgenomicsqlite.so + - uses: actions/upload-artifact@v7 + with: + name: libsqlite3.so.0 + path: libsqlite3.so.0 + + macOS-arm64-dylib: + runs-on: macos-15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: dependencies + run: | + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + brew update || (sleep 2; brew update) + for dep in $BREW_DEPS ; do + brew install $dep || brew upgrade $dep + done + + # setup-python's macOS ARM 3.12.10 build lacked SQLite extension loading. + # Homebrew enables it; 3.12 is our CI baseline, matching Ubuntu 24.04. + # A venv selects that interpreter and keeps pip installs isolated. + "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" + echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.mac-venv/bin:$PATH" + python3 -m pip install $PIP_DEPS + python3 - <<'PYTHON' + import sqlite3, sys + print(sys.executable, sys.version) + print("SQLite", sqlite3.sqlite_version) + sqlite3.connect(":memory:").enable_load_extension(True) + PYTHON + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build environment + run: | + echo "$(brew --prefix)/opt/sqlite/bin" >> "$GITHUB_PATH" + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + # used by rusqlite: + echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + - name: build + run: | + # Shipped-extension baseline: the Apple M1 CPU target (the flags + # compose with the include paths in $CFLAGS / $CXXFLAGS from the + # "build environment" step). + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="$CFLAGS -mcpu=apple-m1" \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS -mcpu=apple-m1" \ + -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ + -B build . + cmake --build build -j 4 + - name: symbols & digests + run: | + otool -L build/libgenomicsqlite.dylib + sha256sum build/libgenomicsqlite.dylib + - name: ctest + run: | + cd build + otool -L test/capi_smoke_test + SQLITE_WEB_LOG=99 ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib + SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib + - name: installed-package smoke test + run: | + python3 -m pip install -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" + - name: prepare artifacts + run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite-macos-aarch64.dylib + path: libgenomicsqlite-macos-aarch64.dylib + + artifacts: + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] + steps: + - uses: actions/checkout@v7 + - name: fetch libgenomicsqlite.so + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite.so + - name: fetch libsqlite3.so.0 + uses: actions/download-artifact@v8 + with: + name: libsqlite3.so.0 + - name: fetch libgenomicsqlite.dylib + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite-macos-aarch64.dylib + - name: detect GIT_REVISION + run: | + cp include/genomicsqlite.h . + echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV + - uses: actions/upload-artifact@v7 + with: + name: GenomicSQLite-${{ env.GIT_REVISION }} + path: | + LICENSE + genomicsqlite.h + libgenomicsqlite.so + libgenomicsqlite-macos-aarch64.dylib + - name: build jar + run: | + cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so + cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib + mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: JDBC smoke test + run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc + - name: digests + run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 + - uses: actions/upload-artifact@v7 + with: + name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} + path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo diff --git a/docs/guide_db.md b/docs/guide_db.md index 7b8abea..ac7ad07 100644 --- a/docs/guide_db.md +++ b/docs/guide_db.md @@ -94,9 +94,9 @@ First complete the [installation instructions](index.md). === "Python" ``` python3 dbconn = genomicsqlite.connect( - db_filename, - read_only=False, - **kwargs # genomicsqlite + sqlite3.connect() arguments + db_filename, + read_only=False, + **kwargs, # genomicsqlite + sqlite3.connect() arguments ) assert isinstance(dbconn, sqlite3.Connection) ``` From aea42451a9d84f91c994d9d5a202599b19f07b7e Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 23:38:37 -1000 Subject: [PATCH 02/11] Phase-2 modernization rebuilt on merged main (tree-identical to conrad/phase2-modernization 172884d) [2/4] --- docs/guide_gri.md | 42 ++++++++--------- docs/guide_helpers.md | 10 ++-- docs/internals.md | 18 ++++--- ruff.toml | 94 +++++++++++++++++++++++++++++++++++++ test/MavenSmokeTest/pom.xml | 22 ++++----- 5 files changed, 142 insertions(+), 44 deletions(-) create mode 100644 ruff.toml diff --git a/docs/guide_gri.md b/docs/guide_gri.md index 8bf7f66..747e0db 100644 --- a/docs/guide_gri.md +++ b/docs/guide_gri.md @@ -21,11 +21,7 @@ The extension provides routines to populate a small `_gri_refseq` table describi === "Python" ``` python3 create_gri_sql = genomicsqlite.create_genomic_range_index_sql( - dbconn, - 'tableName', - 'chromosome', - 'beginPosition', - 'endPosition' + dbconn, "tableName", "chromosome", "beginPosition", "endPosition" ) dbconn.executescript(create_gri_sql) ``` @@ -229,9 +225,7 @@ The following routines support the aforementioned, recommended convention for st === "Python" ``` python3 - refseq_sql = genomicsqlite.put_reference_assembly_sql( - dbconn, 'GRCh38_no_alt_analysis_set' - ) + refseq_sql = genomicsqlite.put_reference_assembly_sql(dbconn, "GRCh38_no_alt_analysis_set") dbconn.executescript(refseq_sql) ``` @@ -307,8 +301,10 @@ Available assemblies: === "Python" ``` python3 refseq_sql = genomicsqlite.put_reference_sequence_sql( - dbconn, 'chr17', 83257441 - # optional: assembly, refget_id, meta (dict), rid + dbconn, + "chr17", + 83257441, + # optional: assembly, refget_id, meta (dict), rid ) dbconn.executescript(refseq_sql) ``` @@ -407,12 +403,13 @@ If the `rid` argument is omitted or -1 then it will be assigned automatically up === "Python" ``` python3 class ReferenceSequence(NamedTuple): - rid: int - name: str - length: int - assembly: Optional[str] - refget_id: Optional[str] - meta: Dict[str, Any] + rid: int + name: str + length: int + assembly: Optional[str] + refget_id: Optional[str] + meta: Dict[str, Any] + refseq_by_rid = genomicsqlite.get_reference_sequences_by_rid(dbconn) # refseq_by_rid: Dict[int, ReferenceSequence] @@ -492,12 +489,13 @@ The optional `assembly` argument restricts the retrieved sequences to those with === "Python" ``` python3 class ReferenceSequence(NamedTuple): - rid: int - name: str - length: int - assembly: Optional[str] - refget_id: Optional[str] - meta: Dict[str, Any] + rid: int + name: str + length: int + assembly: Optional[str] + refget_id: Optional[str] + meta: Dict[str, Any] + refseq_by_name = genomicsqlite.get_reference_sequences_by_name(dbconn) # refseq_by_name: Dict[str, ReferenceSequence] diff --git a/docs/guide_helpers.md b/docs/guide_helpers.md index d1cd9a7..95bc3e3 100644 --- a/docs/guide_helpers.md +++ b/docs/guide_helpers.md @@ -88,8 +88,8 @@ The [UINT collating sequence](https://www.sqlite.org/uintcseq.html) is also bund === "Python" ``` python3 - dbconn = sqlite3.connect('any.db', uri=True) - attach_sql = genomicsqlite.attach_sql(dbconn, 'compressed.db', 'db2') + dbconn = sqlite3.connect("any.db", uri=True) + attach_sql = genomicsqlite.attach_sql(dbconn, "compressed.db", "db2") # attach_sql() also takes configuration keyword arguments like # genomicsqlite.connect() dbconn.executescript(attach_sql) @@ -190,13 +190,13 @@ The [UINT collating sequence](https://www.sqlite.org/uintcseq.html) is also bund === "Python" ``` python3 - dbconn = sqlite3.connect('existing.db', uri=True) - vacuum_sql = genomicsqlite.vacuum_into_sql(dbconn, 'compressed.db') + dbconn = sqlite3.connect("existing.db", uri=True) + vacuum_sql = genomicsqlite.vacuum_into_sql(dbconn, "compressed.db") # vacuum_into_sql() also takes configuration keyword arguments like # genomicsqlite.connect() to control compression level & page sizes dbconn.executescript(vacuum_sql) - dbconn2 = genomicsqlite.connect('compressed.db') + dbconn2 = genomicsqlite.connect("compressed.db") ``` === "Java" diff --git a/docs/internals.md b/docs/internals.md index df34720..01dd569 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -70,13 +70,19 @@ The extension also exposes the routine to generate the SQL subquery, which you c === "Python" ``` python3 query = ( - 'SELECT * FROM tableName WHERE tableName._rowid_ IN ' + - genomicsqlite.genomic_range_rowids_sql(dbconn, 'tableName', - # defaults: - qrid='?1', qbeg='?2', qend='?3', - ceiling=-1, floor=-1) + "SELECT * FROM tableName WHERE tableName._rowid_ IN " + + genomicsqlite.genomic_range_rowids_sql( + dbconn, + "tableName", + # defaults: + qrid="?1", + qbeg="?2", + qend="?3", + ceiling=-1, + floor=-1, + ) ) - cursor = dbconn.execute(query, ('chr12',111803912,111804012)) + cursor = dbconn.execute(query, ("chr12", 111803912, 111804012)) ``` === "SQLiteCpp" diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..413ea41 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,94 @@ +# Ruff configuration -- Python formatting and linting for this repo. +# Adopted 2026-10, replacing black (formatting) and flake8 + pylint +# (linting), as a tool swap at equal enforcement: every check the old +# tools enforced is either carried below or named in the migration +# record (ops/phase2-deps-audit-2026-10-18.md) as having no ruff +# counterpart. +# Line length matches the former `black -l 100` / flake8 +# --max-line-length 100 settings. +line-length = 100 + +[lint] +select = [ + # The flake8 rule families (E/F/W), as configured for this project, + # plus ruff's Pylint family (PL), which carries most of the checks + # pylint enforced here. + "E", "F", "W", "PL", + # Individual rules from other ruff families, each carrying one + # specific check pylint enforced here (pylint code in parentheses): + "ARG001", "ARG002", "ARG003", "ARG004", "ARG005", # unused-argument (W0613) + "A001", "A002", "A003", # redefined-builtin (W0622) + "B006", "B008", # dangerous-default-value (W0102) + "B018", # pointless-statement (W0104) + "B023", # cell-var-from-loop (W0640) + "B025", # duplicate-except (W0705) + "B904", # raise-missing-from (W0707) + "G001", "G002", "G003", "G004", # logging-not-lazy / -format-interpolation / -fstring-interpolation (W1201-W1203) + "ISC001", "ISC002", "ISC004", # implicit-str-concat (W1404) + "I001", # unsorted-imports (isort): the counterpart of pylint's + # wrong-import-order (C0411) and ungrouped-imports (C0412). + # Deliberately not adopted in the initial tool swap (ruff's + # grouping is stricter than pylint's); adopted on Mike's + # ruling, 2026-10-18, as a low-risk cleanup. + "S102", "S307", # exec-used (W0122), eval-used (W0123) + "T100", # forgotten-debug-statement (W1515) + "SIM115", # consider-using-with (R1732), the open() case +] +ignore = [ + # E501 (line too long): line length is policed by `ruff format`, + # not the linter; long strings/comments may legitimately exceed + # it. Carried over from the flake8 --ignore list; also ports the + # pylint line-too-long (C0301) disable. + "E501", + # E722 (bare except): tolerated in this codebase. Carried over + # from the flake8 --ignore list; also ports the pylint + # bare-except (W0702) disable. + "E722", + # The flake8 --ignore list also named W503 (line break before + # binary operator). Ruff removed W503/W504 entirely (they + # conflict with the formatter); it was a relaxation, not an + # enforced check, so nothing is lost. + # + # Ports of this repo's pylint disable list (formerly the -d list + # in .pre-commit-config.yaml), one per entry: + "PLW0603", # global-statement + "PLR0913", # too-many-arguments + "PLR0917", # too-many-positional-arguments (round-4 ruling: the + # positional split-out of the too-many-arguments + # family; the flagged functions are public API) + "PLR0914", # too-many-locals + "PLR0912", # too-many-branches + "PLR0915", # too-many-statements + "PLC0415", # import-outside-toplevel + "PLW1514", # unspecified-encoding + # The remaining pylint disables have no selector to port: + # missing-docstring / missing-module-docstring are not selected + # (ruff's pydocstyle D rules stay off, as under pylint); + # duplicate-code (R0801), redefined-outer-name (W0621) and + # protected-access (W0212) have no ruff counterpart at all; + # broad-except's counterpart (BLE001) is not selected. + # + # Ruff PL rules whose pylint counterparts were NOT part of this + # repo's enforced pylint gate (not enabled under the repo's + # pylint configuration) -- ignored to hold the swap at equal + # enforcement: + "PLC1901", # compare-to-empty-string + "PLC2701", # import-private-name + "PLR2004", # magic-value-comparison + "PLR2044", # empty-comment + "PLR5501", # collapsible-else-if + "PLR6104", # non-augmented-assignment + "PLR6201", # literal-membership + "PLR6301", # no-self-use + "PLW0717", # too-many-statements-in-try-clause + "PLW1641", # eq-without-hash + "PLW2901", # redefined-loop-name + "PLW3201", # bad-dunder-method-name +] + +[lint.per-file-ignores] +# The pylint hook excluded version.py (generated version-stamping +# helper); black and flake8 still applied to it. Port that split: +# version.py is exempt from the pylint-derived rule families below, +# and still checked by E/F/W and formatted like everything else. +"**/version.py" = ["PL", "ARG", "A", "B", "G", "ISC", "S", "T100", "SIM"] diff --git a/test/MavenSmokeTest/pom.xml b/test/MavenSmokeTest/pom.xml index 6b93186..bd6c08e 100644 --- a/test/MavenSmokeTest/pom.xml +++ b/test/MavenSmokeTest/pom.xml @@ -14,8 +14,8 @@ UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 @@ -39,41 +39,41 @@ maven-clean-plugin - 3.1.0 + 3.5.0 maven-resources-plugin - 3.0.2 + 3.5.0 maven-compiler-plugin - 3.8.0 + 3.16.0 maven-surefire-plugin - 2.22.1 + 3.6.0 maven-jar-plugin - 3.0.2 + 3.5.1 maven-install-plugin - 2.5.2 + 3.2.0 maven-deploy-plugin - 2.8.2 + 3.2.0 maven-site-plugin - 3.7.1 + 3.22.0 maven-project-info-reports-plugin - 3.0.0 + 3.9.0 From be57b746ceca5cd9521a4d81a973cb7538c4ded4 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 23:38:53 -1000 Subject: [PATCH 03/11] Phase-2 modernization rebuilt on merged main (tree-identical to conrad/phase2-modernization 172884d) [3/4] --- test/genomicsqlite_smoke_test.py | 20 ++++++++++---------- test/test_bam.py | 9 ++++++--- test/test_gri.py | 5 +++-- test/test_parse_genomic_range.py | 3 ++- test/test_twobit.py | 5 +++-- 5 files changed, 24 insertions(+), 18 deletions(-) mode change 100755 => 100644 test/genomicsqlite_smoke_test.py diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py old mode 100755 new mode 100644 index 9baf123..25ad8f8 --- a/test/genomicsqlite_smoke_test.py +++ b/test/genomicsqlite_smoke_test.py @@ -4,13 +4,13 @@ small test database and check GRI query plan on it. """ -import sys -import time import os -import subprocess -import traceback import platform +import subprocess +import sys import textwrap +import time +import traceback def main(): @@ -126,9 +126,9 @@ def main(): control = "SELECT exons1.id, exons2.id FROM exons1 LEFT JOIN exons2 NOT INDEXED ON NOT (exons2.end < exons1.pos OR exons2.pos > exons1.end) ORDER BY exons1.id, exons2.id" control_results = list(dbconn.execute(control)) assert len(control_results) == 1139, "unexpected control query results" - assert ( - results == control_results - ), "discrepancy between genomic_range_rowids_sql() and control results" + assert results == control_results, ( + "discrepancy between genomic_range_rowids_sql() and control results" + ) if gri_constraints != 3: print("GRI query opcodes:") @@ -154,9 +154,9 @@ def main(): genomic_range_rowids('exons2', exons1.chrom, exons1.pos, exons1.end, _gri_ceiling, _gri_floor) ORDER BY exons1.id, exons2.id """ - assert ( - list(dbconn.execute(query2)) == control_results - ), "discrepancy between genomic_range_rowids() and control results" + assert list(dbconn.execute(query2)) == control_results, ( + "discrepancy between genomic_range_rowids() and control results" + ) dbconn.close() print("\nGenomicSQLite smoke test OK =)\n") diff --git a/test/test_bam.py b/test/test_bam.py index 0748485..6d39c2b 100644 --- a/test/test_bam.py +++ b/test/test_bam.py @@ -1,6 +1,7 @@ import os -import subprocess import sqlite3 +import subprocess + import genomicsqlite HERE = os.path.dirname(__file__) @@ -55,10 +56,12 @@ def test_bam(tmp_path): count = next(con.execute("SELECT COUNT(DISTINCT qname) FROM NA12878_reads_seqs"))[0] assert count == 299205 - mq_hist = dict(con.execute(""" + mq_hist = dict( + con.execute(""" SELECT mq, COUNT(*) as count FROM (SELECT ifnull(json_extract(tags_json, '$.MQ'),0) AS mq FROM NA12878_reads NATURAL JOIN NA12878_reads_tags WHERE (flag & 3840) = 0) GROUP BY mq ORDER BY mq DESC - """)) + """) + ) assert (mq_hist[0], mq_hist[60]) == (2734, 520522) diff --git a/test/test_gri.py b/test/test_gri.py index 3ab2cda..ef1d6a1 100644 --- a/test/test_gri.py +++ b/test/test_gri.py @@ -1,8 +1,9 @@ import os -import sqlite3 import random -import pytest +import sqlite3 + import genomicsqlite +import pytest HERE = os.path.dirname(__file__) BUILD = os.path.abspath(os.path.join(HERE, "..", "build")) diff --git a/test/test_parse_genomic_range.py b/test/test_parse_genomic_range.py index 8b5e508..e561f13 100644 --- a/test/test_parse_genomic_range.py +++ b/test/test_parse_genomic_range.py @@ -1,6 +1,7 @@ import sqlite3 -import pytest + import genomicsqlite +import pytest def test_parse_genomic_range(): diff --git a/test/test_twobit.py b/test/test_twobit.py index f574d98..2c22afc 100644 --- a/test/test_twobit.py +++ b/test/test_twobit.py @@ -1,8 +1,9 @@ -import random import math +import random import sqlite3 -import pytest + import genomicsqlite +import pytest def test_twobit_random(): From 8a324f588d9719c52a2c0a5643569e3fb18ada61 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 23:39:01 -1000 Subject: [PATCH 04/11] Phase-2 modernization rebuilt on merged main (tree-identical to conrad/phase2-modernization 172884d) [4/4] --- test/test_txdb.py | 7 ++++--- test/test_vcf.py | 3 ++- test/test_vcf_lines.py | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/test_txdb.py b/test/test_txdb.py index b96bb52..87a20fc 100644 --- a/test/test_txdb.py +++ b/test/test_txdb.py @@ -1,9 +1,10 @@ import os -import sys -import subprocess import random -import pytest +import subprocess +import sys + import genomicsqlite +import pytest HERE = os.path.dirname(__file__) diff --git a/test/test_vcf.py b/test/test_vcf.py index bed3151..f6c6d90 100644 --- a/test/test_vcf.py +++ b/test/test_vcf.py @@ -1,8 +1,9 @@ import os import sqlite3 import subprocess -import pytest + import genomicsqlite +import pytest HERE = os.path.dirname(__file__) BUILD = os.path.abspath(os.path.join(HERE, "..", "build")) diff --git a/test/test_vcf_lines.py b/test/test_vcf_lines.py index 7ebdb91..d23c90c 100644 --- a/test/test_vcf_lines.py +++ b/test/test_vcf_lines.py @@ -1,6 +1,7 @@ import os import sqlite3 import subprocess + import genomicsqlite import pytest From 72700e5b852cfa2d8ecb31c39e6a5387671cff50 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 23:39:11 -1000 Subject: [PATCH 05/11] Remove live workflow (staged at build.yml pending rename into .github/workflows/) --- .github/workflows/build.yml | 241 ------------------------------------ 1 file changed, 241 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 4db37dc..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,241 +0,0 @@ -name: build -on: [push, pull_request] - -env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip - BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 - PIP_DEPS: pytest pytest-xdist - -jobs: - - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip python3-venv - python3 -m venv "$HOME/.lint-venv" - "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint - echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - - name: pre-commit - run: pre-commit run --all-files - - name: mvn spotless:check - run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - name: rust lint - run: | - cd bindings/rust - ./cargo check --no-default-features - ./cargo fmt -- --check - - test: - runs-on: ubuntu-24.04 - strategy: - matrix: - BUILD_TYPE: [Debug, Release] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build - run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - - name: test - run: env -C build SQLITE_WEB_LOG=99 ctest -V - - linux-x86-64-so: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: build loaders - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) - rm build/libgenomicsqlite.so - sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - - name: build portable .so in docker - run: | - # BuildKit builds only the stages the final target depends on, so the - # two minimum-runtime smoke-test stages -- which nothing depends on -- - # must be built explicitly, or they are silently skipped. - docker build --target alma-smoke-test . - docker build --target rust-smoke-test . - docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - - name: symbols & digests - run: | - readelf -d build/libgenomicsqlite.so - ldd -r build/libgenomicsqlite.so - echo "----" - echo "libgenomicsqlite.so GLIBC version requirements:" - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 - echo "libsqlite3.so.0 GLIBC version requirements:" - objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - echo "----" - sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 - echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" - python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - # The steps below test the portable libraries against the portable SQLite - # they ship with, scoping that environment (build/ first on PATH / - # LD_LIBRARY_PATH) to the tested processes only -- never the runner's - # global loader cache or PATH. - - name: test portable .so - run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so - ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage - env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so - - name: installed-package smoke test - run: | - sudo pip3 install --break-system-packages -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" - - name: prepare artifacts - run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite.so - path: libgenomicsqlite.so - - uses: actions/upload-artifact@v7 - with: - name: libsqlite3.so.0 - path: libsqlite3.so.0 - - macOS-arm64-dylib: - runs-on: macos-15 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: dependencies - run: | - rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 - brew update || (sleep 2; brew update) - for dep in $BREW_DEPS ; do - brew install $dep || brew upgrade $dep - done - - # setup-python's macOS ARM 3.12.10 build lacked SQLite extension loading. - # Homebrew enables it; 3.12 is our CI baseline, matching Ubuntu 24.04. - # A venv selects that interpreter and keeps pip installs isolated. - "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" - echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.mac-venv/bin:$PATH" - python3 -m pip install $PIP_DEPS - python3 - <<'PYTHON' - import sqlite3, sys - print(sys.executable, sys.version) - print("SQLite", sqlite3.sqlite_version) - sqlite3.connect(":memory:").enable_load_extension(True) - PYTHON - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build environment - run: | - echo "$(brew --prefix)/opt/sqlite/bin" >> "$GITHUB_PATH" - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - # used by rusqlite: - echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - - name: build - run: | - # Shipped-extension baseline: the Apple M1 CPU target (the flags - # compose with the include paths in $CFLAGS / $CXXFLAGS from the - # "build environment" step). - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_FLAGS="$CFLAGS -mcpu=apple-m1" \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS -mcpu=apple-m1" \ - -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ - -B build . - cmake --build build -j 4 - - name: symbols & digests - run: | - otool -L build/libgenomicsqlite.dylib - sha256sum build/libgenomicsqlite.dylib - - name: ctest - run: | - cd build - otool -L test/capi_smoke_test - SQLITE_WEB_LOG=99 ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib - - name: installed-package smoke test - run: | - python3 -m pip install -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" - - name: prepare artifacts - run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite-macos-aarch64.dylib - path: libgenomicsqlite-macos-aarch64.dylib - - artifacts: - runs-on: ubuntu-24.04 - needs: [linux-x86-64-so, macOS-arm64-dylib] - steps: - - uses: actions/checkout@v7 - - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite.so - - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v8 - with: - name: libsqlite3.so.0 - - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite-macos-aarch64.dylib - - name: detect GIT_REVISION - run: | - cp include/genomicsqlite.h . - echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v7 - with: - name: GenomicSQLite-${{ env.GIT_REVISION }} - path: | - LICENSE - genomicsqlite.h - libgenomicsqlite.so - libgenomicsqlite-macos-aarch64.dylib - - name: build jar - run: | - cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so - cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib - mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: JDBC smoke test - run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc - - name: digests - run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v7 - with: - name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} - path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo From a46a87e13086ff7c3b68a324620624134db9cdf4 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Tue, 20 Oct 2026 13:37:42 -1000 Subject: [PATCH 06/11] Restore exec bits; move build.yml into .github/workflows/ --- bindings/python/genomicsqlite/__init__.py | 0 bindings/python/release.sh | 0 bindings/python/setup.py | 0 test/genomicsqlite_smoke_test.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bindings/python/genomicsqlite/__init__.py mode change 100644 => 100755 bindings/python/release.sh mode change 100644 => 100755 bindings/python/setup.py mode change 100644 => 100755 test/genomicsqlite_smoke_test.py diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100644 new mode 100755 diff --git a/bindings/python/release.sh b/bindings/python/release.sh old mode 100644 new mode 100755 diff --git a/bindings/python/setup.py b/bindings/python/setup.py old mode 100644 new mode 100755 diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py old mode 100644 new mode 100755 From 66b8e99a0bc595b335f74155145aac2653a27c16 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Tue, 20 Oct 2026 18:09:05 -1000 Subject: [PATCH 07/11] Move build.yml into .github/workflows/ --- build.yml => .github/workflows/build.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build.yml => .github/workflows/build.yml (100%) diff --git a/build.yml b/.github/workflows/build.yml similarity index 100% rename from build.yml rename to .github/workflows/build.yml From 5d34b5ef4e75748b5c1fa262c214ff3ae05bbc60 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Tue, 20 Oct 2026 19:21:50 -1000 Subject: [PATCH 08/11] ruff.toml: run ruff on defaults; documented opt-outs only (Mike's lint policy, 2026-10-20) --- ruff.toml | 102 +++++++----------------------------------------------- 1 file changed, 12 insertions(+), 90 deletions(-) diff --git a/ruff.toml b/ruff.toml index 413ea41..6de4501 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,94 +1,16 @@ -# Ruff configuration -- Python formatting and linting for this repo. -# Adopted 2026-10, replacing black (formatting) and flake8 + pylint -# (linting), as a tool swap at equal enforcement: every check the old -# tools enforced is either carried below or named in the migration -# record (ops/phase2-deps-audit-2026-10-18.md) as having no ruff -# counterpart. -# Line length matches the former `black -l 100` / flake8 -# --max-line-length 100 settings. -line-length = 100 +# Ruff: run on ruff's own defaults (Mike's lint policy, 2026-10-20). +# Opt out of specific rules only where warranted; each opt-out is documented +# inline. No rule sets are selected or opted into. +line-length = 100 # exception to the ruff default (88): matches this codebase's black -l 100 formatting; avoids a whole-tree reformat [lint] -select = [ - # The flake8 rule families (E/F/W), as configured for this project, - # plus ruff's Pylint family (PL), which carries most of the checks - # pylint enforced here. - "E", "F", "W", "PL", - # Individual rules from other ruff families, each carrying one - # specific check pylint enforced here (pylint code in parentheses): - "ARG001", "ARG002", "ARG003", "ARG004", "ARG005", # unused-argument (W0613) - "A001", "A002", "A003", # redefined-builtin (W0622) - "B006", "B008", # dangerous-default-value (W0102) - "B018", # pointless-statement (W0104) - "B023", # cell-var-from-loop (W0640) - "B025", # duplicate-except (W0705) - "B904", # raise-missing-from (W0707) - "G001", "G002", "G003", "G004", # logging-not-lazy / -format-interpolation / -fstring-interpolation (W1201-W1203) - "ISC001", "ISC002", "ISC004", # implicit-str-concat (W1404) - "I001", # unsorted-imports (isort): the counterpart of pylint's - # wrong-import-order (C0411) and ungrouped-imports (C0412). - # Deliberately not adopted in the initial tool swap (ruff's - # grouping is stricter than pylint's); adopted on Mike's - # ruling, 2026-10-18, as a low-risk cleanup. - "S102", "S307", # exec-used (W0122), eval-used (W0123) - "T100", # forgotten-debug-statement (W1515) - "SIM115", # consider-using-with (R1732), the open() case -] ignore = [ - # E501 (line too long): line length is policed by `ruff format`, - # not the linter; long strings/comments may legitimately exceed - # it. Carried over from the flake8 --ignore list; also ports the - # pylint line-too-long (C0301) disable. - "E501", - # E722 (bare except): tolerated in this codebase. Carried over - # from the flake8 --ignore list; also ports the pylint - # bare-except (W0702) disable. - "E722", - # The flake8 --ignore list also named W503 (line break before - # binary operator). Ruff removed W503/W504 entirely (they - # conflict with the formatter); it was a relaxation, not an - # enforced check, so nothing is lost. - # - # Ports of this repo's pylint disable list (formerly the -d list - # in .pre-commit-config.yaml), one per entry: - "PLW0603", # global-statement - "PLR0913", # too-many-arguments - "PLR0917", # too-many-positional-arguments (round-4 ruling: the - # positional split-out of the too-many-arguments - # family; the flagged functions are public API) - "PLR0914", # too-many-locals - "PLR0912", # too-many-branches - "PLR0915", # too-many-statements - "PLC0415", # import-outside-toplevel - "PLW1514", # unspecified-encoding - # The remaining pylint disables have no selector to port: - # missing-docstring / missing-module-docstring are not selected - # (ruff's pydocstyle D rules stay off, as under pylint); - # duplicate-code (R0801), redefined-outer-name (W0621) and - # protected-access (W0212) have no ruff counterpart at all; - # broad-except's counterpart (BLE001) is not selected. - # - # Ruff PL rules whose pylint counterparts were NOT part of this - # repo's enforced pylint gate (not enabled under the repo's - # pylint configuration) -- ignored to hold the swap at equal - # enforcement: - "PLC1901", # compare-to-empty-string - "PLC2701", # import-private-name - "PLR2004", # magic-value-comparison - "PLR2044", # empty-comment - "PLR5501", # collapsible-else-if - "PLR6104", # non-augmented-assignment - "PLR6201", # literal-membership - "PLR6301", # no-self-use - "PLW0717", # too-many-statements-in-try-clause - "PLW1641", # eq-without-hash - "PLW2901", # redefined-loop-name - "PLW3201", # bad-dunder-method-name + "E722", # bare except: deliberate bare-except uses in this codebase, carried from the flake8 era + "UP006", # non-pep585-annotation: binding annotations are evaluated at runtime; package declares python_requires>=3.6 + "UP045", # non-pep604-annotation-optional: same runtime-annotation / python>=3.6 constraint as UP006 + "UP035", # deprecated-import: typing.Dict/Optional etc. retained for the same python>=3.6 annotation constraint + "UP031", # printf-string-formatting: fires only in vendored bindings/python/version.py (third-party, public domain), not hand-edited + "UP009", # utf8-encoding-declaration: fires only in vendored bindings/python/version.py, not hand-edited + "PLE0605", # invalid-all-format: fires only in vendored bindings/python/version.py, not hand-edited + "BLE001", # blind-except: smoke-test harness deliberately catches broad exceptions to degrade gracefully ] - -[lint.per-file-ignores] -# The pylint hook excluded version.py (generated version-stamping -# helper); black and flake8 still applied to it. Port that split: -# version.py is exempt from the pylint-derived rule families below, -# and still checked by E/F/W and formatted like everything else. -"**/version.py" = ["PL", "ARG", "A", "B", "G", "ISC", "S", "T100", "SIM"] From e6bdb5da49f76c1e1aa7dbfa36abce2d2528fe5d Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Tue, 20 Oct 2026 19:22:43 -1000 Subject: [PATCH 09/11] ruff defaults: mechanical code fixes (comprehension conversions, subprocess kwargs, extraneous parens, .values(), startswith tuple, dead shebang) --- bindings/python/genomicsqlite/__init__.py | 12 +++++------ test/check_elf_version_floor.py | 1 - test/genomicsqlite_smoke_test.py | 7 +++--- test/test_gri.py | 26 +++++++++++------------ test/test_txdb.py | 12 +++++------ test/test_vcf.py | 14 ++++++------ test/test_vcf_lines.py | 14 ++++++------ 7 files changed, 40 insertions(+), 46 deletions(-) mode change 100755 => 100644 bindings/python/genomicsqlite/__init__.py mode change 100755 => 100644 test/genomicsqlite_smoke_test.py diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100755 new mode 100644 index 12cf6ec..d20be29 --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -200,7 +200,7 @@ def get_reference_sequences_by_name( con: sqlite3.Connection, assembly: Optional[str] = None, schema: Optional[str] = None ) -> Dict[str, ReferenceSequence]: ans = {} - for _, item in get_reference_sequences_by_rid(con, assembly, schema).items(): + for item in get_reference_sequences_by_rid(con, assembly, schema).values(): assert item.name not in ans, "genomicsqlite: non-unique reference sequence names" ans[item.name] = item return ans @@ -333,12 +333,10 @@ def _cli(argv=None): print( " ".join( ( - ( - (arg if " " not in arg else f"'{arg}'") - + (" \\\n " if len(arg) > 50 else "") - ) - for arg in cmd + (arg if " " not in arg else f"'{arg}'") + + (" \\\n " if len(arg) > 50 else "") ) + for arg in cmd ) ) os.execvp("sqlite3", cmd) @@ -403,7 +401,7 @@ def _compact(dbfilename, argv): # sniff whether input db is currently compressed with zstd_vfs or not con = None - web = dbfilename.startswith("http:") or dbfilename.startswith("https:") + web = dbfilename.startswith(("http:", "https:")) if not web: con = sqlite3.connect(f"file:{urllib.parse.quote(dbfilename)}?mode=ro", uri=True) if next(con.execute("PRAGMA application_id"))[0] != 0x7A737464: diff --git a/test/check_elf_version_floor.py b/test/check_elf_version_floor.py index 04a9e76..0e65b97 100644 --- a/test/check_elf_version_floor.py +++ b/test/check_elf_version_floor.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Assert that ELF shared objects' required symbol versions stay at or below the project's compatibility ceilings -- the manylinux_2_28 floor asserted by CI for diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py old mode 100755 new mode 100644 index 25ad8f8..4e6f16d --- a/test/genomicsqlite_smoke_test.py +++ b/test/genomicsqlite_smoke_test.py @@ -36,9 +36,8 @@ def main(): sysctl = subprocess.run( ["sysctl", "-n", "machdep.cpu.brand_string"], check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + text=True, + capture_output=True, ) if sysctl.returncode == 0 and sysctl.stdout.strip(): print(": " + sysctl.stdout.strip()) @@ -65,7 +64,7 @@ def main(): conn = sqlite3.connect(":memory:") print(next(conn.execute("SELECT sqlite_version()"))[0]) - sqlite_opts = list(row[0] for row in conn.execute("PRAGMA compile_options")) + sqlite_opts = [row[0] for row in conn.execute("PRAGMA compile_options")] print(f" {' '.join(sqlite_opts)}") print("genomicsqlite: ", end="") diff --git a/test/test_gri.py b/test/test_gri.py index ef1d6a1..9911d92 100644 --- a/test/test_gri.py +++ b/test/test_gri.py @@ -63,13 +63,13 @@ def test_indexing(): # for expl in opcodes: # if expl[1] in {"OpenRead", "OpenPseudo", "Column"}: # print(expl) - accessed_cursors = list(opcode[2] for opcode in opcodes if opcode[1] == "Column") - table_rootpages = set( + accessed_cursors = [opcode[2] for opcode in opcodes if opcode[1] == "Column"] + table_rootpages = { row[0] for row in con.execute("SELECT rootpage FROM sqlite_master WHERE type='table'") - ) - table_cursors = set( + } + table_cursors = { opcode[2] for opcode in opcodes if opcode[1] == "OpenRead" and opcode[3] in table_rootpages - ) + } assert sum(1 for cursor in accessed_cursors if cursor not in table_cursors) > 1 # get from idx assert sum(1 for cursor in accessed_cursors if cursor in table_cursors) == 1 # get from table @@ -80,8 +80,8 @@ def test_indexing(): for _ in range(50000): beg = random.randint(43044294 - 10000, 43044294 + 10000) end = beg + random.randint(1, random.choice([10, 100, 1000, 10000])) - ids = list(row[0] for row in con.execute(query, ("chr17", beg, end))) - control_ids = list(row[0] for row in con.execute(control_query, ("chr17", beg, end))) + ids = [row[0] for row in con.execute(query, ("chr17", beg, end))] + control_ids = [row[0] for row in con.execute(control_query, ("chr17", beg, end))] assert ids == control_ids total_results += len(control_ids) assert total_results == 189935 @@ -381,14 +381,12 @@ def test_query_in_sql(tmp_path): assert "USE TEMP B-TREE FOR ORDER BY" not in expl[3] assert next( - ( - expl[3] - for expl in con.execute( - "EXPLAIN QUERY PLAN SELECT _rowid_ FROM genomic_range_rowids('exons',?,?,?) ORDER BY _rowid_ DESC", - ("chr17", 43044294, 43048294), - ) - if "USE TEMP B-TREE FOR ORDER BY" in expl[3] + expl[3] + for expl in con.execute( + "EXPLAIN QUERY PLAN SELECT _rowid_ FROM genomic_range_rowids('exons',?,?,?) ORDER BY _rowid_ DESC", + ("chr17", 43044294, 43048294), ) + if "USE TEMP B-TREE FOR ORDER BY" in expl[3] ) with pytest.raises(sqlite3.OperationalError, match="domain error"): diff --git a/test/test_txdb.py b/test/test_txdb.py index 87a20fc..f56b446 100644 --- a/test/test_txdb.py +++ b/test/test_txdb.py @@ -50,7 +50,7 @@ def test_txdbquery(genomicsqlite_txdb): assert next(conn.execute("PRAGMA page_size"))[0] == 16384 # one query - results = list( + results = [ t[0] for t in conn.execute( "SELECT tx_name FROM transcript WHERE _rowid_ IN " @@ -58,7 +58,7 @@ def test_txdbquery(genomicsqlite_txdb): + " ORDER BY tx_name", ("chr12", 111803912, 111804012), ) - ) + ] print(results) assert results == sorted( ["ENST00000416293.7", "ENST00000261733.7", "ENST00000548536.1", "ENST00000549106.1"] @@ -92,8 +92,8 @@ def test_txdbquery(genomicsqlite_txdb): chrom = random.choice(chroms) beg = random.randint(0, chrom[1] - 65536) end = beg + random.randint(1, random.choice([16, 256, 4096, 65536])) - ids = list(row[0] for row in conn.execute(query, (chrom[0], beg, end))) - control_ids = list(row[0] for row in conn.execute(control_query, (chrom[0], beg, end))) + ids = [row[0] for row in conn.execute(query, (chrom[0], beg, end))] + control_ids = [row[0] for row in conn.execute(control_query, (chrom[0], beg, end))] assert ids == control_ids total_results += len(control_ids) assert total_results in (7341, 2660) @@ -149,7 +149,7 @@ def test_web(): read_only=True, ) - results = list( + results = [ t[0] for t in conn.execute( "SELECT tx_name FROM transcript WHERE _rowid_ IN " @@ -157,7 +157,7 @@ def test_web(): + " ORDER BY tx_name", ("chr12", 111803912, 111804012), ) - ) + ] print(results) sys.stdout.flush() assert results == sorted( diff --git a/test/test_vcf.py b/test/test_vcf.py index f6c6d90..ea456c4 100644 --- a/test/test_vcf.py +++ b/test/test_vcf.py @@ -50,22 +50,22 @@ def test_gnomad_sites_small(tmp_path): indexed += 1 assert indexed == 3 opcodes = list(con.execute("EXPLAIN " + query, rs671)) - accessed_cursors = list(opcode[2] for opcode in opcodes if opcode[1] == "Column") - table_rootpages = set( + accessed_cursors = [opcode[2] for opcode in opcodes if opcode[1] == "Column"] + table_rootpages = { row[0] for row in con.execute("SELECT rootpage FROM sqlite_master WHERE type='table'") - ) - table_cursors = set( + } + table_cursors = { opcode[2] for opcode in opcodes if opcode[1] == "OpenRead" and opcode[3] in table_rootpages - ) + } assert sum(1 for cursor in accessed_cursors if cursor not in table_cursors) > 1 assert sum(1 for cursor in accessed_cursors if cursor in table_cursors) == 1 results = list(con.execute(query, rs671)) - results_rowids = set(vt[0] for vt in results) + results_rowids = {vt[0] for vt in results} assert next(vt for vt in results if vt[1] and "rs671" in vt[1]) control = "SELECT variant_rowid FROM gnomad_variants NATURAL JOIN _gri_refseq WHERE gri_refseq_name = ? AND NOT ((pos+rlen) < ? OR pos > ?)" - control_rowids = set(vt[0] for vt in con.execute(control, rs671)) + control_rowids = {vt[0] for vt in con.execute(control, rs671)} assert len(control_rowids) == 22 assert results_rowids == control_rowids diff --git a/test/test_vcf_lines.py b/test/test_vcf_lines.py index d23c90c..9c96f98 100644 --- a/test/test_vcf_lines.py +++ b/test/test_vcf_lines.py @@ -44,22 +44,22 @@ def test_gnomad_sites_small(tmp_path): indexed += 1 assert indexed == 3 opcodes = list(con.execute("EXPLAIN " + query, rs671)) - accessed_cursors = list(opcode[2] for opcode in opcodes if opcode[1] == "Column") - table_rootpages = set( + accessed_cursors = [opcode[2] for opcode in opcodes if opcode[1] == "Column"] + table_rootpages = { row[0] for row in con.execute("SELECT rootpage FROM sqlite_master WHERE type='table'") - ) - table_cursors = set( + } + table_cursors = { opcode[2] for opcode in opcodes if opcode[1] == "OpenRead" and opcode[3] in table_rootpages - ) + } assert sum(1 for cursor in accessed_cursors if cursor not in table_cursors) > 1 assert sum(1 for cursor in accessed_cursors if cursor in table_cursors) == 1 results = list(con.execute(query, rs671)) - results_rowids = set(vt[0] for vt in results) + results_rowids = {vt[0] for vt in results} assert next(vt for vt in results if vt[1] and "rs671" in vt[1]) control = "SELECT rowid FROM gnomad_vcf_lines WHERE NOT ((POS+rlen) < ? OR POS > ?)" - control_rowids = set(vt[0] for vt in con.execute(control, (rs671[1], rs671[2]))) + control_rowids = {vt[0] for vt in con.execute(control, (rs671[1], rs671[2]))} assert len(control_rowids) == 22 assert results_rowids == control_rowids From 87e11ccd939ce18468733e364ac5ddb6ef76ae79 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Tue, 20 Oct 2026 19:53:26 -1000 Subject: [PATCH 10/11] Restore exec bits stripped by connector push (fixes EXE001 lint) --- bindings/python/genomicsqlite/__init__.py | 0 test/genomicsqlite_smoke_test.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bindings/python/genomicsqlite/__init__.py mode change 100644 => 100755 test/genomicsqlite_smoke_test.py diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100644 new mode 100755 diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py old mode 100644 new mode 100755 From 97cf46502b4d4f04430757b58035828c0d9aa73e Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Tue, 20 Oct 2026 21:30:05 -1000 Subject: [PATCH 11/11] ruff.toml: drop personal name from header comment (PR review feedback) --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 6de4501..5accfef 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -# Ruff: run on ruff's own defaults (Mike's lint policy, 2026-10-20). +# Ruff: run on ruff's own defaults (project lint policy, 2026-10-20). # Opt out of specific rules only where warranted; each opt-out is documented # inline. No rule sets are selected or opted into. line-length = 100 # exception to the ruff default (88): matches this codebase's black -l 100 formatting; avoids a whole-tree reformat