From fbff4200e5b46707ec12ddd3ea6308c640730770 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Sun, 18 Oct 2026 19:36:20 -1000 Subject: [PATCH 01/21] CI resurrection: modernized workflows, AlmaLinux 8 portable build, JDBC/Rust/Python bit-rot fixes (pre-phase-2 state, = conrad/ci-modernization @ 4c4f5de) --- .github/workflows/build.yml | 136 ++++++++++-------- .pre-commit-config.yaml | 2 +- CMakeLists.txt | 9 +- Dockerfile | 39 ++--- bindings/jdbc/genomicsqlite-jdbc/pom.xml | 10 +- .../net/mlin/genomicsqlite/GenomicSQLite.java | 3 +- .../net/mlin/genomicsqlite/JdbcDriver.java | 8 +- bindings/python/genomicsqlite/__init__.py | 34 +++-- bindings/python/setup.py | 2 +- bindings/python/version.py | 1 - bindings/rust/.gitignore | 1 + bindings/rust/Cargo.toml.in | 2 +- bindings/rust/build.rs | 74 ++++++++++ bindings/rust/cargo | 9 +- bindings/rust/src/lib.rs | 86 ++++++++--- loaders/vcf_into_sqlite.cc | 6 +- src/genomicsqlite.cc | 13 +- test/cargo_smoke_test/cargo | 9 +- test/genomicsqlite_smoke_test.py | 1 + test/test_bam.py | 8 +- test/test_ext.py | 6 +- test/test_gri.py | 26 ++-- test/test_txdb.py | 2 +- test/test_vcf_lines.py | 1 - 24 files changed, 329 insertions(+), 159 deletions(-) mode change 100755 => 100644 bindings/python/genomicsqlite/__init__.py mode change 100755 => 100644 bindings/python/setup.py create mode 100644 bindings/rust/build.rs mode change 100755 => 100644 bindings/rust/cargo mode change 100755 => 100644 test/cargo_smoke_test/cargo mode change 100755 => 100644 test/genomicsqlite_smoke_test.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c41822c..b1e7ee9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,79 +2,76 @@ name: build on: [push, pull_request] env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip libcurl4-openssl-dev - BREW_DEPS: zstd samtools pigz coreutils sqlite + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 PIP_DEPS: pytest pytest-xdist jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: deps run: | sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip - sudo pip3 install --system pre-commit $PIP_DEPS black flake8 pylint + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - name: pre-commit run: pre-commit run --all-files - name: mvn spotless:check run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - name: rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy - toolchain: stable - default: true - name: rust lint run: | cd bindings/rust - ./cargo check --no-default-features - ./cargo fmt -- --check + bash ./cargo check --no-default-features + bash ./cargo fmt -- --check test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: BUILD_TYPE: [Debug, Release] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: deps run: | sudo apt-get -qq update sudo apt-get install -y $APT_DEPS - sudo pip3 install --system $PIP_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS - name: rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true + uses: dtolnay/rust-toolchain@stable - name: build run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - name: test run: env -C build SQLITE_WEB_LOG=99 ctest -V linux-x86-64-so: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: build loaders run: | sudo apt-get -qq update sudo apt-get install -y $APT_DEPS - sudo pip3 install --system $PIP_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) rm build/libgenomicsqlite.so sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - name: build portable .so in docker - run: docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /mnt/build/ + run: docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - name: symbols & digests run: | readelf -d build/libgenomicsqlite.so @@ -87,60 +84,83 @@ jobs: objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 echo "----" sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + for f in build/libgenomicsqlite.so build/libsqlite3.so.0; do + max_glibc=$(objdump -t $f | grep -oE 'GLIBC_[0-9.]+' | sort -Vu | tail -n1) + echo "$f max GLIBC need: $max_glibc" + if [ "$(printf 'GLIBC_2.28\n%s\n' "$max_glibc" | sort -V | tail -n1)" != "GLIBC_2.28" ]; then + echo "$f requires $max_glibc, above the glibc 2.28 floor" >&2 + exit 1 + fi + done + max_glibcxx=$(objdump -t build/libgenomicsqlite.so | grep -oE 'GLIBCXX_[0-9.]+' | sort -Vu | tail -n1) + echo "build/libgenomicsqlite.so max GLIBCXX need: $max_glibcxx" + if [ "$(printf 'GLIBCXX_3.4.25\n%s\n' "$max_glibcxx" | sort -V | tail -n1)" != "GLIBCXX_3.4.25" ]; then + echo "build/libgenomicsqlite.so requires $max_glibcxx, above the GLIBCXX 3.4.25 ceiling" >&2 + exit 1 + fi sudo cp build/libsqlite3.so.0 /usr/local/lib sudo ldconfig + # Pair like with like: the ldconfig plant above makes the portable + # libsqlite3.so.0 (3.41.2) shadow the system one, so the runner's own + # sqlite3 shell (built against >= 3.42) dies at startup with + # "undefined symbol: sqlite3_stmt_explain". Put the portable build's + # own shell -- copied out of the docker image into build/ in the + # previous step -- first on PATH for the remaining steps, so the + # portable test exercises the CLI against the exact SQLite vintage + # the portable library ships with. (Later steps do not look up a + # sqlite3 shell by name, so the prepend is harmless to them.) + echo "$(pwd)/build" >> "$GITHUB_PATH" - name: rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true + uses: dtolnay/rust-toolchain@stable - name: test portable .so run: env -C build SQLITE_WEB_LOG=99 ctest -V - name: test rust crate run: | set -e - cp build/libgenomicsqlite.so bindings/rust - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite.so + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + SQLITE_WEB_LOG=99 bash bindings/rust/cargo test --release + bash bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so - name: prepare artifacts run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: libgenomicsqlite.so path: libgenomicsqlite.so - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: libsqlite3.so.0 path: libsqlite3.so.0 - macOS-x86-64-dylib: - runs-on: macOS-11 + macOS-arm64-dylib: + runs-on: macos-15 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - name: dependencies run: | - rm -f /usr/local/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 brew update || (sleep 2; brew update) for dep in $BREW_DEPS ; do brew install $dep || brew upgrade $dep done - pip install $PIP_DEPS + # The tests must run under Homebrew's Python: it is configured with + # --enable-loadable-sqlite-extensions (the binding cannot be imported + # without sqlite3.Connection.enable_load_extension), while the + # image's python.org framework Python is built without it. + echo "$(brew --prefix)/opt/python@3.12/bin" >> "$GITHUB_PATH" + export PATH="$(brew --prefix)/opt/python@3.12/bin:$PATH" + python3 -m pip install --break-system-packages $PIP_DEPS + python3 -c "import sqlite3; assert hasattr(sqlite3.Connection, 'enable_load_extension')" - name: rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true + uses: dtolnay/rust-toolchain@stable - name: build environment run: | - echo "PATH=$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=ivybridge" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=ivybridge" >> $GITHUB_ENV + echo "PATH=$(brew --prefix)/opt/python@3.12/bin:$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV # used by rusqlite: echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV @@ -163,38 +183,38 @@ jobs: - name: test rust crate run: | set -e - cp build/libgenomicsqlite.dylib bindings/rust - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite.dylib + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib + SQLITE_WEB_LOG=99 bash bindings/rust/cargo test --release + bash bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib - name: prepare artifacts run: cp build/libgenomicsqlite.dylib include/genomicsqlite.h . - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: libgenomicsqlite.dylib path: libgenomicsqlite.dylib artifacts: - runs-on: ubuntu-20.04 - needs: [linux-x86-64-so, macOS-x86-64-dylib] + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v8 with: name: libgenomicsqlite.so - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v8 with: name: libsqlite3.so.0 - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v8 with: name: libgenomicsqlite.dylib - name: detect GIT_REVISION run: | cp include/genomicsqlite.h . echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: GenomicSQLite-${{ env.GIT_REVISION }} path: | @@ -208,7 +228,7 @@ jobs: mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - name: digests run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c6a30e0..269465d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,4 +38,4 @@ repos: exclude: version\.py verbose: true entry: env PYTHONPATH=bindings/python pylint - args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] + args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,too-many-positional-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bc5852..6d67d4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,13 @@ if(${CMAKE_VERSION} VERSION_LESS 3.12) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) endif() +# CMake 4 removed compatibility with projects declaring +# cmake_minimum_required() below 3.5; the pinned SQLiteCpp fetched +# below declares 3.1, which makes configure fail outright under +# CMake 4 (e.g. the macos-15 runner image). Raise the policy-version +# floor for fetched subprojects; harmless under older CMake. +set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + include(FetchContent) if(${CMAKE_VERSION} VERSION_LESS 3.14) @@ -73,6 +80,6 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) add_test(NAME smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} python3 ${CMAKE_CURRENT_SOURCE_DIR}/test/genomicsqlite_smoke_test.py) add_test(NAME capi_smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_BINARY_DIR}/test/capi_smoke_test) add_test(NAME JdbcTests COMMAND ${GENOMICSQLITE_TEST_ENV} mvn test -B -f ${CMAKE_CURRENT_SOURCE_DIR}/bindings/jdbc/genomicsqlite-jdbc/pom.xml) - add_test(NAME cargo_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rust/cargo test --no-default-features) + add_test(NAME cargo_test COMMAND ${GENOMICSQLITE_TEST_ENV} bash ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rust/cargo test --no-default-features) add_test(NAME pytest COMMAND ${GENOMICSQLITE_TEST_ENV} python3 -m pytest -xv -n 4 --tb=short ${CMAKE_CURRENT_SOURCE_DIR}/test) endif() diff --git a/Dockerfile b/Dockerfile index 8f5d464..fb9f891 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -# Builds libgenomicsqlite.so in CentOS 7 (+ devtoolset-8) to maximize compatibility. Bakes in -# Zstandard, and also builds a compatible libsqlite3.so.0 for users unable/unwilling to upgrade -# their host's. -FROM centos:7 as builder +# Builds libgenomicsqlite.so in AlmaLinux 8 (glibc 2.28, the manylinux_2_28 baseline) to +# maximize compatibility. Bakes in Zstandard, and also builds a compatible libsqlite3.so.0 +# for users unable/unwilling to upgrade their host's. +FROM almalinux:8 as builder ARG CMAKE_VERSION=3.17.3 ARG SQLITE_VERSION=3410200 @@ -28,8 +28,7 @@ ENV SQLITE_CFLAGS="\ -DSQLITE_ENABLE_SESSION \ " -RUN yum install -y -q wget unzip rsync git libcurl-devel centos-release-scl -RUN yum install -y -q devtoolset-8-gcc devtoolset-8-gcc-c++ devtoolset-8-make +RUN dnf install -y -q wget unzip rsync git libcurl-devel gcc gcc-c++ make RUN mkdir -p /work/GenomicSQLite @@ -38,8 +37,8 @@ WORKDIR /work RUN wget -nv http://www.sqlite.org/2023/sqlite-amalgamation-${SQLITE_VERSION}.zip \ && unzip -o sqlite-amalgamation-${SQLITE_VERSION}.zip WORKDIR /work/sqlite-amalgamation-${SQLITE_VERSION} -RUN bash -c "scl enable devtoolset-8 'gcc -shared -o libsqlite3.so.0 -fPIC -shared -Wl,-soname,libsqlite3.so.0 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c' & pid=$? \ - scl enable devtoolset-8 'gcc -o sqlite3 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c shell.c -lpthread -ldl -lm' && wait $pid" +RUN bash -c "gcc -shared -o libsqlite3.so.0 -fPIC -shared -Wl,-soname,libsqlite3.so.0 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c & pid=$? \ + gcc -o sqlite3 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c shell.c -lpthread -ldl -lm && wait $pid" RUN cp libsqlite3.so.0 /usr/local/lib && cp *.h /usr/local/include && cp sqlite3 /usr/local/bin RUN ln -s /usr/local/lib/libsqlite3.so.0 /usr/local/lib/libsqlite3.so @@ -47,7 +46,7 @@ RUN ln -s /usr/local/lib/libsqlite3.so.0 /usr/local/lib/libsqlite3.so WORKDIR /work RUN wget -nv -O - https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz | tar zx WORKDIR /work/zstd-${ZSTD_VERSION} -RUN scl enable devtoolset-8 "CFLAGS='${CFLAGS} -fPIC' make install -j $(nproc)" +RUN CFLAGS="${CFLAGS} -fPIC" make install -j $(nproc) # CMake WORKDIR /work @@ -61,21 +60,27 @@ RUN ldconfig ADD . /work/GenomicSQLite WORKDIR /work/GenomicSQLite RUN rm -rf build -RUN scl enable devtoolset-8 "cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_WHOLE_ARCHIVE=true -B build . && cmake --build build --target genomicsqlite -j $(nproc)" +# _GLIBCXX_USE_CXX11_ABI=0: the header's C++ API passes std::string across the +# library boundary, so the .so's C++ ABI is part of its public contract. Keep the +# pre-C++11 libstdc++ ABI -- the artifact's historical ABI (the old CentOS +# devtoolset toolchain defaulted to it) and the ABI the CI workflow compiles the +# loader/test executables with. Stock AlmaLinux GCC defaults to ABI=1, which +# silently changes the exported symbols and breaks those executables. +RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="${CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0" -DZSTD_WHOLE_ARCHIVE=true -B build . && cmake --build build --target genomicsqlite -j $(nproc) RUN sqlite3 -cmd '.load build/libgenomicsqlite.so' :memory: 'select sqlite_version(); select genomicsqlite_version()' ################################################################################################### -# Run-up in fresh centos:7, to confirm no dependencies crept into the .so's +# Run-up in fresh almalinux:8, to confirm no dependencies crept into the .so's -FROM centos:7 +FROM almalinux:8 COPY --from=builder /usr/local/bin/sqlite3 /usr/local/bin/ COPY --from=builder /usr/local/include/sqlite3.h /usr/local/include/ -# NOTE: /usr/local/lib is NOT a default ld path in CentOS -- https://unix.stackexchange.com/q/356624 +# NOTE: /usr/local/lib is NOT a default ld path in AlmaLinux -- https://unix.stackexchange.com/q/356624 RUN mkdir -p /work/lib WORKDIR /work COPY --from=builder /usr/local/lib/libsqlite3.so.0 ./lib/ @@ -98,7 +103,7 @@ RUN ./genomicsqlite_capi_smoke_test # ^ we didn't need LD_LIBRARY_PATH even though the .so's aren't at their linktime locations # test rust bindings too -FROM centos:7 +FROM almalinux:8 RUN yum install -y -q gcc git ADD https://sh.rustup.rs /usr/local/bin/rustup-init.sh @@ -111,13 +116,13 @@ COPY --from=builder /usr/local/lib/libsqlite3.so.0 ./lib/ RUN ln -s libsqlite3.so.0 lib/libsqlite3.so ADD ./.git ./.git ADD ./bindings/rust ./rust -COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/ +COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/libgenomicsqlite-linux-x86_64.so RUN SQLITE_WEB_LOG=99 LD_LIBRARY_PATH=$(pwd)/lib LIBRARY_PATH=$(pwd)/lib rust/cargo test --release ################################################################################################### -# Run-up in ubuntu 16.04 +# Run-up in ubuntu 20.04 -FROM ubuntu:16.04 +FROM ubuntu:20.04 COPY --from=builder /usr/local/bin/sqlite3 /usr/local/bin/ COPY --from=builder /usr/local/include/sqlite3.h /usr/local/include/ diff --git a/bindings/jdbc/genomicsqlite-jdbc/pom.xml b/bindings/jdbc/genomicsqlite-jdbc/pom.xml index 5b1cdac..63b4894 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/pom.xml +++ b/bindings/jdbc/genomicsqlite-jdbc/pom.xml @@ -43,9 +43,13 @@ + + 0.9.4-SNAPSHOT UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 @@ -110,7 +114,7 @@ com.diffplug.spotless spotless-maven-plugin - 2.3.1 + 3.10.4 diff --git a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java index c7ae92c..cc353cf 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java +++ b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java @@ -156,7 +156,8 @@ public static String putReferenceSequenceSQL(Connection conn, String name, long public static HashMap getReferenceSequencesByRid( Connection conn, String assembly, String schema) throws SQLException { String sql = - "SELECT _gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id, gri_refseq_meta_json FROM "; + "SELECT _gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id," + + " gri_refseq_meta_json FROM "; if (schema != null && schema.length() > 0) { sql += schema + "."; } diff --git a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java index 3fc605a..fcd501e 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java +++ b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java @@ -40,7 +40,9 @@ public class JdbcDriver extends org.sqlite.JDBC { } } - /** @see java.sql.Driver#acceptsURL(java.lang.String) */ + /** + * @see java.sql.Driver#acceptsURL(java.lang.String) + */ @Override public boolean acceptsURL(String url) { return isValidURL(url); @@ -56,7 +58,9 @@ public static boolean isValidURL(String url) { return url != null && url.toLowerCase().startsWith(PREFIX); } - /** @see java.sql.Driver#connect(java.lang.String, java.util.Properties) */ + /** + * @see java.sql.Driver#connect(java.lang.String, java.util.Properties) + */ @Override public Connection connect(String url, Properties info) throws SQLException { return createConnection(url, info); diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100755 new mode 100644 index ea9f3cf..e91da64 --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -2,6 +2,7 @@ """ GenomicSQLite Python binding """ + import os import sys import platform @@ -252,6 +253,7 @@ def _cli(argv=None): sys.exit(1) import shutil + import tempfile if not shutil.which("sqlite3"): print( @@ -271,30 +273,36 @@ def _cli(argv=None): uri = _execute1(_MEMCONN, "SELECT genomicsqlite_uri(?,?)", (argv[1], cfg)) tuning_sql = _execute1(_MEMCONN, "SELECT genomicsqlite_tuning_sql(?)", (cfg,)) - cmd = [ - "sqlite3", - "-bail", - "-cmd", + # Assemble the shell startup commands into a temporary -init file, instead of passing them + # as -cmd arguments: since SQLite 3.52, the shell quits immediately (with a success exit + # code, without reading any input) whenever -bail is combined with an SQL -cmd argument, + # which silently disabled this wrapper. os.execvp() below replaces this process, so the + # file can't be unlinked afterwards; leave it for the OS temp reaper. + tuning_sql = tuning_sql.strip() + if not tuning_sql.endswith(";"): + # in -init file context, an unterminated statement would absorb the following line + tuning_sql += ";" + init_lines = [ f".load {_DLL}", - "-cmd", f".open {uri}", - "-cmd", ".once /dev/null", - "-cmd", tuning_sql, - "-cmd", '.prompt "GenomicSQLite> "', ] if sys.stdout.isatty() and not next( (arg for arg in argv[2:] if not arg.startswith("-")), False ): - # interactive mode: - cmd += [ - "-cmd", - "SELECT 'GenomicSQLite ' || genomicsqlite_version()", - "-cmd", + # interactive mode: print version banner, then turn on column headers + init_lines += [ + "SELECT 'GenomicSQLite ' || genomicsqlite_version();", ".headers on", ] + with tempfile.NamedTemporaryFile( + "w", prefix="genomicsqlite-init-", suffix=".sql", delete=False + ) as init_file: + init_file.write("\n".join(init_lines) + "\n") + + cmd = ["sqlite3", "-bail", "-init", init_file.name] cmd.append(":memory:") # placeholder so remaining positional args are recognized as such cmd += argv[2:] diff --git a/bindings/python/setup.py b/bindings/python/setup.py old mode 100755 new mode 100644 index 13b2af2..82dfa62 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from os import path -from setuptools import setup, find_packages +from setuptools import setup, find_packages # pylint: disable=import-error from version import get_version HERE = path.abspath(path.dirname(__file__)) diff --git a/bindings/python/version.py b/bindings/python/version.py index ddd2328..5b7cfd8 100644 --- a/bindings/python/version.py +++ b/bindings/python/version.py @@ -48,7 +48,6 @@ import subprocess import sys - RELEASE_VERSION_FILE = "RELEASE-VERSION" # http://www.python.org/dev/peps/pep-0386/ diff --git a/bindings/rust/.gitignore b/bindings/rust/.gitignore index 2d7878f..042eab2 100644 --- a/bindings/rust/.gitignore +++ b/bindings/rust/.gitignore @@ -13,3 +13,4 @@ Cargo.lock # Cargo.toml is generated from Cargo.toml.in by ./cargo Cargo.toml libgenomicsqlite.* +libgenomicsqlite-* diff --git a/bindings/rust/Cargo.toml.in b/bindings/rust/Cargo.toml.in index 7813476..d3979a9 100644 --- a/bindings/rust/Cargo.toml.in +++ b/bindings/rust/Cargo.toml.in @@ -7,7 +7,7 @@ edition = "2018" license = "Apache-2.0" repository = "https://github.com/mlin/GenomicSQLite" documentation = "https://mlin.github.io/GenomicSQLite" -include = ["src/**/*", "Cargo.toml", "libgenomicsqlite.*"] +include = ["src/**/*", "Cargo.toml", "build.rs", "libgenomicsqlite.*", "libgenomicsqlite-*"] [features] # Use default-features=false to prevent bundling a platform-specific libgenomicsqlite into your diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs new file mode 100644 index 0000000..5460adf --- /dev/null +++ b/bindings/rust/build.rs @@ -0,0 +1,74 @@ +// Build script for the genomicsqlite crate: inventory the prebuilt +// libgenomicsqlite libraries bundled next to Cargo.toml. +// +// A bundled library is embedded into the compilation unit only for the +// target it was built for, under a per-OS/architecture file name: +// libgenomicsqlite-linux-x86_64.so +// libgenomicsqlite-linux-aarch64.so +// libgenomicsqlite-macos-aarch64.dylib +// macOS x86_64 is deliberately not a bundled target (macOS CI builds +// ARM-only), so it falls back to the runtime library search like any +// other unsupported target. +// For each candidate file that actually exists in the package, this script +// sets a rustc cfg (bundled_linux_x86_64 etc.) gating the corresponding +// include_bytes! arm in src/lib.rs. A missing file therefore degrades to +// the runtime library search instead of breaking compilation with a +// missing-file error. +use std::env; +use std::path::Path; + +// (target_os, target_arch, file name, cfg name) for each bundled candidate +const CANDIDATES: &[(&str, &str, &str, &str)] = &[ + ( + "linux", + "x86_64", + "libgenomicsqlite-linux-x86_64.so", + "bundled_linux_x86_64", + ), + ( + "linux", + "aarch64", + "libgenomicsqlite-linux-aarch64.so", + "bundled_linux_aarch64", + ), + ( + "macos", + "aarch64", + "libgenomicsqlite-macos-aarch64.dylib", + "bundled_macos_aarch64", + ), +]; + +fn main() { + // Declare the custom cfg names unconditionally, so rustc's check-cfg + // validation accepts them wherever src/lib.rs mentions them. + for cand in CANDIDATES { + println!("cargo:rustc-check-cfg=cfg({})", cand.3); + } + println!("cargo:rustc-check-cfg=cfg(bundled_target)"); + + let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let bundling = env::var_os("CARGO_FEATURE_BUNDLE_LIBGENOMICSQLITE").is_some(); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + + for cand in CANDIDATES { + let path = Path::new(&manifest_dir).join(cand.2); + // Rerun this script if a bundled library appears or disappears. + println!("cargo:rerun-if-changed={}", path.display()); + if path.is_file() { + println!("cargo:rustc-cfg={}", cand.3); + if cand.0 == target_os && cand.1 == target_arch { + // The compilation target has a bundled library, so one of + // the include_bytes! arms in src/lib.rs will be compiled. + println!("cargo:rustc-cfg=bundled_target"); + } + } else if bundling && cand.0 == target_os && cand.1 == target_arch { + println!( + "cargo:warning=genomicsqlite: no bundled {} in the crate for {}-{}; \ + the crate will fall back to searching for a system libgenomicsqlite at runtime", + cand.2, cand.0, cand.1 + ); + } + } +} diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100755 new mode 100644 index 0aaa6bf..9056fd3 --- a/bindings/rust/cargo +++ b/bindings/rust/cargo @@ -4,7 +4,14 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty)" +revision="$(git describe --tags --dirty 2>/dev/null || true)" +if [ -z "$revision" ]; then + # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the + # latest release version with the short commit SHA as a prerelease suffix, + # keeping the generated crate version valid semver. (Update the release + # version here when cutting a new release.) + revision="0.9.4-$(git rev-parse --short HEAD)" +fi revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 96d41cf..1cdc6c7 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -7,9 +7,9 @@ use json::object::Object; use rusqlite::{params, Connection, LoadExtensionGuard, OpenFlags, Result}; use std::collections::HashMap; use std::env; -#[cfg(feature = "bundle_libgenomicsqlite")] +#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] use std::fs::File; -#[cfg(feature = "bundle_libgenomicsqlite")] +#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] use std::io::prelude::*; use std::path::Path; use std::sync::Once; @@ -18,7 +18,7 @@ use tempfile::TempDir; /* Helper functions for bundling libgenomicsqlite.{so,dylib} into the compilation unit */ // extract slice into basename under a temp directory -#[cfg(feature = "bundle_libgenomicsqlite")] +#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] fn extract_libgenomicsqlite_impl( bytes: &[u8], basename: &str, @@ -40,7 +40,7 @@ fn extract_libgenomicsqlite_impl( Ok((libpath, tmp)) } -#[cfg(feature = "bundle_libgenomicsqlite")] +#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { match extract_libgenomicsqlite_impl(bytes, filename) { Ok(ans) => ans, @@ -48,35 +48,76 @@ fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { } } -#[cfg(not(feature = "bundle_libgenomicsqlite"))] +/* Bundled libgenomicsqlite selection. + * + * With the bundle_libgenomicsqlite feature, the crate may embed a prebuilt + * libgenomicsqlite for its target, in a per-OS/architecture file next to + * Cargo.toml: + * libgenomicsqlite-linux-x86_64.so + * libgenomicsqlite-linux-aarch64.so + * libgenomicsqlite-macos-aarch64.dylib + * macOS x86_64 is deliberately not bundled (macOS CI builds ARM-only); + * it takes the fallback path below like any other unsupported target. + * build.rs inventories which of these files the package actually contains + * and sets a matching bundled_* cfg for each one present, so an arm below is + * compiled only when its file exists. On any other target -- or when the + * target's file isn't bundled -- bundle_libgenomicsqlite() returns None and + * open() falls back to the normal library search. + */ + +#[cfg(all( + feature = "bundle_libgenomicsqlite", + target_os = "linux", + target_arch = "x86_64", + bundled_linux_x86_64 +))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { - None + Some(extract_libgenomicsqlite( + include_bytes!("../libgenomicsqlite-linux-x86_64.so"), + "libgenomicsqlite.so", + )) } #[cfg(all( feature = "bundle_libgenomicsqlite", target_os = "linux", - target_arch = "x86_64" + target_arch = "aarch64", + bundled_linux_aarch64 ))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite.so"), - &"libgenomicsqlite.so", + include_bytes!("../libgenomicsqlite-linux-aarch64.so"), + "libgenomicsqlite.so", )) } #[cfg(all( feature = "bundle_libgenomicsqlite", target_os = "macos", - target_arch = "x86_64" + target_arch = "aarch64", + bundled_macos_aarch64 ))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite.dylib"), + include_bytes!("../libgenomicsqlite-macos-aarch64.dylib"), "libgenomicsqlite.dylib", )) } +#[cfg(not(feature = "bundle_libgenomicsqlite"))] +fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { + None +} + +// Feature on, but no bundled library for this target (another OS, an +// architecture the release didn't bundle, or the target's file absent from +// the package -- build.rs sets bundled_target exactly when one of the +// bundled arms above applies): fall back to the normal library search. +#[cfg(all(feature = "bundle_libgenomicsqlite", not(bundled_target)))] +fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { + None +} + // helper for simple queries fn query1str

(conn: &Connection, sql: &str, params: P) -> Result where @@ -112,23 +153,32 @@ pub fn open>(path: P, flags: OpenFlags, config: &Object) -> Resul // once: load libgenomicsqlite extension START.call_once(|| { let mut _tmpdir; - let libgenomicsqlite = match env::var("LIBGENOMICSQLITE") { - Ok(v) => v, + // Decide where to load libgenomicsqlite from, remembering how it was + // obtained, so a load failure can say exactly what was tried. + let (libgenomicsqlite, libgenomicsqlite_source) = match env::var("LIBGENOMICSQLITE") { + Ok(v) => (v, "path from the LIBGENOMICSQLITE environment variable"), Err(_) => match bundle_libgenomicsqlite() { Some((ans, td)) => { _tmpdir = td; - ans + (ans, "copy bundled into the genomicsqlite crate") } - None => "libgenomicsqlite".to_string(), + None => ( + "libgenomicsqlite".to_string(), + "bare library name, to be searched by the system loader \ + (no bundled copy available for this target)", + ), }, }; let memconn = Connection::open_in_memory().unwrap(); unsafe { let _guard = LoadExtensionGuard::new(&memconn).unwrap(); - match memconn.load_extension(libgenomicsqlite.clone(), None) { + match memconn.load_extension(libgenomicsqlite.clone(), None::<&str>) { Err(err) => panic!( - "genomicsqlite failed to load_extension(\"{}\"): {}", - libgenomicsqlite, err + "genomicsqlite: failed to load_extension(\"{}\") [{}]: {}\n\ + genomicsqlite needs a loadable libgenomicsqlite shared library. To provide one:\n\ + - set the LIBGENOMICSQLITE environment variable to the path of a locally built libgenomicsqlite, or\n\ + - install libgenomicsqlite where the system library loader can find it.", + libgenomicsqlite, libgenomicsqlite_source, err ), Ok(()) => (), } diff --git a/loaders/vcf_into_sqlite.cc b/loaders/vcf_into_sqlite.cc index f3b0eb8..359f469 100644 --- a/loaders/vcf_into_sqlite.cc +++ b/loaders/vcf_into_sqlite.cc @@ -67,8 +67,7 @@ string schemaDDL(const string &table_prefix, vector> &info_h } // add SQL comment with original metadata - out << " -- " - << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; + out << " -- " << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; auto desc = hrec.find("Description"); if (desc != hrec.end() && !desc->second.empty()) { out << ",Description=" << desc->second; @@ -109,8 +108,7 @@ string schemaDDL(const string &table_prefix, vector> &info_h } // add SQL comment with original metadata - out << " -- " - << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; + out << " -- " << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; auto desc = hrec.find("Description"); if (desc != hrec.end() && !desc->second.empty()) { out << ",Description=" << desc->second; diff --git a/src/genomicsqlite.cc b/src/genomicsqlite.cc index f94c475..d5a8c07 100644 --- a/src/genomicsqlite.cc +++ b/src/genomicsqlite.cc @@ -639,10 +639,10 @@ string GenomicRangeRowidsSQL(sqlite3 *dbconn, const string &indexed_table, const lvq << "\n UNION ALL\n "; } const string &it = indexed_table; - lvq << "SELECT _rowid_ FROM " << it << " INDEXED BY " << table << "__gri WHERE" - << "\n (" << it << "._gri_rid," << it << "._gri_lvl," << it << "._gri_beg) BETWEEN ((" - << qrid << "),-" << lv << ",(" << qbeg << ")-0x1" << string(lv, '0') << ") AND ((" - << qrid << "),-" << lv << ",(" << qend << ")-0)" // (*) + lvq << "SELECT _rowid_ FROM " << it << " INDEXED BY " << table << "__gri WHERE" << "\n (" + << it << "._gri_rid," << it << "._gri_lvl," << it << "._gri_beg) BETWEEN ((" << qrid + << "),-" << lv << ",(" << qbeg << ")-0x1" << string(lv, '0') << ") AND ((" << qrid + << "),-" << lv << ",(" << qend << ")-0)" // (*) << "\n AND (" << it << "._gri_beg+" << it << "._gri_len) >= (" << qbeg << ")"; /* * (*) For some reason, we have to obfuscate qend a little (such as with unary *+*, or by @@ -1023,9 +1023,8 @@ static string gri_refseq_ddl(const string &schema) { out << "CREATE TABLE IF NOT EXISTS " << schema_prefix << "_gri_refseq" << "(_gri_rid INTEGER NOT NULL PRIMARY KEY, gri_refseq_name TEXT NOT NULL, gri_assembly TEXT," << " gri_refget_id TEXT UNIQUE, gri_refseq_length INTEGER NOT NULL, gri_refseq_meta_json TEXT NOT NULL DEFAULT '{}', " - << "UNIQUE(gri_assembly,gri_refseq_name))" - << ";\nCREATE INDEX IF NOT EXISTS " << schema_prefix << "_gri_refseq_name ON " - << "_gri_refseq(gri_refseq_name)"; + << "UNIQUE(gri_assembly,gri_refseq_name))" << ";\nCREATE INDEX IF NOT EXISTS " + << schema_prefix << "_gri_refseq_name ON " << "_gri_refseq(gri_refseq_name)"; return out.str(); } diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100755 new mode 100644 index 0aaa6bf..9056fd3 --- a/test/cargo_smoke_test/cargo +++ b/test/cargo_smoke_test/cargo @@ -4,7 +4,14 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty)" +revision="$(git describe --tags --dirty 2>/dev/null || true)" +if [ -z "$revision" ]; then + # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the + # latest release version with the short commit SHA as a prerelease suffix, + # keeping the generated crate version valid semver. (Update the release + # version here when cutting a new release.) + revision="0.9.4-$(git rev-parse --short HEAD)" +fi revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py old mode 100755 new mode 100644 index 9264fcf..9baf123 --- a/test/genomicsqlite_smoke_test.py +++ b/test/genomicsqlite_smoke_test.py @@ -3,6 +3,7 @@ Display platform, environment, & version info for debugging GenomicSQLite installation. Load a small test database and check GRI query plan on it. """ + import sys import time import os diff --git a/test/test_bam.py b/test/test_bam.py index 81f0b99..0748485 100644 --- a/test/test_bam.py +++ b/test/test_bam.py @@ -55,14 +55,10 @@ def test_bam(tmp_path): count = next(con.execute("SELECT COUNT(DISTINCT qname) FROM NA12878_reads_seqs"))[0] assert count == 299205 - mq_hist = dict( - con.execute( - """ + mq_hist = dict(con.execute(""" SELECT mq, COUNT(*) as count FROM (SELECT ifnull(json_extract(tags_json, '$.MQ'),0) AS mq FROM NA12878_reads NATURAL JOIN NA12878_reads_tags WHERE (flag & 3840) = 0) GROUP BY mq ORDER BY mq DESC - """ - ) - ) + """)) assert (mq_hist[0], mq_hist[60]) == (2734, 520522) diff --git a/test/test_ext.py b/test/test_ext.py index 79bc99c..29dcd55 100644 --- a/test/test_ext.py +++ b/test/test_ext.py @@ -4,14 +4,12 @@ def test_bundled_extensions(): con = genomicsqlite.connect(":memory:") - con.executescript( - """ + con.executescript(""" CREATE TABLE test(json TEXT); INSERT INTO test(json) VALUES('{"chrom":"chr10"}'); INSERT INTO test(json) VALUES('{"chrom":"chr2"}'); INSERT INTO test(json) VALUES('{"chrom":"chrMT"}'); - """ - ) + """) results = con.execute( "SELECT json_extract(json,'$.chrom') AS chrom FROM test ORDER BY chrom COLLATE UINT" diff --git a/test/test_gri.py b/test/test_gri.py index d34c777..3ab2cda 100644 --- a/test/test_gri.py +++ b/test/test_gri.py @@ -104,7 +104,7 @@ def test_abutment(): query = genomicsqlite.genomic_range_rowids_sql( con, "features", "42", str(pos0), str(pos0 + 123) )[1:-1] - control_query = f"SELECT _rowid_ FROM features NOT INDEXED WHERE rid = 42 AND NOT (end < {pos0} OR beg > {pos0+123}) ORDER BY _rowid_" + control_query = f"SELECT _rowid_ FROM features NOT INDEXED WHERE rid = 42 AND NOT (end < {pos0} OR beg > {pos0 + 123}) ORDER BY _rowid_" assert list(con.execute(query)) == list(con.execute(control_query)) @@ -148,12 +148,10 @@ def fanout(query): assert fanout(query) == 1 - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(44, 1048568, 1048584); INSERT INTO features VALUES(44, 0, 64000) - """ - ) + """) query = genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1] print("\n" + query) assert "-4," in query and "-0x10000)" in query @@ -168,30 +166,24 @@ def fanout(query): == 4 ) - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(43, NULL, 10000000000); INSERT INTO features VALUES(44, 0, NULL) - """ - ) + """) assert fanout(genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1]) == 4 - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(43, 0, 10000000000); INSERT INTO features VALUES(43, 32, 33) - """ - ) + """) query = genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1] print("\n" + query) assert fanout(query) == 10 - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(43, 0, 10000000000); INSERT INTO features VALUES(43, 32, 32) - """ - ) + """) query = genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1] assert fanout(query) == 10 assert len(list(con.execute(query, (43, 32, 33)))) == 4 diff --git a/test/test_txdb.py b/test/test_txdb.py index 3f7bd83..b96bb52 100644 --- a/test/test_txdb.py +++ b/test/test_txdb.py @@ -12,7 +12,7 @@ def txdb(tmp_path): ans = str(tmp_path / "TxDb.Hsapiens.UCSC.hg38.knownGene.sqlite") subprocess.run( - f"pigz -dc {HERE+'/data/TxDb.Hsapiens.UCSC.hg38.knownGene.sqlite.gz'} > {ans}", + f"pigz -dc {HERE + '/data/TxDb.Hsapiens.UCSC.hg38.knownGene.sqlite.gz'} > {ans}", shell=True, check=True, ) diff --git a/test/test_vcf_lines.py b/test/test_vcf_lines.py index dab3b59..7ebdb91 100644 --- a/test/test_vcf_lines.py +++ b/test/test_vcf_lines.py @@ -4,7 +4,6 @@ import genomicsqlite import pytest - HERE = os.path.dirname(__file__) BUILD = os.path.abspath(os.path.join(HERE, "..", "build")) From 799bc8117d59ec6743e08dbc2921fc871c440fd2 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 18 Oct 2026 21:54:58 -1000 Subject: [PATCH 02/21] fix executable bits --- bindings/python/genomicsqlite/__init__.py | 0 bindings/python/setup.py | 0 bindings/rust/cargo | 0 test/cargo_smoke_test/cargo | 0 test/genomicsqlite_smoke_test.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bindings/python/genomicsqlite/__init__.py mode change 100644 => 100755 bindings/python/setup.py mode change 100644 => 100755 bindings/rust/cargo mode change 100644 => 100755 test/cargo_smoke_test/cargo mode change 100644 => 100755 test/genomicsqlite_smoke_test.py diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100644 new mode 100755 diff --git a/bindings/python/setup.py b/bindings/python/setup.py old mode 100644 new mode 100755 diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100644 new mode 100755 diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100644 new mode 100755 diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py old mode 100644 new mode 100755 From 9b06788f0183cf51b9c95850a5f55a9018d5c07b Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Sun, 18 Oct 2026 22:02:24 -1000 Subject: [PATCH 03/21] Address PR #28 review: arch-qualified bundled libraries and selectors - P1: Python/JDBC selectors match OS+CPU exactly and stage the bundled library under its plain basename at load time; installed-package smoke test (test/pkg_smoke.sh) exercises the selector with LIBGENOMICSQLITE unset - P2a: CLI startup via anonymous inherited fd (-cmd '.read /dev/fd/N') instead of a -init tempfile; composes with a user-supplied -init - P2b: name the Docker test stages and build each target explicitly in CI - P2c: cargo wrappers reverted to main's bytes (fetch-depth: 0 checkouts have the tags); direct invocations restored at all call sites - P2d+R1: Rust bundling selects one OS/CPU/ABI path, validates it, and fails the build when a requested bundle is missing or ABI-incompatible - P2e: macOS job uses an explicit Homebrew python@3.12 venv and asserts version + enable_load_extension - R2: portable-library tests scoped to build/ instead of a global ldconfig plant; R3: ELF version-floor assertions extracted to test/check_elf_version_floor.py (build.yml is staged at the branch root in this commit; it moves to .github/workflows/ by rename, which also triggers CI.) --- CMakeLists.txt | 2 +- Dockerfile | 14 +- .../net/mlin/genomicsqlite/JdbcDriver.java | 37 ++- bindings/python/MANIFEST.in | 2 +- bindings/python/conda/genomicsqlite/meta.yaml | 2 +- bindings/python/genomicsqlite/__init__.py | 105 ++++--- bindings/python/release.sh | 6 +- bindings/python/setup.py | 2 +- bindings/rust/build.rs | 149 ++++++---- bindings/rust/cargo | 9 +- bindings/rust/src/lib.rs | 85 ++---- build.yml | 271 ++++++++++++++++++ test/cargo_smoke_test/cargo | 9 +- test/check_elf_version_floor.py | 77 +++++ test/pkg_smoke.sh | 46 +++ 15 files changed, 631 insertions(+), 185 deletions(-) mode change 100755 => 100644 bindings/python/genomicsqlite/__init__.py mode change 100755 => 100644 bindings/python/release.sh mode change 100755 => 100644 bindings/python/setup.py mode change 100755 => 100644 bindings/rust/cargo create mode 100644 build.yml mode change 100755 => 100644 test/cargo_smoke_test/cargo create mode 100644 test/check_elf_version_floor.py create mode 100644 test/pkg_smoke.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d67d4d..6878daf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,6 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) add_test(NAME smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} python3 ${CMAKE_CURRENT_SOURCE_DIR}/test/genomicsqlite_smoke_test.py) add_test(NAME capi_smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_BINARY_DIR}/test/capi_smoke_test) add_test(NAME JdbcTests COMMAND ${GENOMICSQLITE_TEST_ENV} mvn test -B -f ${CMAKE_CURRENT_SOURCE_DIR}/bindings/jdbc/genomicsqlite-jdbc/pom.xml) - add_test(NAME cargo_test COMMAND ${GENOMICSQLITE_TEST_ENV} bash ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rust/cargo test --no-default-features) + add_test(NAME cargo_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rust/cargo test --no-default-features) add_test(NAME pytest COMMAND ${GENOMICSQLITE_TEST_ENV} python3 -m pytest -xv -n 4 --tb=short ${CMAKE_CURRENT_SOURCE_DIR}/test) endif() diff --git a/Dockerfile b/Dockerfile index fb9f891..2d4fb78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,8 +74,11 @@ RUN sqlite3 -cmd '.load build/libgenomicsqlite.so' :memory: 'select sqlite_versi ################################################################################################### # Run-up in fresh almalinux:8, to confirm no dependencies crept into the .so's +# NOTE: this stage is a test stage -- nothing in the final stage depends on it, +# so CI builds it explicitly with `docker build --target alma-smoke-test` +# (BuildKit would otherwise skip it silently). -FROM almalinux:8 +FROM almalinux:8 AS alma-smoke-test COPY --from=builder /usr/local/bin/sqlite3 /usr/local/bin/ COPY --from=builder /usr/local/include/sqlite3.h /usr/local/include/ @@ -102,8 +105,8 @@ RUN mv lib/{libgenomicsqlite.so,libsqlite3.so.0} . && rm -rf lib RUN ./genomicsqlite_capi_smoke_test # ^ we didn't need LD_LIBRARY_PATH even though the .so's aren't at their linktime locations -# test rust bindings too -FROM almalinux:8 +# test rust bindings too (also a test stage: CI builds --target rust-smoke-test) +FROM almalinux:8 AS rust-smoke-test RUN yum install -y -q gcc git ADD https://sh.rustup.rs /usr/local/bin/rustup-init.sh @@ -120,9 +123,10 @@ COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/libgeno RUN SQLITE_WEB_LOG=99 LD_LIBRARY_PATH=$(pwd)/lib LIBRARY_PATH=$(pwd)/lib rust/cargo test --release ################################################################################################### -# Run-up in ubuntu 20.04 +# Run-up in ubuntu 20.04 (final stage: the default docker build target, and the +# image the CI workflow copies the portable libraries out of) -FROM ubuntu:20.04 +FROM ubuntu:20.04 AS ubuntu-smoke-test COPY --from=builder /usr/local/bin/sqlite3 /usr/local/bin/ COPY --from=builder /usr/local/include/sqlite3.h /usr/local/include/ diff --git a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java index fcd501e..0241ab3 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java +++ b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java @@ -160,7 +160,13 @@ static String getLibraryFilename() { String tempDirpath = Files.createTempDirectory("genomicsqlite-jdbc-").toAbsolutePath().toString(); (new File(tempDirpath)).deleteOnExit(); - File extracted = new File(tempDirpath, basename); + // Extract under the PLAIN basename: SQLite derives the extension + // entry point (sqlite3_genomicsqlite_init) from the name of the file + // it's asked to load, so loading the extracted copy under its + // architecture-qualified resource name would look up a mangled entry + // point and fail. + String plainBasename = "libgenomicsqlite" + basename.substring(basename.lastIndexOf('.')); + File extracted = new File(tempDirpath, plainBasename); FileOutputStream writer = new FileOutputStream(extracted.getAbsolutePath()); extracted.deleteOnExit(); byte[] buffer = new byte[65536]; @@ -179,18 +185,27 @@ static String getLibraryFilename() { } static String getLibraryBasename() { + // Resource name of the bundled library for this OS + CPU, in the + // architecture-qualified form the release pipeline packs into the JAR (the + // same naming the Rust crate uses). Matching is exact: an Intel JVM never + // selects the ARM build, nor vice versa. If the JAR carries no resource + // under the returned name (a platform the release doesn't bundle), the + // caller falls back to the system library search. String arch = System.getProperty("os.arch", "unknown").toLowerCase(Locale.ENGLISH); + String archTok = null; if (arch.equals("amd64") || arch.equals("x86_64")) { - String os = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); - String ext = null; - if (os.indexOf("mac") >= 0 || os.indexOf("darwin") >= 0) { - ext = "dylib"; - } else if (os.indexOf("nux") >= 0) { - ext = "so"; - } - if (ext != null) { - return "libgenomicsqlite." + ext; - } + archTok = "x86_64"; + } else if (arch.equals("aarch64") || arch.equals("arm64")) { + archTok = "aarch64"; + } + if (archTok == null) { + return null; + } + String os = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); + if (os.indexOf("mac") >= 0 || os.indexOf("darwin") >= 0) { + return "libgenomicsqlite-macos-" + archTok + ".dylib"; + } else if (os.indexOf("nux") >= 0) { + return "libgenomicsqlite-linux-" + archTok + ".so"; } return null; } diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in index a93b1bf..2d72935 100644 --- a/bindings/python/MANIFEST.in +++ b/bindings/python/MANIFEST.in @@ -1 +1 @@ -include RELEASE-VERSION version.py genomicsqlite/libgenomicsqlite.so genomicsqlite/libgenomicsqlite.dylib +include RELEASE-VERSION version.py genomicsqlite/libgenomicsqlite-* diff --git a/bindings/python/conda/genomicsqlite/meta.yaml b/bindings/python/conda/genomicsqlite/meta.yaml index fbe257f..3ae1048 100644 --- a/bindings/python/conda/genomicsqlite/meta.yaml +++ b/bindings/python/conda/genomicsqlite/meta.yaml @@ -27,7 +27,7 @@ test: imports: - genomicsqlite commands: - - 'ldd -v -r ${CONDA_PREFIX}/lib/python3.*/site-packages/genomicsqlite/libgenomicsqlite.so' # [linux64] + - 'ldd -v -r ${CONDA_PREFIX}/lib/python3.*/site-packages/genomicsqlite/libgenomicsqlite-linux-x86_64.so' # [linux64] - 'wget -nv -O - https://raw.githubusercontent.com/mlin/GenomicSQLite/release/test/genomicsqlite_smoke_test.py | python3 -' # [linux64] - which genomicsqlite diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100755 new mode 100644 index e91da64..369f9c3 --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -15,13 +15,39 @@ # Module initialization -- locate shared library file _DLL = os.environ.get("LIBGENOMICSQLITE", "").strip() # use env var if present -if not _DLL and platform.machine() == "x86_64": - # select platform-appropriate library bundled with this package - _DLL = {"Linux": ".so", "Darwin": ".dylib"}.get(platform.system(), None) - if _DLL: - _DLL = os.path.join(_HERE, "libgenomicsqlite") + _DLL - if not os.path.isfile(_DLL): - _DLL = None +if not _DLL: + # select the platform-appropriate library bundled with this package, which + # the release pipeline ships under an architecture-qualified name (the same + # naming the Rust crate uses). Selection is exact-match on OS + CPU: an + # Intel machine never selects the ARM build, nor vice versa. + _BUNDLED_NAME = { + ("Linux", "x86_64"): "libgenomicsqlite-linux-x86_64.so", + ("Linux", "aarch64"): "libgenomicsqlite-linux-aarch64.so", + ("Darwin", "x86_64"): "libgenomicsqlite-macos-x86_64.dylib", + ("Darwin", "arm64"): "libgenomicsqlite-macos-aarch64.dylib", + }.get((platform.system(), platform.machine())) + if _BUNDLED_NAME: + _bundled_path = os.path.join(_HERE, _BUNDLED_NAME) + if os.path.isfile(_bundled_path): + # SQLite derives the extension entry point + # (sqlite3_genomicsqlite_init) from the name of the file it's asked + # to load, so the bundled library must be staged under the plain + # basename before loading (the Rust crate extracts its embedded + # copy the same way). Stage a hardlink (falling back to a copy) in + # a temp directory, removed at interpreter exit; _DLL must name an + # existing file for the process lifetime because the CLI entry + # point later execs the sqlite3 shell with .load on this path. + import atexit + import shutil + import tempfile + + _STAGE_DIR = tempfile.mkdtemp(prefix="genomicsqlite-") + _DLL = os.path.join(_STAGE_DIR, "libgenomicsqlite" + os.path.splitext(_BUNDLED_NAME)[1]) + try: + os.link(_bundled_path, _DLL) + except OSError: + shutil.copyfile(_bundled_path, _DLL) + atexit.register(shutil.rmtree, _STAGE_DIR, ignore_errors=True) if not _DLL: # otherwise, let SQLite use dlopen() to look for it _DLL = "libgenomicsqlite" @@ -273,18 +299,27 @@ def _cli(argv=None): uri = _execute1(_MEMCONN, "SELECT genomicsqlite_uri(?,?)", (argv[1], cfg)) tuning_sql = _execute1(_MEMCONN, "SELECT genomicsqlite_tuning_sql(?)", (cfg,)) - # Assemble the shell startup commands into a temporary -init file, instead of passing them - # as -cmd arguments: since SQLite 3.52, the shell quits immediately (with a success exit - # code, without reading any input) whenever -bail is combined with an SQL -cmd argument, - # which silently disabled this wrapper. os.execvp() below replaces this process, so the - # file can't be unlinked afterwards; leave it for the OS temp reaper. + # Assemble the shell startup commands, then replay them inside the shell via + # "-cmd .read /dev/fd/N" on an anonymous temporary file whose descriptor is + # inherited across exec: + # * Since SQLite 3.52, the shell quits immediately (with a success exit code, + # without reading any input) whenever -bail is combined with an SQL -cmd + # argument, which silently disabled this wrapper when the startup commands + # were passed as -cmd arguments directly. A dot-command -cmd does not take + # that code path. + # * The anonymous TemporaryFile leaves nothing behind in the temp directory + # (unlike a named -init file, which os.execvp() would strand there). + # * The single -init slot stays free, so a user-supplied -init in argv + # composes with -- instead of displacing -- this startup sequence. tuning_sql = tuning_sql.strip() if not tuning_sql.endswith(";"): - # in -init file context, an unterminated statement would absorb the following line + # in script context, an unterminated statement would absorb the following line tuning_sql += ";" init_lines = [ - f".load {_DLL}", - f".open {uri}", + # quoted: the shell's dot-command tokenizer splits on unquoted whitespace, + # so paths containing spaces would otherwise truncate at the first space + f'.load "{_DLL}"', + f'.open "{uri}"', ".once /dev/null", tuning_sql, '.prompt "GenomicSQLite> "', @@ -297,28 +332,34 @@ def _cli(argv=None): "SELECT 'GenomicSQLite ' || genomicsqlite_version();", ".headers on", ] - with tempfile.NamedTemporaryFile( - "w", prefix="genomicsqlite-init-", suffix=".sql", delete=False - ) as init_file: - init_file.write("\n".join(init_lines) + "\n") - - cmd = ["sqlite3", "-bail", "-init", init_file.name] - cmd.append(":memory:") # placeholder so remaining positional args are recognized as such - cmd += argv[2:] - if "DEBUG" in os.environ: - print( - " ".join( - ( + with tempfile.TemporaryFile("w+") as init_file: + init_file.write("\n".join(init_lines) + "\n") + init_file.flush() + # rewind: on some platforms (notably macOS) reading /dev/fd/N shares this + # descriptor's file offset, so the shell must inherit it positioned at 0 + os.lseek(init_file.fileno(), 0, os.SEEK_SET) + # Python file objects are non-inheritable by default (PEP 446); the + # descriptor must survive the os.execvp() below + os.set_inheritable(init_file.fileno(), True) + + cmd = ["sqlite3", "-bail", "-cmd", f".read /dev/fd/{init_file.fileno()}"] + cmd.append(":memory:") # placeholder so remaining positional args are recognized as such + cmd += argv[2:] + + if "DEBUG" in os.environ: + print( + " ".join( ( - (arg if " " not in arg else f"'{arg}'") - + (" \\\n " if len(arg) > 50 else "") + ( + (arg if " " not in arg else f"'{arg}'") + + (" \\\n " if len(arg) > 50 else "") + ) + for arg in cmd ) - for arg in cmd ) ) - ) - os.execvp("sqlite3", cmd) + os.execvp("sqlite3", cmd) def _compact(dbfilename, argv): diff --git a/bindings/python/release.sh b/bindings/python/release.sh old mode 100755 new mode 100644 index 2a3eff3..1d1cea9 --- a/bindings/python/release.sh +++ b/bindings/python/release.sh @@ -9,13 +9,13 @@ if grep dirty <(git describe --always --dirty); then >&2 echo "Cannot release dirty working tree" exit 1 fi -if [[ ! -f genomicsqlite/libgenomicsqlite.so ]] || [[ ! -f genomicsqlite/libgenomicsqlite.dylib ]]; then - >&2 echo "Download the portable libgenomicsqlite.{so,dylib} to ${HERE}/genomicsqlite/" +if [[ ! -f genomicsqlite/libgenomicsqlite-linux-x86_64.so ]] || [[ ! -f genomicsqlite/libgenomicsqlite-macos-aarch64.dylib ]]; then + >&2 echo "Download the portable libgenomicsqlite-linux-x86_64.so & libgenomicsqlite-macos-aarch64.dylib to ${HERE}/genomicsqlite/" exit 1 fi function cleanup { # force us to put libs on each attempt; ensures a stale version can't be left behind - rm -f genomicsqlite/libgenomicsqlite.* + rm -f genomicsqlite/libgenomicsqlite* } trap cleanup EXIT diff --git a/bindings/python/setup.py b/bindings/python/setup.py old mode 100755 new mode 100644 index 82dfa62..b9fee7b --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -20,7 +20,7 @@ license="Apache", packages=find_packages(), package_data={ - "": ["genomicsqlite/libgenomicsqlite.so", "genomicsqlite/libgenomicsqlite.dylib"] + "": ["genomicsqlite/libgenomicsqlite-*.so", "genomicsqlite/libgenomicsqlite-*.dylib"] }, include_package_data=True, entry_points={"console_scripts": ["genomicsqlite = genomicsqlite:_cli"]}, diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index 5460adf..4a2b752 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -1,74 +1,107 @@ -// Build script for the genomicsqlite crate: inventory the prebuilt -// libgenomicsqlite libraries bundled next to Cargo.toml. +// Build script for the genomicsqlite crate: when the bundle_libgenomicsqlite +// feature is enabled, select the ONE prebuilt libgenomicsqlite matching the +// compilation target (OS + CPU + libc ABI), validate that it's present in the +// package, and hand its file name to src/lib.rs via cargo:rustc-env, where a +// single include_bytes! embeds it. // -// A bundled library is embedded into the compilation unit only for the -// target it was built for, under a per-OS/architecture file name: -// libgenomicsqlite-linux-x86_64.so -// libgenomicsqlite-linux-aarch64.so -// libgenomicsqlite-macos-aarch64.dylib +// Supported bundled targets -- the bundled libraries are the project's +// portable builds, and the Linux ones link glibc, so the target env must be +// gnu (a musl binary must never be handed the glibc library): +// linux + x86_64 + gnu -> libgenomicsqlite-linux-x86_64.so +// linux + aarch64 + gnu -> libgenomicsqlite-linux-aarch64.so +// macos + aarch64 -> libgenomicsqlite-macos-aarch64.dylib // macOS x86_64 is deliberately not a bundled target (macOS CI builds -// ARM-only), so it falls back to the runtime library search like any -// other unsupported target. -// For each candidate file that actually exists in the package, this script -// sets a rustc cfg (bundled_linux_x86_64 etc.) gating the corresponding -// include_bytes! arm in src/lib.rs. A missing file therefore degrades to -// the runtime library search instead of breaking compilation with a -// missing-file error. +// ARM-only), so it falls back to the runtime library search like any other +// unsupported target. +// +// Bundling is a build-time contract: with the feature enabled, a supported +// target whose bundled file is missing FAILS the build -- a release-packaging +// error must not silently degrade into a runtime library search. Likewise, a +// Linux target with a non-glibc ABI (e.g. musl) whose CPU has a bundled +// (glibc) library in the package FAILS with guidance, instead of the library +// being mis-selected or silently ignored. Any other unsupported target, or +// building without the feature (--no-default-features), leaves +// GENOMICSQLITE_BUNDLED_LIB empty, and the crate loads an external +// libgenomicsqlite at runtime (LIBGENOMICSQLITE / system loader search). use std::env; use std::path::Path; -// (target_os, target_arch, file name, cfg name) for each bundled candidate -const CANDIDATES: &[(&str, &str, &str, &str)] = &[ - ( - "linux", - "x86_64", - "libgenomicsqlite-linux-x86_64.so", - "bundled_linux_x86_64", - ), - ( - "linux", - "aarch64", - "libgenomicsqlite-linux-aarch64.so", - "bundled_linux_aarch64", - ), - ( - "macos", - "aarch64", - "libgenomicsqlite-macos-aarch64.dylib", - "bundled_macos_aarch64", - ), -]; - fn main() { - // Declare the custom cfg names unconditionally, so rustc's check-cfg - // validation accepts them wherever src/lib.rs mentions them. - for cand in CANDIDATES { - println!("cargo:rustc-check-cfg=cfg({})", cand.3); - } - println!("cargo:rustc-check-cfg=cfg(bundled_target)"); + // Declare the one custom cfg src/lib.rs uses to gate the bundled-bytes + // path, so rustc's check-cfg validation accepts it. + println!("cargo:rustc-check-cfg=cfg(bundled_lib)"); let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); let bundling = env::var_os("CARGO_FEATURE_BUNDLE_LIBGENOMICSQLITE").is_some(); let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); - for cand in CANDIDATES { - let path = Path::new(&manifest_dir).join(cand.2); - // Rerun this script if a bundled library appears or disappears. - println!("cargo:rerun-if-changed={}", path.display()); - if path.is_file() { - println!("cargo:rustc-cfg={}", cand.3); - if cand.0 == target_os && cand.1 == target_arch { - // The compilation target has a bundled library, so one of - // the include_bytes! arms in src/lib.rs will be compiled. - println!("cargo:rustc-cfg=bundled_target"); + // The single bundled file for this compilation target, if supported. + let bundled_file: Option<&str> = match ( + target_os.as_str(), + target_arch.as_str(), + target_env.as_str(), + ) { + ("linux", "x86_64", "gnu") => Some("libgenomicsqlite-linux-x86_64.so"), + ("linux", "aarch64", "gnu") => Some("libgenomicsqlite-linux-aarch64.so"), + ("macos", "aarch64", _) => Some("libgenomicsqlite-macos-aarch64.dylib"), + _ => None, + }; + + let mut selected = ""; + if bundling { + match bundled_file { + Some(name) => { + let path = Path::new(&manifest_dir).join(name); + // Rerun this script if the bundled library appears or disappears. + println!("cargo:rerun-if-changed={}", path.display()); + if path.is_file() { + selected = name; + println!("cargo:rustc-cfg=bundled_lib"); + } else { + panic!( + "genomicsqlite: the bundle_libgenomicsqlite feature is enabled \ + but the bundled library {} for target {}-{}-{} is missing \ + from the crate. Either package the prebuilt library under \ + that name, or build with --no-default-features to load an \ + external libgenomicsqlite at runtime instead.", + name, target_os, target_arch, target_env + ); + } + } + None => { + // Incompatible-bundle guard: if this is a Linux target with a + // non-glibc ABI and the package carries the glibc library for + // the same CPU, reject the combination outright -- selecting + // it would produce a broken binary, and silently ignoring it + // would mask the ABI mismatch. + if target_os == "linux" && target_env != "gnu" { + let glibc_twin = match target_arch.as_str() { + "x86_64" => Some("libgenomicsqlite-linux-x86_64.so"), + "aarch64" => Some("libgenomicsqlite-linux-aarch64.so"), + _ => None, + }; + if let Some(twin) = glibc_twin { + let path = Path::new(&manifest_dir).join(twin); + println!("cargo:rerun-if-changed={}", path.display()); + if path.is_file() { + panic!( + "genomicsqlite: the bundled {} is built for glibc \ + (linux-gnu), which is incompatible with this \ + linux-{} compilation target. Build with \ + --no-default-features and provide a \ + libgenomicsqlite built for linux-{} instead \ + (LIBGENOMICSQLITE env var / system loader).", + twin, target_env, target_env + ); + } + } + } + // Any other unsupported target: no bundle; the crate loads an + // external libgenomicsqlite at runtime. } - } else if bundling && cand.0 == target_os && cand.1 == target_arch { - println!( - "cargo:warning=genomicsqlite: no bundled {} in the crate for {}-{}; \ - the crate will fall back to searching for a system libgenomicsqlite at runtime", - cand.2, cand.0, cand.1 - ); } } + println!("cargo:rustc-env=GENOMICSQLITE_BUNDLED_LIB={}", selected); } diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100755 new mode 100644 index 9056fd3..0aaa6bf --- a/bindings/rust/cargo +++ b/bindings/rust/cargo @@ -4,14 +4,7 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty 2>/dev/null || true)" -if [ -z "$revision" ]; then - # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the - # latest release version with the short commit SHA as a prerelease suffix, - # keeping the generated crate version valid semver. (Update the release - # version here when cutting a new release.) - revision="0.9.4-$(git rev-parse --short HEAD)" -fi +revision="$(git describe --tags --dirty)" revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 1cdc6c7..1305d76 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -7,9 +7,9 @@ use json::object::Object; use rusqlite::{params, Connection, LoadExtensionGuard, OpenFlags, Result}; use std::collections::HashMap; use std::env; -#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] +#[cfg(feature = "bundle_libgenomicsqlite")] use std::fs::File; -#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] +#[cfg(feature = "bundle_libgenomicsqlite")] use std::io::prelude::*; use std::path::Path; use std::sync::Once; @@ -18,7 +18,7 @@ use tempfile::TempDir; /* Helper functions for bundling libgenomicsqlite.{so,dylib} into the compilation unit */ // extract slice into basename under a temp directory -#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] +#[cfg(feature = "bundle_libgenomicsqlite")] fn extract_libgenomicsqlite_impl( bytes: &[u8], basename: &str, @@ -40,7 +40,7 @@ fn extract_libgenomicsqlite_impl( Ok((libpath, tmp)) } -#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_target))] +#[cfg(feature = "bundle_libgenomicsqlite")] fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { match extract_libgenomicsqlite_impl(bytes, filename) { Ok(ans) => ans, @@ -50,57 +50,31 @@ fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { /* Bundled libgenomicsqlite selection. * - * With the bundle_libgenomicsqlite feature, the crate may embed a prebuilt - * libgenomicsqlite for its target, in a per-OS/architecture file next to - * Cargo.toml: - * libgenomicsqlite-linux-x86_64.so - * libgenomicsqlite-linux-aarch64.so - * libgenomicsqlite-macos-aarch64.dylib - * macOS x86_64 is deliberately not bundled (macOS CI builds ARM-only); - * it takes the fallback path below like any other unsupported target. - * build.rs inventories which of these files the package actually contains - * and sets a matching bundled_* cfg for each one present, so an arm below is - * compiled only when its file exists. On any other target -- or when the - * target's file isn't bundled -- bundle_libgenomicsqlite() returns None and - * open() falls back to the normal library search. + * With the bundle_libgenomicsqlite feature, build.rs selects the one bundled + * library matching the compilation target (OS + CPU + libc ABI), validates + * that it's present in the package, and names it in the + * GENOMICSQLITE_BUNDLED_LIB environment variable (see build.rs for the + * supported-target table and the build-time contract). The single + * include_bytes! below embeds exactly that file -- the bundled_lib cfg is set + * by build.rs only when a file was selected. On any other target + * bundle_libgenomicsqlite() returns None and open() falls back to the normal + * library search. */ -#[cfg(all( - feature = "bundle_libgenomicsqlite", - target_os = "linux", - target_arch = "x86_64", - bundled_linux_x86_64 -))] +#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_lib))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite-linux-x86_64.so"), - "libgenomicsqlite.so", - )) -} - -#[cfg(all( - feature = "bundle_libgenomicsqlite", - target_os = "linux", - target_arch = "aarch64", - bundled_linux_aarch64 -))] -fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { - Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite-linux-aarch64.so"), - "libgenomicsqlite.so", - )) -} - -#[cfg(all( - feature = "bundle_libgenomicsqlite", - target_os = "macos", - target_arch = "aarch64", - bundled_macos_aarch64 -))] -fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { - Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite-macos-aarch64.dylib"), - "libgenomicsqlite.dylib", + include_bytes!(concat!("../", env!("GENOMICSQLITE_BUNDLED_LIB"))), + // SQLite derives the extension entry point (sqlite3_genomicsqlite_init) + // from the name of the file it's asked to load, so the extracted copy + // must carry the plain library basename -- loading a file under its + // arch-qualified package name looks for a mangled entry point + // (sqlite3_genomicsqlitelinuxx_init) and fails. + if env!("GENOMICSQLITE_BUNDLED_LIB").ends_with(".dylib") { + "libgenomicsqlite.dylib" + } else { + "libgenomicsqlite.so" + }, )) } @@ -109,11 +83,10 @@ fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { None } -// Feature on, but no bundled library for this target (another OS, an -// architecture the release didn't bundle, or the target's file absent from -// the package -- build.rs sets bundled_target exactly when one of the -// bundled arms above applies): fall back to the normal library search. -#[cfg(all(feature = "bundle_libgenomicsqlite", not(bundled_target)))] +// Feature on, but no bundled library for this target (an OS/architecture/ABI +// the release doesn't bundle -- build.rs sets bundled_lib exactly when it +// selected a file): fall back to the normal library search. +#[cfg(all(feature = "bundle_libgenomicsqlite", not(bundled_lib)))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { None } diff --git a/build.yml b/build.yml new file mode 100644 index 0000000..bf79af2 --- /dev/null +++ b/build.yml @@ -0,0 +1,271 @@ +name: build +on: [push, pull_request] + +env: + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 + PIP_DEPS: pytest pytest-xdist + +jobs: + + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" + - name: pre-commit + run: pre-commit run --all-files + - name: mvn spotless:check + run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: rust lint + run: | + cd bindings/rust + ./cargo check --no-default-features + ./cargo fmt -- --check + + test: + runs-on: ubuntu-24.04 + strategy: + matrix: + BUILD_TYPE: [Debug, Release] + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build + run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) + - name: test + run: env -C build SQLITE_WEB_LOG=99 ctest -V + + linux-x86-64-so: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: build loaders + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) + rm build/libgenomicsqlite.so + sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so + - name: build portable .so in docker + run: | + # BuildKit builds only the stages the final target depends on, so the + # two minimum-runtime smoke-test stages -- which nothing depends on -- + # must be built explicitly, or they are silently skipped. + docker build --target alma-smoke-test . + docker build --target rust-smoke-test . + docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ + - name: symbols & digests + run: | + readelf -d build/libgenomicsqlite.so + ldd -r build/libgenomicsqlite.so + echo "----" + echo "libgenomicsqlite.so GLIBC version requirements:" + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 + echo "libsqlite3.so.0 GLIBC version requirements:" + objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + echo "----" + sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + # The steps below exercise the portable libraries against the portable + # SQLite they ship with, WITHOUT installing anything into the runner's + # global loader cache or PATH: the portable libsqlite3.so.0 and shell sit + # side by side in build/, and each step scopes them to its own processes + # via LD_LIBRARY_PATH / PATH. (The suite also runs against the host SQLite + # in the separate `test` job.) Planting the portable libsqlite3 into the + # system loader cache instead makes unrelated later tools resolve against + # it -- the runner's own sqlite3 shell, built for a newer SQLite, then + # refuses to start against the older portable library. + - name: test portable .so + run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage + env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so + - name: installed-package smoke test + run: | + sudo pip3 install --break-system-packages -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so build + - name: prepare artifacts + run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite.so + path: libgenomicsqlite.so + - uses: actions/upload-artifact@v7 + with: + name: libsqlite3.so.0 + path: libsqlite3.so.0 + + macOS-arm64-dylib: + runs-on: macos-15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: dependencies + run: | + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + brew update || (sleep 2; brew update) + for dep in $BREW_DEPS ; do + brew install $dep || brew upgrade $dep + done + + # The tests must run under Homebrew's Python 3.12: it is configured with + # --enable-loadable-sqlite-extensions (the binding cannot be imported + # without sqlite3.Connection.enable_load_extension), while the + # image's python.org framework Python is built without it. Select the + # interpreter explicitly: the versioned formula's opt bin provides + # only python3.12 -- the unversioned python3 symlinks live in its + # libexec/bin -- so a PATH entry alone leaves `python3` resolving to + # whatever other Python the image ships. Build a venv from it and put + # the venv on PATH for all subsequent steps. + "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" + echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.mac-venv/bin:$PATH" + python3 -m pip install $PIP_DEPS + python3 -c "import sys, sqlite3; assert sys.version_info[:2] == (3, 12), sys.version; assert hasattr(sqlite3.Connection, 'enable_load_extension')" + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build environment + run: | + echo "PATH=$HOME/.mac-venv/bin:$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + # used by rusqlite: + echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + - name: build + run: | + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ + -B build . + cmake --build build -j 4 + - name: symbols & digests + run: | + otool -L build/libgenomicsqlite.dylib + sha256sum build/libgenomicsqlite.dylib + - name: ctest + run: | + cd build + otool -L test/capi_smoke_test + SQLITE_WEB_LOG=99 ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib + SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib + - name: installed-package smoke test + run: | + python3 -m pip install -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" + - name: prepare artifacts + run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite-macos-aarch64.dylib + path: libgenomicsqlite-macos-aarch64.dylib + + artifacts: + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] + steps: + - uses: actions/checkout@v7 + - name: fetch libgenomicsqlite.so + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite.so + - name: fetch libsqlite3.so.0 + uses: actions/download-artifact@v8 + with: + name: libsqlite3.so.0 + - name: fetch libgenomicsqlite.dylib + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite-macos-aarch64.dylib + - name: detect GIT_REVISION + run: | + cp include/genomicsqlite.h . + echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV + - uses: actions/upload-artifact@v7 + with: + name: GenomicSQLite-${{ env.GIT_REVISION }} + path: | + LICENSE + genomicsqlite.h + libgenomicsqlite.so + libgenomicsqlite-macos-aarch64.dylib + - name: build jar + run: | + cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so + cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib + mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: JDBC smoke test + run: | + set -e + # The JAR must carry both bundled libraries under their + # architecture-qualified resource names, and the driver must open a + # database with LIBGENOMICSQLITE unset (the CTest JDBC tests set the + # override, bypassing the resource selector). + JAR=$(ls bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite-jdbc-*.jar | head -n1) + SQLITE_JDBC_JAR=$(find bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo -name 'sqlite-jdbc-*.jar' | head -n1) + unzip -l "$JAR" | grep libgenomicsqlite-linux-x86_64.so + unzip -l "$JAR" | grep libgenomicsqlite-macos-aarch64.dylib + SMOKE_DIR="$RUNNER_TEMP/jdbc-smoke" + mkdir -p "$SMOKE_DIR" + cat > "$SMOKE_DIR/Smoke.java" <<'EOF' + import java.sql.*; + public class Smoke { + public static void main(String[] args) throws Exception { + Connection conn = DriverManager.getConnection("jdbc:genomicsqlite:" + args[0]); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT genomicsqlite_version()"); + rs.next(); + System.out.println("genomicsqlite_version: " + rs.getString(1)); + st.executeUpdate("CREATE TABLE t(x)"); + st.executeUpdate("INSERT INTO t VALUES('jdbc-ok')"); + conn.close(); + System.out.println("JDBC SMOKE OK"); + } + } + EOF + javac -cp "$JAR" -d "$SMOKE_DIR" "$SMOKE_DIR/Smoke.java" + env -u LIBGENOMICSQLITE java -cp "$JAR:$SQLITE_JDBC_JAR:$SMOKE_DIR" Smoke "$SMOKE_DIR/smoke.db" + - name: digests + run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 + - uses: actions/upload-artifact@v7 + with: + name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} + path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100755 new mode 100644 index 9056fd3..0aaa6bf --- a/test/cargo_smoke_test/cargo +++ b/test/cargo_smoke_test/cargo @@ -4,14 +4,7 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty 2>/dev/null || true)" -if [ -z "$revision" ]; then - # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the - # latest release version with the short commit SHA as a prerelease suffix, - # keeping the generated crate version valid semver. (Update the release - # version here when cutting a new release.) - revision="0.9.4-$(git rev-parse --short HEAD)" -fi +revision="$(git describe --tags --dirty)" revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/test/check_elf_version_floor.py b/test/check_elf_version_floor.py new file mode 100644 index 0000000..04a9e76 --- /dev/null +++ b/test/check_elf_version_floor.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Assert that ELF shared objects' required symbol versions stay at or below the +project's compatibility ceilings -- the manylinux_2_28 floor asserted by CI for +the portable libraries (GLIBC <= 2.28, GLIBCXX <= 3.4.25). + +The check parses the dynamic version needs (the "Version References" of +objdump -p: exactly the (library, version) requirements the dynamic loader +enforces) instead of scraping version strings out of the symbol table. + +Usage: + check_elf_version_floor.py [--glibc X.Y] [--glibcxx X.Y.Z] FILE [FILE...] + +Exit status: 0 if every file is within the ceilings, 1 otherwise (with a +diagnostic naming the file and the offending requirement). +""" + +import argparse +import re +import subprocess +import sys + +NEED_RE = re.compile(r"\b(GLIBC|GLIBCXX)_(\d+(?:\.\d+)*)\s*$") + + +def version_key(text): + return tuple(int(part) for part in text.split(".")) + + +def max_version_needs(path): + """Map each of GLIBC/GLIBCXX to the highest version `path` requires.""" + proc = subprocess.run(["objdump", "-p", path], capture_output=True, text=True, check=True) + needs = {} + in_refs = False + for line in proc.stdout.splitlines(): + if line.startswith("Version References:"): + in_refs = True + continue + if in_refs: + if line and not line[0].isspace(): + break # next top-level section header + match = NEED_RE.search(line) + if match: + lib, ver = match.group(1), match.group(2) + if lib not in needs or version_key(ver) > version_key(needs[lib]): + needs[lib] = ver + return needs + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--glibc", default="2.28", help="GLIBC ceiling [2.28]") + parser.add_argument("--glibcxx", default="3.4.25", help="GLIBCXX ceiling [3.4.25]") + parser.add_argument("files", nargs="+") + args = parser.parse_args() + ceilings = {"GLIBC": args.glibc, "GLIBCXX": args.glibcxx} + + failed = False + for path in args.files: + needs = max_version_needs(path) + summary = ", ".join( + f"max {lib} need {needs.get(lib, 'none')}" for lib in ("GLIBC", "GLIBCXX") + ) + print(f"{path}: {summary}") + for lib, ceiling in ceilings.items(): + if lib in needs and version_key(needs[lib]) > version_key(ceiling): + print( + f" ERROR: {path} requires {lib}_{needs[lib]}, " + f"above the {lib} {ceiling} ceiling", + file=sys.stderr, + ) + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/pkg_smoke.sh b/test/pkg_smoke.sh new file mode 100644 index 0000000..c05dba8 --- /dev/null +++ b/test/pkg_smoke.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Installed-package smoke test: verify the Python package end-to-end the way +# an end user experiences it. Stages a copy of bindings/python, plants the +# built library under its architecture-qualified name, pip-installs into a +# throwaway target dir, then -- from a neutral working directory, with +# LIBGENOMICSQLITE unset -- imports the package, creates a database through +# the API, and reads it back through the CLI wrapper. +# +# This exists because the CTest suite sets LIBGENOMICSQLITE, which bypasses +# the package's bundled-library selector entirely; only an installed-package +# run exercises the selector (and the load-time staging behind it). +# +# $1 python interpreter (pip + setuptools available for it) +# $2 bindings/python source dir +# $3 built library file to plant +# $4 architecture-qualified name to plant it as (e.g. libgenomicsqlite-macos-aarch64.dylib) +# $5 dir containing the sqlite3 shell to use for the CLI check +set -euo pipefail +PY="$1"; PKG_SRC="$2"; LIB="$3"; QNAME="$4"; SHELLDIR="$5" + +WORK="$(mktemp -d)" +SRC="$WORK/pkg-src" +cp -rL "$PKG_SRC" "$SRC" # -L: materialize the sqlite_web_dbi.py symlink +cp "$LIB" "$SRC/genomicsqlite/$QNAME" +# version.py wants git describe (unavailable in the staged copy) or this file +[ -f "$SRC/RELEASE-VERSION" ] || echo "0.0.0" > "$SRC/RELEASE-VERSION" +SITE="$WORK/site" +"$PY" -m pip install --quiet --no-build-isolation --target "$SITE" "$SRC" +echo "installed bundled libraries:"; ls "$SITE/genomicsqlite" | grep libgenomicsqlite + +cd "$WORK" # neutral cwd: the repo tree must not shadow the installed package +DB="$WORK/smoke.db" +env -u LIBGENOMICSQLITE PYTHONPATH="$SITE" "$PY" - "$DB" <<'EOF' +import genomicsqlite, sys +con = genomicsqlite.connect(sys.argv[1]) +print("genomicsqlite_version:", con.execute("SELECT genomicsqlite_version()").fetchone()[0]) +con.execute("CREATE TABLE t(x)") +con.execute("INSERT INTO t VALUES('pkg-ok')") +con.commit(); con.close() +print("PYTHON PACKAGE SMOKE OK") +EOF +OUT="$(env -u LIBGENOMICSQLITE PYTHONPATH="$SITE" PATH="$SHELLDIR:$PATH" \ + "$PY" -m genomicsqlite "$DB" "SELECT 'CLI:' || (SELECT x FROM t)")" +echo "$OUT" +[ "$OUT" = "CLI:pkg-ok" ] || { echo "CLI smoke output mismatch" >&2; exit 1; } +echo "CLI SMOKE OK" From bab5ac4d0bca8a29b886adb7f8f228a28e944b2f Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Sun, 18 Oct 2026 22:06:29 -1000 Subject: [PATCH 04/21] Remove live workflow while the PR #28 review-fix build.yml is staged at the branch root (rename restores it and triggers CI) --- .github/workflows/build.yml | 234 ------------------------------------ 1 file changed, 234 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index b1e7ee9..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,234 +0,0 @@ -name: build -on: [push, pull_request] - -env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip - BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 - PIP_DEPS: pytest pytest-xdist - -jobs: - - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip python3-venv - python3 -m venv "$HOME/.lint-venv" - "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint - echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - - name: pre-commit - run: pre-commit run --all-files - - name: mvn spotless:check - run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - name: rust lint - run: | - cd bindings/rust - bash ./cargo check --no-default-features - bash ./cargo fmt -- --check - - test: - runs-on: ubuntu-24.04 - strategy: - matrix: - BUILD_TYPE: [Debug, Release] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build - run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - - name: test - run: env -C build SQLITE_WEB_LOG=99 ctest -V - - linux-x86-64-so: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: build loaders - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) - rm build/libgenomicsqlite.so - sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - - name: build portable .so in docker - run: docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - - name: symbols & digests - run: | - readelf -d build/libgenomicsqlite.so - ldd -r build/libgenomicsqlite.so - echo "----" - echo "libgenomicsqlite.so GLIBC version requirements:" - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 - echo "libsqlite3.so.0 GLIBC version requirements:" - objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - echo "----" - sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 - echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" - for f in build/libgenomicsqlite.so build/libsqlite3.so.0; do - max_glibc=$(objdump -t $f | grep -oE 'GLIBC_[0-9.]+' | sort -Vu | tail -n1) - echo "$f max GLIBC need: $max_glibc" - if [ "$(printf 'GLIBC_2.28\n%s\n' "$max_glibc" | sort -V | tail -n1)" != "GLIBC_2.28" ]; then - echo "$f requires $max_glibc, above the glibc 2.28 floor" >&2 - exit 1 - fi - done - max_glibcxx=$(objdump -t build/libgenomicsqlite.so | grep -oE 'GLIBCXX_[0-9.]+' | sort -Vu | tail -n1) - echo "build/libgenomicsqlite.so max GLIBCXX need: $max_glibcxx" - if [ "$(printf 'GLIBCXX_3.4.25\n%s\n' "$max_glibcxx" | sort -V | tail -n1)" != "GLIBCXX_3.4.25" ]; then - echo "build/libgenomicsqlite.so requires $max_glibcxx, above the GLIBCXX 3.4.25 ceiling" >&2 - exit 1 - fi - sudo cp build/libsqlite3.so.0 /usr/local/lib - sudo ldconfig - # Pair like with like: the ldconfig plant above makes the portable - # libsqlite3.so.0 (3.41.2) shadow the system one, so the runner's own - # sqlite3 shell (built against >= 3.42) dies at startup with - # "undefined symbol: sqlite3_stmt_explain". Put the portable build's - # own shell -- copied out of the docker image into build/ in the - # previous step -- first on PATH for the remaining steps, so the - # portable test exercises the CLI against the exact SQLite vintage - # the portable library ships with. (Later steps do not look up a - # sqlite3 shell by name, so the prepend is harmless to them.) - echo "$(pwd)/build" >> "$GITHUB_PATH" - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: test portable .so - run: env -C build SQLITE_WEB_LOG=99 ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so - SQLITE_WEB_LOG=99 bash bindings/rust/cargo test --release - bash bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so - - name: prepare artifacts - run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite.so - path: libgenomicsqlite.so - - uses: actions/upload-artifact@v7 - with: - name: libsqlite3.so.0 - path: libsqlite3.so.0 - - macOS-arm64-dylib: - runs-on: macos-15 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: dependencies - run: | - rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 - brew update || (sleep 2; brew update) - for dep in $BREW_DEPS ; do - brew install $dep || brew upgrade $dep - done - - # The tests must run under Homebrew's Python: it is configured with - # --enable-loadable-sqlite-extensions (the binding cannot be imported - # without sqlite3.Connection.enable_load_extension), while the - # image's python.org framework Python is built without it. - echo "$(brew --prefix)/opt/python@3.12/bin" >> "$GITHUB_PATH" - export PATH="$(brew --prefix)/opt/python@3.12/bin:$PATH" - python3 -m pip install --break-system-packages $PIP_DEPS - python3 -c "import sqlite3; assert hasattr(sqlite3.Connection, 'enable_load_extension')" - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build environment - run: | - echo "PATH=$(brew --prefix)/opt/python@3.12/bin:$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - # used by rusqlite: - echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - - name: build - run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ - -B build . - cmake --build build -j 4 - - name: symbols & digests - run: | - otool -L build/libgenomicsqlite.dylib - sha256sum build/libgenomicsqlite.dylib - - name: ctest - run: | - cd build - otool -L test/capi_smoke_test - SQLITE_WEB_LOG=99 ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib - SQLITE_WEB_LOG=99 bash bindings/rust/cargo test --release - bash bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib - - name: prepare artifacts - run: cp build/libgenomicsqlite.dylib include/genomicsqlite.h . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite.dylib - path: libgenomicsqlite.dylib - - artifacts: - runs-on: ubuntu-24.04 - needs: [linux-x86-64-so, macOS-arm64-dylib] - steps: - - uses: actions/checkout@v7 - - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite.so - - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v8 - with: - name: libsqlite3.so.0 - - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite.dylib - - name: detect GIT_REVISION - run: | - cp include/genomicsqlite.h . - echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v7 - with: - name: GenomicSQLite-${{ env.GIT_REVISION }} - path: | - LICENSE - genomicsqlite.h - libgenomicsqlite.so - libgenomicsqlite.dylib - - name: build jar - run: | - cp libgenomicsqlite.so libgenomicsqlite.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/ - mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: digests - run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v7 - with: - name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} - path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo From 447736c15c49213329135691304d6367dfa33adc Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 18 Oct 2026 22:12:18 -1000 Subject: [PATCH 05/21] fix executable bits --- bindings/python/genomicsqlite/__init__.py | 0 bindings/python/setup.py | 0 bindings/rust/cargo | 0 test/cargo_smoke_test/cargo | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bindings/python/genomicsqlite/__init__.py mode change 100644 => 100755 bindings/python/setup.py mode change 100644 => 100755 bindings/rust/cargo mode change 100644 => 100755 test/cargo_smoke_test/cargo diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100644 new mode 100755 diff --git a/bindings/python/setup.py b/bindings/python/setup.py old mode 100644 new mode 100755 diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100644 new mode 100755 diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100644 new mode 100755 From e1ec04aee6952746bd76ea999d92176cbef67a03 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Sun, 18 Oct 2026 22:13:03 -1000 Subject: [PATCH 06/21] Rename build.yml to .github/workflows/build.yml --- build.yml => .github/workflows/build.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build.yml => .github/workflows/build.yml (100%) diff --git a/build.yml b/.github/workflows/build.yml similarity index 100% rename from build.yml rename to .github/workflows/build.yml From 78194cd6c9a4a595c467abf3980beb34fc973c83 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Sun, 18 Oct 2026 23:16:31 -1000 Subject: [PATCH 07/21] Fix tag-less checkout CI failures; address PR #28 re-review - cargo wrappers: restore the tag-less git describe fallback (the fork has no tags for checkout to fetch) - Dockerfile rust stage: safe.directory for the build-context .git - python: load the bundled library from its qualified name with the entry point named explicitly; delete the temp-dir staging (the CLI's exec leaked one staging dir per call) - pkg_smoke: canonicalize the shell dir before cd; assert the CLI leaves no staging dirs - artifacts: extract the JDBC smoke test to test/jdbc_smoke_test.sh; resolve the runtime classpath with dependency:build-classpath --- Dockerfile | 4 + bindings/python/genomicsqlite/__init__.py | 41 ++-- bindings/rust/cargo | 9 +- build.yml | 242 ++++++++++++++++++++++ test/cargo_smoke_test/cargo | 9 +- test/jdbc_smoke_test.sh | 83 ++++++++ test/pkg_smoke.sh | 25 ++- 7 files changed, 388 insertions(+), 25 deletions(-) mode change 100755 => 100644 bindings/python/genomicsqlite/__init__.py mode change 100755 => 100644 bindings/rust/cargo create mode 100644 build.yml mode change 100755 => 100644 test/cargo_smoke_test/cargo create mode 100644 test/jdbc_smoke_test.sh diff --git a/Dockerfile b/Dockerfile index 2d4fb78..d0856b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,6 +120,10 @@ RUN ln -s libsqlite3.so.0 lib/libsqlite3.so ADD ./.git ./.git ADD ./bindings/rust ./rust COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/libgenomicsqlite-linux-x86_64.so +# git in the cargo wrapper must accept the context's .git even if the build +# host checked it out under a different uid than root; the wrapper itself +# tolerates a tag-less .git +RUN git config --global --add safe.directory /work RUN SQLITE_WEB_LOG=99 LD_LIBRARY_PATH=$(pwd)/lib LIBRARY_PATH=$(pwd)/lib rust/cargo test --release ################################################################################################### diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100755 new mode 100644 index 369f9c3..470aefa --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -29,32 +29,26 @@ if _BUNDLED_NAME: _bundled_path = os.path.join(_HERE, _BUNDLED_NAME) if os.path.isfile(_bundled_path): - # SQLite derives the extension entry point - # (sqlite3_genomicsqlite_init) from the name of the file it's asked - # to load, so the bundled library must be staged under the plain - # basename before loading (the Rust crate extracts its embedded - # copy the same way). Stage a hardlink (falling back to a copy) in - # a temp directory, removed at interpreter exit; _DLL must name an - # existing file for the process lifetime because the CLI entry - # point later execs the sqlite3 shell with .load on this path. - import atexit - import shutil - import tempfile - - _STAGE_DIR = tempfile.mkdtemp(prefix="genomicsqlite-") - _DLL = os.path.join(_STAGE_DIR, "libgenomicsqlite" + os.path.splitext(_BUNDLED_NAME)[1]) - try: - os.link(_bundled_path, _DLL) - except OSError: - shutil.copyfile(_bundled_path, _DLL) - atexit.register(shutil.rmtree, _STAGE_DIR, ignore_errors=True) + # Load the bundled library directly from its qualified name. + # SQLite normally derives the extension entry point + # (sqlite3_genomicsqlite_init) from the name of the file it's + # asked to load, which the qualified name would defeat; the loads + # therefore name the entry point explicitly -- the second + # argument of the load_extension() SQL function below, and of the + # shell's .load command in the CLI entry point. (An earlier + # version instead staged a copy under the plain basename in a + # temp directory, but the CLI's os.execvp() never ran the atexit + # cleanup, leaking one staged directory per CLI call.) + _DLL = _bundled_path if not _DLL: # otherwise, let SQLite use dlopen() to look for it _DLL = "libgenomicsqlite" # open a dummy connection to :memory: just for loading the extension. +# SELECT load_extension(?,?) is the parameterized form; unlike +# Connection.load_extension() it takes the entry point as an argument. _MEMCONN = sqlite3.connect(":memory:") _MEMCONN.enable_load_extension(True) -_MEMCONN.load_extension(_DLL) +_MEMCONN.execute("SELECT load_extension(?,?)", (_DLL, "sqlite3_genomicsqlite_init")) # now that it's been loaded, the extension will automatically enable itself on any new connections. @@ -317,8 +311,11 @@ def _cli(argv=None): tuning_sql += ";" init_lines = [ # quoted: the shell's dot-command tokenizer splits on unquoted whitespace, - # so paths containing spaces would otherwise truncate at the first space - f'.load "{_DLL}"', + # so paths containing spaces would otherwise truncate at the first space. + # The entry point is named explicitly because _DLL may be the bundled + # library's architecture-qualified filename, from which the shell could + # not derive it. + f'.load "{_DLL}" sqlite3_genomicsqlite_init', f'.open "{uri}"', ".once /dev/null", tuning_sql, diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100755 new mode 100644 index 0aaa6bf..9056fd3 --- a/bindings/rust/cargo +++ b/bindings/rust/cargo @@ -4,7 +4,14 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty)" +revision="$(git describe --tags --dirty 2>/dev/null || true)" +if [ -z "$revision" ]; then + # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the + # latest release version with the short commit SHA as a prerelease suffix, + # keeping the generated crate version valid semver. (Update the release + # version here when cutting a new release.) + revision="0.9.4-$(git rev-parse --short HEAD)" +fi revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/build.yml b/build.yml new file mode 100644 index 0000000..cb04f33 --- /dev/null +++ b/build.yml @@ -0,0 +1,242 @@ +name: build +on: [push, pull_request] + +env: + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 + PIP_DEPS: pytest pytest-xdist + +jobs: + + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" + - name: pre-commit + run: pre-commit run --all-files + - name: mvn spotless:check + run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: rust lint + run: | + cd bindings/rust + ./cargo check --no-default-features + ./cargo fmt -- --check + + test: + runs-on: ubuntu-24.04 + strategy: + matrix: + BUILD_TYPE: [Debug, Release] + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build + run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) + - name: test + run: env -C build SQLITE_WEB_LOG=99 ctest -V + + linux-x86-64-so: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: build loaders + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) + rm build/libgenomicsqlite.so + sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so + - name: build portable .so in docker + run: | + # BuildKit builds only the stages the final target depends on, so the + # two minimum-runtime smoke-test stages -- which nothing depends on -- + # must be built explicitly, or they are silently skipped. + docker build --target alma-smoke-test . + docker build --target rust-smoke-test . + docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ + - name: symbols & digests + run: | + readelf -d build/libgenomicsqlite.so + ldd -r build/libgenomicsqlite.so + echo "----" + echo "libgenomicsqlite.so GLIBC version requirements:" + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 + echo "libsqlite3.so.0 GLIBC version requirements:" + objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + echo "----" + sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + # The steps below exercise the portable libraries against the portable + # SQLite they ship with, WITHOUT installing anything into the runner's + # global loader cache or PATH: the portable libsqlite3.so.0 and shell sit + # side by side in build/, and each step scopes them to its own processes + # via LD_LIBRARY_PATH / PATH. (The suite also runs against the host SQLite + # in the separate `test` job.) Planting the portable libsqlite3 into the + # system loader cache instead makes unrelated later tools resolve against + # it -- the runner's own sqlite3 shell, built for a newer SQLite, then + # refuses to start against the older portable library. + - name: test portable .so + run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage + env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so + - name: installed-package smoke test + run: | + sudo pip3 install --break-system-packages -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" + - name: prepare artifacts + run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite.so + path: libgenomicsqlite.so + - uses: actions/upload-artifact@v7 + with: + name: libsqlite3.so.0 + path: libsqlite3.so.0 + + macOS-arm64-dylib: + runs-on: macos-15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: dependencies + run: | + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + brew update || (sleep 2; brew update) + for dep in $BREW_DEPS ; do + brew install $dep || brew upgrade $dep + done + + # The tests must run under Homebrew's Python 3.12: it is configured with + # --enable-loadable-sqlite-extensions (the binding cannot be imported + # without sqlite3.Connection.enable_load_extension), while the + # image's python.org framework Python is built without it. Select the + # interpreter explicitly: the versioned formula's opt bin provides + # only python3.12 -- the unversioned python3 symlinks live in its + # libexec/bin -- so a PATH entry alone leaves `python3` resolving to + # whatever other Python the image ships. Build a venv from it and put + # the venv on PATH for all subsequent steps. + "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" + echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.mac-venv/bin:$PATH" + python3 -m pip install $PIP_DEPS + python3 -c "import sys, sqlite3; assert sys.version_info[:2] == (3, 12), sys.version; assert hasattr(sqlite3.Connection, 'enable_load_extension')" + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build environment + run: | + echo "PATH=$HOME/.mac-venv/bin:$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + # used by rusqlite: + echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + - name: build + run: | + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ + -B build . + cmake --build build -j 4 + - name: symbols & digests + run: | + otool -L build/libgenomicsqlite.dylib + sha256sum build/libgenomicsqlite.dylib + - name: ctest + run: | + cd build + otool -L test/capi_smoke_test + SQLITE_WEB_LOG=99 ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib + SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib + - name: installed-package smoke test + run: | + python3 -m pip install -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" + - name: prepare artifacts + run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite-macos-aarch64.dylib + path: libgenomicsqlite-macos-aarch64.dylib + + artifacts: + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] + steps: + - uses: actions/checkout@v7 + - name: fetch libgenomicsqlite.so + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite.so + - name: fetch libsqlite3.so.0 + uses: actions/download-artifact@v8 + with: + name: libsqlite3.so.0 + - name: fetch libgenomicsqlite.dylib + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite-macos-aarch64.dylib + - name: detect GIT_REVISION + run: | + cp include/genomicsqlite.h . + echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV + - uses: actions/upload-artifact@v7 + with: + name: GenomicSQLite-${{ env.GIT_REVISION }} + path: | + LICENSE + genomicsqlite.h + libgenomicsqlite.so + libgenomicsqlite-macos-aarch64.dylib + - name: build jar + run: | + cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so + cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib + mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: JDBC smoke test + run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc + - name: digests + run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 + - uses: actions/upload-artifact@v7 + with: + name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} + path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100755 new mode 100644 index 0aaa6bf..9056fd3 --- a/test/cargo_smoke_test/cargo +++ b/test/cargo_smoke_test/cargo @@ -4,7 +4,14 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty)" +revision="$(git describe --tags --dirty 2>/dev/null || true)" +if [ -z "$revision" ]; then + # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the + # latest release version with the short commit SHA as a prerelease suffix, + # keeping the generated crate version valid semver. (Update the release + # version here when cutting a new release.) + revision="0.9.4-$(git rev-parse --short HEAD)" +fi revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/test/jdbc_smoke_test.sh b/test/jdbc_smoke_test.sh new file mode 100644 index 0000000..696f16a --- /dev/null +++ b/test/jdbc_smoke_test.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# JDBC smoke test: verify the packaged GenomicSQLite JDBC driver end-to-end +# the way an end user experiences it. The driver JAR (built by the preceding +# `mvn deploy` step, with both platform libraries planted in its resources) +# must open a database with LIBGENOMICSQLITE unset -- the CTest JDBC tests +# set that override, which bypasses the driver's bundled-resource selector. +# +# The driver's runtime dependencies (sqlite-jdbc) are NOT inside the JAR and +# NOT in the deploy target repo either -- `mvn deploy` publishes only this +# project's own JAR/POM there. Resolve them here with +# dependency:build-classpath, which emits the complete runtime classpath. +# (An earlier inline version of this step searched target/mvn-repo for the +# sqlite-jdbc JAR, found nothing, and the smoke failed with "No suitable +# driver found".) +# +# $1 JDBC module directory [bindings/jdbc/genomicsqlite-jdbc] +# +# Run from the repository root, after the JAR build (mvn deploy) step. +set -euo pipefail + +MODULE_DIR="${1:-bindings/jdbc/genomicsqlite-jdbc}" +POM="$MODULE_DIR/pom.xml" +[ -f "$POM" ] || { + echo "jdbc_smoke_test: no POM at $POM -- run from the repository root" >&2 + exit 1 +} + +JAR="$(ls "$MODULE_DIR"/target/genomicsqlite-jdbc-*.jar 2>/dev/null | head -n1 || true)" +[ -n "$JAR" ] || { + echo "jdbc_smoke_test: no genomicsqlite-jdbc-*.jar in $MODULE_DIR/target -- did the mvn deploy step run?" >&2 + exit 1 +} + +# The JAR must carry both bundled libraries under their +# architecture-qualified resource names. +for res in libgenomicsqlite-linux-x86_64.so libgenomicsqlite-macos-aarch64.dylib; do + unzip -l "$JAR" | grep -q "$res" || { + echo "jdbc_smoke_test: $JAR lacks bundled resource $res" >&2 + exit 1 + } +done + +WORK="$(mktemp -d)" +CP_FILE="$WORK/classpath.txt" +mvn -B -f "$POM" org.apache.maven.plugins:maven-dependency-plugin:3.11.0:build-classpath \ + -DincludeScope=runtime -Dmdep.outputFile="$CP_FILE" +[ -s "$CP_FILE" ] || { + echo "jdbc_smoke_test: dependency:build-classpath wrote no classpath to $CP_FILE" >&2 + exit 1 +} +DEPS_CP="$(cat "$CP_FILE")" +case "$DEPS_CP" in + *sqlite-jdbc*) ;; + *) + echo "jdbc_smoke_test: runtime classpath lacks sqlite-jdbc: $DEPS_CP" >&2 + exit 1 + ;; +esac + +cat > "$WORK/Smoke.java" <<'EOF' +import java.sql.*; +public class Smoke { + public static void main(String[] args) throws Exception { + Connection conn = DriverManager.getConnection("jdbc:genomicsqlite:" + args[0]); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT genomicsqlite_version()"); + rs.next(); + System.out.println("genomicsqlite_version: " + rs.getString(1)); + st.executeUpdate("CREATE TABLE t(x)"); + st.executeUpdate("INSERT INTO t VALUES('jdbc-ok')"); + conn.close(); + System.out.println("JDBC SMOKE OK"); + } +} +EOF +javac -cp "$JAR" -d "$WORK" "$WORK/Smoke.java" +OUT="$(env -u LIBGENOMICSQLITE java -cp "$JAR:$DEPS_CP:$WORK" Smoke "$WORK/smoke.db")" +echo "$OUT" +grep -q "JDBC SMOKE OK" <<<"$OUT" || { + echo "jdbc_smoke_test: smoke program did not report success" >&2 + exit 1 +} +echo "JDBC SMOKE TEST OK" diff --git a/test/pkg_smoke.sh b/test/pkg_smoke.sh index c05dba8..cc0f999 100644 --- a/test/pkg_smoke.sh +++ b/test/pkg_smoke.sh @@ -8,7 +8,8 @@ # # This exists because the CTest suite sets LIBGENOMICSQLITE, which bypasses # the package's bundled-library selector entirely; only an installed-package -# run exercises the selector (and the load-time staging behind it). +# run exercises the selector, including loading the library from its +# architecture-qualified filename via the explicit entry point. # # $1 python interpreter (pip + setuptools available for it) # $2 bindings/python source dir @@ -17,6 +18,11 @@ # $5 dir containing the sqlite3 shell to use for the CLI check set -euo pipefail PY="$1"; PKG_SRC="$2"; LIB="$3"; QNAME="$4"; SHELLDIR="$5" +# Canonicalize $5 to an absolute path BEFORE the cd into the scratch dir +# below: a relative $5 (the Linux call site passes "build") would otherwise +# re-resolve under the scratch dir, silently putting the wrong sqlite3 +# shell on PATH for the CLI check. +SHELLDIR="$(cd "$SHELLDIR" && pwd)" WORK="$(mktemp -d)" SRC="$WORK/pkg-src" @@ -39,8 +45,25 @@ con.execute("INSERT INTO t VALUES('pkg-ok')") con.commit(); con.close() print("PYTHON PACKAGE SMOKE OK") EOF +# Regression check: the installed CLI must not leave genomicsqlite-* staging +# directories behind in the temp dir. (The bundled-library staging that an +# earlier selector design used leaked exactly one per CLI call, because the +# CLI's os.execvp() never ran the atexit cleanup.) +count_staging_dirs() { + local n=0 d + for d in "${TMPDIR:-/tmp}"/genomicsqlite-*; do + [ -d "$d" ] && n=$((n + 1)) + done + echo "$n" +} +STAGING_BEFORE="$(count_staging_dirs)" OUT="$(env -u LIBGENOMICSQLITE PYTHONPATH="$SITE" PATH="$SHELLDIR:$PATH" \ "$PY" -m genomicsqlite "$DB" "SELECT 'CLI:' || (SELECT x FROM t)")" echo "$OUT" [ "$OUT" = "CLI:pkg-ok" ] || { echo "CLI smoke output mismatch" >&2; exit 1; } +STAGING_AFTER="$(count_staging_dirs)" +[ "$STAGING_AFTER" = "$STAGING_BEFORE" ] || { + echo "CLI leaked $((STAGING_AFTER - STAGING_BEFORE)) genomicsqlite-* staging dir(s) in ${TMPDIR:-/tmp}" >&2 + exit 1 +} echo "CLI SMOKE OK" From 97df40ffbd2722b6f1d07597c73f2dd3165f3d76 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Sun, 18 Oct 2026 23:16:42 -1000 Subject: [PATCH 08/21] Stage updated workflow at branch root for rename (connector cannot push workflow files directly) --- .github/workflows/build.yml | 271 ------------------------------------ 1 file changed, 271 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index bf79af2..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,271 +0,0 @@ -name: build -on: [push, pull_request] - -env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip - BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 - PIP_DEPS: pytest pytest-xdist - -jobs: - - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip python3-venv - python3 -m venv "$HOME/.lint-venv" - "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint - echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - - name: pre-commit - run: pre-commit run --all-files - - name: mvn spotless:check - run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - name: rust lint - run: | - cd bindings/rust - ./cargo check --no-default-features - ./cargo fmt -- --check - - test: - runs-on: ubuntu-24.04 - strategy: - matrix: - BUILD_TYPE: [Debug, Release] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build - run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - - name: test - run: env -C build SQLITE_WEB_LOG=99 ctest -V - - linux-x86-64-so: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: build loaders - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) - rm build/libgenomicsqlite.so - sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - - name: build portable .so in docker - run: | - # BuildKit builds only the stages the final target depends on, so the - # two minimum-runtime smoke-test stages -- which nothing depends on -- - # must be built explicitly, or they are silently skipped. - docker build --target alma-smoke-test . - docker build --target rust-smoke-test . - docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - - name: symbols & digests - run: | - readelf -d build/libgenomicsqlite.so - ldd -r build/libgenomicsqlite.so - echo "----" - echo "libgenomicsqlite.so GLIBC version requirements:" - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 - echo "libsqlite3.so.0 GLIBC version requirements:" - objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - echo "----" - sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 - echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" - python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - # The steps below exercise the portable libraries against the portable - # SQLite they ship with, WITHOUT installing anything into the runner's - # global loader cache or PATH: the portable libsqlite3.so.0 and shell sit - # side by side in build/, and each step scopes them to its own processes - # via LD_LIBRARY_PATH / PATH. (The suite also runs against the host SQLite - # in the separate `test` job.) Planting the portable libsqlite3 into the - # system loader cache instead makes unrelated later tools resolve against - # it -- the runner's own sqlite3 shell, built for a newer SQLite, then - # refuses to start against the older portable library. - - name: test portable .so - run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so - ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage - env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so - - name: installed-package smoke test - run: | - sudo pip3 install --break-system-packages -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so build - - name: prepare artifacts - run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite.so - path: libgenomicsqlite.so - - uses: actions/upload-artifact@v7 - with: - name: libsqlite3.so.0 - path: libsqlite3.so.0 - - macOS-arm64-dylib: - runs-on: macos-15 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: dependencies - run: | - rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 - brew update || (sleep 2; brew update) - for dep in $BREW_DEPS ; do - brew install $dep || brew upgrade $dep - done - - # The tests must run under Homebrew's Python 3.12: it is configured with - # --enable-loadable-sqlite-extensions (the binding cannot be imported - # without sqlite3.Connection.enable_load_extension), while the - # image's python.org framework Python is built without it. Select the - # interpreter explicitly: the versioned formula's opt bin provides - # only python3.12 -- the unversioned python3 symlinks live in its - # libexec/bin -- so a PATH entry alone leaves `python3` resolving to - # whatever other Python the image ships. Build a venv from it and put - # the venv on PATH for all subsequent steps. - "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" - echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.mac-venv/bin:$PATH" - python3 -m pip install $PIP_DEPS - python3 -c "import sys, sqlite3; assert sys.version_info[:2] == (3, 12), sys.version; assert hasattr(sqlite3.Connection, 'enable_load_extension')" - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build environment - run: | - echo "PATH=$HOME/.mac-venv/bin:$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - # used by rusqlite: - echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - - name: build - run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ - -B build . - cmake --build build -j 4 - - name: symbols & digests - run: | - otool -L build/libgenomicsqlite.dylib - sha256sum build/libgenomicsqlite.dylib - - name: ctest - run: | - cd build - otool -L test/capi_smoke_test - SQLITE_WEB_LOG=99 ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib - - name: installed-package smoke test - run: | - python3 -m pip install -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" - - name: prepare artifacts - run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite-macos-aarch64.dylib - path: libgenomicsqlite-macos-aarch64.dylib - - artifacts: - runs-on: ubuntu-24.04 - needs: [linux-x86-64-so, macOS-arm64-dylib] - steps: - - uses: actions/checkout@v7 - - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite.so - - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v8 - with: - name: libsqlite3.so.0 - - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite-macos-aarch64.dylib - - name: detect GIT_REVISION - run: | - cp include/genomicsqlite.h . - echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v7 - with: - name: GenomicSQLite-${{ env.GIT_REVISION }} - path: | - LICENSE - genomicsqlite.h - libgenomicsqlite.so - libgenomicsqlite-macos-aarch64.dylib - - name: build jar - run: | - cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so - cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib - mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: JDBC smoke test - run: | - set -e - # The JAR must carry both bundled libraries under their - # architecture-qualified resource names, and the driver must open a - # database with LIBGENOMICSQLITE unset (the CTest JDBC tests set the - # override, bypassing the resource selector). - JAR=$(ls bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite-jdbc-*.jar | head -n1) - SQLITE_JDBC_JAR=$(find bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo -name 'sqlite-jdbc-*.jar' | head -n1) - unzip -l "$JAR" | grep libgenomicsqlite-linux-x86_64.so - unzip -l "$JAR" | grep libgenomicsqlite-macos-aarch64.dylib - SMOKE_DIR="$RUNNER_TEMP/jdbc-smoke" - mkdir -p "$SMOKE_DIR" - cat > "$SMOKE_DIR/Smoke.java" <<'EOF' - import java.sql.*; - public class Smoke { - public static void main(String[] args) throws Exception { - Connection conn = DriverManager.getConnection("jdbc:genomicsqlite:" + args[0]); - Statement st = conn.createStatement(); - ResultSet rs = st.executeQuery("SELECT genomicsqlite_version()"); - rs.next(); - System.out.println("genomicsqlite_version: " + rs.getString(1)); - st.executeUpdate("CREATE TABLE t(x)"); - st.executeUpdate("INSERT INTO t VALUES('jdbc-ok')"); - conn.close(); - System.out.println("JDBC SMOKE OK"); - } - } - EOF - javac -cp "$JAR" -d "$SMOKE_DIR" "$SMOKE_DIR/Smoke.java" - env -u LIBGENOMICSQLITE java -cp "$JAR:$SQLITE_JDBC_JAR:$SMOKE_DIR" Smoke "$SMOKE_DIR/smoke.db" - - name: digests - run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v7 - with: - name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} - path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo From 0b3916c64a571c41663d19c4d916c1df16bea099 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Mon, 19 Oct 2026 07:24:54 -1000 Subject: [PATCH 09/21] fix executable bits --- bindings/python/genomicsqlite/__init__.py | 0 bindings/python/release.sh | 0 bindings/rust/cargo | 0 test/cargo_smoke_test/cargo | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bindings/python/genomicsqlite/__init__.py mode change 100644 => 100755 bindings/python/release.sh mode change 100644 => 100755 bindings/rust/cargo mode change 100644 => 100755 test/cargo_smoke_test/cargo diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100644 new mode 100755 diff --git a/bindings/python/release.sh b/bindings/python/release.sh old mode 100644 new mode 100755 diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100644 new mode 100755 diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100644 new mode 100755 From 53f101f1accdb586416b8d9fe2ff4e397e0550a5 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 17:27:52 +0000 Subject: [PATCH 10/21] Rename build.yml to .github/workflows/build.yml --- build.yml => .github/workflows/build.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build.yml => .github/workflows/build.yml (100%) diff --git a/build.yml b/.github/workflows/build.yml similarity index 100% rename from build.yml rename to .github/workflows/build.yml From 6fab91f209a4a2d41eb93acb9da7b64fbd32c14f Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 07:54:09 -1000 Subject: [PATCH 11/21] Trigger full CI re-run (macOS runner capacity failure in run 39294337754) --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 269465d..abf7065 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,3 +39,4 @@ repos: verbose: true entry: env PYTHONPATH=bindings/python pylint args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,too-many-positional-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] +# CI re-run trigger 2026-10-19 (run 39294337754 lost its macOS runner to GitHub capacity) From a35eb76fc02a12309ef61c81e3a6fb9196855494 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 16:41:58 -1000 Subject: [PATCH 12/21] Adopt mlin:codex/sqlite-loading items 1-2 (PR #28 follow-up) - src/genomicsqlite.cc: add the standard sqlite3_extension_init entry point forwarding to sqlite3_genomicsqlite_init, so loading no longer depends on the library filename; the named entry point stays for callers that already specify it - python: plain load_extension(_DLL) and .load with no explicit initializer (architecture-qualified _DLL selection unchanged) - test/extension_loading.py: new CTest-registered loading matrix (4 filenames x default/legacy init x Python/shell) - test/pkg_smoke.sh: drop the obsolete staging-dir scan; EXIT trap cleans the script's own scratch dir - macOS job: tested Homebrew python@3.12 venv setup (shorter comment; enable_load_extension(True) check with version logging; GITHUB_PATH adds the SQLite shell instead of reconstructing PATH) - docs: note the interpreter loadable-extensions requirement (build.yml is staged at the branch root in this commit; it moves to .github/workflows/ by rename, which also triggers CI.) --- CMakeLists.txt | 1 + bindings/python/genomicsqlite/__init__.py | 19 +- build.yml | 241 ++++++++++++++++++++++ docs/index.md | 2 + src/genomicsqlite.cc | 7 + test/extension_loading.py | 54 +++++ test/pkg_smoke.sh | 20 +- 7 files changed, 309 insertions(+), 35 deletions(-) mode change 100755 => 100644 bindings/python/genomicsqlite/__init__.py create mode 100644 build.yml create mode 100644 test/extension_loading.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 6878daf..d095a63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) enable_testing() set(GENOMICSQLITE_TEST_ENV env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/bindings/python:${PYTHONPATH} LIBGENOMICSQLITE=${CMAKE_BINARY_DIR}/libgenomicsqlite SQLITE_WEB_LOG=99) add_test(NAME smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} python3 ${CMAKE_CURRENT_SOURCE_DIR}/test/genomicsqlite_smoke_test.py) + add_test(NAME extension_loading COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/test/extension_loading.py $) add_test(NAME capi_smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_BINARY_DIR}/test/capi_smoke_test) add_test(NAME JdbcTests COMMAND ${GENOMICSQLITE_TEST_ENV} mvn test -B -f ${CMAKE_CURRENT_SOURCE_DIR}/bindings/jdbc/genomicsqlite-jdbc/pom.xml) add_test(NAME cargo_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rust/cargo test --no-default-features) diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100755 new mode 100644 index 470aefa..c1ec17e --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -29,26 +29,14 @@ if _BUNDLED_NAME: _bundled_path = os.path.join(_HERE, _BUNDLED_NAME) if os.path.isfile(_bundled_path): - # Load the bundled library directly from its qualified name. - # SQLite normally derives the extension entry point - # (sqlite3_genomicsqlite_init) from the name of the file it's - # asked to load, which the qualified name would defeat; the loads - # therefore name the entry point explicitly -- the second - # argument of the load_extension() SQL function below, and of the - # shell's .load command in the CLI entry point. (An earlier - # version instead staged a copy under the plain basename in a - # temp directory, but the CLI's os.execvp() never ran the atexit - # cleanup, leaking one staged directory per CLI call.) _DLL = _bundled_path if not _DLL: # otherwise, let SQLite use dlopen() to look for it _DLL = "libgenomicsqlite" # open a dummy connection to :memory: just for loading the extension. -# SELECT load_extension(?,?) is the parameterized form; unlike -# Connection.load_extension() it takes the entry point as an argument. _MEMCONN = sqlite3.connect(":memory:") _MEMCONN.enable_load_extension(True) -_MEMCONN.execute("SELECT load_extension(?,?)", (_DLL, "sqlite3_genomicsqlite_init")) +_MEMCONN.load_extension(_DLL) # now that it's been loaded, the extension will automatically enable itself on any new connections. @@ -312,10 +300,7 @@ def _cli(argv=None): init_lines = [ # quoted: the shell's dot-command tokenizer splits on unquoted whitespace, # so paths containing spaces would otherwise truncate at the first space. - # The entry point is named explicitly because _DLL may be the bundled - # library's architecture-qualified filename, from which the shell could - # not derive it. - f'.load "{_DLL}" sqlite3_genomicsqlite_init', + f'.load "{_DLL}"', f'.open "{uri}"', ".once /dev/null", tuning_sql, diff --git a/build.yml b/build.yml new file mode 100644 index 0000000..42bff54 --- /dev/null +++ b/build.yml @@ -0,0 +1,241 @@ +name: build +on: [push, pull_request] + +env: + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 + PIP_DEPS: pytest pytest-xdist + +jobs: + + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" + - name: pre-commit + run: pre-commit run --all-files + - name: mvn spotless:check + run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: rust lint + run: | + cd bindings/rust + ./cargo check --no-default-features + ./cargo fmt -- --check + + test: + runs-on: ubuntu-24.04 + strategy: + matrix: + BUILD_TYPE: [Debug, Release] + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build + run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) + - name: test + run: env -C build SQLITE_WEB_LOG=99 ctest -V + + linux-x86-64-so: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: build loaders + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) + rm build/libgenomicsqlite.so + sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so + - name: build portable .so in docker + run: | + # BuildKit builds only the stages the final target depends on, so the + # two minimum-runtime smoke-test stages -- which nothing depends on -- + # must be built explicitly, or they are silently skipped. + docker build --target alma-smoke-test . + docker build --target rust-smoke-test . + docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ + - name: symbols & digests + run: | + readelf -d build/libgenomicsqlite.so + ldd -r build/libgenomicsqlite.so + echo "----" + echo "libgenomicsqlite.so GLIBC version requirements:" + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 + echo "libsqlite3.so.0 GLIBC version requirements:" + objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + echo "----" + sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + # The steps below exercise the portable libraries against the portable + # SQLite they ship with, WITHOUT installing anything into the runner's + # global loader cache or PATH: the portable libsqlite3.so.0 and shell sit + # side by side in build/, and each step scopes them to its own processes + # via LD_LIBRARY_PATH / PATH. (The suite also runs against the host SQLite + # in the separate `test` job.) Planting the portable libsqlite3 into the + # system loader cache instead makes unrelated later tools resolve against + # it -- the runner's own sqlite3 shell, built for a newer SQLite, then + # refuses to start against the older portable library. + - name: test portable .so + run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage + env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so + - name: installed-package smoke test + run: | + sudo pip3 install --break-system-packages -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" + - name: prepare artifacts + run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite.so + path: libgenomicsqlite.so + - uses: actions/upload-artifact@v7 + with: + name: libsqlite3.so.0 + path: libsqlite3.so.0 + + macOS-arm64-dylib: + runs-on: macos-15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: dependencies + run: | + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + brew update || (sleep 2; brew update) + for dep in $BREW_DEPS ; do + brew install $dep || brew upgrade $dep + done + + # setup-python's macOS ARM 3.12.10 build lacked SQLite extension loading. + # Homebrew enables it; 3.12 is our CI baseline, matching Ubuntu 24.04. + # A venv selects that interpreter and keeps pip installs isolated. + "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" + echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.mac-venv/bin:$PATH" + python3 -m pip install $PIP_DEPS + python3 - <<'PYTHON' + import sqlite3, sys + print(sys.executable, sys.version) + print("SQLite", sqlite3.sqlite_version) + sqlite3.connect(":memory:").enable_load_extension(True) + PYTHON + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build environment + run: | + echo "$(brew --prefix)/opt/sqlite/bin" >> "$GITHUB_PATH" + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + # used by rusqlite: + echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + - name: build + run: | + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ + -B build . + cmake --build build -j 4 + - name: symbols & digests + run: | + otool -L build/libgenomicsqlite.dylib + sha256sum build/libgenomicsqlite.dylib + - name: ctest + run: | + cd build + otool -L test/capi_smoke_test + SQLITE_WEB_LOG=99 ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib + SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib + - name: installed-package smoke test + run: | + python3 -m pip install -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" + - name: prepare artifacts + run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite-macos-aarch64.dylib + path: libgenomicsqlite-macos-aarch64.dylib + + artifacts: + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] + steps: + - uses: actions/checkout@v7 + - name: fetch libgenomicsqlite.so + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite.so + - name: fetch libsqlite3.so.0 + uses: actions/download-artifact@v8 + with: + name: libsqlite3.so.0 + - name: fetch libgenomicsqlite.dylib + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite-macos-aarch64.dylib + - name: detect GIT_REVISION + run: | + cp include/genomicsqlite.h . + echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV + - uses: actions/upload-artifact@v7 + with: + name: GenomicSQLite-${{ env.GIT_REVISION }} + path: | + LICENSE + genomicsqlite.h + libgenomicsqlite.so + libgenomicsqlite-macos-aarch64.dylib + - name: build jar + run: | + cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so + cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib + mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: JDBC smoke test + run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc + - name: digests + run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 + - uses: actions/upload-artifact@v7 + with: + name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} + path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo diff --git a/docs/index.md b/docs/index.md index ee95659..5e2d785 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,6 +45,8 @@ It's usually easiest to obtain the extension as a pre-compiled shared library (L # -or- conda install -c mlin genomicsqlite ``` + Python's `sqlite3` module must support [loadable extensions](https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension). Some macOS Python builds omit this support; Homebrew Python enables it. This is an interpreter build option, independent of the Python version. + The package loads a bundled shared library by default. To override the bundled file, set environment variable `LIBGENOMICSQLITE` to a filename. === "JVM" diff --git a/src/genomicsqlite.cc b/src/genomicsqlite.cc index d5a8c07..63ea5ee 100644 --- a/src/genomicsqlite.cc +++ b/src/genomicsqlite.cc @@ -1863,3 +1863,10 @@ extern "C" int sqlite3_genomicsqlite_init(sqlite3 *db, char **pzErrMsg, } return SQLITE_OK_LOAD_PERMANENTLY; } + +// SQLite checks this standard entry point before deriving one from the filename. +// Keep the named entry point above for callers that already use it explicitly. +extern "C" int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { + return sqlite3_genomicsqlite_init(db, pzErrMsg, pApi); +} diff --git a/test/extension_loading.py b/test/extension_loading.py new file mode 100644 index 0000000..9cd864e --- /dev/null +++ b/test/extension_loading.py @@ -0,0 +1,54 @@ +"""Check that native extension loading is independent of the library filename.""" + +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +from pathlib import Path + + +def main(): + library = Path(sys.argv[1]).resolve() + # Each load runs in a fresh process: GenomicSQLite registers itself for all + # subsequent connections and remains loaded until process exit. + if len(sys.argv) == 3: + con = sqlite3.connect(":memory:") + con.enable_load_extension(True) + if sys.argv[2] == "default": + con.load_extension(str(library)) + else: + con.execute("SELECT load_extension(?,?)", (str(library), sys.argv[2])) + con.close() + con = sqlite3.connect(":memory:") + assert con.execute("SELECT genomicsqlite_version()").fetchone()[0] + return + + with tempfile.TemporaryDirectory() as scratch: + for basename in ( + "libgenomicsqlite", + "libgenomicsqlite-linux-x86_64", + "libgenomicsqlite-macos-aarch64", + "arbitrary name", + ): + renamed = Path(scratch) / (basename + library.suffix) + shutil.copyfile(library, renamed) + for entrypoint in ("default", "sqlite3_genomicsqlite_init"): + subprocess.run([sys.executable, __file__, str(renamed), entrypoint], check=True) + load = f'.load "{renamed}"' + if entrypoint != "default": + load += " " + entrypoint + result = subprocess.run( + ["sqlite3", ":memory:"], + input=load + "\nSELECT genomicsqlite_version();\n", + text=True, + capture_output=True, + check=True, + ) + assert result.stdout.strip(), result.stderr + assert not result.stderr, result.stderr + print(basename, entrypoint, "Python and shell OK") + + +if __name__ == "__main__": + main() diff --git a/test/pkg_smoke.sh b/test/pkg_smoke.sh index cc0f999..30933b1 100644 --- a/test/pkg_smoke.sh +++ b/test/pkg_smoke.sh @@ -9,7 +9,7 @@ # This exists because the CTest suite sets LIBGENOMICSQLITE, which bypasses # the package's bundled-library selector entirely; only an installed-package # run exercises the selector, including loading the library from its -# architecture-qualified filename via the explicit entry point. +# architecture-qualified filename. # # $1 python interpreter (pip + setuptools available for it) # $2 bindings/python source dir @@ -25,6 +25,7 @@ PY="$1"; PKG_SRC="$2"; LIB="$3"; QNAME="$4"; SHELLDIR="$5" SHELLDIR="$(cd "$SHELLDIR" && pwd)" WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT SRC="$WORK/pkg-src" cp -rL "$PKG_SRC" "$SRC" # -L: materialize the sqlite_web_dbi.py symlink cp "$LIB" "$SRC/genomicsqlite/$QNAME" @@ -45,25 +46,8 @@ con.execute("INSERT INTO t VALUES('pkg-ok')") con.commit(); con.close() print("PYTHON PACKAGE SMOKE OK") EOF -# Regression check: the installed CLI must not leave genomicsqlite-* staging -# directories behind in the temp dir. (The bundled-library staging that an -# earlier selector design used leaked exactly one per CLI call, because the -# CLI's os.execvp() never ran the atexit cleanup.) -count_staging_dirs() { - local n=0 d - for d in "${TMPDIR:-/tmp}"/genomicsqlite-*; do - [ -d "$d" ] && n=$((n + 1)) - done - echo "$n" -} -STAGING_BEFORE="$(count_staging_dirs)" OUT="$(env -u LIBGENOMICSQLITE PYTHONPATH="$SITE" PATH="$SHELLDIR:$PATH" \ "$PY" -m genomicsqlite "$DB" "SELECT 'CLI:' || (SELECT x FROM t)")" echo "$OUT" [ "$OUT" = "CLI:pkg-ok" ] || { echo "CLI smoke output mismatch" >&2; exit 1; } -STAGING_AFTER="$(count_staging_dirs)" -[ "$STAGING_AFTER" = "$STAGING_BEFORE" ] || { - echo "CLI leaked $((STAGING_AFTER - STAGING_BEFORE)) genomicsqlite-* staging dir(s) in ${TMPDIR:-/tmp}" >&2 - exit 1 -} echo "CLI SMOKE OK" From aa7fa1c12c727bb478ab39535714658e89faf971 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 16:42:10 -1000 Subject: [PATCH 13/21] Remove live workflow while the adopted build.yml is staged at the branch root (rename restores it and triggers CI) --- .github/workflows/build.yml | 242 ------------------------------------ 1 file changed, 242 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index cb04f33..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,242 +0,0 @@ -name: build -on: [push, pull_request] - -env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip - BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 - PIP_DEPS: pytest pytest-xdist - -jobs: - - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip python3-venv - python3 -m venv "$HOME/.lint-venv" - "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint - echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - - name: pre-commit - run: pre-commit run --all-files - - name: mvn spotless:check - run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - name: rust lint - run: | - cd bindings/rust - ./cargo check --no-default-features - ./cargo fmt -- --check - - test: - runs-on: ubuntu-24.04 - strategy: - matrix: - BUILD_TYPE: [Debug, Release] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build - run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - - name: test - run: env -C build SQLITE_WEB_LOG=99 ctest -V - - linux-x86-64-so: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: build loaders - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) - rm build/libgenomicsqlite.so - sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - - name: build portable .so in docker - run: | - # BuildKit builds only the stages the final target depends on, so the - # two minimum-runtime smoke-test stages -- which nothing depends on -- - # must be built explicitly, or they are silently skipped. - docker build --target alma-smoke-test . - docker build --target rust-smoke-test . - docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - - name: symbols & digests - run: | - readelf -d build/libgenomicsqlite.so - ldd -r build/libgenomicsqlite.so - echo "----" - echo "libgenomicsqlite.so GLIBC version requirements:" - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 - echo "libsqlite3.so.0 GLIBC version requirements:" - objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - echo "----" - sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 - echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" - python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - # The steps below exercise the portable libraries against the portable - # SQLite they ship with, WITHOUT installing anything into the runner's - # global loader cache or PATH: the portable libsqlite3.so.0 and shell sit - # side by side in build/, and each step scopes them to its own processes - # via LD_LIBRARY_PATH / PATH. (The suite also runs against the host SQLite - # in the separate `test` job.) Planting the portable libsqlite3 into the - # system loader cache instead makes unrelated later tools resolve against - # it -- the runner's own sqlite3 shell, built for a newer SQLite, then - # refuses to start against the older portable library. - - name: test portable .so - run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so - ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage - env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so - - name: installed-package smoke test - run: | - sudo pip3 install --break-system-packages -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" - - name: prepare artifacts - run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite.so - path: libgenomicsqlite.so - - uses: actions/upload-artifact@v7 - with: - name: libsqlite3.so.0 - path: libsqlite3.so.0 - - macOS-arm64-dylib: - runs-on: macos-15 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: dependencies - run: | - rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 - brew update || (sleep 2; brew update) - for dep in $BREW_DEPS ; do - brew install $dep || brew upgrade $dep - done - - # The tests must run under Homebrew's Python 3.12: it is configured with - # --enable-loadable-sqlite-extensions (the binding cannot be imported - # without sqlite3.Connection.enable_load_extension), while the - # image's python.org framework Python is built without it. Select the - # interpreter explicitly: the versioned formula's opt bin provides - # only python3.12 -- the unversioned python3 symlinks live in its - # libexec/bin -- so a PATH entry alone leaves `python3` resolving to - # whatever other Python the image ships. Build a venv from it and put - # the venv on PATH for all subsequent steps. - "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" - echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.mac-venv/bin:$PATH" - python3 -m pip install $PIP_DEPS - python3 -c "import sys, sqlite3; assert sys.version_info[:2] == (3, 12), sys.version; assert hasattr(sqlite3.Connection, 'enable_load_extension')" - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build environment - run: | - echo "PATH=$HOME/.mac-venv/bin:$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - # used by rusqlite: - echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - - name: build - run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ - -B build . - cmake --build build -j 4 - - name: symbols & digests - run: | - otool -L build/libgenomicsqlite.dylib - sha256sum build/libgenomicsqlite.dylib - - name: ctest - run: | - cd build - otool -L test/capi_smoke_test - SQLITE_WEB_LOG=99 ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib - - name: installed-package smoke test - run: | - python3 -m pip install -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" - - name: prepare artifacts - run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite-macos-aarch64.dylib - path: libgenomicsqlite-macos-aarch64.dylib - - artifacts: - runs-on: ubuntu-24.04 - needs: [linux-x86-64-so, macOS-arm64-dylib] - steps: - - uses: actions/checkout@v7 - - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite.so - - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v8 - with: - name: libsqlite3.so.0 - - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite-macos-aarch64.dylib - - name: detect GIT_REVISION - run: | - cp include/genomicsqlite.h . - echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v7 - with: - name: GenomicSQLite-${{ env.GIT_REVISION }} - path: | - LICENSE - genomicsqlite.h - libgenomicsqlite.so - libgenomicsqlite-macos-aarch64.dylib - - name: build jar - run: | - cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so - cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib - mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: JDBC smoke test - run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc - - name: digests - run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v7 - with: - name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} - path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo From 328e632b36bb1551b991ecf8d42606c3adfaae02 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 16:48:34 -1000 Subject: [PATCH 14/21] Restore fail-fast git describe in the cargo wrappers (fork now has the tags) --- bindings/rust/cargo | 9 +-------- test/cargo_smoke_test/cargo | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) mode change 100755 => 100644 bindings/rust/cargo mode change 100755 => 100644 test/cargo_smoke_test/cargo diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100755 new mode 100644 index 9056fd3..0aaa6bf --- a/bindings/rust/cargo +++ b/bindings/rust/cargo @@ -4,14 +4,7 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty 2>/dev/null || true)" -if [ -z "$revision" ]; then - # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the - # latest release version with the short commit SHA as a prerelease suffix, - # keeping the generated crate version valid semver. (Update the release - # version here when cutting a new release.) - revision="0.9.4-$(git rev-parse --short HEAD)" -fi +revision="$(git describe --tags --dirty)" revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100755 new mode 100644 index 9056fd3..0aaa6bf --- a/test/cargo_smoke_test/cargo +++ b/test/cargo_smoke_test/cargo @@ -4,14 +4,7 @@ set -e cd `dirname $0` -revision="$(git describe --tags --dirty 2>/dev/null || true)" -if [ -z "$revision" ]; then - # Tag-less checkout (e.g. actions/checkout fetches no tags): fall back to the - # latest release version with the short commit SHA as a prerelease suffix, - # keeping the generated crate version valid semver. (Update the release - # version here when cutting a new release.) - revision="0.9.4-$(git rev-parse --short HEAD)" -fi +revision="$(git describe --tags --dirty)" revision="${revision#v}" # strip 'v' prefix >&2 echo "$revision" From 6ca012afcd1c468ba43aa4a6e01980c091a64497 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Mon, 19 Oct 2026 18:35:42 -1000 Subject: [PATCH 15/21] fix executable bits --- bindings/python/genomicsqlite/__init__.py | 0 bindings/rust/cargo | 0 test/cargo_smoke_test/cargo | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bindings/python/genomicsqlite/__init__.py mode change 100644 => 100755 bindings/rust/cargo mode change 100644 => 100755 test/cargo_smoke_test/cargo diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py old mode 100644 new mode 100755 diff --git a/bindings/rust/cargo b/bindings/rust/cargo old mode 100644 new mode 100755 diff --git a/test/cargo_smoke_test/cargo b/test/cargo_smoke_test/cargo old mode 100644 new mode 100755 From 2e5ede4f5a732cd591db1296ced4905022b12f13 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 18:36:44 -1000 Subject: [PATCH 16/21] Rename build.yml to .github/workflows/build.yml --- build.yml => .github/workflows/build.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build.yml => .github/workflows/build.yml (100%) diff --git a/build.yml b/.github/workflows/build.yml similarity index 100% rename from build.yml rename to .github/workflows/build.yml From c2349a1742c62c5b2389b6d44e9d5fac2a4d4a08 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 20:31:10 -1000 Subject: [PATCH 17/21] PR #28 refinements: macOS 11.0/M1 baseline for the shipped extension; extraction-basename and comment cleanups - macOS job: configure the extension build with CMAKE_OSX_DEPLOYMENT_TARGET=11.0 and -mcpu=apple-m1; check Homebrew libzstd.a against that target (source build + LIBZSTD_A fallback if its objects require a newer macOS); assert the shipped dylib declares minos 11.0 in the symbols & digests step - Rust/JDBC: extract the bundled library under its architecture-qualified name -- the built binaries export sqlite3_extension_init, so the plain-basename renaming workaround and its comments are obsolete - Rust: merge the two identical None bundle_libgenomicsqlite() impls under the single complement cfg - Trim the selector and portable-runtime comments to their contracts; remove the Dockerfile claim that the cargo wrapper tolerates a tag-less checkout (the restored git describe is deliberately fail-fast); the safe.directory ownership note stays (build.yml is staged at the branch root in this commit; it moves to .github/workflows/ by rename, which also triggers CI.) --- Dockerfile | 3 +- .../net/mlin/genomicsqlite/JdbcDriver.java | 11 +- bindings/rust/build.rs | 19 +- bindings/rust/src/lib.rs | 38 +-- build.yml | 267 ++++++++++++++++++ 5 files changed, 291 insertions(+), 47 deletions(-) create mode 100644 build.yml diff --git a/Dockerfile b/Dockerfile index d0856b9..faec530 100644 --- a/Dockerfile +++ b/Dockerfile @@ -121,8 +121,7 @@ ADD ./.git ./.git ADD ./bindings/rust ./rust COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/libgenomicsqlite-linux-x86_64.so # git in the cargo wrapper must accept the context's .git even if the build -# host checked it out under a different uid than root; the wrapper itself -# tolerates a tag-less .git +# host checked it out under a different uid than root RUN git config --global --add safe.directory /work RUN SQLITE_WEB_LOG=99 LD_LIBRARY_PATH=$(pwd)/lib LIBRARY_PATH=$(pwd)/lib rust/cargo test --release diff --git a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java index 0241ab3..f74fe01 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java +++ b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java @@ -160,13 +160,10 @@ static String getLibraryFilename() { String tempDirpath = Files.createTempDirectory("genomicsqlite-jdbc-").toAbsolutePath().toString(); (new File(tempDirpath)).deleteOnExit(); - // Extract under the PLAIN basename: SQLite derives the extension - // entry point (sqlite3_genomicsqlite_init) from the name of the file - // it's asked to load, so loading the extracted copy under its - // architecture-qualified resource name would look up a mangled entry - // point and fail. - String plainBasename = "libgenomicsqlite" + basename.substring(basename.lastIndexOf('.')); - File extracted = new File(tempDirpath, plainBasename); + // Extract under the resource's own architecture-qualified name: the + // bundled binaries export the standard sqlite3_extension_init entry + // point, which SQLite tries before guessing one from the file name. + File extracted = new File(tempDirpath, basename); FileOutputStream writer = new FileOutputStream(extracted.getAbsolutePath()); extracted.deleteOnExit(); byte[] buffer = new byte[65536]; diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index 4a2b752..0f9efcf 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -4,23 +4,16 @@ // package, and hand its file name to src/lib.rs via cargo:rustc-env, where a // single include_bytes! embeds it. // -// Supported bundled targets -- the bundled libraries are the project's -// portable builds, and the Linux ones link glibc, so the target env must be -// gnu (a musl binary must never be handed the glibc library): +// Supported bundled targets (the Linux libraries link glibc, so the target +// env must be gnu): // linux + x86_64 + gnu -> libgenomicsqlite-linux-x86_64.so // linux + aarch64 + gnu -> libgenomicsqlite-linux-aarch64.so // macos + aarch64 -> libgenomicsqlite-macos-aarch64.dylib -// macOS x86_64 is deliberately not a bundled target (macOS CI builds -// ARM-only), so it falls back to the runtime library search like any other -// unsupported target. // -// Bundling is a build-time contract: with the feature enabled, a supported -// target whose bundled file is missing FAILS the build -- a release-packaging -// error must not silently degrade into a runtime library search. Likewise, a -// Linux target with a non-glibc ABI (e.g. musl) whose CPU has a bundled -// (glibc) library in the package FAILS with guidance, instead of the library -// being mis-selected or silently ignored. Any other unsupported target, or -// building without the feature (--no-default-features), leaves +// Bundling is a build-time contract: a supported target whose bundled file is +// missing FAILS the build, as does a non-glibc Linux target whose CPU has a +// bundled (glibc) library in the package. Any other target, or building +// without the feature (--no-default-features), leaves // GENOMICSQLITE_BUNDLED_LIB empty, and the crate loads an external // libgenomicsqlite at runtime (LIBGENOMICSQLITE / system loader search). use std::env; diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 1305d76..5f96d1a 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -51,12 +51,10 @@ fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { /* Bundled libgenomicsqlite selection. * * With the bundle_libgenomicsqlite feature, build.rs selects the one bundled - * library matching the compilation target (OS + CPU + libc ABI), validates - * that it's present in the package, and names it in the - * GENOMICSQLITE_BUNDLED_LIB environment variable (see build.rs for the - * supported-target table and the build-time contract). The single - * include_bytes! below embeds exactly that file -- the bundled_lib cfg is set - * by build.rs only when a file was selected. On any other target + * library matching the compilation target, validates that it's present in + * the package, and names it in GENOMICSQLITE_BUNDLED_LIB (see build.rs for + * the supported-target table and build-time contract); the single + * include_bytes! below embeds exactly that file. With no selection, * bundle_libgenomicsqlite() returns None and open() falls back to the normal * library search. */ @@ -65,28 +63,18 @@ fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { Some(extract_libgenomicsqlite( include_bytes!(concat!("../", env!("GENOMICSQLITE_BUNDLED_LIB"))), - // SQLite derives the extension entry point (sqlite3_genomicsqlite_init) - // from the name of the file it's asked to load, so the extracted copy - // must carry the plain library basename -- loading a file under its - // arch-qualified package name looks for a mangled entry point - // (sqlite3_genomicsqlitelinuxx_init) and fails. - if env!("GENOMICSQLITE_BUNDLED_LIB").ends_with(".dylib") { - "libgenomicsqlite.dylib" - } else { - "libgenomicsqlite.so" - }, + // The bundled binaries export the standard sqlite3_extension_init + // entry point, which SQLite tries before guessing an entry point + // from the file name -- so the extracted copy loads under its + // architecture-qualified package name as-is. + env!("GENOMICSQLITE_BUNDLED_LIB"), )) } -#[cfg(not(feature = "bundle_libgenomicsqlite"))] -fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { - None -} - -// Feature on, but no bundled library for this target (an OS/architecture/ABI -// the release doesn't bundle -- build.rs sets bundled_lib exactly when it -// selected a file): fall back to the normal library search. -#[cfg(all(feature = "bundle_libgenomicsqlite", not(bundled_lib)))] +// The complement of the bundled build above: the feature is off, or the +// compilation target is not one the release bundles. Fall back to the normal +// library search. +#[cfg(not(all(feature = "bundle_libgenomicsqlite", bundled_lib)))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { None } diff --git a/build.yml b/build.yml new file mode 100644 index 0000000..81677ac --- /dev/null +++ b/build.yml @@ -0,0 +1,267 @@ +name: build +on: [push, pull_request] + +env: + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 + PIP_DEPS: pytest pytest-xdist + +jobs: + + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" + - name: pre-commit + run: pre-commit run --all-files + - name: mvn spotless:check + run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: rust lint + run: | + cd bindings/rust + ./cargo check --no-default-features + ./cargo fmt -- --check + + test: + runs-on: ubuntu-24.04 + strategy: + matrix: + BUILD_TYPE: [Debug, Release] + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: deps + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build + run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) + - name: test + run: env -C build SQLITE_WEB_LOG=99 ctest -V + + linux-x86-64-so: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: build loaders + run: | + sudo apt-get -qq update + sudo apt-get install -y $APT_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) + rm build/libgenomicsqlite.so + sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so + - name: build portable .so in docker + run: | + # BuildKit builds only the stages the final target depends on, so the + # two minimum-runtime smoke-test stages -- which nothing depends on -- + # must be built explicitly, or they are silently skipped. + docker build --target alma-smoke-test . + docker build --target rust-smoke-test . + docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ + - name: symbols & digests + run: | + readelf -d build/libgenomicsqlite.so + ldd -r build/libgenomicsqlite.so + echo "----" + echo "libgenomicsqlite.so GLIBC version requirements:" + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 + echo "libsqlite3.so.0 GLIBC version requirements:" + objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 + echo "----" + sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + # The steps below test the portable libraries against the portable SQLite + # they ship with, scoping that environment (build/ first on PATH / + # LD_LIBRARY_PATH) to the tested processes only -- never the runner's + # global loader cache or PATH. + - name: test portable .so + run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage + env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so + - name: installed-package smoke test + run: | + sudo pip3 install --break-system-packages -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" + - name: prepare artifacts + run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite.so + path: libgenomicsqlite.so + - uses: actions/upload-artifact@v7 + with: + name: libsqlite3.so.0 + path: libsqlite3.so.0 + + macOS-arm64-dylib: + runs-on: macos-15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: dependencies + run: | + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + brew update || (sleep 2; brew update) + for dep in $BREW_DEPS ; do + brew install $dep || brew upgrade $dep + done + + # setup-python's macOS ARM 3.12.10 build lacked SQLite extension loading. + # Homebrew enables it; 3.12 is our CI baseline, matching Ubuntu 24.04. + # A venv selects that interpreter and keeps pip installs isolated. + "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" + echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.mac-venv/bin:$PATH" + python3 -m pip install $PIP_DEPS + python3 - <<'PYTHON' + import sqlite3, sys + print(sys.executable, sys.version) + print("SQLite", sqlite3.sqlite_version) + sqlite3.connect(":memory:").enable_load_extension(True) + PYTHON + - name: libzstd deployment target + run: | + # The shipped extension statically links Homebrew's libzstd.a, so its + # objects must not require a newer macOS than the 11.0 deployment + # target below. If any does (or the archive's minimum can't be + # established), build zstd from source for 11.0/apple-m1 instead and + # point LIBZSTD_A at that archive. + ZSTD_A="$(brew --prefix)/lib/libzstd.a" + ZSTD_MINOS=$(otool -l "$ZSTD_A" | awk '$1=="cmd" && ($2=="LC_BUILD_VERSION" || $2=="LC_VERSION_MIN_MACOSX") {b=1; next} $1=="cmd" {b=0} b && ($1=="minos" || $1=="version") {print $2; b=0}' | sort -Vu | tail -n1) + echo "Homebrew libzstd.a maximum minimum-macOS: ${ZSTD_MINOS:-none found}" + if [ -n "$ZSTD_MINOS" ] && [ "$(printf '11.0\n%s\n' "$ZSTD_MINOS" | sort -V | tail -n1)" = "11.0" ]; then + echo "LIBZSTD_A=$ZSTD_A" >> "$GITHUB_ENV" + else + ZSTD_VER=$(brew list --versions zstd | awk '{print $NF}') + curl -fsL "https://github.com/facebook/zstd/releases/download/v${ZSTD_VER}/zstd-${ZSTD_VER}.tar.gz" | tar xz + (cd "zstd-${ZSTD_VER}" && MACOSX_DEPLOYMENT_TARGET=11.0 CFLAGS="-O3 -fPIC -mcpu=apple-m1" make -C lib libzstd.a -j 4) + echo "LIBZSTD_A=$PWD/zstd-${ZSTD_VER}/lib/libzstd.a" >> "$GITHUB_ENV" + fi + - name: rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: build environment + run: | + echo "$(brew --prefix)/opt/sqlite/bin" >> "$GITHUB_PATH" + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + # used by rusqlite: + echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV + - name: build + run: | + # Shipped-extension baseline: macOS 11.0 deployment target and the + # Apple M1 CPU (the flags compose with the include paths in $CFLAGS / + # $CXXFLAGS from the "build environment" step). + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \ + -DCMAKE_C_FLAGS="$CFLAGS -mcpu=apple-m1" \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS -mcpu=apple-m1" \ + -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ + -DLIBZSTD_A="$LIBZSTD_A" \ + -B build . + cmake --build build -j 4 + - name: symbols & digests + run: | + otool -L build/libgenomicsqlite.dylib + sha256sum build/libgenomicsqlite.dylib + # Verify the shipped dylib declares the 11.0 deployment target; its + # otool -L dependencies above should be system libcurl, libc++, and + # libSystem only. + DYLIB_MINOS=$(otool -l build/libgenomicsqlite.dylib | awk '$1=="cmd" && ($2=="LC_BUILD_VERSION" || $2=="LC_VERSION_MIN_MACOSX") {b=1; next} $1=="cmd" {b=0} b && ($1=="minos" || $1=="version") {print $2; b=0}') + echo "libgenomicsqlite.dylib declared minimum macOS: $DYLIB_MINOS" + test "$DYLIB_MINOS" = "11.0" + - name: ctest + run: | + cd build + otool -L test/capi_smoke_test + SQLITE_WEB_LOG=99 ctest -V + - name: test rust crate + run: | + set -e + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib + SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib + - name: installed-package smoke test + run: | + python3 -m pip install -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" + - name: prepare artifacts + run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . + - uses: actions/upload-artifact@v7 + with: + name: libgenomicsqlite-macos-aarch64.dylib + path: libgenomicsqlite-macos-aarch64.dylib + + artifacts: + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] + steps: + - uses: actions/checkout@v7 + - name: fetch libgenomicsqlite.so + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite.so + - name: fetch libsqlite3.so.0 + uses: actions/download-artifact@v8 + with: + name: libsqlite3.so.0 + - name: fetch libgenomicsqlite.dylib + uses: actions/download-artifact@v8 + with: + name: libgenomicsqlite-macos-aarch64.dylib + - name: detect GIT_REVISION + run: | + cp include/genomicsqlite.h . + echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV + - uses: actions/upload-artifact@v7 + with: + name: GenomicSQLite-${{ env.GIT_REVISION }} + path: | + LICENSE + genomicsqlite.h + libgenomicsqlite.so + libgenomicsqlite-macos-aarch64.dylib + - name: build jar + run: | + cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so + cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib + mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: JDBC smoke test + run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc + - name: digests + run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 + - uses: actions/upload-artifact@v7 + with: + name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} + path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo From b591e9d79548ad93955dbde68f0da424b563f9e7 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 20:31:30 -1000 Subject: [PATCH 18/21] Remove live workflow while the PR #28 refinement build.yml is staged at the branch root (rename restores it and triggers CI) --- .github/workflows/build.yml | 241 ------------------------------------ 1 file changed, 241 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 42bff54..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,241 +0,0 @@ -name: build -on: [push, pull_request] - -env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip - BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 - PIP_DEPS: pytest pytest-xdist - -jobs: - - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip python3-venv - python3 -m venv "$HOME/.lint-venv" - "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint - echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - - name: pre-commit - run: pre-commit run --all-files - - name: mvn spotless:check - run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - name: rust lint - run: | - cd bindings/rust - ./cargo check --no-default-features - ./cargo fmt -- --check - - test: - runs-on: ubuntu-24.04 - strategy: - matrix: - BUILD_TYPE: [Debug, Release] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: deps - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build - run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - - name: test - run: env -C build SQLITE_WEB_LOG=99 ctest -V - - linux-x86-64-so: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: build loaders - run: | - sudo apt-get -qq update - sudo apt-get install -y $APT_DEPS - sudo pip3 install --break-system-packages $PIP_DEPS - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) - rm build/libgenomicsqlite.so - sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - - name: build portable .so in docker - run: | - # BuildKit builds only the stages the final target depends on, so the - # two minimum-runtime smoke-test stages -- which nothing depends on -- - # must be built explicitly, or they are silently skipped. - docker build --target alma-smoke-test . - docker build --target rust-smoke-test . - docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - - name: symbols & digests - run: | - readelf -d build/libgenomicsqlite.so - ldd -r build/libgenomicsqlite.so - echo "----" - echo "libgenomicsqlite.so GLIBC version requirements:" - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - objdump -t build/libgenomicsqlite.so | grep -o 'GLIBCXX_.*' | sort -Vr | head -n1 - echo "libsqlite3.so.0 GLIBC version requirements:" - objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 - echo "----" - sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 - echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" - python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - # The steps below exercise the portable libraries against the portable - # SQLite they ship with, WITHOUT installing anything into the runner's - # global loader cache or PATH: the portable libsqlite3.so.0 and shell sit - # side by side in build/, and each step scopes them to its own processes - # via LD_LIBRARY_PATH / PATH. (The suite also runs against the host SQLite - # in the separate `test` job.) Planting the portable libsqlite3 into the - # system loader cache instead makes unrelated later tools resolve against - # it -- the runner's own sqlite3 shell, built for a newer SQLite, then - # refuses to start against the older portable library. - - name: test portable .so - run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so - ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage - env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so - - name: installed-package smoke test - run: | - sudo pip3 install --break-system-packages -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" - - name: prepare artifacts - run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite.so - path: libgenomicsqlite.so - - uses: actions/upload-artifact@v7 - with: - name: libsqlite3.so.0 - path: libsqlite3.so.0 - - macOS-arm64-dylib: - runs-on: macos-15 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: dependencies - run: | - rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 - brew update || (sleep 2; brew update) - for dep in $BREW_DEPS ; do - brew install $dep || brew upgrade $dep - done - - # setup-python's macOS ARM 3.12.10 build lacked SQLite extension loading. - # Homebrew enables it; 3.12 is our CI baseline, matching Ubuntu 24.04. - # A venv selects that interpreter and keeps pip installs isolated. - "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" - echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.mac-venv/bin:$PATH" - python3 -m pip install $PIP_DEPS - python3 - <<'PYTHON' - import sqlite3, sys - print(sys.executable, sys.version) - print("SQLite", sqlite3.sqlite_version) - sqlite3.connect(":memory:").enable_load_extension(True) - PYTHON - - name: rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: build environment - run: | - echo "$(brew --prefix)/opt/sqlite/bin" >> "$GITHUB_PATH" - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - # used by rusqlite: - echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV - echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - - name: build - run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ - -B build . - cmake --build build -j 4 - - name: symbols & digests - run: | - otool -L build/libgenomicsqlite.dylib - sha256sum build/libgenomicsqlite.dylib - - name: ctest - run: | - cd build - otool -L test/capi_smoke_test - SQLITE_WEB_LOG=99 ctest -V - - name: test rust crate - run: | - set -e - cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib - - name: installed-package smoke test - run: | - python3 -m pip install -q setuptools wheel - bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" - - name: prepare artifacts - run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . - - uses: actions/upload-artifact@v7 - with: - name: libgenomicsqlite-macos-aarch64.dylib - path: libgenomicsqlite-macos-aarch64.dylib - - artifacts: - runs-on: ubuntu-24.04 - needs: [linux-x86-64-so, macOS-arm64-dylib] - steps: - - uses: actions/checkout@v7 - - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite.so - - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v8 - with: - name: libsqlite3.so.0 - - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v8 - with: - name: libgenomicsqlite-macos-aarch64.dylib - - name: detect GIT_REVISION - run: | - cp include/genomicsqlite.h . - echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v7 - with: - name: GenomicSQLite-${{ env.GIT_REVISION }} - path: | - LICENSE - genomicsqlite.h - libgenomicsqlite.so - libgenomicsqlite-macos-aarch64.dylib - - name: build jar - run: | - cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so - cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib - mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - - name: JDBC smoke test - run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc - - name: digests - run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v7 - with: - name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} - path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo From 1ccafdd41b793f90f3e181800c61ac17a001c674 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 20:53:34 -1000 Subject: [PATCH 19/21] PR #28: macOS baseline back to 15 per Mike's reconsideration; keep explicit -mcpu=apple-m1 Mike checked Homebrew's libzstd.a carries the same baseline, so the 11.0 deployment target, the libzstd deployment-target step with its LIBZSTD_A source-build fallback, and the minos assertion are removed again. The macOS job is now the pre-refinement version plus the explicit M1 flags. Everything else from the refinement round stands. (build.yml remains staged at the branch root; rename restores it and triggers CI.) --- build.yml | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/build.yml b/build.yml index 81677ac..4db37dc 100644 --- a/build.yml +++ b/build.yml @@ -149,24 +149,6 @@ jobs: print("SQLite", sqlite3.sqlite_version) sqlite3.connect(":memory:").enable_load_extension(True) PYTHON - - name: libzstd deployment target - run: | - # The shipped extension statically links Homebrew's libzstd.a, so its - # objects must not require a newer macOS than the 11.0 deployment - # target below. If any does (or the archive's minimum can't be - # established), build zstd from source for 11.0/apple-m1 instead and - # point LIBZSTD_A at that archive. - ZSTD_A="$(brew --prefix)/lib/libzstd.a" - ZSTD_MINOS=$(otool -l "$ZSTD_A" | awk '$1=="cmd" && ($2=="LC_BUILD_VERSION" || $2=="LC_VERSION_MIN_MACOSX") {b=1; next} $1=="cmd" {b=0} b && ($1=="minos" || $1=="version") {print $2; b=0}' | sort -Vu | tail -n1) - echo "Homebrew libzstd.a maximum minimum-macOS: ${ZSTD_MINOS:-none found}" - if [ -n "$ZSTD_MINOS" ] && [ "$(printf '11.0\n%s\n' "$ZSTD_MINOS" | sort -V | tail -n1)" = "11.0" ]; then - echo "LIBZSTD_A=$ZSTD_A" >> "$GITHUB_ENV" - else - ZSTD_VER=$(brew list --versions zstd | awk '{print $NF}') - curl -fsL "https://github.com/facebook/zstd/releases/download/v${ZSTD_VER}/zstd-${ZSTD_VER}.tar.gz" | tar xz - (cd "zstd-${ZSTD_VER}" && MACOSX_DEPLOYMENT_TARGET=11.0 CFLAGS="-O3 -fPIC -mcpu=apple-m1" make -C lib libzstd.a -j 4) - echo "LIBZSTD_A=$PWD/zstd-${ZSTD_VER}/lib/libzstd.a" >> "$GITHUB_ENV" - fi - name: rust toolchain uses: dtolnay/rust-toolchain@stable - name: build environment @@ -180,27 +162,19 @@ jobs: echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - name: build run: | - # Shipped-extension baseline: macOS 11.0 deployment target and the - # Apple M1 CPU (the flags compose with the include paths in $CFLAGS / - # $CXXFLAGS from the "build environment" step). + # Shipped-extension baseline: the Apple M1 CPU target (the flags + # compose with the include paths in $CFLAGS / $CXXFLAGS from the + # "build environment" step). cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \ -DCMAKE_C_FLAGS="$CFLAGS -mcpu=apple-m1" \ -DCMAKE_CXX_FLAGS="$CXXFLAGS -mcpu=apple-m1" \ -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ - -DLIBZSTD_A="$LIBZSTD_A" \ -B build . cmake --build build -j 4 - name: symbols & digests run: | otool -L build/libgenomicsqlite.dylib sha256sum build/libgenomicsqlite.dylib - # Verify the shipped dylib declares the 11.0 deployment target; its - # otool -L dependencies above should be system libcurl, libc++, and - # libSystem only. - DYLIB_MINOS=$(otool -l build/libgenomicsqlite.dylib | awk '$1=="cmd" && ($2=="LC_BUILD_VERSION" || $2=="LC_VERSION_MIN_MACOSX") {b=1; next} $1=="cmd" {b=0} b && ($1=="minos" || $1=="version") {print $2; b=0}') - echo "libgenomicsqlite.dylib declared minimum macOS: $DYLIB_MINOS" - test "$DYLIB_MINOS" = "11.0" - name: ctest run: | cd build From bff979acf4223d03f78d242d24185f5bf51a75d8 Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Tue, 20 Oct 2026 06:54:56 +0000 Subject: [PATCH 20/21] Rename build.yml to .github/workflows/build.yml --- build.yml => .github/workflows/build.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build.yml => .github/workflows/build.yml (100%) diff --git a/build.yml b/.github/workflows/build.yml similarity index 100% rename from build.yml rename to .github/workflows/build.yml From 2a5fdd00c43069609b5638ffb94c514ab03c851e Mon Sep 17 00:00:00 2001 From: "Wid L. Hacker" Date: Mon, 19 Oct 2026 21:09:30 -1000 Subject: [PATCH 21/21] README: supplied binaries are Linux x86-64 and macOS Apple Silicon (macOS 15+) Correct the stale line 25 platform claim from PR #28 approval review 5581776926. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 313a0b9..352593f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Our **[Colab notebook](https://colab.research.google.com/drive/1OlHPOcRQBhDmEnS1 **Start Here 👉 [full documentation site](https://mlin.github.io/GenomicSQLite/)** -We supply the extension [prepackaged](https://github.com/mlin/GenomicSQLite/releases) for Linux and macOS on x86-64. An up-to-date version of SQLite itself is also required, as specified in the docs. +We supply the extension [prepackaged](https://github.com/mlin/GenomicSQLite/releases) for Linux x86-64 and macOS Apple Silicon (macOS 15+). An up-to-date version of SQLite itself is also required, as specified in the docs. Programming language support: