diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c41822c..4db37dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,33 +2,33 @@ name: build on: [push, pull_request] env: - APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip libcurl4-openssl-dev - BREW_DEPS: zstd samtools pigz coreutils sqlite + APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip + BREW_DEPS: zstd samtools pigz coreutils sqlite python@3.12 PIP_DEPS: pytest pytest-xdist jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: deps run: | sudo apt-get -qq update - sudo apt-get install -y clang-format cppcheck python3-pip - sudo pip3 install --system pre-commit $PIP_DEPS black flake8 pylint + sudo apt-get install -y clang-format cppcheck python3-pip python3-venv + python3 -m venv "$HOME/.lint-venv" + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - name: pre-commit run: pre-commit run --all-files - name: mvn spotless:check run: mvn spotless:check -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml - name: rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy - toolchain: stable - default: true - name: rust lint run: | cd bindings/rust @@ -36,45 +36,48 @@ jobs: ./cargo fmt -- --check test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: BUILD_TYPE: [Debug, Release] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: deps run: | sudo apt-get -qq update sudo apt-get install -y $APT_DEPS - sudo pip3 install --system $PIP_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS - name: rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true + uses: dtolnay/rust-toolchain@stable - name: build run: cmake -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} -B build . && cmake --build build -j $(nproc) - name: test run: env -C build SQLITE_WEB_LOG=99 ctest -V linux-x86-64-so: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: build loaders run: | sudo apt-get -qq update sudo apt-get install -y $APT_DEPS - sudo pip3 install --system $PIP_DEPS + sudo pip3 install --break-system-packages $PIP_DEPS cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' -B build . && cmake --build build -j $(nproc) rm build/libgenomicsqlite.so sudo apt-get remove -y libzstd-dev # should be linked into libgenomicsqlite.so - name: build portable .so in docker - run: docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /mnt/build/ + run: | + # BuildKit builds only the stages the final target depends on, so the + # two minimum-runtime smoke-test stages -- which nothing depends on -- + # must be built explicitly, or they are silently skipped. + docker build --target alma-smoke-test . + docker build --target rust-smoke-test . + docker build -t genomicsqlite . && docker run -v $(pwd):/mnt --rm genomicsqlite cp /usr/local/lib/{libgenomicsqlite.so,libsqlite3.so.0} /usr/local/bin/sqlite3 /mnt/build/ - name: symbols & digests run: | readelf -d build/libgenomicsqlite.so @@ -87,67 +90,84 @@ jobs: objdump -t build/libsqlite3.so.0 | grep -o 'GLIBC_.*' | sort -Vr | head -n1 echo "----" sha256sum build/libgenomicsqlite.so build/libsqlite3.so.0 - sudo cp build/libsqlite3.so.0 /usr/local/lib - sudo ldconfig + echo "asserting manylinux_2_28 floor (GLIBC <= 2.28, GLIBCXX <= 3.4.25):" + python3 test/check_elf_version_floor.py build/libgenomicsqlite.so build/libsqlite3.so.0 - name: rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true + uses: dtolnay/rust-toolchain@stable + # The steps below test the portable libraries against the portable SQLite + # they ship with, scoping that environment (build/ first on PATH / + # LD_LIBRARY_PATH) to the tested processes only -- never the runner's + # global loader cache or PATH. - name: test portable .so - run: env -C build SQLITE_WEB_LOG=99 ctest -V + run: env -C build SQLITE_WEB_LOG=99 PATH="$(pwd)/build:$PATH" LD_LIBRARY_PATH="$(pwd)/build" ctest -V - name: test rust crate run: | set -e - cp build/libgenomicsqlite.so bindings/rust - SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite.so + cp build/libgenomicsqlite.so bindings/rust/libgenomicsqlite-linux-x86_64.so + ln -sf libsqlite3.so.0 build/libsqlite3.so # linker name, scoped to build/ like the Dockerfile rust stage + env LD_LIBRARY_PATH="$(pwd)/build" LIBRARY_PATH="$(pwd)/build" SQLITE_WEB_LOG=99 bindings/rust/cargo test --release + bindings/rust/cargo package --list | grep libgenomicsqlite-linux-x86_64.so + - name: installed-package smoke test + run: | + sudo pip3 install --break-system-packages -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.so libgenomicsqlite-linux-x86_64.so "$(pwd)/build" - name: prepare artifacts run: cp build/{libgenomicsqlite.so,libsqlite3.so.0} . - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: libgenomicsqlite.so path: libgenomicsqlite.so - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: libsqlite3.so.0 path: libsqlite3.so.0 - macOS-x86-64-dylib: - runs-on: macOS-11 + macOS-arm64-dylib: + runs-on: macos-15 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - name: dependencies run: | - rm -f /usr/local/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 + rm -f $(brew --prefix)/bin/2to3 # https://github.com/pypa/pipenv/issues/3831 brew update || (sleep 2; brew update) for dep in $BREW_DEPS ; do brew install $dep || brew upgrade $dep done - pip install $PIP_DEPS + # setup-python's macOS ARM 3.12.10 build lacked SQLite extension loading. + # Homebrew enables it; 3.12 is our CI baseline, matching Ubuntu 24.04. + # A venv selects that interpreter and keeps pip installs isolated. + "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$HOME/.mac-venv" + echo "$HOME/.mac-venv/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.mac-venv/bin:$PATH" + python3 -m pip install $PIP_DEPS + python3 - <<'PYTHON' + import sqlite3, sys + print(sys.executable, sys.version) + print("SQLite", sqlite3.sqlite_version) + sqlite3.connect(":memory:").enable_load_extension(True) + PYTHON - name: rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true + uses: dtolnay/rust-toolchain@stable - name: build environment run: | - echo "PATH=$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV - echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=ivybridge" >> $GITHUB_ENV - echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=ivybridge" >> $GITHUB_ENV + echo "$(brew --prefix)/opt/sqlite/bin" >> "$GITHUB_PATH" + echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV # used by rusqlite: echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV echo "SQLITE3_LIB_DIR=$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV - name: build run: | + # Shipped-extension baseline: the Apple M1 CPU target (the flags + # compose with the include paths in $CFLAGS / $CXXFLAGS from the + # "build environment" step). cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="$CFLAGS -mcpu=apple-m1" \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS -mcpu=apple-m1" \ -DCMAKE_PREFIX_PATH=$(brew --prefix)/opt/sqlite \ -B build . cmake --build build -j 4 @@ -163,52 +183,59 @@ jobs: - name: test rust crate run: | set -e - cp build/libgenomicsqlite.dylib bindings/rust + cp build/libgenomicsqlite.dylib bindings/rust/libgenomicsqlite-macos-aarch64.dylib SQLITE_WEB_LOG=99 bindings/rust/cargo test --release - bindings/rust/cargo package --list | grep libgenomicsqlite.dylib + bindings/rust/cargo package --list | grep libgenomicsqlite-macos-aarch64.dylib + - name: installed-package smoke test + run: | + python3 -m pip install -q setuptools wheel + bash test/pkg_smoke.sh python3 bindings/python build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib "$(brew --prefix)/opt/sqlite/bin" - name: prepare artifacts - run: cp build/libgenomicsqlite.dylib include/genomicsqlite.h . - - uses: actions/upload-artifact@v2 + run: cp build/libgenomicsqlite.dylib libgenomicsqlite-macos-aarch64.dylib && cp include/genomicsqlite.h . + - uses: actions/upload-artifact@v7 with: - name: libgenomicsqlite.dylib - path: libgenomicsqlite.dylib + name: libgenomicsqlite-macos-aarch64.dylib + path: libgenomicsqlite-macos-aarch64.dylib artifacts: - runs-on: ubuntu-20.04 - needs: [linux-x86-64-so, macOS-x86-64-dylib] + runs-on: ubuntu-24.04 + needs: [linux-x86-64-so, macOS-arm64-dylib] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: fetch libgenomicsqlite.so - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v8 with: name: libgenomicsqlite.so - name: fetch libsqlite3.so.0 - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v8 with: name: libsqlite3.so.0 - name: fetch libgenomicsqlite.dylib - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v8 with: - name: libgenomicsqlite.dylib + name: libgenomicsqlite-macos-aarch64.dylib - name: detect GIT_REVISION run: | cp include/genomicsqlite.h . echo "GIT_REVISION=$(git describe --tags --always)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: GenomicSQLite-${{ env.GIT_REVISION }} path: | LICENSE genomicsqlite.h libgenomicsqlite.so - libgenomicsqlite.dylib + libgenomicsqlite-macos-aarch64.dylib - name: build jar run: | - cp libgenomicsqlite.so libgenomicsqlite.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/ + cp libgenomicsqlite.so bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-linux-x86_64.so + cp libgenomicsqlite-macos-aarch64.dylib bindings/jdbc/genomicsqlite-jdbc/src/main/resources/libgenomicsqlite-macos-aarch64.dylib mvn deploy -Drevision=${{ env.GIT_REVISION }} -B -f bindings/jdbc/genomicsqlite-jdbc/pom.xml + - name: JDBC smoke test + run: bash test/jdbc_smoke_test.sh bindings/jdbc/genomicsqlite-jdbc - name: digests run: sha256sum *genomicsqlite* bindings/jdbc/genomicsqlite-jdbc/target/genomicsqlite*.jar libsqlite3.so.0 - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v7 with: name: genomicsqlite-jdbc-${{ env.GIT_REVISION }} path: bindings/jdbc/genomicsqlite-jdbc/target/mvn-repo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c6a30e0..abf7065 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,4 +38,5 @@ repos: exclude: version\.py verbose: true entry: env PYTHONPATH=bindings/python pylint - args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] + args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,too-many-positional-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] +# CI re-run trigger 2026-10-19 (run 39294337754 lost its macOS runner to GitHub capacity) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bc5852..d095a63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,13 @@ if(${CMAKE_VERSION} VERSION_LESS 3.12) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) endif() +# CMake 4 removed compatibility with projects declaring +# cmake_minimum_required() below 3.5; the pinned SQLiteCpp fetched +# below declares 3.1, which makes configure fail outright under +# CMake 4 (e.g. the macos-15 runner image). Raise the policy-version +# floor for fetched subprojects; harmless under older CMake. +set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + include(FetchContent) if(${CMAKE_VERSION} VERSION_LESS 3.14) @@ -71,6 +78,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) enable_testing() set(GENOMICSQLITE_TEST_ENV env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/bindings/python:${PYTHONPATH} LIBGENOMICSQLITE=${CMAKE_BINARY_DIR}/libgenomicsqlite SQLITE_WEB_LOG=99) add_test(NAME smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} python3 ${CMAKE_CURRENT_SOURCE_DIR}/test/genomicsqlite_smoke_test.py) + add_test(NAME extension_loading COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/test/extension_loading.py $) add_test(NAME capi_smoke_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_BINARY_DIR}/test/capi_smoke_test) add_test(NAME JdbcTests COMMAND ${GENOMICSQLITE_TEST_ENV} mvn test -B -f ${CMAKE_CURRENT_SOURCE_DIR}/bindings/jdbc/genomicsqlite-jdbc/pom.xml) add_test(NAME cargo_test COMMAND ${GENOMICSQLITE_TEST_ENV} ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rust/cargo test --no-default-features) diff --git a/Dockerfile b/Dockerfile index 8f5d464..faec530 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -# Builds libgenomicsqlite.so in CentOS 7 (+ devtoolset-8) to maximize compatibility. Bakes in -# Zstandard, and also builds a compatible libsqlite3.so.0 for users unable/unwilling to upgrade -# their host's. -FROM centos:7 as builder +# Builds libgenomicsqlite.so in AlmaLinux 8 (glibc 2.28, the manylinux_2_28 baseline) to +# maximize compatibility. Bakes in Zstandard, and also builds a compatible libsqlite3.so.0 +# for users unable/unwilling to upgrade their host's. +FROM almalinux:8 as builder ARG CMAKE_VERSION=3.17.3 ARG SQLITE_VERSION=3410200 @@ -28,8 +28,7 @@ ENV SQLITE_CFLAGS="\ -DSQLITE_ENABLE_SESSION \ " -RUN yum install -y -q wget unzip rsync git libcurl-devel centos-release-scl -RUN yum install -y -q devtoolset-8-gcc devtoolset-8-gcc-c++ devtoolset-8-make +RUN dnf install -y -q wget unzip rsync git libcurl-devel gcc gcc-c++ make RUN mkdir -p /work/GenomicSQLite @@ -38,8 +37,8 @@ WORKDIR /work RUN wget -nv http://www.sqlite.org/2023/sqlite-amalgamation-${SQLITE_VERSION}.zip \ && unzip -o sqlite-amalgamation-${SQLITE_VERSION}.zip WORKDIR /work/sqlite-amalgamation-${SQLITE_VERSION} -RUN bash -c "scl enable devtoolset-8 'gcc -shared -o libsqlite3.so.0 -fPIC -shared -Wl,-soname,libsqlite3.so.0 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c' & pid=$? \ - scl enable devtoolset-8 'gcc -o sqlite3 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c shell.c -lpthread -ldl -lm' && wait $pid" +RUN bash -c "gcc -shared -o libsqlite3.so.0 -fPIC -shared -Wl,-soname,libsqlite3.so.0 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c & pid=$? \ + gcc -o sqlite3 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c shell.c -lpthread -ldl -lm && wait $pid" RUN cp libsqlite3.so.0 /usr/local/lib && cp *.h /usr/local/include && cp sqlite3 /usr/local/bin RUN ln -s /usr/local/lib/libsqlite3.so.0 /usr/local/lib/libsqlite3.so @@ -47,7 +46,7 @@ RUN ln -s /usr/local/lib/libsqlite3.so.0 /usr/local/lib/libsqlite3.so WORKDIR /work RUN wget -nv -O - https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz | tar zx WORKDIR /work/zstd-${ZSTD_VERSION} -RUN scl enable devtoolset-8 "CFLAGS='${CFLAGS} -fPIC' make install -j $(nproc)" +RUN CFLAGS="${CFLAGS} -fPIC" make install -j $(nproc) # CMake WORKDIR /work @@ -61,21 +60,30 @@ RUN ldconfig ADD . /work/GenomicSQLite WORKDIR /work/GenomicSQLite RUN rm -rf build -RUN scl enable devtoolset-8 "cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_WHOLE_ARCHIVE=true -B build . && cmake --build build --target genomicsqlite -j $(nproc)" +# _GLIBCXX_USE_CXX11_ABI=0: the header's C++ API passes std::string across the +# library boundary, so the .so's C++ ABI is part of its public contract. Keep the +# pre-C++11 libstdc++ ABI -- the artifact's historical ABI (the old CentOS +# devtoolset toolchain defaulted to it) and the ABI the CI workflow compiles the +# loader/test executables with. Stock AlmaLinux GCC defaults to ABI=1, which +# silently changes the exported symbols and breaks those executables. +RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="${CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0" -DZSTD_WHOLE_ARCHIVE=true -B build . && cmake --build build --target genomicsqlite -j $(nproc) RUN sqlite3 -cmd '.load build/libgenomicsqlite.so' :memory: 'select sqlite_version(); select genomicsqlite_version()' ################################################################################################### -# Run-up in fresh centos:7, to confirm no dependencies crept into the .so's +# Run-up in fresh almalinux:8, to confirm no dependencies crept into the .so's +# NOTE: this stage is a test stage -- nothing in the final stage depends on it, +# so CI builds it explicitly with `docker build --target alma-smoke-test` +# (BuildKit would otherwise skip it silently). -FROM centos:7 +FROM almalinux:8 AS alma-smoke-test COPY --from=builder /usr/local/bin/sqlite3 /usr/local/bin/ COPY --from=builder /usr/local/include/sqlite3.h /usr/local/include/ -# NOTE: /usr/local/lib is NOT a default ld path in CentOS -- https://unix.stackexchange.com/q/356624 +# NOTE: /usr/local/lib is NOT a default ld path in AlmaLinux -- https://unix.stackexchange.com/q/356624 RUN mkdir -p /work/lib WORKDIR /work COPY --from=builder /usr/local/lib/libsqlite3.so.0 ./lib/ @@ -97,8 +105,8 @@ RUN mv lib/{libgenomicsqlite.so,libsqlite3.so.0} . && rm -rf lib RUN ./genomicsqlite_capi_smoke_test # ^ we didn't need LD_LIBRARY_PATH even though the .so's aren't at their linktime locations -# test rust bindings too -FROM centos:7 +# test rust bindings too (also a test stage: CI builds --target rust-smoke-test) +FROM almalinux:8 AS rust-smoke-test RUN yum install -y -q gcc git ADD https://sh.rustup.rs /usr/local/bin/rustup-init.sh @@ -111,13 +119,17 @@ COPY --from=builder /usr/local/lib/libsqlite3.so.0 ./lib/ RUN ln -s libsqlite3.so.0 lib/libsqlite3.so ADD ./.git ./.git ADD ./bindings/rust ./rust -COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/ +COPY --from=builder /work/GenomicSQLite/build/libgenomicsqlite.so ./rust/libgenomicsqlite-linux-x86_64.so +# git in the cargo wrapper must accept the context's .git even if the build +# host checked it out under a different uid than root +RUN git config --global --add safe.directory /work RUN SQLITE_WEB_LOG=99 LD_LIBRARY_PATH=$(pwd)/lib LIBRARY_PATH=$(pwd)/lib rust/cargo test --release ################################################################################################### -# Run-up in ubuntu 16.04 +# Run-up in ubuntu 20.04 (final stage: the default docker build target, and the +# image the CI workflow copies the portable libraries out of) -FROM ubuntu:16.04 +FROM ubuntu:20.04 AS ubuntu-smoke-test COPY --from=builder /usr/local/bin/sqlite3 /usr/local/bin/ COPY --from=builder /usr/local/include/sqlite3.h /usr/local/include/ diff --git a/README.md b/README.md index 313a0b9..352593f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Our **[Colab notebook](https://colab.research.google.com/drive/1OlHPOcRQBhDmEnS1 **Start Here 👉 [full documentation site](https://mlin.github.io/GenomicSQLite/)** -We supply the extension [prepackaged](https://github.com/mlin/GenomicSQLite/releases) for Linux and macOS on x86-64. An up-to-date version of SQLite itself is also required, as specified in the docs. +We supply the extension [prepackaged](https://github.com/mlin/GenomicSQLite/releases) for Linux x86-64 and macOS Apple Silicon (macOS 15+). An up-to-date version of SQLite itself is also required, as specified in the docs. Programming language support: diff --git a/bindings/jdbc/genomicsqlite-jdbc/pom.xml b/bindings/jdbc/genomicsqlite-jdbc/pom.xml index 5b1cdac..63b4894 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/pom.xml +++ b/bindings/jdbc/genomicsqlite-jdbc/pom.xml @@ -43,9 +43,13 @@ + + 0.9.4-SNAPSHOT UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 @@ -110,7 +114,7 @@ com.diffplug.spotless spotless-maven-plugin - 2.3.1 + 3.10.4 diff --git a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java index c7ae92c..cc353cf 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java +++ b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/GenomicSQLite.java @@ -156,7 +156,8 @@ public static String putReferenceSequenceSQL(Connection conn, String name, long public static HashMap getReferenceSequencesByRid( Connection conn, String assembly, String schema) throws SQLException { String sql = - "SELECT _gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id, gri_refseq_meta_json FROM "; + "SELECT _gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id," + + " gri_refseq_meta_json FROM "; if (schema != null && schema.length() > 0) { sql += schema + "."; } diff --git a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java index 3fc605a..f74fe01 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java +++ b/bindings/jdbc/genomicsqlite-jdbc/src/main/java/net/mlin/genomicsqlite/JdbcDriver.java @@ -40,7 +40,9 @@ public class JdbcDriver extends org.sqlite.JDBC { } } - /** @see java.sql.Driver#acceptsURL(java.lang.String) */ + /** + * @see java.sql.Driver#acceptsURL(java.lang.String) + */ @Override public boolean acceptsURL(String url) { return isValidURL(url); @@ -56,7 +58,9 @@ public static boolean isValidURL(String url) { return url != null && url.toLowerCase().startsWith(PREFIX); } - /** @see java.sql.Driver#connect(java.lang.String, java.util.Properties) */ + /** + * @see java.sql.Driver#connect(java.lang.String, java.util.Properties) + */ @Override public Connection connect(String url, Properties info) throws SQLException { return createConnection(url, info); @@ -156,6 +160,9 @@ static String getLibraryFilename() { String tempDirpath = Files.createTempDirectory("genomicsqlite-jdbc-").toAbsolutePath().toString(); (new File(tempDirpath)).deleteOnExit(); + // Extract under the resource's own architecture-qualified name: the + // bundled binaries export the standard sqlite3_extension_init entry + // point, which SQLite tries before guessing one from the file name. File extracted = new File(tempDirpath, basename); FileOutputStream writer = new FileOutputStream(extracted.getAbsolutePath()); extracted.deleteOnExit(); @@ -175,18 +182,27 @@ static String getLibraryFilename() { } static String getLibraryBasename() { + // Resource name of the bundled library for this OS + CPU, in the + // architecture-qualified form the release pipeline packs into the JAR (the + // same naming the Rust crate uses). Matching is exact: an Intel JVM never + // selects the ARM build, nor vice versa. If the JAR carries no resource + // under the returned name (a platform the release doesn't bundle), the + // caller falls back to the system library search. String arch = System.getProperty("os.arch", "unknown").toLowerCase(Locale.ENGLISH); + String archTok = null; if (arch.equals("amd64") || arch.equals("x86_64")) { - String os = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); - String ext = null; - if (os.indexOf("mac") >= 0 || os.indexOf("darwin") >= 0) { - ext = "dylib"; - } else if (os.indexOf("nux") >= 0) { - ext = "so"; - } - if (ext != null) { - return "libgenomicsqlite." + ext; - } + archTok = "x86_64"; + } else if (arch.equals("aarch64") || arch.equals("arm64")) { + archTok = "aarch64"; + } + if (archTok == null) { + return null; + } + String os = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); + if (os.indexOf("mac") >= 0 || os.indexOf("darwin") >= 0) { + return "libgenomicsqlite-macos-" + archTok + ".dylib"; + } else if (os.indexOf("nux") >= 0) { + return "libgenomicsqlite-linux-" + archTok + ".so"; } return null; } diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in index a93b1bf..2d72935 100644 --- a/bindings/python/MANIFEST.in +++ b/bindings/python/MANIFEST.in @@ -1 +1 @@ -include RELEASE-VERSION version.py genomicsqlite/libgenomicsqlite.so genomicsqlite/libgenomicsqlite.dylib +include RELEASE-VERSION version.py genomicsqlite/libgenomicsqlite-* diff --git a/bindings/python/conda/genomicsqlite/meta.yaml b/bindings/python/conda/genomicsqlite/meta.yaml index fbe257f..3ae1048 100644 --- a/bindings/python/conda/genomicsqlite/meta.yaml +++ b/bindings/python/conda/genomicsqlite/meta.yaml @@ -27,7 +27,7 @@ test: imports: - genomicsqlite commands: - - 'ldd -v -r ${CONDA_PREFIX}/lib/python3.*/site-packages/genomicsqlite/libgenomicsqlite.so' # [linux64] + - 'ldd -v -r ${CONDA_PREFIX}/lib/python3.*/site-packages/genomicsqlite/libgenomicsqlite-linux-x86_64.so' # [linux64] - 'wget -nv -O - https://raw.githubusercontent.com/mlin/GenomicSQLite/release/test/genomicsqlite_smoke_test.py | python3 -' # [linux64] - which genomicsqlite diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py index ea9f3cf..c1ec17e 100755 --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -2,6 +2,7 @@ """ GenomicSQLite Python binding """ + import os import sys import platform @@ -14,13 +15,21 @@ # Module initialization -- locate shared library file _DLL = os.environ.get("LIBGENOMICSQLITE", "").strip() # use env var if present -if not _DLL and platform.machine() == "x86_64": - # select platform-appropriate library bundled with this package - _DLL = {"Linux": ".so", "Darwin": ".dylib"}.get(platform.system(), None) - if _DLL: - _DLL = os.path.join(_HERE, "libgenomicsqlite") + _DLL - if not os.path.isfile(_DLL): - _DLL = None +if not _DLL: + # select the platform-appropriate library bundled with this package, which + # the release pipeline ships under an architecture-qualified name (the same + # naming the Rust crate uses). Selection is exact-match on OS + CPU: an + # Intel machine never selects the ARM build, nor vice versa. + _BUNDLED_NAME = { + ("Linux", "x86_64"): "libgenomicsqlite-linux-x86_64.so", + ("Linux", "aarch64"): "libgenomicsqlite-linux-aarch64.so", + ("Darwin", "x86_64"): "libgenomicsqlite-macos-x86_64.dylib", + ("Darwin", "arm64"): "libgenomicsqlite-macos-aarch64.dylib", + }.get((platform.system(), platform.machine())) + if _BUNDLED_NAME: + _bundled_path = os.path.join(_HERE, _BUNDLED_NAME) + if os.path.isfile(_bundled_path): + _DLL = _bundled_path if not _DLL: # otherwise, let SQLite use dlopen() to look for it _DLL = "libgenomicsqlite" @@ -252,6 +261,7 @@ def _cli(argv=None): sys.exit(1) import shutil + import tempfile if not shutil.which("sqlite3"): print( @@ -271,46 +281,67 @@ def _cli(argv=None): uri = _execute1(_MEMCONN, "SELECT genomicsqlite_uri(?,?)", (argv[1], cfg)) tuning_sql = _execute1(_MEMCONN, "SELECT genomicsqlite_tuning_sql(?)", (cfg,)) - cmd = [ - "sqlite3", - "-bail", - "-cmd", - f".load {_DLL}", - "-cmd", - f".open {uri}", - "-cmd", + # Assemble the shell startup commands, then replay them inside the shell via + # "-cmd .read /dev/fd/N" on an anonymous temporary file whose descriptor is + # inherited across exec: + # * Since SQLite 3.52, the shell quits immediately (with a success exit code, + # without reading any input) whenever -bail is combined with an SQL -cmd + # argument, which silently disabled this wrapper when the startup commands + # were passed as -cmd arguments directly. A dot-command -cmd does not take + # that code path. + # * The anonymous TemporaryFile leaves nothing behind in the temp directory + # (unlike a named -init file, which os.execvp() would strand there). + # * The single -init slot stays free, so a user-supplied -init in argv + # composes with -- instead of displacing -- this startup sequence. + tuning_sql = tuning_sql.strip() + if not tuning_sql.endswith(";"): + # in script context, an unterminated statement would absorb the following line + tuning_sql += ";" + init_lines = [ + # quoted: the shell's dot-command tokenizer splits on unquoted whitespace, + # so paths containing spaces would otherwise truncate at the first space. + f'.load "{_DLL}"', + f'.open "{uri}"', ".once /dev/null", - "-cmd", tuning_sql, - "-cmd", '.prompt "GenomicSQLite> "', ] if sys.stdout.isatty() and not next( (arg for arg in argv[2:] if not arg.startswith("-")), False ): - # interactive mode: - cmd += [ - "-cmd", - "SELECT 'GenomicSQLite ' || genomicsqlite_version()", - "-cmd", + # interactive mode: print version banner, then turn on column headers + init_lines += [ + "SELECT 'GenomicSQLite ' || genomicsqlite_version();", ".headers on", ] - cmd.append(":memory:") # placeholder so remaining positional args are recognized as such - cmd += argv[2:] - if "DEBUG" in os.environ: - print( - " ".join( - ( + with tempfile.TemporaryFile("w+") as init_file: + init_file.write("\n".join(init_lines) + "\n") + init_file.flush() + # rewind: on some platforms (notably macOS) reading /dev/fd/N shares this + # descriptor's file offset, so the shell must inherit it positioned at 0 + os.lseek(init_file.fileno(), 0, os.SEEK_SET) + # Python file objects are non-inheritable by default (PEP 446); the + # descriptor must survive the os.execvp() below + os.set_inheritable(init_file.fileno(), True) + + cmd = ["sqlite3", "-bail", "-cmd", f".read /dev/fd/{init_file.fileno()}"] + cmd.append(":memory:") # placeholder so remaining positional args are recognized as such + cmd += argv[2:] + + if "DEBUG" in os.environ: + print( + " ".join( ( - (arg if " " not in arg else f"'{arg}'") - + (" \\\n " if len(arg) > 50 else "") + ( + (arg if " " not in arg else f"'{arg}'") + + (" \\\n " if len(arg) > 50 else "") + ) + for arg in cmd ) - for arg in cmd ) ) - ) - os.execvp("sqlite3", cmd) + os.execvp("sqlite3", cmd) def _compact(dbfilename, argv): diff --git a/bindings/python/release.sh b/bindings/python/release.sh index 2a3eff3..1d1cea9 100755 --- a/bindings/python/release.sh +++ b/bindings/python/release.sh @@ -9,13 +9,13 @@ if grep dirty <(git describe --always --dirty); then >&2 echo "Cannot release dirty working tree" exit 1 fi -if [[ ! -f genomicsqlite/libgenomicsqlite.so ]] || [[ ! -f genomicsqlite/libgenomicsqlite.dylib ]]; then - >&2 echo "Download the portable libgenomicsqlite.{so,dylib} to ${HERE}/genomicsqlite/" +if [[ ! -f genomicsqlite/libgenomicsqlite-linux-x86_64.so ]] || [[ ! -f genomicsqlite/libgenomicsqlite-macos-aarch64.dylib ]]; then + >&2 echo "Download the portable libgenomicsqlite-linux-x86_64.so & libgenomicsqlite-macos-aarch64.dylib to ${HERE}/genomicsqlite/" exit 1 fi function cleanup { # force us to put libs on each attempt; ensures a stale version can't be left behind - rm -f genomicsqlite/libgenomicsqlite.* + rm -f genomicsqlite/libgenomicsqlite* } trap cleanup EXIT diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 13b2af2..b9fee7b 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from os import path -from setuptools import setup, find_packages +from setuptools import setup, find_packages # pylint: disable=import-error from version import get_version HERE = path.abspath(path.dirname(__file__)) @@ -20,7 +20,7 @@ license="Apache", packages=find_packages(), package_data={ - "": ["genomicsqlite/libgenomicsqlite.so", "genomicsqlite/libgenomicsqlite.dylib"] + "": ["genomicsqlite/libgenomicsqlite-*.so", "genomicsqlite/libgenomicsqlite-*.dylib"] }, include_package_data=True, entry_points={"console_scripts": ["genomicsqlite = genomicsqlite:_cli"]}, diff --git a/bindings/python/version.py b/bindings/python/version.py index ddd2328..5b7cfd8 100644 --- a/bindings/python/version.py +++ b/bindings/python/version.py @@ -48,7 +48,6 @@ import subprocess import sys - RELEASE_VERSION_FILE = "RELEASE-VERSION" # http://www.python.org/dev/peps/pep-0386/ diff --git a/bindings/rust/.gitignore b/bindings/rust/.gitignore index 2d7878f..042eab2 100644 --- a/bindings/rust/.gitignore +++ b/bindings/rust/.gitignore @@ -13,3 +13,4 @@ Cargo.lock # Cargo.toml is generated from Cargo.toml.in by ./cargo Cargo.toml libgenomicsqlite.* +libgenomicsqlite-* diff --git a/bindings/rust/Cargo.toml.in b/bindings/rust/Cargo.toml.in index 7813476..d3979a9 100644 --- a/bindings/rust/Cargo.toml.in +++ b/bindings/rust/Cargo.toml.in @@ -7,7 +7,7 @@ edition = "2018" license = "Apache-2.0" repository = "https://github.com/mlin/GenomicSQLite" documentation = "https://mlin.github.io/GenomicSQLite" -include = ["src/**/*", "Cargo.toml", "libgenomicsqlite.*"] +include = ["src/**/*", "Cargo.toml", "build.rs", "libgenomicsqlite.*", "libgenomicsqlite-*"] [features] # Use default-features=false to prevent bundling a platform-specific libgenomicsqlite into your diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs new file mode 100644 index 0000000..0f9efcf --- /dev/null +++ b/bindings/rust/build.rs @@ -0,0 +1,100 @@ +// Build script for the genomicsqlite crate: when the bundle_libgenomicsqlite +// feature is enabled, select the ONE prebuilt libgenomicsqlite matching the +// compilation target (OS + CPU + libc ABI), validate that it's present in the +// package, and hand its file name to src/lib.rs via cargo:rustc-env, where a +// single include_bytes! embeds it. +// +// Supported bundled targets (the Linux libraries link glibc, so the target +// env must be gnu): +// linux + x86_64 + gnu -> libgenomicsqlite-linux-x86_64.so +// linux + aarch64 + gnu -> libgenomicsqlite-linux-aarch64.so +// macos + aarch64 -> libgenomicsqlite-macos-aarch64.dylib +// +// Bundling is a build-time contract: a supported target whose bundled file is +// missing FAILS the build, as does a non-glibc Linux target whose CPU has a +// bundled (glibc) library in the package. Any other target, or building +// without the feature (--no-default-features), leaves +// GENOMICSQLITE_BUNDLED_LIB empty, and the crate loads an external +// libgenomicsqlite at runtime (LIBGENOMICSQLITE / system loader search). +use std::env; +use std::path::Path; + +fn main() { + // Declare the one custom cfg src/lib.rs uses to gate the bundled-bytes + // path, so rustc's check-cfg validation accepts it. + println!("cargo:rustc-check-cfg=cfg(bundled_lib)"); + + let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let bundling = env::var_os("CARGO_FEATURE_BUNDLE_LIBGENOMICSQLITE").is_some(); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + + // The single bundled file for this compilation target, if supported. + let bundled_file: Option<&str> = match ( + target_os.as_str(), + target_arch.as_str(), + target_env.as_str(), + ) { + ("linux", "x86_64", "gnu") => Some("libgenomicsqlite-linux-x86_64.so"), + ("linux", "aarch64", "gnu") => Some("libgenomicsqlite-linux-aarch64.so"), + ("macos", "aarch64", _) => Some("libgenomicsqlite-macos-aarch64.dylib"), + _ => None, + }; + + let mut selected = ""; + if bundling { + match bundled_file { + Some(name) => { + let path = Path::new(&manifest_dir).join(name); + // Rerun this script if the bundled library appears or disappears. + println!("cargo:rerun-if-changed={}", path.display()); + if path.is_file() { + selected = name; + println!("cargo:rustc-cfg=bundled_lib"); + } else { + panic!( + "genomicsqlite: the bundle_libgenomicsqlite feature is enabled \ + but the bundled library {} for target {}-{}-{} is missing \ + from the crate. Either package the prebuilt library under \ + that name, or build with --no-default-features to load an \ + external libgenomicsqlite at runtime instead.", + name, target_os, target_arch, target_env + ); + } + } + None => { + // Incompatible-bundle guard: if this is a Linux target with a + // non-glibc ABI and the package carries the glibc library for + // the same CPU, reject the combination outright -- selecting + // it would produce a broken binary, and silently ignoring it + // would mask the ABI mismatch. + if target_os == "linux" && target_env != "gnu" { + let glibc_twin = match target_arch.as_str() { + "x86_64" => Some("libgenomicsqlite-linux-x86_64.so"), + "aarch64" => Some("libgenomicsqlite-linux-aarch64.so"), + _ => None, + }; + if let Some(twin) = glibc_twin { + let path = Path::new(&manifest_dir).join(twin); + println!("cargo:rerun-if-changed={}", path.display()); + if path.is_file() { + panic!( + "genomicsqlite: the bundled {} is built for glibc \ + (linux-gnu), which is incompatible with this \ + linux-{} compilation target. Build with \ + --no-default-features and provide a \ + libgenomicsqlite built for linux-{} instead \ + (LIBGENOMICSQLITE env var / system loader).", + twin, target_env, target_env + ); + } + } + } + // Any other unsupported target: no bundle; the crate loads an + // external libgenomicsqlite at runtime. + } + } + } + println!("cargo:rustc-env=GENOMICSQLITE_BUNDLED_LIB={}", selected); +} diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 96d41cf..5f96d1a 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -48,33 +48,35 @@ fn extract_libgenomicsqlite(bytes: &[u8], filename: &str) -> (String, TempDir) { } } -#[cfg(not(feature = "bundle_libgenomicsqlite"))] -fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { - None -} - -#[cfg(all( - feature = "bundle_libgenomicsqlite", - target_os = "linux", - target_arch = "x86_64" -))] +/* Bundled libgenomicsqlite selection. + * + * With the bundle_libgenomicsqlite feature, build.rs selects the one bundled + * library matching the compilation target, validates that it's present in + * the package, and names it in GENOMICSQLITE_BUNDLED_LIB (see build.rs for + * the supported-target table and build-time contract); the single + * include_bytes! below embeds exactly that file. With no selection, + * bundle_libgenomicsqlite() returns None and open() falls back to the normal + * library search. + */ + +#[cfg(all(feature = "bundle_libgenomicsqlite", bundled_lib))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite.so"), - &"libgenomicsqlite.so", + include_bytes!(concat!("../", env!("GENOMICSQLITE_BUNDLED_LIB"))), + // The bundled binaries export the standard sqlite3_extension_init + // entry point, which SQLite tries before guessing an entry point + // from the file name -- so the extracted copy loads under its + // architecture-qualified package name as-is. + env!("GENOMICSQLITE_BUNDLED_LIB"), )) } -#[cfg(all( - feature = "bundle_libgenomicsqlite", - target_os = "macos", - target_arch = "x86_64" -))] +// The complement of the bundled build above: the feature is off, or the +// compilation target is not one the release bundles. Fall back to the normal +// library search. +#[cfg(not(all(feature = "bundle_libgenomicsqlite", bundled_lib)))] fn bundle_libgenomicsqlite() -> Option<(String, TempDir)> { - Some(extract_libgenomicsqlite( - include_bytes!("../libgenomicsqlite.dylib"), - "libgenomicsqlite.dylib", - )) + None } // helper for simple queries @@ -112,23 +114,32 @@ pub fn open>(path: P, flags: OpenFlags, config: &Object) -> Resul // once: load libgenomicsqlite extension START.call_once(|| { let mut _tmpdir; - let libgenomicsqlite = match env::var("LIBGENOMICSQLITE") { - Ok(v) => v, + // Decide where to load libgenomicsqlite from, remembering how it was + // obtained, so a load failure can say exactly what was tried. + let (libgenomicsqlite, libgenomicsqlite_source) = match env::var("LIBGENOMICSQLITE") { + Ok(v) => (v, "path from the LIBGENOMICSQLITE environment variable"), Err(_) => match bundle_libgenomicsqlite() { Some((ans, td)) => { _tmpdir = td; - ans + (ans, "copy bundled into the genomicsqlite crate") } - None => "libgenomicsqlite".to_string(), + None => ( + "libgenomicsqlite".to_string(), + "bare library name, to be searched by the system loader \ + (no bundled copy available for this target)", + ), }, }; let memconn = Connection::open_in_memory().unwrap(); unsafe { let _guard = LoadExtensionGuard::new(&memconn).unwrap(); - match memconn.load_extension(libgenomicsqlite.clone(), None) { + match memconn.load_extension(libgenomicsqlite.clone(), None::<&str>) { Err(err) => panic!( - "genomicsqlite failed to load_extension(\"{}\"): {}", - libgenomicsqlite, err + "genomicsqlite: failed to load_extension(\"{}\") [{}]: {}\n\ + genomicsqlite needs a loadable libgenomicsqlite shared library. To provide one:\n\ + - set the LIBGENOMICSQLITE environment variable to the path of a locally built libgenomicsqlite, or\n\ + - install libgenomicsqlite where the system library loader can find it.", + libgenomicsqlite, libgenomicsqlite_source, err ), Ok(()) => (), } diff --git a/docs/index.md b/docs/index.md index ee95659..5e2d785 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,6 +45,8 @@ It's usually easiest to obtain the extension as a pre-compiled shared library (L # -or- conda install -c mlin genomicsqlite ``` + Python's `sqlite3` module must support [loadable extensions](https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension). Some macOS Python builds omit this support; Homebrew Python enables it. This is an interpreter build option, independent of the Python version. + The package loads a bundled shared library by default. To override the bundled file, set environment variable `LIBGENOMICSQLITE` to a filename. === "JVM" diff --git a/loaders/vcf_into_sqlite.cc b/loaders/vcf_into_sqlite.cc index f3b0eb8..359f469 100644 --- a/loaders/vcf_into_sqlite.cc +++ b/loaders/vcf_into_sqlite.cc @@ -67,8 +67,7 @@ string schemaDDL(const string &table_prefix, vector> &info_h } // add SQL comment with original metadata - out << " -- " - << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; + out << " -- " << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; auto desc = hrec.find("Description"); if (desc != hrec.end() && !desc->second.empty()) { out << ",Description=" << desc->second; @@ -109,8 +108,7 @@ string schemaDDL(const string &table_prefix, vector> &info_h } // add SQL comment with original metadata - out << " -- " - << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; + out << " -- " << "Number=" << hrec["Number"] << ",Type=" << hrec["Type"]; auto desc = hrec.find("Description"); if (desc != hrec.end() && !desc->second.empty()) { out << ",Description=" << desc->second; diff --git a/src/genomicsqlite.cc b/src/genomicsqlite.cc index f94c475..63ea5ee 100644 --- a/src/genomicsqlite.cc +++ b/src/genomicsqlite.cc @@ -639,10 +639,10 @@ string GenomicRangeRowidsSQL(sqlite3 *dbconn, const string &indexed_table, const lvq << "\n UNION ALL\n "; } const string &it = indexed_table; - lvq << "SELECT _rowid_ FROM " << it << " INDEXED BY " << table << "__gri WHERE" - << "\n (" << it << "._gri_rid," << it << "._gri_lvl," << it << "._gri_beg) BETWEEN ((" - << qrid << "),-" << lv << ",(" << qbeg << ")-0x1" << string(lv, '0') << ") AND ((" - << qrid << "),-" << lv << ",(" << qend << ")-0)" // (*) + lvq << "SELECT _rowid_ FROM " << it << " INDEXED BY " << table << "__gri WHERE" << "\n (" + << it << "._gri_rid," << it << "._gri_lvl," << it << "._gri_beg) BETWEEN ((" << qrid + << "),-" << lv << ",(" << qbeg << ")-0x1" << string(lv, '0') << ") AND ((" << qrid + << "),-" << lv << ",(" << qend << ")-0)" // (*) << "\n AND (" << it << "._gri_beg+" << it << "._gri_len) >= (" << qbeg << ")"; /* * (*) For some reason, we have to obfuscate qend a little (such as with unary *+*, or by @@ -1023,9 +1023,8 @@ static string gri_refseq_ddl(const string &schema) { out << "CREATE TABLE IF NOT EXISTS " << schema_prefix << "_gri_refseq" << "(_gri_rid INTEGER NOT NULL PRIMARY KEY, gri_refseq_name TEXT NOT NULL, gri_assembly TEXT," << " gri_refget_id TEXT UNIQUE, gri_refseq_length INTEGER NOT NULL, gri_refseq_meta_json TEXT NOT NULL DEFAULT '{}', " - << "UNIQUE(gri_assembly,gri_refseq_name))" - << ";\nCREATE INDEX IF NOT EXISTS " << schema_prefix << "_gri_refseq_name ON " - << "_gri_refseq(gri_refseq_name)"; + << "UNIQUE(gri_assembly,gri_refseq_name))" << ";\nCREATE INDEX IF NOT EXISTS " + << schema_prefix << "_gri_refseq_name ON " << "_gri_refseq(gri_refseq_name)"; return out.str(); } @@ -1864,3 +1863,10 @@ extern "C" int sqlite3_genomicsqlite_init(sqlite3 *db, char **pzErrMsg, } return SQLITE_OK_LOAD_PERMANENTLY; } + +// SQLite checks this standard entry point before deriving one from the filename. +// Keep the named entry point above for callers that already use it explicitly. +extern "C" int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { + return sqlite3_genomicsqlite_init(db, pzErrMsg, pApi); +} diff --git a/test/check_elf_version_floor.py b/test/check_elf_version_floor.py new file mode 100644 index 0000000..04a9e76 --- /dev/null +++ b/test/check_elf_version_floor.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Assert that ELF shared objects' required symbol versions stay at or below the +project's compatibility ceilings -- the manylinux_2_28 floor asserted by CI for +the portable libraries (GLIBC <= 2.28, GLIBCXX <= 3.4.25). + +The check parses the dynamic version needs (the "Version References" of +objdump -p: exactly the (library, version) requirements the dynamic loader +enforces) instead of scraping version strings out of the symbol table. + +Usage: + check_elf_version_floor.py [--glibc X.Y] [--glibcxx X.Y.Z] FILE [FILE...] + +Exit status: 0 if every file is within the ceilings, 1 otherwise (with a +diagnostic naming the file and the offending requirement). +""" + +import argparse +import re +import subprocess +import sys + +NEED_RE = re.compile(r"\b(GLIBC|GLIBCXX)_(\d+(?:\.\d+)*)\s*$") + + +def version_key(text): + return tuple(int(part) for part in text.split(".")) + + +def max_version_needs(path): + """Map each of GLIBC/GLIBCXX to the highest version `path` requires.""" + proc = subprocess.run(["objdump", "-p", path], capture_output=True, text=True, check=True) + needs = {} + in_refs = False + for line in proc.stdout.splitlines(): + if line.startswith("Version References:"): + in_refs = True + continue + if in_refs: + if line and not line[0].isspace(): + break # next top-level section header + match = NEED_RE.search(line) + if match: + lib, ver = match.group(1), match.group(2) + if lib not in needs or version_key(ver) > version_key(needs[lib]): + needs[lib] = ver + return needs + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--glibc", default="2.28", help="GLIBC ceiling [2.28]") + parser.add_argument("--glibcxx", default="3.4.25", help="GLIBCXX ceiling [3.4.25]") + parser.add_argument("files", nargs="+") + args = parser.parse_args() + ceilings = {"GLIBC": args.glibc, "GLIBCXX": args.glibcxx} + + failed = False + for path in args.files: + needs = max_version_needs(path) + summary = ", ".join( + f"max {lib} need {needs.get(lib, 'none')}" for lib in ("GLIBC", "GLIBCXX") + ) + print(f"{path}: {summary}") + for lib, ceiling in ceilings.items(): + if lib in needs and version_key(needs[lib]) > version_key(ceiling): + print( + f" ERROR: {path} requires {lib}_{needs[lib]}, " + f"above the {lib} {ceiling} ceiling", + file=sys.stderr, + ) + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/extension_loading.py b/test/extension_loading.py new file mode 100644 index 0000000..9cd864e --- /dev/null +++ b/test/extension_loading.py @@ -0,0 +1,54 @@ +"""Check that native extension loading is independent of the library filename.""" + +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +from pathlib import Path + + +def main(): + library = Path(sys.argv[1]).resolve() + # Each load runs in a fresh process: GenomicSQLite registers itself for all + # subsequent connections and remains loaded until process exit. + if len(sys.argv) == 3: + con = sqlite3.connect(":memory:") + con.enable_load_extension(True) + if sys.argv[2] == "default": + con.load_extension(str(library)) + else: + con.execute("SELECT load_extension(?,?)", (str(library), sys.argv[2])) + con.close() + con = sqlite3.connect(":memory:") + assert con.execute("SELECT genomicsqlite_version()").fetchone()[0] + return + + with tempfile.TemporaryDirectory() as scratch: + for basename in ( + "libgenomicsqlite", + "libgenomicsqlite-linux-x86_64", + "libgenomicsqlite-macos-aarch64", + "arbitrary name", + ): + renamed = Path(scratch) / (basename + library.suffix) + shutil.copyfile(library, renamed) + for entrypoint in ("default", "sqlite3_genomicsqlite_init"): + subprocess.run([sys.executable, __file__, str(renamed), entrypoint], check=True) + load = f'.load "{renamed}"' + if entrypoint != "default": + load += " " + entrypoint + result = subprocess.run( + ["sqlite3", ":memory:"], + input=load + "\nSELECT genomicsqlite_version();\n", + text=True, + capture_output=True, + check=True, + ) + assert result.stdout.strip(), result.stderr + assert not result.stderr, result.stderr + print(basename, entrypoint, "Python and shell OK") + + +if __name__ == "__main__": + main() diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py index 9264fcf..9baf123 100755 --- a/test/genomicsqlite_smoke_test.py +++ b/test/genomicsqlite_smoke_test.py @@ -3,6 +3,7 @@ Display platform, environment, & version info for debugging GenomicSQLite installation. Load a small test database and check GRI query plan on it. """ + import sys import time import os diff --git a/test/jdbc_smoke_test.sh b/test/jdbc_smoke_test.sh new file mode 100644 index 0000000..696f16a --- /dev/null +++ b/test/jdbc_smoke_test.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# JDBC smoke test: verify the packaged GenomicSQLite JDBC driver end-to-end +# the way an end user experiences it. The driver JAR (built by the preceding +# `mvn deploy` step, with both platform libraries planted in its resources) +# must open a database with LIBGENOMICSQLITE unset -- the CTest JDBC tests +# set that override, which bypasses the driver's bundled-resource selector. +# +# The driver's runtime dependencies (sqlite-jdbc) are NOT inside the JAR and +# NOT in the deploy target repo either -- `mvn deploy` publishes only this +# project's own JAR/POM there. Resolve them here with +# dependency:build-classpath, which emits the complete runtime classpath. +# (An earlier inline version of this step searched target/mvn-repo for the +# sqlite-jdbc JAR, found nothing, and the smoke failed with "No suitable +# driver found".) +# +# $1 JDBC module directory [bindings/jdbc/genomicsqlite-jdbc] +# +# Run from the repository root, after the JAR build (mvn deploy) step. +set -euo pipefail + +MODULE_DIR="${1:-bindings/jdbc/genomicsqlite-jdbc}" +POM="$MODULE_DIR/pom.xml" +[ -f "$POM" ] || { + echo "jdbc_smoke_test: no POM at $POM -- run from the repository root" >&2 + exit 1 +} + +JAR="$(ls "$MODULE_DIR"/target/genomicsqlite-jdbc-*.jar 2>/dev/null | head -n1 || true)" +[ -n "$JAR" ] || { + echo "jdbc_smoke_test: no genomicsqlite-jdbc-*.jar in $MODULE_DIR/target -- did the mvn deploy step run?" >&2 + exit 1 +} + +# The JAR must carry both bundled libraries under their +# architecture-qualified resource names. +for res in libgenomicsqlite-linux-x86_64.so libgenomicsqlite-macos-aarch64.dylib; do + unzip -l "$JAR" | grep -q "$res" || { + echo "jdbc_smoke_test: $JAR lacks bundled resource $res" >&2 + exit 1 + } +done + +WORK="$(mktemp -d)" +CP_FILE="$WORK/classpath.txt" +mvn -B -f "$POM" org.apache.maven.plugins:maven-dependency-plugin:3.11.0:build-classpath \ + -DincludeScope=runtime -Dmdep.outputFile="$CP_FILE" +[ -s "$CP_FILE" ] || { + echo "jdbc_smoke_test: dependency:build-classpath wrote no classpath to $CP_FILE" >&2 + exit 1 +} +DEPS_CP="$(cat "$CP_FILE")" +case "$DEPS_CP" in + *sqlite-jdbc*) ;; + *) + echo "jdbc_smoke_test: runtime classpath lacks sqlite-jdbc: $DEPS_CP" >&2 + exit 1 + ;; +esac + +cat > "$WORK/Smoke.java" <<'EOF' +import java.sql.*; +public class Smoke { + public static void main(String[] args) throws Exception { + Connection conn = DriverManager.getConnection("jdbc:genomicsqlite:" + args[0]); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT genomicsqlite_version()"); + rs.next(); + System.out.println("genomicsqlite_version: " + rs.getString(1)); + st.executeUpdate("CREATE TABLE t(x)"); + st.executeUpdate("INSERT INTO t VALUES('jdbc-ok')"); + conn.close(); + System.out.println("JDBC SMOKE OK"); + } +} +EOF +javac -cp "$JAR" -d "$WORK" "$WORK/Smoke.java" +OUT="$(env -u LIBGENOMICSQLITE java -cp "$JAR:$DEPS_CP:$WORK" Smoke "$WORK/smoke.db")" +echo "$OUT" +grep -q "JDBC SMOKE OK" <<<"$OUT" || { + echo "jdbc_smoke_test: smoke program did not report success" >&2 + exit 1 +} +echo "JDBC SMOKE TEST OK" diff --git a/test/pkg_smoke.sh b/test/pkg_smoke.sh new file mode 100644 index 0000000..30933b1 --- /dev/null +++ b/test/pkg_smoke.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Installed-package smoke test: verify the Python package end-to-end the way +# an end user experiences it. Stages a copy of bindings/python, plants the +# built library under its architecture-qualified name, pip-installs into a +# throwaway target dir, then -- from a neutral working directory, with +# LIBGENOMICSQLITE unset -- imports the package, creates a database through +# the API, and reads it back through the CLI wrapper. +# +# This exists because the CTest suite sets LIBGENOMICSQLITE, which bypasses +# the package's bundled-library selector entirely; only an installed-package +# run exercises the selector, including loading the library from its +# architecture-qualified filename. +# +# $1 python interpreter (pip + setuptools available for it) +# $2 bindings/python source dir +# $3 built library file to plant +# $4 architecture-qualified name to plant it as (e.g. libgenomicsqlite-macos-aarch64.dylib) +# $5 dir containing the sqlite3 shell to use for the CLI check +set -euo pipefail +PY="$1"; PKG_SRC="$2"; LIB="$3"; QNAME="$4"; SHELLDIR="$5" +# Canonicalize $5 to an absolute path BEFORE the cd into the scratch dir +# below: a relative $5 (the Linux call site passes "build") would otherwise +# re-resolve under the scratch dir, silently putting the wrong sqlite3 +# shell on PATH for the CLI check. +SHELLDIR="$(cd "$SHELLDIR" && pwd)" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +SRC="$WORK/pkg-src" +cp -rL "$PKG_SRC" "$SRC" # -L: materialize the sqlite_web_dbi.py symlink +cp "$LIB" "$SRC/genomicsqlite/$QNAME" +# version.py wants git describe (unavailable in the staged copy) or this file +[ -f "$SRC/RELEASE-VERSION" ] || echo "0.0.0" > "$SRC/RELEASE-VERSION" +SITE="$WORK/site" +"$PY" -m pip install --quiet --no-build-isolation --target "$SITE" "$SRC" +echo "installed bundled libraries:"; ls "$SITE/genomicsqlite" | grep libgenomicsqlite + +cd "$WORK" # neutral cwd: the repo tree must not shadow the installed package +DB="$WORK/smoke.db" +env -u LIBGENOMICSQLITE PYTHONPATH="$SITE" "$PY" - "$DB" <<'EOF' +import genomicsqlite, sys +con = genomicsqlite.connect(sys.argv[1]) +print("genomicsqlite_version:", con.execute("SELECT genomicsqlite_version()").fetchone()[0]) +con.execute("CREATE TABLE t(x)") +con.execute("INSERT INTO t VALUES('pkg-ok')") +con.commit(); con.close() +print("PYTHON PACKAGE SMOKE OK") +EOF +OUT="$(env -u LIBGENOMICSQLITE PYTHONPATH="$SITE" PATH="$SHELLDIR:$PATH" \ + "$PY" -m genomicsqlite "$DB" "SELECT 'CLI:' || (SELECT x FROM t)")" +echo "$OUT" +[ "$OUT" = "CLI:pkg-ok" ] || { echo "CLI smoke output mismatch" >&2; exit 1; } +echo "CLI SMOKE OK" diff --git a/test/test_bam.py b/test/test_bam.py index 81f0b99..0748485 100644 --- a/test/test_bam.py +++ b/test/test_bam.py @@ -55,14 +55,10 @@ def test_bam(tmp_path): count = next(con.execute("SELECT COUNT(DISTINCT qname) FROM NA12878_reads_seqs"))[0] assert count == 299205 - mq_hist = dict( - con.execute( - """ + mq_hist = dict(con.execute(""" SELECT mq, COUNT(*) as count FROM (SELECT ifnull(json_extract(tags_json, '$.MQ'),0) AS mq FROM NA12878_reads NATURAL JOIN NA12878_reads_tags WHERE (flag & 3840) = 0) GROUP BY mq ORDER BY mq DESC - """ - ) - ) + """)) assert (mq_hist[0], mq_hist[60]) == (2734, 520522) diff --git a/test/test_ext.py b/test/test_ext.py index 79bc99c..29dcd55 100644 --- a/test/test_ext.py +++ b/test/test_ext.py @@ -4,14 +4,12 @@ def test_bundled_extensions(): con = genomicsqlite.connect(":memory:") - con.executescript( - """ + con.executescript(""" CREATE TABLE test(json TEXT); INSERT INTO test(json) VALUES('{"chrom":"chr10"}'); INSERT INTO test(json) VALUES('{"chrom":"chr2"}'); INSERT INTO test(json) VALUES('{"chrom":"chrMT"}'); - """ - ) + """) results = con.execute( "SELECT json_extract(json,'$.chrom') AS chrom FROM test ORDER BY chrom COLLATE UINT" diff --git a/test/test_gri.py b/test/test_gri.py index d34c777..3ab2cda 100644 --- a/test/test_gri.py +++ b/test/test_gri.py @@ -104,7 +104,7 @@ def test_abutment(): query = genomicsqlite.genomic_range_rowids_sql( con, "features", "42", str(pos0), str(pos0 + 123) )[1:-1] - control_query = f"SELECT _rowid_ FROM features NOT INDEXED WHERE rid = 42 AND NOT (end < {pos0} OR beg > {pos0+123}) ORDER BY _rowid_" + control_query = f"SELECT _rowid_ FROM features NOT INDEXED WHERE rid = 42 AND NOT (end < {pos0} OR beg > {pos0 + 123}) ORDER BY _rowid_" assert list(con.execute(query)) == list(con.execute(control_query)) @@ -148,12 +148,10 @@ def fanout(query): assert fanout(query) == 1 - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(44, 1048568, 1048584); INSERT INTO features VALUES(44, 0, 64000) - """ - ) + """) query = genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1] print("\n" + query) assert "-4," in query and "-0x10000)" in query @@ -168,30 +166,24 @@ def fanout(query): == 4 ) - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(43, NULL, 10000000000); INSERT INTO features VALUES(44, 0, NULL) - """ - ) + """) assert fanout(genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1]) == 4 - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(43, 0, 10000000000); INSERT INTO features VALUES(43, 32, 33) - """ - ) + """) query = genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1] print("\n" + query) assert fanout(query) == 10 - con.executescript( - """ + con.executescript(""" INSERT INTO features VALUES(43, 0, 10000000000); INSERT INTO features VALUES(43, 32, 32) - """ - ) + """) query = genomicsqlite.genomic_range_rowids_sql(con, "features")[1:-1] assert fanout(query) == 10 assert len(list(con.execute(query, (43, 32, 33)))) == 4 diff --git a/test/test_txdb.py b/test/test_txdb.py index 3f7bd83..b96bb52 100644 --- a/test/test_txdb.py +++ b/test/test_txdb.py @@ -12,7 +12,7 @@ def txdb(tmp_path): ans = str(tmp_path / "TxDb.Hsapiens.UCSC.hg38.knownGene.sqlite") subprocess.run( - f"pigz -dc {HERE+'/data/TxDb.Hsapiens.UCSC.hg38.knownGene.sqlite.gz'} > {ans}", + f"pigz -dc {HERE + '/data/TxDb.Hsapiens.UCSC.hg38.knownGene.sqlite.gz'} > {ans}", shell=True, check=True, ) diff --git a/test/test_vcf_lines.py b/test/test_vcf_lines.py index dab3b59..7ebdb91 100644 --- a/test/test_vcf_lines.py +++ b/test/test_vcf_lines.py @@ -4,7 +4,6 @@ import genomicsqlite import pytest - HERE = os.path.dirname(__file__) BUILD = os.path.abspath(os.path.join(HERE, "..", "build"))