Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/nightly-pypi-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
python-version: 3.12

- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0

- name: Set version
id: set-version
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pypi-build-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
3.14

- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0

- name: Set version with RC
shell: bash
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/python-ci-docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
with:
python-version: 3.12
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
- name: Build docs
run: make docs-build
- name: Run linters
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/python-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
with:
python-version: ${{ matrix.python }}
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install system dependencies
Expand Down Expand Up @@ -95,7 +95,7 @@ jobs:
with:
python-version: ${{ matrix.python }}
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install
Expand All @@ -113,7 +113,7 @@ jobs:
with:
python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
# Why this exists:
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/python-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
with:
python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install system dependencies
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
with:
python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install system dependencies
Expand Down Expand Up @@ -111,7 +111,7 @@ jobs:
with:
python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install system dependencies
Expand Down Expand Up @@ -141,7 +141,7 @@ jobs:
with:
python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install system dependencies
Expand Down Expand Up @@ -171,7 +171,7 @@ jobs:
with:
python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
- name: Install dependencies
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/python-release-docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
with:
python-version: 3.12
- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
- name: Build docs
run: make docs-build
- name: Copy
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/python-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ jobs:
python-version: 3.12

- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: false

Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/sidequery-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ on:
version:
description: Distribution version
required: true
default: 0.12.0.post1
default: 0.12.0.post2
type: string

permissions:
Expand All @@ -36,7 +36,7 @@ jobs:
build:
uses: ./.github/workflows/pypi-build-artifacts.yml
with:
VERSION: ${{ inputs.version || '0.12.0.post1' }}
VERSION: ${{ inputs.version || '0.12.0.post2' }}

publish:
needs: build
Expand All @@ -50,7 +50,7 @@ jobs:
- name: Download release artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: pypi-release-candidate-${{ inputs.version || '0.12.0.post1' }}
name: pypi-release-candidate-${{ inputs.version || '0.12.0.post2' }}
path: dist

- name: Publish to PyPI
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/svn-build-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
3.14

- name: Install UV
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0

# Publish the source distribution with the version that's in
# the repository, otherwise the tests will fail
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ test-integration-rebuild: ## Rebuild integration Docker services from scratch
docker compose -f dev/docker-compose-integration.yml build --no-cache

test-s3: ## Run tests marked with @pytest.mark.s3
sh ./dev/run-minio.sh
sh ./dev/run-s3.sh
$(TEST_RUNNER) pytest tests/ -m s3 $(PYTEST_ARGS)

test-adls: ## Run tests marked with @pytest.mark.adls
Expand Down
54 changes: 32 additions & 22 deletions dev/docker-compose-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ services:
depends_on:
- rest
- hive
- minio
- object-store
ports:
- 15002:15002 # Spark Connect
- 4040:4040 # Spark UI
Expand All @@ -39,7 +39,7 @@ services:
links:
- rest:rest
- hive:hive
- minio:minio
- object-store:object-store
healthcheck:
test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"]
interval: 30s
Expand All @@ -59,41 +59,51 @@ services:
- AWS_REGION=us-east-1
- CATALOG_WAREHOUSE=s3://warehouse/
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
- CATALOG_S3_ENDPOINT=http://minio:9000
- CATALOG_S3_ENDPOINT=http://object-store:9000
- CATALOG_JDBC_STRICT__MODE=true
minio:
image: minio/minio
container_name: pyiceberg-minio
object-store:
image: rustfs/rustfs:1.0.0-rc.5
container_name: pyiceberg-object-store
networks:
iceberg_net:
aliases:
- warehouse.minio
- warehouse.object-store
ports:
- 9001:9001
- 9000:9000
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
- MINIO_DOMAIN=minio
command: ["server", "/data", "--console-address", ":9001"]
mc:
image: minio/mc
container_name: pyiceberg-mc
- RUSTFS_ACCESS_KEY=admin
- RUSTFS_SECRET_KEY=password
- RUSTFS_VOLUMES=/data
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CONSOLE_ADDRESS=:9001
# Resolves virtual-hosted-style `warehouse.object-store` to the warehouse
# bucket. The port is required until rustfs/rustfs#7051 ships.
- RUSTFS_SERVER_DOMAINS=object-store:9000
healthcheck:
test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s
aws-cli:
image: amazon/aws-cli:2.36.40
container_name: pyiceberg-aws-cli
networks:
iceberg_net:
depends_on:
- minio
object-store:
condition: service_healthy
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
entrypoint: >
/bin/sh -c "
until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
/usr/bin/mc mb minio/warehouse;
/usr/bin/mc policy set public minio/warehouse;
tail -f /dev/null
"
- AWS_ENDPOINT_URL=http://object-store:9000
entrypoint: /bin/sh
command:
- -c
- "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
hive:
image: pyiceberg-hive:latest
build:
Expand Down
46 changes: 26 additions & 20 deletions dev/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,37 @@
# under the License.

services:
minio:
image: minio/minio
container_name: pyiceberg-minio
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
- MINIO_DOMAIN=minio
object-store:
image: rustfs/rustfs:1.0.0-rc.5
container_name: pyiceberg-object-store
ports:
- 9001:9001
- 9000:9000
command: ["server", "/data", "--console-address", ":9001"]
mc:
environment:
- RUSTFS_ACCESS_KEY=admin
- RUSTFS_SECRET_KEY=password
- RUSTFS_VOLUMES=/data
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CONSOLE_ADDRESS=:9001
healthcheck:
test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s
aws-cli:
image: amazon/aws-cli:2.36.40
container_name: pyiceberg-aws-cli
depends_on:
- minio
image: minio/mc
container_name: pyiceberg-mc
object-store:
condition: service_healthy
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
entrypoint: >
/bin/sh -c "
until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
/usr/bin/mc rm -r --force minio/warehouse;
/usr/bin/mc mb minio/warehouse;
/usr/bin/mc policy set public minio/warehouse;
exit 0;
"
- AWS_ENDPOINT_URL=http://object-store:9000
entrypoint: /bin/sh
command:
- -c
- "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
17 changes: 4 additions & 13 deletions dev/hive/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,10 @@ ARG MAVEN_MIRROR=https://repo1.maven.org/maven2

USER root

# Install curl (separate layer - rarely changes)
RUN apt-get update -qq && \
apt-get -qq -y install --no-install-recommends curl && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# Download JARs with retry logic (slow layer - only changes when versions change)
RUN curl -fsSL --retry 3 --retry-delay 5 \
-o /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar \
"${MAVEN_MIRROR}/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" && \
curl -fsSL --retry 3 --retry-delay 5 \
-o /opt/hive/lib/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar \
"${MAVEN_MIRROR}/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar"
# Download JARs (slow layer - only changes when versions change). ADD fetches them without curl, because the
# image's Debian 11 security repository moved to archive.debian.org and apt-get can no longer install curl.
ADD --chmod=644 ${MAVEN_MIRROR}/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar /opt/hive/lib/
ADD --chmod=644 ${MAVEN_MIRROR}/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar /opt/hive/lib/

# Copy configuration last (changes more frequently than JARs)
COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml
Expand Down
2 changes: 1 addition & 1 deletion dev/hive/core-site.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
</property>
<property>
<name>fs.s3a.endpoint</name>
<value>http://minio:9000</value>
<value>http://object-store:9000</value>
</property>
<property>
<name>fs.s3a.access.key</name>
Expand Down
11 changes: 3 additions & 8 deletions dev/run-minio.sh → dev/run-s3.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,9 @@

set -ex

if [ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]; then
echo "Minio backend running"
if [ $(docker ps -q --filter "name=pyiceberg-object-store" --filter "health=healthy" ) ]; then
echo "S3 backend running"
else
docker compose -f dev/docker-compose.yml kill
docker compose -f dev/docker-compose.yml up -d
while [ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]
do
echo "Waiting for Minio"
sleep 1
done
docker compose -f dev/docker-compose.yml up -d --wait
fi
6 changes: 3 additions & 3 deletions dev/spark/spark-defaults.conf
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest
spark.sql.catalog.rest.uri http://rest:8181
spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO
spark.sql.catalog.rest.warehouse s3://warehouse/rest/
spark.sql.catalog.rest.s3.endpoint http://minio:9000
spark.sql.catalog.rest.s3.endpoint http://object-store:9000
spark.sql.catalog.rest.cache-enabled false

# Configure Iceberg Hive catalog
Expand All @@ -32,13 +32,13 @@ spark.sql.catalog.hive.type hive
spark.sql.catalog.hive.uri thrift://hive:9083
spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO
spark.sql.catalog.hive.warehouse s3://warehouse/hive/
spark.sql.catalog.hive.s3.endpoint http://minio:9000
spark.sql.catalog.hive.s3.endpoint http://object-store:9000

# Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore
spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog
spark.sql.catalog.spark_catalog.type hive
spark.sql.catalog.spark_catalog.uri thrift://hive:9083
spark.hadoop.fs.s3a.endpoint http://minio:9000
spark.hadoop.fs.s3a.endpoint http://object-store:9000
spark.sql.catalogImplementation hive
spark.sql.warehouse.dir s3a://warehouse/hive/

Expand Down
Loading
Loading