From 2b566034e9a2ab9297a3ca6ff7f2a1c688f44e2f Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Wed, 30 Sep 2026 06:18:27 +0000 Subject: [PATCH 01/11] feat: support geospatial results on the kernel path Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CHANGELOG.md | 3 + CONNECTION_PARAMETERS.md | 1 + KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/client.py | 31 +++++++ src/databricks/sql/client.py | 8 ++ src/databricks/sql/session.py | 1 + tests/unit/test_kernel_client.py | 92 ++++++++++++++++++++- tests/unit/test_kernel_result_set.py | 40 ++++++++- tests/unit/test_session.py | 2 + 9 files changed, 177 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bce9e16e4..bf9065694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Release History +# Unreleased +- Add the kernel-only `geospatial_as_string` connection option. GEOMETRY / GEOGRAPHY results are exposed as EWKT strings when true or `{"srid": int, "wkb": bytes}` values when false. + # 4.6.0 (2026-09-24) - Upgrade Databricks SQL Kernel to 1.1.0; the kernel dependency is now stable and no longer experimental. - Transparently auto-recover Thrift connections to Reyden / Real-Time warehouses: when a warehouse rejects the default Thrift protocol (SQLSTATE `KP001`), the session is re-opened on the kernel backend and the warehouse is remembered so later connections skip Thrift. Applies only when no backend was chosen explicitly. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index eaf69c57b..a146fbbc3 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -153,6 +153,7 @@ to change without notice. | `max_download_threads` | `int` | ✅ | ❌ | `10` | Worker threads for cloud-fetch downloads. Not forwarded to the kernel. | | `enable_query_result_lz4_compression` | `bool` | ✅ | ❌ | `True` | LZ4-compress result payloads. Not forwarded; the kernel handles compression internally. | | `_disable_pandas` | `bool` | ✅ | ✅ | `False` | Skip the pandas-based Arrow→row deserialization and materialize rows directly with PyArrow. This is a **Python-side** result-conversion toggle, not a wire option: the kernel returns results as Arrow (`RecordBatch`es) and the connector runs the *same* `_convert_arrow_table` for both backends, so the flag is honored on the kernel path too. Affects only row fetches (`fetchone`/`fetchmany`/`fetchall`); the `fetch*_arrow` methods return the Arrow table unchanged regardless of this flag. | +| `geospatial_as_string` | `bool \| None` | ❌ | ✅ | `None` (kernel default: `True`) | Return GEOMETRY / GEOGRAPHY as EWKT strings when `True`, or as `{"srid": int, "wkb": bytes}` values when `False`. `None` leaves the kernel default in force. This is a local result conversion and is never forwarded to SEA. | | `_use_arrow_native_complex_types` | `bool` | ✅ | ✅ | `True` | Return `ARRAY`/`MAP`/`STRUCT` as native Arrow types instead of JSON strings. Forwarded to the kernel. | | `_use_arrow_native_decimals` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `DECIMAL`: `True` → native Arrow `decimal128`, `False` → Arrow string. **No value-level effect**, though: the connector unconditionally re-casts the column back to `decimal128` (`convert_decimals_in_arrow_table`, `thrift_backend.py`), so both `fetchall()` and `fetchall_arrow()` yield `Decimal` / `decimal128(p,s)` either way (verified live). Not forwarded to the kernel, which always returns native Arrow decimals. | | `_use_arrow_native_timestamps` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `TIMESTAMP`: `True` → native Arrow timestamp (→ Python `datetime`), `False` → Arrow string (→ Python **`str`**). **Unlike decimals there is no re-cast**, so `False` genuinely surfaces strings — and `cursor.description` still reports the type code as `'timestamp'`, a mismatch to watch for (verified live). Note the connector always also sends the `spark.thriftserver.arrowBasedRowSet.timestampAsString=false` conf, but the `timestampAsArrow=False` flag wins. Not forwarded to the kernel, which always returns native Arrow timestamps. | diff --git a/KERNEL_REV b/KERNEL_REV index f22f21685..f6a0e2687 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -80f2aee7d884994d7b0af9a9ea6078872859a9cd +ad3bc6993bca95b810839feade77ccd0ab98ece5 diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index ab1ef0dd7..771fb72ee 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -169,6 +169,29 @@ def _kernel_session_accepts_kwarg(name: str) -> bool: return name in params +def _kernel_geospatial_kwargs(value: Optional[bool]) -> Dict[str, bool]: + """Build the optional geospatial result-representation kwarg. + + ``None`` deliberately omits the option so the installed kernel owns its + default. An explicit value must never be silently ignored: older kernel + wheels do not declare ``geospatial_as_string`` and would otherwise return a + different public value shape than the caller requested. + """ + if value is None: + return {} + if not isinstance(value, bool): + raise ValueError( + "geospatial_as_string must be a bool or None; " + f"got {type(value).__name__}" + ) + if not _kernel_session_accepts_kwarg("geospatial_as_string"): + raise NotSupportedError( + "geospatial_as_string requires a newer databricks-sql-kernel " + "wheel that exposes geospatial result representation support." + ) + return {"geospatial_as_string": value} + + def _kernel_telemetry_kwargs(options: Dict[str, Any]) -> Dict[str, Any]: """Build phase-7 telemetry/system kwargs for ``databricks_sql_kernel.Session``. @@ -262,6 +285,12 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") + # Client-side result representation for GEOMETRY / GEOGRAPHY. None + # leaves the kernel default in force (EWKT strings); False requests the + # canonical Arrow struct and surfaces as ``{"srid": int, "wkb": + # bytes}`` through pyarrow. This is intentionally separate from + # ``session_configuration``: it is never forwarded to SEA. + self._geospatial_as_string = kwargs.get("geospatial_as_string") # Kernel telemetry phase 7 adds binding/runtime identity and # telemetry config kwargs directly to ``databricks_sql_kernel.Session``. self._telemetry_options = kwargs.get("telemetry_options") or {} @@ -379,6 +408,7 @@ def open_session( # kernel's ``retry_*`` kwargs. Empty when at defaults. retry_kwargs = _kernel_retry_kwargs(self._retry_options) telemetry_kwargs = _kernel_telemetry_kwargs(self._telemetry_options) + geospatial_kwargs = _kernel_geospatial_kwargs(self._geospatial_as_string) max_connections_kwargs: Dict[str, Any] = {} if _kernel_session_accepts_kwarg("max_connections"): max_connections_kwargs["max_connections"] = self._max_connections @@ -426,6 +456,7 @@ def open_session( **tls_kwargs, **retry_kwargs, **telemetry_kwargs, + **geospatial_kwargs, **max_connections_kwargs, **http_headers_kwargs, ) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 3e06aa39b..0cb914b2f 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -194,6 +194,14 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + :param geospatial_as_string: `bool | None`, optional (default is None) + Kernel backend only. Controls the public representation of + ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns + EWKT strings (for example ``"SRID=4326;POINT(1 2)"``); + ``False`` returns ``{"srid": int, "wkb": bytes}``; and + ``None`` uses the kernel default (currently EWKT strings). + The conversion is local to the kernel/driver and this option is + never sent to the SQL Execution API. :param use_hybrid_disposition: `bool`, optional (default is False) Use the hybrid disposition instead of the inline disposition. :param server_hostname: Databricks instance host name. diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index db512dd0b..b95768005 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -314,6 +314,7 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, + geospatial_as_string=kwargs.get("geospatial_as_string"), telemetry_options=kernel_telemetry_options, ) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 3eecb0145..d9b04fab6 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -392,6 +392,94 @@ def fake_session(**kw): assert captured["max_connections"] == max_connections +@pytest.mark.parametrize("as_string", [True, False]) +def test_open_session_passes_geospatial_representation_to_kernel( + monkeypatch, as_string +): + captured = {} + + def fake_session(*, geospatial_as_string=None, **kw): + captured["geospatial_as_string"] = geospatial_as_string + sess = MagicMock() + sess.session_id = "sess-id" + return sess + + monkeypatch.setattr(kernel_client._kernel, "Session", fake_session) + c = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + geospatial_as_string=as_string, + ) + + c.open_session(session_configuration=None, catalog=None, schema=None) + + assert captured["geospatial_as_string"] is as_string + + +def test_open_session_omits_unset_geospatial_representation(monkeypatch): + captured = {} + + def fake_session(**kw): + captured.update(kw) + sess = MagicMock() + sess.session_id = "sess-id" + return sess + + monkeypatch.setattr(kernel_client._kernel, "Session", fake_session) + c = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + ) + + c.open_session(session_configuration=None, catalog=None, schema=None) + + assert "geospatial_as_string" not in captured + + +def test_open_session_rejects_explicit_geospatial_option_with_old_kernel( + monkeypatch, +): + def fake_session_without_geospatial( + host, + http_path, + *, + catalog=None, + schema=None, + session_conf=None, + complex_types_as_json=False, + intervals_as_string=False, + request_timeout_secs=None, + auth_type=None, + access_token=None, + ): + sess = MagicMock() + sess.session_id = "sess-id" + return sess + + monkeypatch.setattr( + kernel_client._kernel, "Session", fake_session_without_geospatial + ) + c = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + geospatial_as_string=False, + ) + + with pytest.raises(NotSupportedError, match="newer databricks-sql-kernel"): + c.open_session(session_configuration=None, catalog=None, schema=None) + + +def test_geospatial_option_rejects_non_bool(): + with pytest.raises(ValueError, match="must be a bool or None"): + kernel_client._kernel_geospatial_kwargs("false") + + def test_open_session_passes_phase_7_telemetry_kwargs_to_kernel(monkeypatch): """Kernel telemetry phase 7 added binding/runtime identity and telemetry config kwargs to ``databricks_sql_kernel.Session``.""" @@ -589,7 +677,9 @@ def raise_value_error(_obj): kwargs = kernel_client._kernel_telemetry_kwargs( {"enable_telemetry": True, "telemetry_batch_size": 17} ) - assert kwargs == {}, f"expected no phase-7 kwargs when signature unreadable, got {kwargs}" + assert ( + kwargs == {} + ), f"expected no phase-7 kwargs when signature unreadable, got {kwargs}" def test_execute_command_forwards_parameters_to_bind_param(): diff --git a/tests/unit/test_kernel_result_set.py b/tests/unit/test_kernel_result_set.py index 9ec69380a..81ef6a91b 100644 --- a/tests/unit/test_kernel_result_set.py +++ b/tests/unit/test_kernel_result_set.py @@ -43,11 +43,12 @@ def close(self): self.closed = True -def _make_rs(handle) -> KernelResultSet: +def _make_rs(handle, *, disable_pandas=False) -> KernelResultSet: # The base ResultSet __init__ takes a `connection` ref it never # actually dereferences during these buffer tests, so a Mock is # fine. connection = MagicMock() + connection.disable_pandas = disable_pandas backend = MagicMock() return KernelResultSet( connection=connection, @@ -94,6 +95,43 @@ def test_fetchall_arrow_drains_all_batches(int_schema): assert rs.has_more_rows is False +def test_geospatial_string_and_binary_values_keep_logical_type(): + wkb = bytes.fromhex("0101000000000000000000F03F0000000000000040") + geo_metadata = { + b"databricks.type_name": b"GEOMETRY", + b"databricks.type_text": b"GEOMETRY(ANY)", + } + + string_schema = pa.schema([pa.field("g", pa.string(), metadata=geo_metadata)]) + string_batch = pa.RecordBatch.from_arrays( + [pa.array(["SRID=4326;POINT(1 2)", None], type=pa.string())], + schema=string_schema, + ) + string_rows = _make_rs( + _FakeKernelHandle(string_schema, [string_batch]), disable_pandas=True + ).fetchall() + assert [row[0] for row in string_rows] == ["SRID=4326;POINT(1 2)", None] + + binary_type = pa.struct( + [ + pa.field("srid", pa.int32(), nullable=False), + pa.field("wkb", pa.binary(), nullable=False), + ] + ) + binary_schema = pa.schema([pa.field("g", binary_type, metadata=geo_metadata)]) + binary_batch = pa.RecordBatch.from_arrays( + [pa.array([{"srid": 4326, "wkb": wkb}, None], type=binary_type)], + schema=binary_schema, + ) + binary_rs = _make_rs( + _FakeKernelHandle(binary_schema, [binary_batch]), disable_pandas=True + ) + assert binary_rs.description[0][1] == "geometry" + binary_rows = binary_rs.fetchall() + assert binary_rows[0][0] == {"srid": 4326, "wkb": wkb} + assert binary_rows[1][0] is None + + def test_fetchmany_arrow_slices_within_batch(int_schema): handle = _FakeKernelHandle(int_schema, [_batch(int_schema, [10, 20, 30, 40])]) rs = _make_rs(handle) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 69522bc60..ff090ad5f 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -476,6 +476,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _retry_stop_after_attempts_duration=600.0, _socket_timeout=12.5, _pool_maxsize=41, + geospatial_as_string=False, ) try: _, kwargs = mock_kernel_client.call_args @@ -486,6 +487,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): assert opts["retry_stop_after_attempts_duration"] == 600.0 assert kwargs["request_timeout_secs"] == 12.5 assert kwargs["max_connections"] == 41 + assert kwargs["geospatial_as_string"] is False finally: conn.close() From 57c96fc35b198ceafa68103c6753369744bb1a13 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:29:27 +0000 Subject: [PATCH 02/11] feat: align geospatial support configuration Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 2 +- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/client.py | 40 +++++++++++---------- src/databricks/sql/client.py | 9 +++-- src/databricks/sql/session.py | 4 ++- tests/unit/test_kernel_client.py | 21 +++++------ tests/unit/test_session.py | 16 +++++++-- 8 files changed, 56 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9065694..6bbd37a9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Add the kernel-only `geospatial_as_string` connection option. GEOMETRY / GEOGRAPHY results are exposed as EWKT strings when true or `{"srid": int, "wkb": bytes}` values when false. +- Add the default-enabled, kernel-only `enable_geospatial_support` connection option. GEOMETRY / GEOGRAPHY results are exposed as `{"srid": int, "wkb": bytes}` values when true or WKT / EWKT strings when false. # 4.6.0 (2026-09-24) - Upgrade Databricks SQL Kernel to 1.1.0; the kernel dependency is now stable and no longer experimental. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index a146fbbc3..f9861a7a3 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -153,7 +153,7 @@ to change without notice. | `max_download_threads` | `int` | ✅ | ❌ | `10` | Worker threads for cloud-fetch downloads. Not forwarded to the kernel. | | `enable_query_result_lz4_compression` | `bool` | ✅ | ❌ | `True` | LZ4-compress result payloads. Not forwarded; the kernel handles compression internally. | | `_disable_pandas` | `bool` | ✅ | ✅ | `False` | Skip the pandas-based Arrow→row deserialization and materialize rows directly with PyArrow. This is a **Python-side** result-conversion toggle, not a wire option: the kernel returns results as Arrow (`RecordBatch`es) and the connector runs the *same* `_convert_arrow_table` for both backends, so the flag is honored on the kernel path too. Affects only row fetches (`fetchone`/`fetchmany`/`fetchall`); the `fetch*_arrow` methods return the Arrow table unchanged regardless of this flag. | -| `geospatial_as_string` | `bool \| None` | ❌ | ✅ | `None` (kernel default: `True`) | Return GEOMETRY / GEOGRAPHY as EWKT strings when `True`, or as `{"srid": int, "wkb": bytes}` values when `False`. `None` leaves the kernel default in force. This is a local result conversion and is never forwarded to SEA. | +| `enable_geospatial_support` | `bool` | ❌ | ✅ | `True` | Return GEOMETRY / GEOGRAPHY as `{"srid": int, "wkb": bytes}` values when `True`, or as WKT / EWKT strings when `False`. This is a local result conversion and is never forwarded to SEA. | | `_use_arrow_native_complex_types` | `bool` | ✅ | ✅ | `True` | Return `ARRAY`/`MAP`/`STRUCT` as native Arrow types instead of JSON strings. Forwarded to the kernel. | | `_use_arrow_native_decimals` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `DECIMAL`: `True` → native Arrow `decimal128`, `False` → Arrow string. **No value-level effect**, though: the connector unconditionally re-casts the column back to `decimal128` (`convert_decimals_in_arrow_table`, `thrift_backend.py`), so both `fetchall()` and `fetchall_arrow()` yield `Decimal` / `decimal128(p,s)` either way (verified live). Not forwarded to the kernel, which always returns native Arrow decimals. | | `_use_arrow_native_timestamps` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `TIMESTAMP`: `True` → native Arrow timestamp (→ Python `datetime`), `False` → Arrow string (→ Python **`str`**). **Unlike decimals there is no re-cast**, so `False` genuinely surfaces strings — and `cursor.description` still reports the type code as `'timestamp'`, a mismatch to watch for (verified live). Note the connector always also sends the `spark.thriftserver.arrowBasedRowSet.timestampAsString=false` conf, but the `timestampAsArrow=False` flag wins. Not forwarded to the kernel, which always returns native Arrow timestamps. | diff --git a/KERNEL_REV b/KERNEL_REV index f6a0e2687..a07683fb2 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ad3bc6993bca95b810839feade77ccd0ab98ece5 +b7e9310b27be16a6c42490e58c4b5c4a525320bc diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 771fb72ee..b98543db2 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -169,27 +169,25 @@ def _kernel_session_accepts_kwarg(name: str) -> bool: return name in params -def _kernel_geospatial_kwargs(value: Optional[bool]) -> Dict[str, bool]: - """Build the optional geospatial result-representation kwarg. +def _kernel_geospatial_kwargs(value: bool) -> Dict[str, bool]: + """Build the default-enabled native geospatial-support kwarg. - ``None`` deliberately omits the option so the installed kernel owns its - default. An explicit value must never be silently ignored: older kernel - wheels do not declare ``geospatial_as_string`` and would otherwise return a - different public value shape than the caller requested. + The value is always passed explicitly so the driver and kernel contracts + cannot drift. Older kernel wheels do not declare + ``enable_geospatial_support`` and must fail clearly rather than silently + return a different public value shape. """ - if value is None: - return {} if not isinstance(value, bool): raise ValueError( - "geospatial_as_string must be a bool or None; " + "enable_geospatial_support must be a bool; " f"got {type(value).__name__}" ) - if not _kernel_session_accepts_kwarg("geospatial_as_string"): + if not _kernel_session_accepts_kwarg("enable_geospatial_support"): raise NotSupportedError( - "geospatial_as_string requires a newer databricks-sql-kernel " + "enable_geospatial_support requires a newer databricks-sql-kernel " "wheel that exposes geospatial result representation support." ) - return {"geospatial_as_string": value} + return {"enable_geospatial_support": value} def _kernel_telemetry_kwargs(options: Dict[str, Any]) -> Dict[str, Any]: @@ -285,12 +283,14 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") - # Client-side result representation for GEOMETRY / GEOGRAPHY. None - # leaves the kernel default in force (EWKT strings); False requests the - # canonical Arrow struct and surfaces as ``{"srid": int, "wkb": - # bytes}`` through pyarrow. This is intentionally separate from - # ``session_configuration``: it is never forwarded to SEA. - self._geospatial_as_string = kwargs.get("geospatial_as_string") + # Default-enabled native GEOMETRY / GEOGRAPHY support. True requests + # the canonical Arrow struct and surfaces as ``{"srid": int, "wkb": + # bytes}`` through pyarrow; False requests WKT / EWKT strings. This is + # intentionally separate from ``session_configuration``: it is never + # forwarded to SEA. + self._enable_geospatial_support = kwargs.get( + "enable_geospatial_support", True + ) # Kernel telemetry phase 7 adds binding/runtime identity and # telemetry config kwargs directly to ``databricks_sql_kernel.Session``. self._telemetry_options = kwargs.get("telemetry_options") or {} @@ -408,7 +408,9 @@ def open_session( # kernel's ``retry_*`` kwargs. Empty when at defaults. retry_kwargs = _kernel_retry_kwargs(self._retry_options) telemetry_kwargs = _kernel_telemetry_kwargs(self._telemetry_options) - geospatial_kwargs = _kernel_geospatial_kwargs(self._geospatial_as_string) + geospatial_kwargs = _kernel_geospatial_kwargs( + self._enable_geospatial_support + ) max_connections_kwargs: Dict[str, Any] = {} if _kernel_session_accepts_kwarg("max_connections"): max_connections_kwargs["max_connections"] = self._max_connections diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 0cb914b2f..d5b63e9e4 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -194,13 +194,12 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. - :param geospatial_as_string: `bool | None`, optional (default is None) + :param enable_geospatial_support: `bool`, optional (default is True) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns - EWKT strings (for example ``"SRID=4326;POINT(1 2)"``); - ``False`` returns ``{"srid": int, "wkb": bytes}``; and - ``None`` uses the kernel default (currently EWKT strings). - The conversion is local to the kernel/driver and this option is + ``{"srid": int, "wkb": bytes}``; ``False`` returns WKT / EWKT + strings (for example ``"SRID=4326;POINT(1 2)"``). The + conversion is local to the kernel/driver and this option is never sent to the SQL Execution API. :param use_hybrid_disposition: `bool`, optional (default is False) Use the hybrid disposition instead of the inline disposition. diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index b95768005..262a8eb02 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -314,7 +314,9 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, - geospatial_as_string=kwargs.get("geospatial_as_string"), + enable_geospatial_support=kwargs.get( + "enable_geospatial_support", True + ), telemetry_options=kernel_telemetry_options, ) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index d9b04fab6..553563061 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -392,14 +392,14 @@ def fake_session(**kw): assert captured["max_connections"] == max_connections -@pytest.mark.parametrize("as_string", [True, False]) +@pytest.mark.parametrize("enabled", [True, False]) def test_open_session_passes_geospatial_representation_to_kernel( - monkeypatch, as_string + monkeypatch, enabled ): captured = {} - def fake_session(*, geospatial_as_string=None, **kw): - captured["geospatial_as_string"] = geospatial_as_string + def fake_session(*, enable_geospatial_support=True, **kw): + captured["enable_geospatial_support"] = enable_geospatial_support sess = MagicMock() sess.session_id = "sess-id" return sess @@ -410,15 +410,15 @@ def fake_session(*, geospatial_as_string=None, **kw): http_path="/sql/1.0/warehouses/abc", auth_provider=AccessTokenAuthProvider("dapi-test"), ssl_options=None, - geospatial_as_string=as_string, + enable_geospatial_support=enabled, ) c.open_session(session_configuration=None, catalog=None, schema=None) - assert captured["geospatial_as_string"] is as_string + assert captured["enable_geospatial_support"] is enabled -def test_open_session_omits_unset_geospatial_representation(monkeypatch): +def test_open_session_enables_geospatial_support_by_default(monkeypatch): captured = {} def fake_session(**kw): @@ -437,7 +437,7 @@ def fake_session(**kw): c.open_session(session_configuration=None, catalog=None, schema=None) - assert "geospatial_as_string" not in captured + assert captured["enable_geospatial_support"] is True def test_open_session_rejects_explicit_geospatial_option_with_old_kernel( @@ -468,7 +468,7 @@ def fake_session_without_geospatial( http_path="/sql/1.0/warehouses/abc", auth_provider=AccessTokenAuthProvider("dapi-test"), ssl_options=None, - geospatial_as_string=False, + enable_geospatial_support=False, ) with pytest.raises(NotSupportedError, match="newer databricks-sql-kernel"): @@ -476,7 +476,7 @@ def fake_session_without_geospatial( def test_geospatial_option_rejects_non_bool(): - with pytest.raises(ValueError, match="must be a bool or None"): + with pytest.raises(ValueError, match="must be a bool"): kernel_client._kernel_geospatial_kwargs("false") @@ -578,6 +578,7 @@ def fake_session_without_optional_kwargs( catalog=None, schema=None, session_conf=None, + enable_geospatial_support=True, complex_types_as_json=False, intervals_as_string=False, request_timeout_secs=None, diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index ff090ad5f..36249706c 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -476,7 +476,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _retry_stop_after_attempts_duration=600.0, _socket_timeout=12.5, _pool_maxsize=41, - geospatial_as_string=False, + enable_geospatial_support=False, ) try: _, kwargs = mock_kernel_client.call_args @@ -487,7 +487,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): assert opts["retry_stop_after_attempts_duration"] == 600.0 assert kwargs["request_timeout_secs"] == 12.5 assert kwargs["max_connections"] == 41 - assert kwargs["geospatial_as_string"] is False + assert kwargs["enable_geospatial_support"] is False finally: conn.close() @@ -793,6 +793,18 @@ def test_connect_use_kernel_instantiates_real_kernel_backend(self): finally: conn.close() + def test_enable_geospatial_support_matches_real_kernel_signature(self): + self._real_kernel_or_skip() + + from databricks.sql.backend.kernel.client import _kernel_geospatial_kwargs + + assert _kernel_geospatial_kwargs(True) == { + "enable_geospatial_support": True + } + assert _kernel_geospatial_kwargs(False) == { + "enable_geospatial_support": False + } + class TestReydenThriftFallback: """Transparent auto-recovery from a Reyden / Real-Time warehouse rejecting From 98141fd9e01b39dd1ac48815e3f12e4f5ac9b5dc Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:46:50 +0000 Subject: [PATCH 03/11] test: handle pending kernel geospatial release Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/backend/kernel/client.py | 7 ++---- src/databricks/sql/session.py | 4 +--- tests/unit/test_session.py | 25 ++++++++++++++------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index b98543db2..607f81519 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -179,8 +179,7 @@ def _kernel_geospatial_kwargs(value: bool) -> Dict[str, bool]: """ if not isinstance(value, bool): raise ValueError( - "enable_geospatial_support must be a bool; " - f"got {type(value).__name__}" + "enable_geospatial_support must be a bool; " f"got {type(value).__name__}" ) if not _kernel_session_accepts_kwarg("enable_geospatial_support"): raise NotSupportedError( @@ -288,9 +287,7 @@ def __init__( # bytes}`` through pyarrow; False requests WKT / EWKT strings. This is # intentionally separate from ``session_configuration``: it is never # forwarded to SEA. - self._enable_geospatial_support = kwargs.get( - "enable_geospatial_support", True - ) + self._enable_geospatial_support = kwargs.get("enable_geospatial_support", True) # Kernel telemetry phase 7 adds binding/runtime identity and # telemetry config kwargs directly to ``databricks_sql_kernel.Session``. self._telemetry_options = kwargs.get("telemetry_options") or {} diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 262a8eb02..2aca3244c 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -314,9 +314,7 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, - enable_geospatial_support=kwargs.get( - "enable_geospatial_support", True - ), + enable_geospatial_support=kwargs.get("enable_geospatial_support", True), telemetry_options=kernel_telemetry_options, ) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 36249706c..cab7a85b4 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -796,14 +796,23 @@ def test_connect_use_kernel_instantiates_real_kernel_backend(self): def test_enable_geospatial_support_matches_real_kernel_signature(self): self._real_kernel_or_skip() - from databricks.sql.backend.kernel.client import _kernel_geospatial_kwargs - - assert _kernel_geospatial_kwargs(True) == { - "enable_geospatial_support": True - } - assert _kernel_geospatial_kwargs(False) == { - "enable_geospatial_support": False - } + from databricks.sql.backend.kernel.client import ( + _kernel_geospatial_kwargs, + _kernel_session_accepts_kwarg, + ) + + # The ordinary kernel unit-test tier installs the latest published + # wheel, which may lag the KERNEL_REV source pin while the matching + # kernel change is still in flight. Validate the compatibility error + # in that case; once the wheel includes the typed option, validate + # both values against its real PyO3 signature. + if not _kernel_session_accepts_kwarg("enable_geospatial_support"): + with pytest.raises(NotSupportedError, match="newer databricks-sql-kernel"): + _kernel_geospatial_kwargs(True) + return + + assert _kernel_geospatial_kwargs(True) == {"enable_geospatial_support": True} + assert _kernel_geospatial_kwargs(False) == {"enable_geospatial_support": False} class TestReydenThriftFallback: From 8ac910d8fd195af9d524a91397565528652f58a2 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:52:18 +0000 Subject: [PATCH 04/11] test: import kernel compatibility error Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- tests/unit/test_session.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index cab7a85b4..09ed0f70b 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -800,6 +800,7 @@ def test_enable_geospatial_support_matches_real_kernel_signature(self): _kernel_geospatial_kwargs, _kernel_session_accepts_kwarg, ) + from databricks.sql.exc import NotSupportedError # The ordinary kernel unit-test tier installs the latest published # wheel, which may lag the KERNEL_REV source pin while the matching From 76bb73a117fd470468f6052771c0bad3da9e6907 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:10:52 +0000 Subject: [PATCH 05/11] fix(thrift): report geospatial cursor metadata Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/backend/thrift_backend.py | 15 ++++--- tests/unit/test_thrift_backend.py | 42 +++++++++++++++++++- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/databricks/sql/backend/thrift_backend.py b/src/databricks/sql/backend/thrift_backend.py index fcf363cf0..4cf929d62 100644 --- a/src/databricks/sql/backend/thrift_backend.py +++ b/src/databricks/sql/backend/thrift_backend.py @@ -3,6 +3,7 @@ import errno import logging import math +import re import time import threading from typing import Dict, List, Optional, Union, Any, TYPE_CHECKING @@ -788,16 +789,20 @@ def _col_to_description(col, field=None, host_url=None): else: precision, scale = None, None - # Extract variant type from field if available + # Extract logical types that Thrift surfaces as strings from Arrow metadata. if field is not None: try: - # Check for variant type in metadata if field.metadata and b"Spark:DataType:SqlName" in field.metadata: sql_type = field.metadata.get(b"Spark:DataType:SqlName") - if sql_type == b"VARIANT": - cleaned_type = "variant" + logical_type = re.fullmatch( + rb"(variant|geometry|geography)(?:\([^()]*\))?", + sql_type.strip(), + re.IGNORECASE, + ) + if logical_type: + cleaned_type = logical_type.group(1).decode("ascii").lower() except Exception as e: - logger.debug(f"Could not extract variant type from field: {e}") + logger.debug(f"Could not extract logical type from field: {e}") return col.columnName, cleaned_type, None, None, precision, scale, None diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py index ddcb378ff..d90db9309 100644 --- a/tests/unit/test_thrift_backend.py +++ b/tests/unit/test_thrift_backend.py @@ -2534,12 +2534,37 @@ def test_execute_command_sets_complex_type_fields_correctly( def test_col_to_description(self): test_cases = [ ("variant_col", {b"Spark:DataType:SqlName": b"VARIANT"}, "variant"), + ( + "geometry_col", + {b"Spark:DataType:SqlName": b"GEOMETRY(0)"}, + "geometry", + ), + ( + "geography_col", + {b"Spark:DataType:SqlName": b"GEOGRAPHY(4326)"}, + "geography", + ), + ( + "geometry_any_col", + {b"Spark:DataType:SqlName": b"geometry(ANY)"}, + "geometry", + ), + ( + "unqualified_geography_col", + {b"Spark:DataType:SqlName": b"GEOGRAPHY"}, + "geography", + ), ("normal_col", {}, "string"), ( "weird_field", {b"Spark:DataType:SqlName": b"Some unexpected value"}, "string", ), + ( + "malformed_geometry_col", + {b"Spark:DataType:SqlName": b"GEOMETRY("}, + "string", + ), ("missing_field", None, "string"), # None field case ] @@ -2575,12 +2600,27 @@ def test_hive_schema_to_description(self): [ ("regular_col", ttypes.TTypeId.STRING_TYPE), ("variant_col", ttypes.TTypeId.STRING_TYPE), + ("geometry_col", ttypes.TTypeId.STRING_TYPE), + ("geography_col", ttypes.TTypeId.STRING_TYPE), ], [ ("regular_col", {}), ("variant_col", {b"Spark:DataType:SqlName": b"VARIANT"}), + ( + "geometry_col", + {b"Spark:DataType:SqlName": b"GEOMETRY(0)"}, + ), + ( + "geography_col", + {b"Spark:DataType:SqlName": b"GEOGRAPHY(4326)"}, + ), + ], + [ + ("regular_col", "string"), + ("variant_col", "variant"), + ("geometry_col", "geometry"), + ("geography_col", "geography"), ], - [("regular_col", "string"), ("variant_col", "variant")], ), ( [("regular_col", ttypes.TTypeId.STRING_TYPE)], From 8e7680908b0e752719c2acb8b2912aaa674a54ce Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:57:59 +0000 Subject: [PATCH 06/11] fix: preserve geospatial result metadata and shapes Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/backend/thrift_backend.py | 16 ++++++++++++---- tests/unit/test_kernel_result_set.py | 12 +++++++++--- tests/unit/test_thrift_backend.py | 11 +++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/databricks/sql/backend/thrift_backend.py b/src/databricks/sql/backend/thrift_backend.py index 4cf929d62..84e8ee210 100644 --- a/src/databricks/sql/backend/thrift_backend.py +++ b/src/databricks/sql/backend/thrift_backend.py @@ -808,12 +808,16 @@ def _col_to_description(col, field=None, host_url=None): @staticmethod def _hive_schema_to_description(t_table_schema, schema_bytes=None, host_url=None): + arrow_fields = [] field_dict = {} if pyarrow and schema_bytes: try: arrow_schema = pyarrow.ipc.read_schema(pyarrow.py_buffer(schema_bytes)) - # Build a dictionary mapping column names to fields - for field in arrow_schema: + arrow_fields = list(arrow_schema) + # Retain a name fallback for malformed/partial schemas, but use + # result-column position whenever it is available. SQL permits + # duplicate aliases, for which a name lookup is ambiguous. + for field in arrow_fields: field_dict[field.name] = field except Exception as e: logger.debug(f"Could not parse arrow schema: {e}") @@ -821,10 +825,14 @@ def _hive_schema_to_description(t_table_schema, schema_bytes=None, host_url=None return [ ThriftDatabricksClient._col_to_description( col, - field_dict.get(col.columnName) if field_dict else None, + ( + arrow_fields[index] + if index < len(arrow_fields) + else field_dict.get(col.columnName) + ), host_url, ) - for col in t_table_schema.columns + for index, col in enumerate(t_table_schema.columns) ] def _results_message_to_execute_response( diff --git a/tests/unit/test_kernel_result_set.py b/tests/unit/test_kernel_result_set.py index 81ef6a91b..cfa2d3f2e 100644 --- a/tests/unit/test_kernel_result_set.py +++ b/tests/unit/test_kernel_result_set.py @@ -95,7 +95,8 @@ def test_fetchall_arrow_drains_all_batches(int_schema): assert rs.has_more_rows is False -def test_geospatial_string_and_binary_values_keep_logical_type(): +@pytest.mark.parametrize("disable_pandas", [False, True]) +def test_geospatial_string_and_binary_values_keep_logical_type(disable_pandas): wkb = bytes.fromhex("0101000000000000000000F03F0000000000000040") geo_metadata = { b"databricks.type_name": b"GEOMETRY", @@ -108,7 +109,8 @@ def test_geospatial_string_and_binary_values_keep_logical_type(): schema=string_schema, ) string_rows = _make_rs( - _FakeKernelHandle(string_schema, [string_batch]), disable_pandas=True + _FakeKernelHandle(string_schema, [string_batch]), + disable_pandas=disable_pandas, ).fetchall() assert [row[0] for row in string_rows] == ["SRID=4326;POINT(1 2)", None] @@ -124,10 +126,14 @@ def test_geospatial_string_and_binary_values_keep_logical_type(): schema=binary_schema, ) binary_rs = _make_rs( - _FakeKernelHandle(binary_schema, [binary_batch]), disable_pandas=True + _FakeKernelHandle(binary_schema, [binary_batch]), + disable_pandas=disable_pandas, ) assert binary_rs.description[0][1] == "geometry" binary_rows = binary_rs.fetchall() + assert isinstance(binary_rows[0][0], dict) + assert isinstance(binary_rows[0][0]["srid"], int) + assert isinstance(binary_rows[0][0]["wkb"], bytes) assert binary_rows[0][0] == {"srid": 4326, "wkb": wkb} assert binary_rows[1][0] is None diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py index d90db9309..6583c94fe 100644 --- a/tests/unit/test_thrift_backend.py +++ b/tests/unit/test_thrift_backend.py @@ -2627,6 +2627,17 @@ def test_hive_schema_to_description(self): None, # No arrow schema [("regular_col", "string")], ), + ( + [ + ("g", ttypes.TTypeId.STRING_TYPE), + ("g", ttypes.TTypeId.STRING_TYPE), + ], + [ + ("g", {}), + ("g", {b"Spark:DataType:SqlName": b"GEOMETRY(0)"}), + ], + [("g", "string"), ("g", "geometry")], + ), ] for columns, arrow_fields, expected_types in test_cases: From 54fbd76cbc52c7969bfb452322f9797b85d6b888 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:04:37 +0000 Subject: [PATCH 07/11] chore: update geospatial kernel pin Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a07683fb2..0c44de9b0 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -b7e9310b27be16a6c42490e58c4b5c4a525320bc +9c2e2e6182fc7285bcfff39b2177941bc804897c From 66a5f73794ef932d047e726d2fd1381e653ecfb1 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:33:50 +0000 Subject: [PATCH 08/11] fix: default Python geospatial results to strings Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CHANGELOG.md | 2 +- src/databricks/sql/backend/kernel/client.py | 6 ++-- src/databricks/sql/client.py | 2 +- src/databricks/sql/session.py | 4 ++- tests/unit/test_kernel_client.py | 4 +-- tests/unit/test_session.py | 36 +++++++++++++++++++++ 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2458d426..85ea1a2f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Add the default-enabled, kernel-only `enable_geospatial_support` connection option. GEOMETRY / GEOGRAPHY results are exposed as `{"srid": int, "wkb": bytes}` values when true or WKT / EWKT strings when false. +- Add the kernel-only `enable_geospatial_support` connection option (default `False`). GEOMETRY / GEOGRAPHY results are exposed as `{"srid": int, "wkb": bytes}` values when true or WKT / EWKT strings when false. - Add the `escape_catalog_name` connection parameter (default `False`) to opt in to escaping unescaped `_` in metadata catalog names on Thrift and kernel connections. Existing escapes and `%` wildcards are preserved. # 4.6.0 (2026-09-24) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index f70e2811e..32ef2b566 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -171,7 +171,7 @@ def _kernel_session_accepts_kwarg(name: str) -> bool: def _kernel_geospatial_kwargs(value: bool) -> Dict[str, bool]: - """Build the default-enabled native geospatial-support kwarg. + """Build the explicit native geospatial-support kwarg. The value is always passed explicitly so the driver and kernel contracts cannot drift. Older kernel wheels do not declare @@ -283,12 +283,12 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") - # Default-enabled native GEOMETRY / GEOGRAPHY support. True requests + # Opt-in native GEOMETRY / GEOGRAPHY support. True requests # the canonical Arrow struct and surfaces as ``{"srid": int, "wkb": # bytes}`` through pyarrow; False requests WKT / EWKT strings. This is # intentionally separate from ``session_configuration``: it is never # forwarded to SEA. - self._enable_geospatial_support = kwargs.get("enable_geospatial_support", True) + self._enable_geospatial_support = kwargs.get("enable_geospatial_support", False) # Kernel telemetry phase 7 adds binding/runtime identity and # telemetry config kwargs directly to ``databricks_sql_kernel.Session``. self._telemetry_options = kwargs.get("telemetry_options") or {} diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 6d95941cb..940b12775 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -194,7 +194,7 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. - :param enable_geospatial_support: `bool`, optional (default is True) + :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns ``{"srid": int, "wkb": bytes}``; ``False`` returns WKT / EWKT diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 3a6a0809e..365b06f8f 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -332,7 +332,9 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, - enable_geospatial_support=kwargs.get("enable_geospatial_support", True), + enable_geospatial_support=kwargs.get( + "enable_geospatial_support", False + ), telemetry_options=kernel_telemetry_options, escape_catalog_name=kwargs.get("escape_catalog_name", False), ) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index b68218fb3..88547afe5 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -418,7 +418,7 @@ def fake_session(*, enable_geospatial_support=True, **kw): assert captured["enable_geospatial_support"] is enabled -def test_open_session_enables_geospatial_support_by_default(monkeypatch): +def test_open_session_disables_native_geospatial_support_by_default(monkeypatch): captured = {} def fake_session(**kw): @@ -437,7 +437,7 @@ def fake_session(**kw): c.open_session(session_configuration=None, catalog=None, schema=None) - assert captured["enable_geospatial_support"] is True + assert captured["enable_geospatial_support"] is False def test_open_session_rejects_explicit_geospatial_option_with_old_kernel( diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index c5e87a2c8..a9f774208 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -877,6 +877,42 @@ def _connect(self, **overrides): args.update(overrides) return databricks.sql.connect(**args) + @pytest.mark.parametrize("route", ["explicit", "recovery", "cached"]) + @pytest.mark.parametrize( + "geospatial_kwargs, expected", + [ + pytest.param({}, False, id="default"), + pytest.param({"enable_geospatial_support": False}, False, id="disabled"), + pytest.param({"enable_geospatial_support": True}, True, id="enabled"), + ], + ) + @patch("%s.session.ThriftDatabricksClient" % PACKAGE) + def test_kernel_routing_preserves_geospatial_representation( + self, mock_thrift, route, geospatial_kwargs, expected + ): + from databricks.sql.backend import reyden_warehouse_cache + + mock_thrift.return_value.open_session.side_effect = self._reject() + if route == "cached": + reyden_warehouse_cache.mark_reyden(self.HOST, "wh-reyden") + if route == "explicit": + geospatial_kwargs = {**geospatial_kwargs, "use_kernel": True} + + with self._fake_kernel() as mock_kernel: + conn = self._connect(**geospatial_kwargs) + try: + assert conn.session.use_kernel is True + assert ( + mock_kernel.call_args.kwargs["enable_geospatial_support"] + is expected + ) + if route == "recovery": + mock_thrift.return_value.open_session.assert_called_once() + else: + mock_thrift.return_value.open_session.assert_not_called() + finally: + conn.close() + @patch("%s.session.ThriftDatabricksClient" % PACKAGE) def test_thrift_rejection_recovers_onto_kernel(self, mock_thrift): mock_thrift.return_value.open_session.side_effect = self._reject() From dd8a639b5402abb8eb5b142bbad6b374d6084da5 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:50:47 +0000 Subject: [PATCH 09/11] docs: correct geospatial support default Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CONNECTION_PARAMETERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 81efde93d..37736ed1a 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -153,7 +153,7 @@ to change without notice. | `max_download_threads` | `int` | ✅ | ❌ | `10` | Worker threads for cloud-fetch downloads. Not forwarded to the kernel. | | `enable_query_result_lz4_compression` | `bool` | ✅ | ❌ | `True` | LZ4-compress result payloads. Not forwarded; the kernel handles compression internally. | | `_disable_pandas` | `bool` | ✅ | ✅ | `False` | Skip the pandas-based Arrow→row deserialization and materialize rows directly with PyArrow. This is a **Python-side** result-conversion toggle, not a wire option: the kernel returns results as Arrow (`RecordBatch`es) and the connector runs the *same* `_convert_arrow_table` for both backends, so the flag is honored on the kernel path too. Affects only row fetches (`fetchone`/`fetchmany`/`fetchall`); the `fetch*_arrow` methods return the Arrow table unchanged regardless of this flag. | -| `enable_geospatial_support` | `bool` | ❌ | ✅ | `True` | Return GEOMETRY / GEOGRAPHY as `{"srid": int, "wkb": bytes}` values when `True`, or as WKT / EWKT strings when `False`. This is a local result conversion and is never forwarded to SEA. | +| `enable_geospatial_support` | `bool` | ❌ | ✅ | `False` | Return GEOMETRY / GEOGRAPHY as `{"srid": int, "wkb": bytes}` values when `True`, or as WKT / EWKT strings when `False`. This is a local result conversion and is never forwarded to SEA. | | `_use_arrow_native_complex_types` | `bool` | ✅ | ✅ | `True` | Return `ARRAY`/`MAP`/`STRUCT` as native Arrow types instead of JSON strings. Forwarded to the kernel. | | `_use_arrow_native_decimals` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `DECIMAL`: `True` → native Arrow `decimal128`, `False` → Arrow string. **No value-level effect**, though: the connector unconditionally re-casts the column back to `decimal128` (`convert_decimals_in_arrow_table`, `thrift_backend.py`), so both `fetchall()` and `fetchall_arrow()` yield `Decimal` / `decimal128(p,s)` either way (verified live). Not forwarded to the kernel, which always returns native Arrow decimals. | | `_use_arrow_native_timestamps` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `TIMESTAMP`: `True` → native Arrow timestamp (→ Python `datetime`), `False` → Arrow string (→ Python **`str`**). **Unlike decimals there is no re-cast**, so `False` genuinely surfaces strings — and `cursor.description` still reports the type code as `'timestamp'`, a mismatch to watch for (verified live). Note the connector always also sends the `spark.thriftserver.arrowBasedRowSet.timestampAsString=false` conf, but the `timestampAsArrow=False` flag wins. Not forwarded to the kernel, which always returns native Arrow timestamps. | From 3c800a507b23817cc764aa7c87964dda5a5e71e1 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:22:38 +0000 Subject: [PATCH 10/11] test: match kernel table-type filters by exact case Align Kernel E2E with case-sensitive filtering from databricks-sql-kernel#393. Check uppercase matches and lowercase/mixed-case empty results, and update stale filter comments. Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/backend/kernel/client.py | 8 +++---- tests/e2e/test_kernel_backend.py | 26 ++++++++++++++------- tests/unit/test_kernel_client.py | 2 +- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 32ef2b566..c53fcde5a 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -1045,11 +1045,9 @@ def get_tables( if self._kernel_session is None: raise InterfaceError("get_tables requires an open session.") try: - # ``table_types`` is filtered kernel-side (the kernel applies - # it to the reshaped result, case-insensitively as of the - # batch-3 kernel change), so we forward it and let the kernel - # do the work — no connector-side drain + refilter. Passing it - # through preserves streaming for large schemas. + # The kernel filters ``table_types`` with case-sensitive + # matching for Thrift parity. Forward the values unchanged + # to preserve those semantics and streaming for large schemas. stream = self._kernel_session.metadata().list_tables( catalog=escape_metadata_catalog_name( _catalog_or_none(catalog_name), self._escape_catalog_name diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index 0b767e1c3..ae0301fb6 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -383,12 +383,12 @@ def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): assert cur.fetchall() == [] -def test_tables_table_types_filter_is_case_insensitive(conn): - """Lowercase ``table_types=['view']`` / uppercase ``['TABLE']`` - each match the right object regardless of case — locks the - kernel-side case-insensitive ``table_types`` match (batch-3 kernel - B2) end-to-end plus the connector drain removal (the filter now - runs kernel-side, not client-side). +def test_tables_table_types_filter_is_case_sensitive(conn): + """Table types match exact case, consistent with Thrift. + + Uppercase ``['VIEW']`` / ``['TABLE']`` match the corresponding + object; lowercase and mixed-case values match no objects. The + kernel applies the filter to values forwarded by the connector. Self-contained: creates a table + a view over it in the session's default (writable) schema, scopes the lookup to a unique name @@ -411,12 +411,12 @@ def _names_and_types(): ni, ti = cols.index("TABLE_NAME"), cols.index("TABLE_TYPE") return {(r[ni], r[ti]) for r in rows} - # Lowercase 'view' must match the VIEW (and only it). + # Uppercase 'VIEW' must match the VIEW (and only it). cur.tables( catalog_name=cat, schema_name=sch, table_name=name_pat, - table_types=["view"], + table_types=["VIEW"], ) assert _names_and_types() == {(vw, "VIEW")} @@ -428,6 +428,16 @@ def _names_and_types(): table_types=["TABLE"], ) assert _names_and_types() == {(tbl, "TABLE")} + + # Lowercase and mixed-case requests must not match either type. + for table_type in ("view", "View", "table", "Table"): + cur.tables( + catalog_name=cat, + schema_name=sch, + table_name=name_pat, + table_types=[table_type], + ) + assert _names_and_types() == set(), table_type finally: cur.execute(f"DROP VIEW IF EXISTS {vw}") cur.execute(f"DROP TABLE IF EXISTS {tbl}") diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 88547afe5..cd90b9398 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -1626,7 +1626,7 @@ def _make_tables_stream() -> MagicMock: def test_get_tables_forwards_table_types_to_kernel(): """``table_types`` is forwarded verbatim to the kernel's - ``list_tables`` (which filters case-insensitively) — the + ``list_tables`` (which filters case-sensitively) — the connector no longer drains + refilters client-side. The stream flows back through the normal ``KernelResultSet`` path unchanged.""" c = _make_client() From 8d3c3db3814203650002e03868d509227791e161 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:54:08 +0000 Subject: [PATCH 11/11] ci: refresh merge-queue check records Refresh status contexts after GitHub reported the Kernel E2E requirement as expected despite matching successful checks. The source tree is unchanged from the validated fix in 3c800a50. Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com>