diff --git a/.nextchanges/bundles/postgres-data-apis.md b/.nextchanges/bundles/postgres-data-apis.md new file mode 100644 index 00000000000..503541f6280 --- /dev/null +++ b/.nextchanges/bundles/postgres-data-apis.md @@ -0,0 +1 @@ +* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. ([#7160](https://github.com/databricks/cli/pull/7160)) diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml b/acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml new file mode 100644 index 00000000000..978fa248926 --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + postgres_data_apis: + data_api1: + parent: projects/test-project/branches/production/databases/test-database + db_schemas: + - public + db_max_rows: 500 diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml b/acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt new file mode 100644 index 00000000000..6d40b177023 --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt @@ -0,0 +1,94 @@ + +=== Enable the Data API on a database outside the bundle +=== Deploy fails because the Data API is already enabled +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot create resources.postgres_data_apis.data_api1: Data API is already enabled for this database (409 ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-project/branches/production/databases/test-database/data-api +HTTP Status: 409 Conflict +API error_code: ALREADY_EXISTS +API message: Data API is already enabled for this database + +Files: 2 uploaded, 0 deleted + +=== Bind adopts the existing Data API +>>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve +Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Postgres Data APIs: + data_api1: + Name: projects/test-project/branches/production/databases/test-database/data-api + +>>> [CLI] bundle plan +update postgres_data_apis.data_api1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated postgres_data_apis.data_api1 +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api +{ + "name": "projects/test-project/branches/production/databases/test-database/data-api", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 500 + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Unbind leaves the Data API enabled +>>> [CLI] bundle deployment unbind data_api1 + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Postgres Data APIs: + data_api1: + Name: + +>>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api +{ + "name": "projects/test-project/branches/production/databases/test-database/data-api" +} + +=== Bind again; destroy turns off the Data API +>>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve +Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.data_api1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +>>> musterr [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api +Error: Data API is not enabled for this database + +>>> [CLI] postgres get-database projects/test-project/branches/production/databases/test-database +{ + "name": "projects/test-project/branches/production/databases/test-database" +} diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/script b/acceptance/bundle/deployment/bind/postgres_data_api/script new file mode 100644 index 00000000000..e1e57ea0f20 --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/script @@ -0,0 +1,31 @@ +DATABASE_NAME="projects/test-project/branches/production/databases/test-database" +DATA_API_NAME="${DATABASE_NAME}/data-api" + +title "Enable the Data API on a database outside the bundle" +$CLI postgres create-project test-project --json '{"spec": {"display_name": "Test Project", "pg_version": 16}}' &> LOG.setup +$CLI postgres create-database projects/test-project/branches/production --database-id test-database --json '{"spec": {"postgres_database": "app_db", "role": "projects/test-project/branches/production/roles/owner"}}' >> LOG.setup 2>&1 +$CLI postgres create-data-api "${DATABASE_NAME}" --json '{"spec": {"db_schemas": ["public"]}}' >> LOG.setup 2>&1 + +title "Deploy fails because the Data API is already enabled" +trace musterr $CLI bundle deploy + +title "Bind adopts the existing Data API" +# Bind records the config as state. The plan compares it with the spec read back +# from status, so the configured db_max_rows is applied on the next deploy. +trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve +trace $CLI bundle summary +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${DATA_API_NAME}" | jq '{name, status: (.status | {db_schemas, db_max_rows})}' +trace $CLI bundle plan + +title "Unbind leaves the Data API enabled" +trace $CLI bundle deployment unbind data_api1 +trace $CLI bundle summary +trace $CLI postgres get-data-api "${DATA_API_NAME}" | jq '{name}' + +title "Bind again; destroy turns off the Data API" +trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve +trace $CLI bundle destroy --auto-approve +trace musterr $CLI postgres get-data-api "${DATA_API_NAME}" +trace $CLI postgres get-database "${DATABASE_NAME}" | jq '{name}' diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/test.toml b/acceptance/bundle/deployment/bind/postgres_data_api/test.toml new file mode 100644 index 00000000000..976a421dd9d --- /dev/null +++ b/acceptance/bundle/deployment/bind/postgres_data_api/test.toml @@ -0,0 +1,5 @@ +Cloud = false + +Ignore = [ + ".databricks" +] diff --git a/acceptance/bundle/invariant/auto-migrate/test.toml b/acceptance/bundle/invariant/auto-migrate/test.toml index e0be4eb605f..5e52234be9b 100644 --- a/acceptance/bundle/invariant/auto-migrate/test.toml +++ b/acceptance/bundle/invariant/auto-migrate/test.toml @@ -41,6 +41,8 @@ EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmp EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] # Features are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"] +# The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it. +EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl new file mode 100644 index 00000000000..2474d6b5995 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl @@ -0,0 +1,38 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + # The Data API needs a read-write endpoint on the branch; the "production" + # branch created with the project comes with one. + postgres_roles: + owner: + parent: ${resources.postgres_projects.project.name}/branches/production + role_id: test-role-$UNIQUE_NAME + postgres_role: app_role + + postgres_databases: + database: + parent: ${resources.postgres_projects.project.name}/branches/production + database_id: test-database-$UNIQUE_NAME + postgres_database: app_db + role: ${resources.postgres_roles.owner.name} + + postgres_data_apis: + foo: + parent: ${resources.postgres_databases.database.name} + db_schemas: + - public + db_max_rows: 1000 # CAN_REMOVE + server_cors_allowed_origins: # CAN_REMOVE + - https://app.example.com # CAN_REMOVE + jwt_cache_max_lifetime: 300s # CAN_REMOVE + openapi_mode: OPEN_API_MODE_IGNORE_PRIVILEGES # CAN_REMOVE + # Removing a bool converges like the other fields: the server keeps the + # last value and status reports it. + db_aggregates_enabled: false # CAN_REMOVE + server_timing_enabled: true # CAN_REMOVE diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 15890459519..150d61a2c4b 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -43,6 +43,9 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # secret resource is not supported on v0.293.0 EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secret.yml.tmpl"] +# postgres_data_apis resource is not supported on v0.293.0 +EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] + # $${...} escaping is not supported on v0.293.0: the seed deploy fails with # 'invalid dependency "${batch_id}", no such node ""' (issue #6480). EnvMatrixExclude.no_escaped_refs = ["INPUT_CONFIG=job_escaped_refs.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index d9bed152760..18735ec8919 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index d9bed152760..18735ec8919 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a4c41dfba3d..0894d465c0b 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -41,6 +41,8 @@ EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmp EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] # Features are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"] +# The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it. +EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 39825248867..d44a9a95ab7 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 2427b6a553f..33899e103f5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -55,6 +55,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", + "postgres_data_api.yml.tmpl", "postgres_database.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", @@ -94,6 +95,7 @@ EnvMatrixExclude.no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CON EnvMatrixExclude.no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] EnvMatrixExclude.no_postgres_role_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_role.yml.tmpl"] EnvMatrixExclude.no_postgres_database_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_database.yml.tmpl"] +EnvMatrixExclude.no_postgres_data_api_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP, # so exclude these from all cloud runs (same as the postgres resources above). EnvMatrixExclude.no_database_instance_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_instance.yml.tmpl"] @@ -119,6 +121,8 @@ EnvMatrixExclude.no_model_service_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG # Deployment history has no "features." resource-key prefix, so the deploy fails before the feature # is created; see dmsResourceTypes in libs/testserver/bundledeployments.go. EnvMatrixExclude.no_feature_dms = ["DMS=true", "INPUT_CONFIG=feature.yml.tmpl"] +# Same for the "postgres_data_apis." prefix. +EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] # Fake SQL endpoint for local tests [[Server]] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 978c468c0ac..2af472ca296 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3995,6 +3995,43 @@ resources.postgres_catalogs.*.status.project string REMOTE resources.postgres_catalogs.*.uid string REMOTE resources.postgres_catalogs.*.update_time *time.Time REMOTE resources.postgres_catalogs.*.url string INPUT +resources.postgres_data_apis.*.create_time *time.Time REMOTE +resources.postgres_data_apis.*.db_aggregates_enabled bool ALL +resources.postgres_data_apis.*.db_extra_search_path []string ALL +resources.postgres_data_apis.*.db_extra_search_path[*] string ALL +resources.postgres_data_apis.*.db_max_rows int ALL +resources.postgres_data_apis.*.db_schemas []string ALL +resources.postgres_data_apis.*.db_schemas[*] string ALL +resources.postgres_data_apis.*.id string INPUT +resources.postgres_data_apis.*.jwt_cache_max_lifetime *duration.Duration ALL +resources.postgres_data_apis.*.jwt_role_claim_key string ALL +resources.postgres_data_apis.*.lifecycle resources.Lifecycle INPUT +resources.postgres_data_apis.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_data_apis.*.modified_status string INPUT +resources.postgres_data_apis.*.name string REMOTE +resources.postgres_data_apis.*.openapi_mode postgres.OpenApiMode ALL +resources.postgres_data_apis.*.parent string ALL +resources.postgres_data_apis.*.server_cors_allowed_origins []string ALL +resources.postgres_data_apis.*.server_cors_allowed_origins[*] string ALL +resources.postgres_data_apis.*.server_timing_enabled bool ALL +resources.postgres_data_apis.*.status *postgres.DataApiDataApiStatus REMOTE +resources.postgres_data_apis.*.status.available_schemas []string REMOTE +resources.postgres_data_apis.*.status.available_schemas[*] string REMOTE +resources.postgres_data_apis.*.status.db_aggregates_enabled bool REMOTE +resources.postgres_data_apis.*.status.db_extra_search_path []string REMOTE +resources.postgres_data_apis.*.status.db_extra_search_path[*] string REMOTE +resources.postgres_data_apis.*.status.db_max_rows int REMOTE +resources.postgres_data_apis.*.status.db_schemas []string REMOTE +resources.postgres_data_apis.*.status.db_schemas[*] string REMOTE +resources.postgres_data_apis.*.status.jwt_cache_max_lifetime *duration.Duration REMOTE +resources.postgres_data_apis.*.status.jwt_role_claim_key string REMOTE +resources.postgres_data_apis.*.status.openapi_mode postgres.OpenApiMode REMOTE +resources.postgres_data_apis.*.status.server_cors_allowed_origins []string REMOTE +resources.postgres_data_apis.*.status.server_cors_allowed_origins[*] string REMOTE +resources.postgres_data_apis.*.status.server_timing_enabled bool REMOTE +resources.postgres_data_apis.*.status.url string REMOTE +resources.postgres_data_apis.*.update_time *time.Time REMOTE +resources.postgres_data_apis.*.url string INPUT resources.postgres_databases.*.create_time *time.Time REMOTE resources.postgres_databases.*.database_id string ALL resources.postgres_databases.*.id string INPUT diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..6a340faa555 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: deploy-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API" + pg_version: 16 + + # The Data API needs a read-write endpoint on the branch; the "production" + # branch created with the project comes with one. + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json new file mode 100644 index 00000000000..3de94b89854 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.deploy.json @@ -0,0 +1,50 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Data API", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles", + "q": { + "role_id": "app-owner" + }, + "body": { + "spec": { + "postgres_role": "app_owner" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases", + "q": { + "database_id": "my-database" + }, + "body": { + "spec": { + "postgres_database": "app_db", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles/app-owner" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "body": { + "spec": { + "db_max_rows": 100, + "db_schemas": [ + "public" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json new file mode 100644 index 00000000000..9d3a7aaf9da --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.requests.remove.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api" +} diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml new file mode 100644 index 00000000000..a026907d175 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/output.txt b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt new file mode 100644 index 00000000000..17e51e228f9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt @@ -0,0 +1,124 @@ + +>>> [CLI] bundle validate +Name: deploy-pg-data-api-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-pg-data-api-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default +Resources: + Postgres Data APIs: + my_data_api: + Name: + Postgres databases: + my_database: + Name: + Postgres projects: + my_project: + Name: Test Project for Data API + Postgres roles: + owner: + Name: + +>>> [CLI] bundle plan +create postgres_data_apis.my_data_api +create postgres_databases.my_database +create postgres_projects.my_project +create postgres_roles.owner + +Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy enables the Data API +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +>>> [CLI] bundle summary +Name: deploy-pg-data-api-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default +Resources: + Postgres Data APIs: + my_data_api: + Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api + Postgres databases: + my_database: + Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database + Postgres projects: + my_project: + Name: Test Project for Data API + Postgres roles: + owner: + Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles/app-owner + +=== Plan after deploy shows no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== Removing the resource turns off the Data API but keeps the database +>>> [CLI] bundle plan +delete postgres_data_apis.my_data_api + +Plan: 0 to add, 0 to change, 1 to delete, 3 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default/files... +Deleted postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 3 unchanged + +>>> musterr [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +Error: Data API is not enabled for this database + +>>> [CLI] postgres get-database projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/script b/acceptance/bundle/resources/postgres_data_apis/basic/script new file mode 100644 index 00000000000..39212785c46 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/basic/script @@ -0,0 +1,33 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +database_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production/databases/my-database" +data_api_name="${database_name}/data-api" + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle validate +trace $CLI bundle summary +trace $CLI bundle plan + +title "Deploy enables the Data API" +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields +trace $CLI bundle summary + +title "Plan after deploy shows no changes" +trace $CLI bundle plan + +title "Removing the resource turns off the Data API but keeps the database" +sed '/postgres_data_apis:/,$d' databricks.yml > databricks.yml.new && mv databricks.yml.new databricks.yml +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy --auto-approve +print_postgres_requests remove +trace musterr $CLI postgres get-data-api "${data_api_name}" +trace $CLI postgres get-database "${database_name}" | jq '{name}' diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl new file mode 100644 index 00000000000..bd1c6aa3878 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: oob-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API Out of Band" + pg_version: 16 + + # The Data API needs a read-write endpoint on the branch; the "production" + # branch created with the project comes with one. + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml new file mode 100644 index 00000000000..a026907d175 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt new file mode 100644 index 00000000000..a5c18b4ffd6 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt @@ -0,0 +1,120 @@ + +=== Deploy enables the Data API +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +=== Change the spec out of band +>>> [CLI] postgres update-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api spec --json {"spec": {"db_schemas": ["public"], "db_max_rows": 7}} +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 7, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +=== Plan detects the change from status, and deploy restores the config +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== Disable the Data API out of band +>>> [CLI] postgres delete-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api + +=== Plan detects the Data API is gone and re-creates it +>>> [CLI] bundle plan +create postgres_data_apis.my_data_api + +Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/script b/acceptance/bundle/resources/postgres_data_apis/out-of-band/script new file mode 100644 index 00000000000..ab1d205a1b7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/script @@ -0,0 +1,29 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +database_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production/databases/my-database" +data_api_name="${database_name}/data-api" + +title "Deploy enables the Data API" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields + +title "Change the spec out of band" +trace $CLI postgres update-data-api "${data_api_name}" spec --json '{"spec": {"db_schemas": ["public"], "db_max_rows": 7}}' | data_api_fields + +title "Plan detects the change from status, and deploy restores the config" +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields +trace $CLI bundle plan + +title "Disable the Data API out of band" +trace $CLI postgres delete-data-api "${data_api_name}" + +title "Plan detects the Data API is gone and re-creates it" +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml b/acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..5b535dc1ef2 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/databricks.yml.tmpl @@ -0,0 +1,37 @@ +bundle: + name: recreate-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API Recreate" + pg_version: 16 + + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + first: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: first-database + postgres_database: first_db + role: ${resources.postgres_roles.owner.id} + second: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: second-database + postgres_database: second_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.first.name} + db_schemas: + - public + db_max_rows: 100 diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json b/acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json new file mode 100644 index 00000000000..33930f3cd6c --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/out.requests.recreate.json @@ -0,0 +1,16 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api", + "body": { + "spec": { + "db_max_rows": 100, + "db_schemas": [ + "public" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml new file mode 100644 index 00000000000..a026907d175 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt new file mode 100644 index 00000000000..0e3258e4ae8 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt @@ -0,0 +1,79 @@ + +=== Deploy the Data API on the first database +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.first +Created postgres_databases.second +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +=== Moving the Data API to another database recreates it +>>> update_file.py databricks.yml postgres_databases.first.name postgres_databases.second.name + +>>> [CLI] bundle plan +recreate postgres_data_apis.my_data_api + +Plan: 1 to add, 0 to change, 1 to delete, 4 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default/files... +Recreated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 4 unchanged + +>>> musterr [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/first-database/data-api +Error: Data API is not enabled for this database + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/second-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 100, + "server_cors_allowed_origins": [ + "none" + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.first + delete resources.postgres_databases.second + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.first + delete resources.postgres_databases.second + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 5 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/script b/acceptance/bundle/resources/postgres_data_apis/recreate/script new file mode 100644 index 00000000000..b28dd1a5c52 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/script @@ -0,0 +1,21 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +branch_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production" + +title "Deploy the Data API on the first database" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy +trace $CLI postgres get-data-api "${branch_name}/databases/first-database/data-api" | data_api_fields + +title "Moving the Data API to another database recreates it" +trace update_file.py databricks.yml 'postgres_databases.first.name' 'postgres_databases.second.name' +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests recreate +trace musterr $CLI postgres get-data-api "${branch_name}/databases/first-database/data-api" +trace $CLI postgres get-data-api "${branch_name}/databases/second-database/data-api" | data_api_fields diff --git a/acceptance/bundle/resources/postgres_data_apis/script.prepare b/acceptance/bundle/resources/postgres_data_apis/script.prepare new file mode 100644 index 00000000000..b965bc2df9d --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/script.prepare @@ -0,0 +1,13 @@ +# data_api_fields is the allow-list of Data API fields the acceptance tests +# assert on. Anything the backend returns outside this list is dropped, so a new +# field in the API response does not break the golden. The status reports the +# configuration in effect, server defaults included. +data_api_fields() { + jq '{name, parent, status: (.status | {db_schemas, db_max_rows, server_cors_allowed_origins})}' +} + +# print_postgres_requests writes the recorded postgres write requests to +# out.requests..json. +print_postgres_requests() { + print_requests.py '//postgres' > "out.requests.$1.json" +} diff --git a/acceptance/bundle/resources/postgres_data_apis/test.toml b/acceptance/bundle/resources/postgres_data_apis/test.toml new file mode 100644 index 00000000000..1555d272216 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/test.toml @@ -0,0 +1,20 @@ +Cloud = true + +# Lakebase v2 (postgres) is only available in AWS as of January 2026 +CloudEnvs.gcp = false +CloudEnvs.azure = false + +# Deployment history has no "postgres_data_apis." resource-key prefix, so the deploy fails before the +# Data API is created. See dmsResourceTypes in libs/testserver/bundledeployments.go, which mirrors Universe. +EnvMatrixExclude.no_data_api_dms = ["DMS=true"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Normalize endpoint UIDs (ep-xxx-yyy format, supports both word-based and hex-based UIDs) +Old = 'ep-[a-z0-9-]+' +New = '[ENDPOINT_UID]' +Order = 1 diff --git a/acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl new file mode 100644 index 00000000000..5dd17ca68c9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: update-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API Update" + pg_version: 16 + + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 + server_cors_allowed_origins: + - https://app.example.com diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json b/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json new file mode 100644 index 00000000000..66214332aee --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.plan.update.json @@ -0,0 +1,31 @@ +{ + "action": "update", + "changes": { + "db_aggregates_enabled": { + "action": "skip", + "reason": "backend_default", + "remote": true + }, + "db_max_rows": { + "action": "update", + "old": 100, + "new": 500, + "remote": 100 + }, + "jwt_role_claim_key": { + "action": "skip", + "reason": "backend_default", + "remote": ".sub" + }, + "openapi_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "OPEN_API_MODE_DISABLED" + }, + "server_timing_enabled": { + "action": "skip", + "reason": "empty", + "remote": false + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json new file mode 100644 index 00000000000..615bb91c67f --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.remove_field.json @@ -0,0 +1,15 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "q": { + "update_mask": "spec" + }, + "body": { + "spec": { + "db_max_rows": 500, + "db_schemas": [ + "public" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json new file mode 100644 index 00000000000..8affc021c9e --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.requests.update.json @@ -0,0 +1,18 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "q": { + "update_mask": "spec" + }, + "body": { + "spec": { + "db_max_rows": 500, + "db_schemas": [ + "public" + ], + "server_cors_allowed_origins": [ + "https://app.example.com" + ] + } + } +} diff --git a/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml new file mode 100644 index 00000000000..a026907d175 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/update/output.txt b/acceptance/bundle/resources/postgres_data_apis/update/output.txt new file mode 100644 index 00000000000..c2aecd05327 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/output.txt @@ -0,0 +1,91 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== Change a spec field +>>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 + +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 500, + "server_cors_allowed_origins": [ + "https://app.example.com" + ] + } +} + +=== Remove a spec field: update_mask=spec keeps a field the request omits, so it keeps its last value +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database", + "status": { + "db_schemas": [ + "public" + ], + "db_max_rows": 500, + "server_cors_allowed_origins": [ + "https://app.example.com" + ] + } +} + +=== Plan after update shows no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/update/script b/acceptance/bundle/resources/postgres_data_apis/update/script new file mode 100644 index 00000000000..8403ad4a6a0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/update/script @@ -0,0 +1,32 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +data_api_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production/databases/my-database/data-api" + +title "Initial deployment" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Change a spec field" +trace update_file.py databricks.yml "db_max_rows: 100" "db_max_rows: 500" +trace $CLI bundle plan +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_data_apis.my_data_api" | {action, changes}' > out.plan.update.json +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests update +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields + +title "Remove a spec field: update_mask=spec keeps a field the request omits, so it keeps its last value" +sed '/server_cors_allowed_origins:/,$d' databricks.yml > databricks.yml.new && mv databricks.yml.new databricks.yml +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" +trace $CLI bundle deploy +print_postgres_requests remove_field +trace $CLI postgres get-data-api "${data_api_name}" | data_api_fields + +title "Plan after update shows no changes" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl new file mode 100644 index 00000000000..33ea8827068 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/databricks.yml.tmpl @@ -0,0 +1,39 @@ +bundle: + name: url-ref-pg-data-api-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Data API URL" + pg_version: 16 + + postgres_roles: + owner: + parent: ${resources.postgres_projects.my_project.id}/branches/production + role_id: app-owner + postgres_role: app_owner + + postgres_databases: + my_database: + parent: ${resources.postgres_projects.my_project.id}/branches/production + database_id: my-database + postgres_database: app_db + role: ${resources.postgres_roles.owner.id} + + postgres_data_apis: + my_data_api: + parent: ${resources.postgres_databases.my_database.name} + db_schemas: + - public + db_max_rows: 100 + + jobs: + my_job: + name: data-api-url-$UNIQUE_NAME + parameters: + - name: data_api_url + default: ${resources.postgres_data_apis.my_data_api.status.url} diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml b/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml new file mode 100644 index 00000000000..a026907d175 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt new file mode 100644 index 00000000000..5ff1e94437e --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt @@ -0,0 +1,70 @@ + +=== The first deploy resolves status.url into the job parameter +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default/files... +Created jobs.my_job +Created postgres_data_apis.my_data_api +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] postgres get-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api +{ + "url": "https://[RW_ENDPOINT_HOST]/api/2.0/workspace/[WORKSPACE_ID]/rest/app_db" +} + +>>> [CLI] jobs get [MY_JOB_ID] +[ + { + "default": "https://[RW_ENDPOINT_HOST]/api/2.0/workspace/[WORKSPACE_ID]/rest/app_db", + "name": "data_api_url" + } +] + +=== An in-place update of the Data API does not update the job that references status.url +>>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 + +>>> [CLI] bundle plan +update postgres_data_apis.my_data_api + +Plan: 0 to add, 1 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default/files... +Updated postgres_data_apis.my_data_api +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 4 unchanged + +>>> [CLI] jobs get [MY_JOB_ID] +[ + { + "default": "https://[RW_ENDPOINT_HOST]/api/2.0/workspace/[WORKSPACE_ID]/rest/app_db", + "name": "data_api_url" + } +] + +=== Plan after update shows no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + delete resources.postgres_data_apis.my_data_api + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default + +Destroy: 5 deleted diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/script b/acceptance/bundle/resources/postgres_data_apis/url_reference/script new file mode 100644 index 00000000000..e422c0626b7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/script @@ -0,0 +1,28 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +branch_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/production" +data_api_name="${branch_name}/databases/my-database/data-api" + +envsubst < databricks.yml.tmpl > databricks.yml + +title "The first deploy resolves status.url into the job parameter" +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" +# The URL is served from the branch's read-write endpoint host. +add_repl "$($CLI postgres get-endpoint "${branch_name}/endpoints/primary" | jq -r .status.hosts.host)" RW_ENDPOINT_HOST +trace $CLI postgres get-data-api "${data_api_name}" | jq '{url: .status.url}' +trace $CLI jobs get "$(read_id.py my_job)" | jq '.settings.parameters' + +title "An in-place update of the Data API does not update the job that references status.url" +trace update_file.py databricks.yml "db_max_rows: 100" "db_max_rows: 500" +trace $CLI bundle plan +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" +trace $CLI jobs get "$(read_id.py my_job)" | jq '.settings.parameters' + +title "Plan after update shows no changes" +trace $CLI bundle plan diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 4ead9785e06..bbc2f045be5 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -32,6 +32,7 @@ var unsupportedResources = []string{ "database_catalogs", "synced_database_tables", "postgres_branches", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_catalogs", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index e9ea418d1d6..8ff6a717c4e 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -324,6 +324,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "postgres_data_api1": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/project1/branches/branch1/databases/database1", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "vs_endpoint1": { CreateEndpoint: vectorsearch.CreateEndpoint{ diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 200939f2dd0..5662465c564 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -62,6 +62,7 @@ func allResourceTypes(t *testing.T) []string { "pipelines", "postgres_branches", "postgres_catalogs", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_projects", @@ -195,6 +196,7 @@ var allowList = []string{ "model_provider_services", "postgres_branches", "postgres_catalogs", + "postgres_data_apis", "postgres_databases", "postgres_endpoints", "postgres_projects", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 75fef099133..8077c76dfd1 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -47,6 +47,7 @@ type Resources struct { PostgresRoles map[string]*resources.PostgresRole `json:"postgres_roles,omitempty"` PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` PostgresSnapshotSchedules map[string]*resources.PostgresSnapshotSchedule `json:"postgres_snapshot_schedules,omitempty"` + PostgresDataApis map[string]*resources.PostgresDataApi `json:"postgres_data_apis,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` @@ -144,6 +145,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_roles"], r.PostgresRoles), collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["postgres_snapshot_schedules"], r.PostgresSnapshotSchedules), + collectResourceMap(descriptions["postgres_data_apis"], r.PostgresDataApis), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), @@ -232,6 +234,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_roles": (&resources.PostgresRole{}).ResourceDescription(), "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "postgres_snapshot_schedules": (&resources.PostgresSnapshotSchedule{}).ResourceDescription(), + "postgres_data_apis": (&resources.PostgresDataApi{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "internal_immutable_snapshots": (&resources.Snapshot{}).ResourceDescription(), diff --git a/bundle/config/resources/postgres_data_api.go b/bundle/config/resources/postgres_data_api.go new file mode 100644 index 00000000000..1d434eb03c7 --- /dev/null +++ b/bundle/config/resources/postgres_data_api.go @@ -0,0 +1,80 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type PostgresDataApiConfig struct { + postgres.DataApiDataApiSpec + + // Parent is the database whose Data API this resource manages. Format: + // "projects/{project_id}/branches/{branch_id}/databases/{database_id}". The + // Data API has no ID of its own: at most one exists per database, and its + // resource name (and this resource's ID) is "{parent}/data-api". + Parent string `json:"parent"` +} + +func (c *PostgresDataApiConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c PostgresDataApiConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type PostgresDataApi struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + PostgresDataApiConfig +} + +func (d *PostgresDataApi) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, d) +} + +func (d PostgresDataApi) MarshalJSON() ([]byte, error) { + return marshal.Marshal(d) +} + +func (d *PostgresDataApi) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Postgres.GetDataApi(ctx, postgres.GetDataApiRequest{Name: name}) + if apierr.IsMissing(err) { + log.Debugf(ctx, "postgres data api %s does not exist", name) + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (d *PostgresDataApi) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "postgres_data_api", + PluralName: "postgres_data_apis", + SingularTitle: "Postgres Data API", + PluralTitle: "Postgres Data APIs", + } +} + +func (d *PostgresDataApi) GetName() string { + // The Data API has no name field of its own; its resource name is its ID + // ("{parent}/data-api"). + return d.ID +} + +func (d *PostgresDataApi) GetURL() (string, bool) { + // The IDs in the API do not (yet) map to IDs in the web UI. + return "", false +} + +func (d *PostgresDataApi) InitializeURL(_ url.URL) { + // The IDs in the API do not (yet) map to IDs in the web UI. +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 963e7efd4ff..d85f7e09cee 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -133,6 +133,7 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. "job_runs": true, "postgres_branches": true, + "postgres_data_apis": true, "postgres_databases": true, "postgres_endpoints": true, "postgres_projects": true, @@ -368,6 +369,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "my_postgres_data_api": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/my-postgres-project/branches/my-postgres-branch/databases/my-postgres-database", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -431,6 +439,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetRole(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetSyncedTable(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetSnapshotSchedule(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockPostgresAPI().EXPECT().GetDataApi(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetRole(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 88f08048235..25039ca99fe 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -30,6 +30,7 @@ var SupportedResources = map[string]any{ "postgres_roles": (*ResourcePostgresRole)(nil), "postgres_synced_tables": (*ResourcePostgresSyncedTable)(nil), "postgres_snapshot_schedules": (*ResourcePostgresSnapshotSchedule)(nil), + "postgres_data_apis": (*ResourcePostgresDataApi)(nil), "alerts": (*ResourceAlert)(nil), "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 1b0da0602a2..58d664c3ac1 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -890,6 +890,46 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "postgres_data_apis": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + // Creating the project implicitly provisions the "production" branch and + // its primary read-write endpoint, which the Data API requires. + _, err := client.Postgres.CreateProject(ctx, postgres.CreateProjectRequest{ + ProjectId: "test-project-for-data-api", + Project: postgres.Project{ + Spec: &postgres.ProjectSpec{ + DisplayName: "Test Project for Data API", + PgVersion: 16, + }, + }, + }) + if err != nil { + return nil, err + } + + waiter, err := client.Postgres.CreateDatabase(ctx, postgres.CreateDatabaseRequest{ + Parent: "projects/test-project-for-data-api/branches/production", + DatabaseId: "test-database", + Database: postgres.Database{ + Spec: &postgres.DatabaseDatabaseSpec{ + PostgresDatabase: "app_db", + Role: "projects/test-project-for-data-api/branches/production/roles/owner", + }, + }, + }) + if err != nil { + return nil, err + } + database, err := waiter.Wait(ctx) + if err != nil { + return nil, err + } + + return &resources.PostgresDataApi{ + Parent: database.Name, + DbSchemas: []string{"public"}, + }, nil + }, + "postgres_roles": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { // Create parent project first _, err := client.Postgres.CreateProject(ctx, postgres.CreateProjectRequest{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 317a04442c4..c60fba878b5 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -46,6 +46,8 @@ postgres_branches: postgres.BranchSpec postgres_catalogs: postgres.CatalogCatalogSpec +postgres_data_apis: postgres.DataApiDataApiSpec + postgres_databases: postgres.DatabaseDatabaseStatus postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index ba16fd72c6a..7832c70e58f 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -30,3 +30,9 @@ postgres_catalogs: postgres.CatalogCatalogSpec postgres_synced_tables: postgres.SyncedTableSyncedTableSpec postgres_roles: postgres.RoleRoleSpec + +# No generated rules for postgres_data_apis. The API marks spec INPUT_ONLY, which +# would generate ignore_remote_changes for every spec field, but DoRead fills +# the spec from status, which echoes the configuration in effect. The field +# rules are hand-written in configs/postgres_data_apis.yml instead. +postgres_data_apis: null diff --git a/bundle/direct/dresources/configs/postgres_data_apis.yml b/bundle/direct/dresources/configs/postgres_data_apis.yml new file mode 100644 index 00000000000..fdef02f93a0 --- /dev/null +++ b/bundle/direct/dresources/configs/postgres_data_apis.yml @@ -0,0 +1,32 @@ +provided_id_fields: + # parent composes the Data API's hierarchical name ("{parent}/data-api"); + # changing it targets a different database's Data API, so it recreates + # (delete + create). + - field: parent + reason: id_field +stable_output_fields: + # name ("{parent}/data-api") is backend-assigned at creation and immutable; + # see the stable_output_fields doc in config.go and ES-2202624. + - field: name + # url is built from the read-write endpoint host, the workspace and the + # database's Postgres name, none of which a Data API update can change, so a + # ${resources.postgres_data_apis..status.url} reference resolves during + # an in-place update instead of planning a no-op update of its dependents. + - field: status.url +backend_defaults: + # DoRead fills the spec from status, which reports every setting in effect. + # A field the config omits is left to the server: create applies the server + # default, and update_mask=spec keeps the current value of any field the + # request omits, so a field removed from the config keeps its last value. + # No allowed values are listed because that last value can be anything. + # Server defaults on create: + - field: db_aggregates_enabled # true + - field: db_schemas # ["public"] + - field: jwt_role_claim_key # ".sub" + - field: openapi_mode # OPEN_API_MODE_DISABLED + - field: server_cors_allowed_origins # ["none"] + # Unset by default: + - field: db_extra_search_path + - field: db_max_rows + - field: jwt_cache_max_lifetime + - field: server_timing_enabled diff --git a/bundle/direct/dresources/postgres_data_api.go b/bundle/direct/dresources/postgres_data_api.go new file mode 100644 index 00000000000..14f9f0e6066 --- /dev/null +++ b/bundle/direct/dresources/postgres_data_api.go @@ -0,0 +1,186 @@ +package dresources + +import ( + "context" + "slices" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +// dataApiUpdateMask masks the whole spec. With this mask the API sets every +// field the request carries and keeps the current value of a field it omits, +// so a field removed from the config keeps its last value. Leaf paths would +// reset an omitted field instead, but resetting db_max_rows or +// jwt_role_claim_key that way leaves the Data API unable to serve requests. +var dataApiUpdateMask = []string{"spec"} + +// dataApiBoolFields are the spec's bool fields. Status omits a bool that is +// false and was never set, so the remote spec forces them: a missing bool in +// status means false, not unknown. +var dataApiBoolFields = []string{"DbAggregatesEnabled", "ServerTimingEnabled"} + +// PostgresDataApiRemote is the return type for DoRead. It embeds +// DataApiDataApiSpec so that all paths in StateType are valid paths in +// RemoteType. GET does not return spec (it is INPUT_ONLY), so the embedded spec +// is filled from status, which reports the configuration in effect. +type PostgresDataApiRemote struct { + postgres.DataApiDataApiSpec + + Parent string `json:"parent,omitempty"` + + Name string `json:"name,omitempty"` + Status *postgres.DataApiDataApiStatus `json:"status,omitempty"` + CreateTime *sdktime.Time `json:"create_time,omitempty"` + UpdateTime *sdktime.Time `json:"update_time,omitempty"` +} + +// Custom marshaler needed because embedded DataApiDataApiSpec has its own +// MarshalJSON which would otherwise take over and ignore the additional fields. +func (s *PostgresDataApiRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PostgresDataApiRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +type ResourcePostgresDataApi struct { + client *databricks.WorkspaceClient +} + +type PostgresDataApiState = resources.PostgresDataApiConfig + +func (*ResourcePostgresDataApi) New(client *databricks.WorkspaceClient) *ResourcePostgresDataApi { + return &ResourcePostgresDataApi{client: client} +} + +func (*ResourcePostgresDataApi) PrepareState(input *resources.PostgresDataApi) *PostgresDataApiState { + return &PostgresDataApiState{ + DataApiDataApiSpec: input.DataApiDataApiSpec, + Parent: input.Parent, + } +} + +func (*ResourcePostgresDataApi) RemapState(remote *PostgresDataApiRemote) *PostgresDataApiState { + return &PostgresDataApiState{ + DataApiDataApiSpec: remote.DataApiDataApiSpec, + Parent: remote.Parent, + } +} + +func makePostgresDataApiRemote(dataApi *postgres.DataApi) *PostgresDataApiRemote { + return &PostgresDataApiRemote{ + DataApiDataApiSpec: dataApiSpecFromStatus(dataApi.Status), + Parent: dataApi.Parent, + Name: dataApi.Name, + Status: dataApi.Status, + CreateTime: dataApi.CreateTime, + UpdateTime: dataApi.UpdateTime, + } +} + +// dataApiSpecFromStatus returns the spec in effect, as reported by status. The +// spec on the resource cannot be used: GET omits it, and the one in an operation +// response leaves out the server defaults. +func dataApiSpecFromStatus(status *postgres.DataApiDataApiStatus) postgres.DataApiDataApiSpec { + if status == nil { + var spec postgres.DataApiDataApiSpec + return spec + } + return postgres.DataApiDataApiSpec{ + DbAggregatesEnabled: status.DbAggregatesEnabled, + DbExtraSearchPath: status.DbExtraSearchPath, + DbMaxRows: status.DbMaxRows, + DbSchemas: status.DbSchemas, + JwtCacheMaxLifetime: status.JwtCacheMaxLifetime, + JwtRoleClaimKey: status.JwtRoleClaimKey, + OpenapiMode: status.OpenapiMode, + ServerCorsAllowedOrigins: status.ServerCorsAllowedOrigins, + ServerTimingEnabled: status.ServerTimingEnabled, + ForceSendFields: slices.Clone(dataApiBoolFields), + } +} + +func (r *ResourcePostgresDataApi) DoRead(ctx context.Context, id string) (*PostgresDataApiRemote, error) { + dataApi, err := r.client.Postgres.GetDataApi(ctx, postgres.GetDataApiRequest{Name: id}) + if err != nil { + return nil, err + } + return makePostgresDataApiRemote(dataApi), nil +} + +// DoCreate enables the Data API on the parent database. If it is already +// enabled the API returns ALREADY_EXISTS; adopt it with `bundle deployment bind`. +func (r *ResourcePostgresDataApi) DoCreate(ctx context.Context, config *PostgresDataApiState) (string, *PostgresDataApiRemote, error) { + waiter, err := r.client.Postgres.CreateDataApi(ctx, postgres.CreateDataApiRequest{ + Parent: config.Parent, + DataApi: postgres.DataApi{ + Spec: &config.DataApiDataApiSpec, + + // Output-only fields; the parent is carried in the request path. + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, + ForceSendFields: nil, + }, + }) + if err != nil { + return "", nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return "", nil, err + } + + remote := makePostgresDataApiRemote(result) + return remote.Name, remote, nil +} + +func (r *ResourcePostgresDataApi) DoUpdate(ctx context.Context, id string, config *PostgresDataApiState, _ *PlanEntry) (*PostgresDataApiRemote, error) { + waiter, err := r.client.Postgres.UpdateDataApi(ctx, postgres.UpdateDataApiRequest{ + Name: id, + DataApi: postgres.DataApi{ + Spec: &config.DataApiDataApiSpec, + + // Output-only fields; the name is carried in the request path. + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, + ForceSendFields: nil, + }, + UpdateMask: fieldmask.FieldMask{ + Paths: dataApiUpdateMask, + }, + }) + if err != nil { + return nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return nil, err + } + return makePostgresDataApiRemote(result), nil +} + +// DoDelete disables the Data API. The parent database and its data are not +// affected. +func (r *ResourcePostgresDataApi) DoDelete(ctx context.Context, id string, _ *PostgresDataApiState) error { + waiter, err := r.client.Postgres.DeleteDataApi(ctx, postgres.DeleteDataApiRequest{ + Name: id, + }) + if err != nil { + return err + } + return waiter.Wait(ctx) +} diff --git a/bundle/direct/dresources/postgres_data_api_test.go b/bundle/direct/dresources/postgres_data_api_test.go new file mode 100644 index 00000000000..0fa5aa50ba2 --- /dev/null +++ b/bundle/direct/dresources/postgres_data_api_test.go @@ -0,0 +1,69 @@ +package dresources + +import ( + "testing" + "time" + + "github.com/databricks/databricks-sdk-go/common/types/duration" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" +) + +func TestDataApiSpecFromStatus(t *testing.T) { + jwtCache := duration.New(5 * time.Minute) + + tests := []struct { + name string + status *postgres.DataApiDataApiStatus + want postgres.DataApiDataApiSpec + }{ + { + name: "no status", + status: nil, + want: postgres.DataApiDataApiSpec{}, + }, + { + name: "every spec field is read back; url and available_schemas are not", + status: &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public", "api"}, + DbAggregatesEnabled: true, + DbExtraSearchPath: []string{"extensions"}, + DbMaxRows: 100, + DbSchemas: []string{"api"}, + JwtCacheMaxLifetime: jwtCache, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeIgnorePrivileges, + ServerCorsAllowedOrigins: []string{"https://app.example.com"}, + ServerTimingEnabled: true, + Url: "https://ep.example.com/rest/app_db", + }, + want: postgres.DataApiDataApiSpec{ + DbAggregatesEnabled: true, + DbExtraSearchPath: []string{"extensions"}, + DbMaxRows: 100, + DbSchemas: []string{"api"}, + JwtCacheMaxLifetime: jwtCache, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeIgnorePrivileges, + ServerCorsAllowedOrigins: []string{"https://app.example.com"}, + ServerTimingEnabled: true, + ForceSendFields: []string{"DbAggregatesEnabled", "ServerTimingEnabled"}, + }, + }, + { + // Status omits a false bool that was never set; it still means false. + name: "omitted bools are false", + status: &postgres.DataApiDataApiStatus{DbSchemas: []string{"public"}}, + want: postgres.DataApiDataApiSpec{ + DbSchemas: []string{"public"}, + ForceSendFields: []string{"DbAggregatesEnabled", "ServerTimingEnabled"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dataApiSpecFromStatus(tt.status)) + }) + } +} diff --git a/bundle/internal/annotation/preview.go b/bundle/internal/annotation/preview.go index b4be7348526..33392256e48 100644 --- a/bundle/internal/annotation/preview.go +++ b/bundle/internal/annotation/preview.go @@ -29,6 +29,7 @@ func PreviewTag(stage clijson.LaunchStage) string { var launchStageOverrides = map[string]clijson.LaunchStage{ "github.com/databricks/cli/bundle/config/resources.PostgresBranch": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresCatalog": clijson.LaunchStagePublicBeta, + "github.com/databricks/cli/bundle/config/resources.PostgresDataApi": clijson.LaunchStagePrivatePreview, // The Data API is at Private Preview, below its siblings' Beta. "github.com/databricks/cli/bundle/config/resources.PostgresDatabase": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresEndpoint": clijson.LaunchStagePublicBeta, "github.com/databricks/cli/bundle/config/resources.PostgresProject": clijson.LaunchStagePublicBeta, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1a5c4ccbfdd..c35d73319ff 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1622,6 +1622,23 @@ resources: "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "postgres_data_apis": + "description": |- + The Postgres Data API definitions for the bundle, where each key is the name of the Data API. Each entry enables and configures the Data API (a REST interface) of a database on a Lakebase Autoscaling branch. Removing an entry turns off the Data API; the database and its data are not affected. + "$type": + "description": |- + The Data API (a PostgREST-compatible REST interface) of a Postgres database on a Lakebase Autoscaling branch. A database has at most one Data API. Its serving URL is available as `status.url`; reference it as `${resources.postgres_data_apis..status.url}`. Durations use the protobuf seconds format, such as `300s`. + "$fields": + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "openapi_mode": + "$type": + "description": |- + Controls how the Data API exposes the OpenAPI documentation endpoint. `OPEN_API_MODE_IGNORE_PRIVILEGES` serves the OpenAPI document without filtering it by the caller's privileges; `OPEN_API_MODE_DISABLED` turns the endpoint off. + "parent": + "description": |- + The database whose Data API this manages. Format: projects/{project_id}/branches/{branch_id}/databases/{database_id} "postgres_databases": "description": |- The Postgres database definitions for the bundle, where each key is the name of the database. Each entry defines a database on a Lakebase Autoscaling branch. diff --git a/bundle/internal/schema/parser_test.go b/bundle/internal/schema/parser_test.go index c89e7b8f832..9369e777177 100644 --- a/bundle/internal/schema/parser_test.go +++ b/bundle/internal/schema/parser_test.go @@ -72,8 +72,9 @@ func TestExtractAnnotationsNestedEmbeddedSDKType(t *testing.T) { } // TestPostgresResourcesHaveLaunchStageOverride asserts every Postgres resource -// type is pinned to Beta via a launch-stage override, so an upstream stage -// change cannot silently alter its label (see annotation.OverrideLaunchStage). +// type is pinned to its expected stage (Beta, or Private Preview for the Data +// API) via a launch-stage override, so an upstream stage change cannot silently +// alter its label (see annotation.OverrideLaunchStage). // // postgresResources is spelled out explicitly so the expected set is visible on // the page, but it is also cross-checked against the Postgres resources actually @@ -81,21 +82,22 @@ func TestExtractAnnotationsNestedEmbeddedSDKType(t *testing.T) { // completeness check, which is the reminder to add it here AND to // launchStageOverrides in bundle/internal/annotation/preview.go. func TestPostgresResourcesHaveLaunchStageOverride(t *testing.T) { - postgresResources := []reflect.Type{ - reflect.TypeFor[resources.PostgresBranch](), - reflect.TypeFor[resources.PostgresCatalog](), - reflect.TypeFor[resources.PostgresDatabase](), - reflect.TypeFor[resources.PostgresEndpoint](), - reflect.TypeFor[resources.PostgresProject](), - reflect.TypeFor[resources.PostgresRole](), - reflect.TypeFor[resources.PostgresSnapshotSchedule](), - reflect.TypeFor[resources.PostgresSyncedTable](), + postgresResources := map[reflect.Type]clijson.LaunchStage{ + reflect.TypeFor[resources.PostgresBranch](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresCatalog](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresDataApi](): clijson.LaunchStagePrivatePreview, + reflect.TypeFor[resources.PostgresDatabase](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresEndpoint](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresProject](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresRole](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresSnapshotSchedule](): clijson.LaunchStagePublicBeta, + reflect.TypeFor[resources.PostgresSyncedTable](): clijson.LaunchStagePublicBeta, } // The list must cover every Postgres resource on config.Resources, so a // newly added one can't slip through unpinned. listed := make(map[reflect.Type]bool, len(postgresResources)) - for _, typ := range postgresResources { + for typ := range postgresResources { listed[typ] = true } for _, f := range reflect.VisibleFields(reflect.TypeFor[config.Resources]()) { @@ -112,13 +114,13 @@ func TestPostgresResourcesHaveLaunchStageOverride(t *testing.T) { "resource %s (%s) is not covered by this test; add it to postgresResources", f.Name, getPath(elem)) } - for _, typ := range postgresResources { + for typ, want := range postgresResources { typePath := getPath(typ) // A configured type returns its override for any input; an unconfigured // one returns the input unchanged, so a GA input that survives as GA // (rather than becoming Beta) means the override is missing. stage := annotation.OverrideLaunchStage(typePath, clijson.LaunchStageGA) - assert.Equalf(t, clijson.LaunchStagePublicBeta, stage, + assert.Equalf(t, want, stage, "resource %s has no launch-stage override; add it to launchStageOverrides in bundle/internal/annotation/preview.go", typePath) } } diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index e67e605689d..65ef4a8af2b 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -266,6 +266,8 @@ var EnumFields = map[string][]string{ "resources.pipelines.*.permissions[*].level": {"CAN_MANAGE", "CAN_RUN", "CAN_VIEW", "IS_OWNER"}, "resources.pipelines.*.restart_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, + "resources.postgres_data_apis.*.openapi_mode": {"OPEN_API_MODE_DISABLED", "OPEN_API_MODE_IGNORE_PRIVILEGES"}, + "resources.postgres_endpoints.*.endpoint_type": {"ENDPOINT_TYPE_READ_ONLY", "ENDPOINT_TYPE_READ_WRITE"}, "resources.postgres_projects.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index f43a07dddce..458659f8da4 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -351,6 +351,8 @@ var RequiredFields = map[string][]string{ "resources.postgres_catalogs.*": {"postgres_database", "catalog_id"}, + "resources.postgres_data_apis.*": {"parent"}, + "resources.postgres_databases.*": {"role", "database_id", "parent"}, "resources.postgres_endpoints.*": {"endpoint_type", "endpoint_id", "parent"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 76a608b5275..3023edbaed1 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2457,6 +2457,88 @@ } ] }, + "resources.PostgresDataApi": { + "oneOf": [ + { + "type": "object", + "description": "[Private Preview] The Data API (a PostgREST-compatible REST interface) of a Postgres database on a Lakebase Autoscaling branch. A database has at most one Data API. Its serving URL is available as `status.url`; reference it as `${resources.postgres_data_apis.\u003ckey\u003e.status.url}`. Durations use the protobuf seconds format, such as `300s`.", + "properties": { + "db_aggregates_enabled": { + "description": "[Private Preview] Enable aggregate functions (count, sum, avg, etc.) in Data API responses.\nDefault: true.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "db_extra_search_path": { + "description": "[Private Preview] Additional schemas to include in the PostgreSQL search path.\nEach entry must be a valid PostgreSQL schema name.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "db_max_rows": { + "description": "[Private Preview] Maximum number of rows returned in a single Data API response.\nMust be a positive integer.", + "$ref": "#/$defs/int", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "db_schemas": { + "description": "[Private Preview] Database schemas exposed through the Data API.\nEach entry must be a valid PostgreSQL schema name (1-63 chars, [a-zA-Z_][a-zA-Z0-9_$]*).\nMaximum 100 entries. Default: [\"public\"].", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "jwt_cache_max_lifetime": { + "description": "[Private Preview] Maximum lifetime for cached JWT tokens. Zero duration disables caching.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/common/types/duration.Duration", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "jwt_role_claim_key": { + "description": "[Private Preview] JSON path to the role claim in JWT tokens (e.g., \".sub\").\nDefault: \".sub\".", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "openapi_mode": { + "description": "[Private Preview] OpenAPI documentation mode for the Data API endpoint.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.OpenApiMode", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "parent": { + "description": "The database whose Data API this manages. Format: projects/{project_id}/branches/{branch_id}/databases/{database_id}", + "$ref": "#/$defs/string" + }, + "server_cors_allowed_origins": { + "description": "[Private Preview] Allowed origins for CORS requests.\nEach entry should be a valid origin URL, or use \"*\" to allow all origins.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "server_timing_enabled": { + "description": "[Private Preview] Enable the Server-Timing header in Data API responses.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "parent" + ], + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresDatabase": { "oneOf": [ { @@ -4200,6 +4282,10 @@ "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" }, + "postgres_data_apis": { + "description": "The Postgres Data API definitions for the bundle, where each key is the name of the Data API. Each entry enables and configures the Data API (a REST interface) of a database on a Lakebase Autoscaling branch. Removing an entry turns off the Data API; the database and its data are not affected.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresDataApi" + }, "postgres_databases": { "description": "The Postgres database definitions for the bundle, where each key is the name of the database. Each entry defines a database on a Lakebase Autoscaling branch.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresDatabase" @@ -17293,6 +17379,26 @@ } ] }, + "postgres.OpenApiMode": { + "oneOf": [ + { + "type": "string", + "description": "Controls how the Data API exposes the OpenAPI documentation endpoint. `OPEN_API_MODE_IGNORE_PRIVILEGES` serves the OpenAPI document without filtering it by the caller's privileges; `OPEN_API_MODE_DISABLED` turns the endpoint off.", + "enum": [ + "OPEN_API_MODE_IGNORE_PRIVILEGES", + "OPEN_API_MODE_DISABLED" + ], + "enumDescriptions": [ + "[Private Preview]", + "[Private Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "postgres.ProjectCustomTag": { "oneOf": [ { @@ -20236,6 +20342,20 @@ } ] }, + "resources.PostgresDataApi": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresDataApi" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresDatabase": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 4ec77578d2b..d43f9cac7df 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -60,6 +60,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_roles.test_postgres_role": {ID: "projects/test-project/branches/main/roles/test-role"}, "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.postgres_snapshot_schedules.test_postgres_snapshot_schedule": {ID: "projects/test-project/branches/main/snapshot-schedule"}, + "resources.postgres_data_apis.test_postgres_data_api": {ID: "projects/test-project/branches/main/databases/test-db/data-api"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, @@ -154,6 +155,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "projects/test-project/branches/main/snapshot-schedule", config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ModifiedStatus) + assert.Equal(t, "projects/test-project/branches/main/databases/test-db/data-api", config.Resources.PostgresDataApis["test_postgres_data_api"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresDataApis["test_postgres_data_api"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -415,6 +419,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "test_postgres_data_api": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/test-project/branches/main/databases/test-db", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -566,6 +577,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresSnapshotSchedules["test_postgres_snapshot_schedule"].ModifiedStatus) + assert.Empty(t, config.Resources.PostgresDataApis["test_postgres_data_api"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresDataApis["test_postgres_data_api"].ModifiedStatus) + assert.Empty(t, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -976,6 +990,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + PostgresDataApis: map[string]*resources.PostgresDataApi{ + "test_postgres_data_api": { + PostgresDataApiConfig: resources.PostgresDataApiConfig{ + Parent: "projects/test-project/branches/main/databases/test-db", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 78234c5ebfa..16f4ee75260 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -265,6 +265,7 @@ type FakeWorkspace struct { PostgresRoles map[string]postgres.Role PostgresSyncedTables map[string]postgres.SyncedTable PostgresSnapshotSchedules map[string]postgres.SnapshotSchedule + PostgresDataApis map[string]postgres.DataApi PostgresOperations map[string]postgres.Operation // Branches and endpoints that the server provisioned implicitly together @@ -545,6 +546,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresRoles: map[string]postgres.Role{}, PostgresSyncedTables: map[string]postgres.SyncedTable{}, PostgresSnapshotSchedules: map[string]postgres.SnapshotSchedule{}, + PostgresDataApis: map[string]postgres.DataApi{}, PostgresOperations: map[string]postgres.Operation{}, postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e1a14b9800b..e8ecdd45141 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -1396,6 +1396,11 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresOperationGet(name) }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api/operations/{operation_id}", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api/operations/" + req.Vars["operation_id"] + return req.Workspace.PostgresOperationGet(name) + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles/{role_id}/operations/{operation_id}", func(req Request) any { name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/roles/" + req.Vars["role_id"] + "/operations/" + req.Vars["operation_id"] return req.Workspace.PostgresOperationGet(name) @@ -1547,6 +1552,27 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresDatabaseDelete(name) }) + // Postgres Data API (a per-database singleton): + server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + return req.Workspace.PostgresDataApiCreate(req, parent) + }) + + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api" + return req.Workspace.PostgresDataApiGet(name) + }) + + server.Handle("PATCH", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api" + return req.Workspace.PostgresDataApiUpdate(req, name) + }) + + server.Handle("DELETE", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api", func(req Request) any { + name := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] + "/databases/" + req.Vars["database_id"] + "/data-api" + return req.Workspace.PostgresDataApiDelete(name) + }) + // Postgres Roles: server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req Request) any { parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 2f59389aeb1..b8d73ec685f 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "reflect" "regexp" "slices" "strings" @@ -1360,6 +1361,264 @@ func (s *FakeWorkspace) PostgresDatabaseDelete(name string) Response { } delete(s.PostgresDatabases, name) + // The Data API is a sub-resource of the database and goes away with it. + delete(s.PostgresDataApis, name+dataApiSuffix) + + return Response{ + Body: s.createOperationLocked(name, nil), + } +} + +// dataApiSuffix is the final segment of a Data API's resource name: +// "projects/{project_id}/branches/{branch_id}/databases/{database_id}/data-api". +const dataApiSuffix = "/data-api" + +// dataApiUpdateMaskPaths is the update_mask the fake accepts: the whole spec, +// which is what the CLI sends. The real API also accepts leaf paths, which reset +// an omitted field instead of keeping it; the CLI never sends them, so the fake +// does not model them. +var dataApiUpdateMaskPaths = []string{"spec"} + +// dataApiNotEnabledMessage is the message the real API answers a get, update, or +// delete with when the database's Data API is not enabled. +const dataApiNotEnabledMessage = "Data API is not enabled for this database" + +// dataApiNotEnabledResponse is the 404 the real API returns for a Data API that +// is not enabled. +func dataApiNotEnabledResponse() Response { + return postgresErrorResponse(404, "NOT_FOUND", dataApiNotEnabledMessage) +} + +// readWriteEndpointLocked returns the read-write endpoint of a branch (caller +// must hold lock). +func (s *FakeWorkspace) readWriteEndpointLocked(branchName string) (postgres.Endpoint, bool) { + prefix := branchName + "/endpoints/" + for name, endpoint := range s.PostgresEndpoints { + if strings.HasPrefix(name, prefix) && endpoint.Status != nil && endpoint.Status.EndpointType == postgres.EndpointTypeEndpointTypeReadWrite { + return endpoint, true + } + } + return postgres.Endpoint{}, false +} + +// withDataApiURLLocked returns the Data API with status.url set the way the real +// API reports it: https://{read-write endpoint host}/api/2.0/workspace/{workspace +// ID}/rest/{Postgres database name}. It is derived from the current endpoint and +// database on every response rather than stored. The caller must hold the lock +// and have passed dataApiPreconditionLocked. +func (s *FakeWorkspace) withDataApiURLLocked(dataApi postgres.DataApi) postgres.DataApi { + branchName, _, _ := strings.Cut(dataApi.Parent, "/databases/") + endpoint, _ := s.readWriteEndpointLocked(branchName) + database := s.PostgresDatabases[dataApi.Parent] + + status := *dataApi.Status + status.Url = fmt.Sprintf("https://%s/api/2.0/workspace/%d/rest/%s", endpoint.Status.Hosts.Host, TestWorkspaceID, database.Status.PostgresDatabase) + dataApi.Status = &status + return dataApi +} + +// dataApiPreconditionLocked returns the error the API answers a Data API create, +// get, or update with when its database is missing (404) or the database's +// branch has no read-write endpoint (400), which the backend requires to serve +// the Data API. Returns nil when the request may proceed (caller must hold lock). +func (s *FakeWorkspace) dataApiPreconditionLocked(databaseName string) *Response { + // Project and branch deletion do not cascade to databases in this fake, so + // check the ancestors first, as PostgresDatabaseGet does. + branchName, _, _ := strings.Cut(databaseName, "/databases/") + projectName, _, _ := strings.Cut(branchName, "/branches/") + if _, exists := s.PostgresProjects[projectName]; !exists { + resp := postgresNotFoundResponse("project") + return &resp + } + if _, exists := s.PostgresBranches[branchName]; !exists { + resp := postgresNotFoundResponse("branch") + return &resp + } + if _, exists := s.PostgresDatabases[databaseName]; !exists { + resp := postgresNotFoundResponse("database") + return &resp + } + if _, ok := s.readWriteEndpointLocked(branchName); !ok { + resp := postgresErrorResponse(400, "BAD_REQUEST", "Data API requires a read-write endpoint on branch "+branchName) + return &resp + } + return nil +} + +// dataApiDefaultStatus is the status the real API reports right after enabling +// the Data API with only db_schemas set: the server defaults, which the status +// echoes because it reports every setting in effect. +func dataApiDefaultStatus() *postgres.DataApiDataApiStatus { + return &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public"}, + DbAggregatesEnabled: true, + DbSchemas: []string{"public"}, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeDisabled, + ServerCorsAllowedOrigins: []string{"none"}, + } +} + +// applyDataApiSpec returns the status after applying the fields the spec sets, +// the way the real API applies a create or an update_mask=spec update: a field +// the request omits keeps its current value. +func applyDataApiSpec(current *postgres.DataApiDataApiStatus, spec *postgres.DataApiDataApiSpec) *postgres.DataApiDataApiStatus { + status := *current + status.ForceSendFields = slices.Clone(current.ForceSendFields) + if spec == nil { + return &status + } + applyDataApiBool(&status, spec, "DbAggregatesEnabled", &status.DbAggregatesEnabled, spec.DbAggregatesEnabled) + applyDataApiBool(&status, spec, "ServerTimingEnabled", &status.ServerTimingEnabled, spec.ServerTimingEnabled) + if spec.DbExtraSearchPath != nil { + status.DbExtraSearchPath = spec.DbExtraSearchPath + } + if spec.DbMaxRows != 0 { + status.DbMaxRows = spec.DbMaxRows + } + if spec.DbSchemas != nil { + status.DbSchemas = spec.DbSchemas + } + if spec.JwtCacheMaxLifetime != nil { + status.JwtCacheMaxLifetime = spec.JwtCacheMaxLifetime + } + if spec.JwtRoleClaimKey != "" { + status.JwtRoleClaimKey = spec.JwtRoleClaimKey + } + if spec.OpenapiMode != "" { + status.OpenapiMode = spec.OpenapiMode + } + if spec.ServerCorsAllowedOrigins != nil { + status.ServerCorsAllowedOrigins = spec.ServerCorsAllowedOrigins + } + return &status +} + +// applyDataApiBool sets a bool status field when the spec sets it. A false sent +// explicitly is recorded in the spec's ForceSendFields, so it counts as set. Once +// set, the real API reports the bool even when it is false, so the status forces +// it too; a bool that was never set and is false is omitted. +func applyDataApiBool(status *postgres.DataApiDataApiStatus, spec *postgres.DataApiDataApiSpec, field string, dst *bool, value bool) { + if !value && !slices.Contains(spec.ForceSendFields, field) { + return + } + *dst = value + if !slices.Contains(status.ForceSendFields, field) { + status.ForceSendFields = append(status.ForceSendFields, field) + } +} + +// PostgresDataApiCreate enables the Data API on a database. At most one Data API +// exists per database, so a second create returns ALREADY_EXISTS. +func (s *FakeWorkspace) PostgresDataApiCreate(req Request, parent string) Response { + defer s.LockUnlock()() + + if errResp := s.dataApiPreconditionLocked(parent); errResp != nil { + return *errResp + } + + name := parent + dataApiSuffix + if _, exists := s.PostgresDataApis[name]; exists { + return postgresErrorResponse(409, "ALREADY_EXISTS", "Data API is already enabled for this database") + } + + var dataApi postgres.DataApi + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &dataApi); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + // The real API rejects enabling the Data API with an empty spec. A field sent + // explicitly with its zero value is recorded in ForceSendFields, so it counts. + if dataApi.Spec == nil || reflect.ValueOf(*dataApi.Spec).IsZero() { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", "Field 'data_api' is required and must contain at least one subfield with a non-default value!") + } + + now := nowTime() + dataApi.Name = name + dataApi.Parent = parent + dataApi.CreateTime = now + dataApi.UpdateTime = now + dataApi.Status = applyDataApiSpec(dataApiDefaultStatus(), dataApi.Spec) + // spec is INPUT_ONLY: GET only returns status. + dataApi.Spec = nil + + s.PostgresDataApis[name] = dataApi + + return Response{ + Body: s.createOperationLocked(name, s.withDataApiURLLocked(dataApi)), + } +} + +// PostgresDataApiGet retrieves the Data API of a database. +func (s *FakeWorkspace) PostgresDataApiGet(name string) Response { + defer s.LockUnlock()() + + if errResp := s.dataApiPreconditionLocked(strings.TrimSuffix(name, dataApiSuffix)); errResp != nil { + return *errResp + } + + dataApi, exists := s.PostgresDataApis[name] + if !exists { + return dataApiNotEnabledResponse() + } + + return Response{ + Body: s.withDataApiURLLocked(dataApi), + } +} + +// PostgresDataApiUpdate updates the Data API configuration of a database. +func (s *FakeWorkspace) PostgresDataApiUpdate(req Request, name string) Response { + if resp := validateUpdateMask(req, dataApiUpdateMaskPaths, nil); resp != nil { + return *resp + } + + defer s.LockUnlock()() + + if errResp := s.dataApiPreconditionLocked(strings.TrimSuffix(name, dataApiSuffix)); errResp != nil { + return *errResp + } + + dataApi, exists := s.PostgresDataApis[name] + if !exists { + return dataApiNotEnabledResponse() + } + + var updateDataApi postgres.DataApi + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &updateDataApi); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + // The "spec" mask keeps the current value of a field the request omits. + dataApi.Status = applyDataApiSpec(dataApi.Status, updateDataApi.Spec) + dataApi.UpdateTime = nowTime() + s.PostgresDataApis[name] = dataApi + + return Response{ + Body: s.createOperationLocked(name, s.withDataApiURLLocked(dataApi)), + } +} + +// PostgresDataApiDelete disables the Data API of a database. The database and +// its data are not affected. +func (s *FakeWorkspace) PostgresDataApiDelete(name string) Response { + defer s.LockUnlock()() + + if _, exists := s.PostgresDataApis[name]; !exists { + return dataApiNotEnabledResponse() + } + + delete(s.PostgresDataApis, name) return Response{ Body: s.createOperationLocked(name, nil), @@ -1740,6 +1999,8 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) resourceType = "SyncedTable" case strings.HasSuffix(resourceName, "/snapshot-schedule"): resourceType = "SnapshotSchedule" + case strings.HasSuffix(resourceName, dataApiSuffix): + resourceType = "DataApi" case strings.Contains(resourceName, "/endpoints/"): resourceType = "Endpoint" case strings.Contains(resourceName, "/databases/"): diff --git a/libs/testserver/postgres_data_api_test.go b/libs/testserver/postgres_data_api_test.go new file mode 100644 index 00000000000..d577b3967d1 --- /dev/null +++ b/libs/testserver/postgres_data_api_test.go @@ -0,0 +1,140 @@ +package testserver + +import ( + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testDataApiBranch = "projects/p/branches/production" + testDataApiDatabase = testDataApiBranch + "/databases/db" + testDataApiName = testDataApiDatabase + "/data-api" +) + +// dataApiWorkspace returns a workspace with a database on the project's root +// branch, which comes with a read-write endpoint. +func dataApiWorkspace(t *testing.T) *FakeWorkspace { + workspace := NewFakeWorkspace("http://test.test", "token") + resp := workspace.PostgresProjectCreate(Request{}, "p") + require.Equal(t, 0, resp.StatusCode) + resp = workspace.PostgresDatabaseCreate(Request{Body: []byte(`{"spec": {"postgres_database": "app_db", "role": "` + testDataApiBranch + `/roles/owner"}}`)}, testDataApiBranch, "db", false) + require.Equal(t, 0, resp.StatusCode) + return workspace +} + +func updateDataApi(workspace *FakeWorkspace, mask, body string) Response { + return workspace.PostgresDataApiUpdate(Request{ + URL: &url.URL{RawQuery: url.Values{"update_mask": {mask}}.Encode()}, + Body: []byte(body), + }, testDataApiName) +} + +func TestPostgresDataApiCreateGetUpdateDelete(t *testing.T) { + workspace := dataApiWorkspace(t) + + resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"], "db_max_rows": 100}}`)}, testDataApiDatabase) + require.Equal(t, 0, resp.StatusCode) + + resp = workspace.PostgresDataApiGet(testDataApiName) + require.Equal(t, 0, resp.StatusCode) + dataApi := resp.Body.(postgres.DataApi) + assert.Equal(t, testDataApiName, dataApi.Name) + assert.Equal(t, testDataApiDatabase, dataApi.Parent) + assert.Nil(t, dataApi.Spec) + require.NotNil(t, dataApi.Status) + assert.Equal(t, []string{"public"}, dataApi.Status.DbSchemas) + assert.Equal(t, 100, dataApi.Status.DbMaxRows) + host := workspace.PostgresEndpoints[testDataApiBranch+"/endpoints/primary"].Status.Hosts.Host + assert.Equal(t, "https://"+host+"/api/2.0/workspace/900800700600/rest/app_db", dataApi.Status.Url) + + // The "spec" mask keeps the current value of a field the request omits, so + // db_max_rows stays. + resp = updateDataApi(workspace, "spec", `{"spec": {"db_schemas": ["api"]}}`) + require.Equal(t, 0, resp.StatusCode) + dataApi = workspace.PostgresDataApis[testDataApiName] + assert.Equal(t, []string{"api"}, dataApi.Status.DbSchemas) + assert.Equal(t, 100, dataApi.Status.DbMaxRows) + + resp = workspace.PostgresDataApiDelete(testDataApiName) + require.Equal(t, 0, resp.StatusCode) + assert.Equal(t, dataApiNotEnabledResponse(), workspace.PostgresDataApiGet(testDataApiName)) + assert.Equal(t, dataApiNotEnabledResponse(), workspace.PostgresDataApiDelete(testDataApiName)) +} + +func TestPostgresDataApiStatusReportsDefaults(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_max_rows": 10}}`)}, testDataApiDatabase).StatusCode) + + status := workspace.PostgresDataApis[testDataApiName].Status + assert.Equal(t, &postgres.DataApiDataApiStatus{ + AvailableSchemas: []string{"public"}, + DbAggregatesEnabled: true, + DbMaxRows: 10, + DbSchemas: []string{"public"}, + JwtRoleClaimKey: ".sub", + OpenapiMode: postgres.OpenApiModeOpenApiModeDisabled, + ServerCorsAllowedOrigins: []string{"none"}, + }, status) +} + +func TestPostgresDataApiUpdateBools(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase).StatusCode) + + // An omitted bool keeps its value; an explicit false is applied and then + // reported, unlike a false that was never set. + require.Equal(t, 0, updateDataApi(workspace, "spec", `{"spec": {"db_schemas": ["public"]}}`).StatusCode) + status := workspace.PostgresDataApis[testDataApiName].Status + assert.True(t, status.DbAggregatesEnabled) + assert.Empty(t, status.ForceSendFields) + + require.Equal(t, 0, updateDataApi(workspace, "spec", `{"spec": {"db_aggregates_enabled": false, "server_timing_enabled": false}}`).StatusCode) + status = workspace.PostgresDataApis[testDataApiName].Status + assert.False(t, status.DbAggregatesEnabled) + assert.ElementsMatch(t, []string{"DbAggregatesEnabled", "ServerTimingEnabled"}, status.ForceSendFields) +} + +func TestPostgresDataApiCreateErrors(t *testing.T) { + workspace := dataApiWorkspace(t) + spec := Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)} + + resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {}}`)}, testDataApiDatabase) + assert.Equal(t, postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", "Field 'data_api' is required and must contain at least one subfield with a non-default value!"), resp) + + resp = workspace.PostgresDataApiCreate(spec, testDataApiBranch+"/databases/missing") + assert.Equal(t, 404, resp.StatusCode) + + require.Equal(t, 0, workspace.PostgresDataApiCreate(spec, testDataApiDatabase).StatusCode) + resp = workspace.PostgresDataApiCreate(spec, testDataApiDatabase) + assert.Equal(t, postgresErrorResponse(409, "ALREADY_EXISTS", "Data API is already enabled for this database"), resp) +} + +func TestPostgresDataApiRequiresReadWriteEndpoint(t *testing.T) { + workspace := dataApiWorkspace(t) + delete(workspace.PostgresEndpoints, testDataApiBranch+"/endpoints/primary") + + resp := workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase) + assert.Equal(t, 400, resp.StatusCode) + assert.Equal(t, 400, workspace.PostgresDataApiGet(testDataApiName).StatusCode) +} + +func TestPostgresDataApiUpdateRejectsLeafMask(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase).StatusCode) + + resp := updateDataApi(workspace, "spec.db_max_rows", `{"spec": {"db_max_rows": 5}}`) + assert.Equal(t, 400, resp.StatusCode) +} + +func TestPostgresDataApiGoneWithDatabase(t *testing.T) { + workspace := dataApiWorkspace(t) + require.Equal(t, 0, workspace.PostgresDataApiCreate(Request{Body: []byte(`{"spec": {"db_schemas": ["public"]}}`)}, testDataApiDatabase).StatusCode) + + require.Equal(t, 0, workspace.PostgresDatabaseDelete(testDataApiDatabase).StatusCode) + assert.NotContains(t, workspace.PostgresDataApis, testDataApiName) + assert.Equal(t, 404, workspace.PostgresDataApiGet(testDataApiName).StatusCode) +}