diff --git a/acceptance/bundle/lifecycle/started/databricks.yml b/acceptance/bundle/lifecycle/started/databricks.yml new file mode 100644 index 0000000000..313b1cb8de --- /dev/null +++ b/acceptance/bundle/lifecycle/started/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test_lifecycle_started + +resources: + jobs: + my_job: + name: my_job + lifecycle: + started: true diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/lifecycle/started/output.txt b/acceptance/bundle/lifecycle/started/output.txt new file mode 100644 index 0000000000..0b4912bfe3 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/output.txt @@ -0,0 +1,7 @@ +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 + +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/lifecycle/started/script b/acceptance/bundle/lifecycle/started/script new file mode 100644 index 0000000000..4fbc2b517c --- /dev/null +++ b/acceptance/bundle/lifecycle/started/script @@ -0,0 +1 @@ +errcode $CLI bundle plan diff --git a/acceptance/bundle/lifecycle/started/test.toml b/acceptance/bundle/lifecycle/started/test.toml new file mode 100644 index 0000000000..d1240963e0 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/test.toml @@ -0,0 +1,7 @@ +Local = true +Cloud = false + +Ignore = [".databricks"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index a942c70c64..56d1198a72 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -100,14 +100,14 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.active_instances int ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL -resources.apps.*.config *resources.AppConfig INPUT -resources.apps.*.config.command []string INPUT -resources.apps.*.config.command[*] string INPUT -resources.apps.*.config.env []resources.AppEnvVar INPUT -resources.apps.*.config.env[*] resources.AppEnvVar INPUT -resources.apps.*.config.env[*].name string INPUT -resources.apps.*.config.env[*].value string INPUT -resources.apps.*.config.env[*].value_from string INPUT +resources.apps.*.config *resources.AppConfig INPUT STATE +resources.apps.*.config.command []string INPUT STATE +resources.apps.*.config.command[*] string INPUT STATE +resources.apps.*.config.env []resources.AppEnvVar INPUT STATE +resources.apps.*.config.env[*] resources.AppEnvVar INPUT STATE +resources.apps.*.config.env[*].name string INPUT STATE +resources.apps.*.config.env[*].value string INPUT STATE +resources.apps.*.config.env[*].value_from string INPUT STATE resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -119,18 +119,20 @@ resources.apps.*.effective_user_api_scopes[*] string ALL resources.apps.*.git_repository *apps.GitRepository ALL resources.apps.*.git_repository.provider string ALL resources.apps.*.git_repository.url string ALL -resources.apps.*.git_source *apps.GitSource INPUT -resources.apps.*.git_source.branch string INPUT -resources.apps.*.git_source.commit string INPUT -resources.apps.*.git_source.git_repository *apps.GitRepository INPUT -resources.apps.*.git_source.git_repository.provider string INPUT -resources.apps.*.git_source.git_repository.url string INPUT -resources.apps.*.git_source.resolved_commit string INPUT -resources.apps.*.git_source.source_code_path string INPUT -resources.apps.*.git_source.tag string INPUT +resources.apps.*.git_source *apps.GitSource INPUT STATE +resources.apps.*.git_source.branch string INPUT STATE +resources.apps.*.git_source.commit string INPUT STATE +resources.apps.*.git_source.git_repository *apps.GitRepository INPUT STATE +resources.apps.*.git_source.git_repository.provider string INPUT STATE +resources.apps.*.git_source.git_repository.url string INPUT STATE +resources.apps.*.git_source.resolved_commit string INPUT STATE +resources.apps.*.git_source.source_code_path string INPUT STATE +resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL resources.apps.*.lifecycle resources.Lifecycle INPUT +resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT +resources.apps.*.lifecycle.started *bool INPUT resources.apps.*.modified_status string INPUT resources.apps.*.name string ALL resources.apps.*.oauth2_app_client_id string ALL @@ -210,8 +212,9 @@ resources.apps.*.resources[*].uc_securable.securable_type apps.AppResourceUcSecu resources.apps.*.service_principal_client_id string ALL resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL -resources.apps.*.source_code_path string INPUT +resources.apps.*.source_code_path string INPUT STATE resources.apps.*.space string ALL +resources.apps.*.started bool STATE resources.apps.*.telemetry_export_destinations []apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*] apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*].unity_catalog *apps.UnityCatalog ALL @@ -396,7 +399,9 @@ resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE resources.clusters.*.lifecycle resources.Lifecycle INPUT +resources.clusters.*.lifecycle resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT +resources.clusters.*.lifecycle.started *bool INPUT resources.clusters.*.modified_status string INPUT resources.clusters.*.node_type_id string ALL resources.clusters.*.num_workers int ALL @@ -527,6 +532,7 @@ resources.clusters.*.spec.workload_type.clients.notebooks bool REMOTE resources.clusters.*.ssh_public_keys []string ALL resources.clusters.*.ssh_public_keys[*] string ALL resources.clusters.*.start_time int64 REMOTE +resources.clusters.*.started bool STATE resources.clusters.*.state compute.State REMOTE resources.clusters.*.state_message string REMOTE resources.clusters.*.terminated_time int64 REMOTE @@ -2875,7 +2881,9 @@ resources.sql_warehouses.*.id string INPUT REMOTE resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT +resources.sql_warehouses.*.lifecycle resources.LifecycleWithStarted INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT +resources.sql_warehouses.*.lifecycle.started *bool INPUT resources.sql_warehouses.*.max_num_clusters int ALL resources.sql_warehouses.*.min_num_clusters int ALL resources.sql_warehouses.*.modified_status string INPUT @@ -2894,6 +2902,7 @@ resources.sql_warehouses.*.permissions[*].level resources.SqlWarehousePermission resources.sql_warehouses.*.permissions[*].service_principal_name string INPUT resources.sql_warehouses.*.permissions[*].user_name string INPUT resources.sql_warehouses.*.spot_instance_policy sql.SpotInstancePolicy ALL +resources.sql_warehouses.*.started bool STATE resources.sql_warehouses.*.state sql.State REMOTE resources.sql_warehouses.*.tags *sql.EndpointTags ALL resources.sql_warehouses.*.tags.custom_tags []sql.EndpointTagPair ALL diff --git a/acceptance/bundle/resources/apps/lifecycle-started/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl new file mode 100644 index 0000000000..ee78dfc150 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt new file mode 100644 index 0000000000..c1ed9e6fce --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt @@ -0,0 +1,75 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps", + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +>>> errcode [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt new file mode 100644 index 0000000000..18744fe287 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt @@ -0,0 +1,48 @@ + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +Error: Resource apps.App not found: [UNIQUE_NAME] + +Exit code: 1 + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps stop [UNIQUE_NAME] +Error: Not Found + +Exit code: 1 + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt new file mode 100644 index 0000000000..8061857d58 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt new file mode 100644 index 0000000000..8061857d58 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml new file mode 100644 index 0000000000..e4c769f3b4 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt new file mode 100644 index 0000000000..1b50a6a148 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -0,0 +1,10 @@ + +>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION + +>>> update_file.py databricks.yml started: true started: false + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 + +>>> update_file.py databricks.yml started: false started: true + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script new file mode 100644 index 0000000000..5bc75f99f9 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve &> out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt + rm -f out.requests.txt +} +trap cleanup EXIT + +{ trace errcode $CLI bundle deploy; } &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +{ trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true + +trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +rm -f out.requests.txt +{ trace errcode $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true + +trace update_file.py databricks.yml "started: true" "started: false" +trace update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 +{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true + +trace update_file.py databricks.yml "started: false" "started: true" +trace update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 +{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +{ trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/test.toml b/acceptance/bundle/resources/apps/lifecycle-started/test.toml new file mode 100644 index 0000000000..79997a76d4 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/bundle/appdeploy/app.go b/bundle/appdeploy/app.go new file mode 100644 index 0000000000..e73922d490 --- /dev/null +++ b/bundle/appdeploy/app.go @@ -0,0 +1,110 @@ +package appdeploy + +import ( + "context" + "fmt" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" + sdkapps "github.com/databricks/databricks-sdk-go/service/apps" +) + +func logProgress(ctx context.Context, msg string) { + if msg == "" { + return + } + cmdio.LogString(ctx, "✓ "+msg) +} + +// BuildDeployment constructs an AppDeployment from the app's source code path, inline config and git source. +func BuildDeployment(sourcePath string, config *resources.AppConfig, gitSource *sdkapps.GitSource) sdkapps.AppDeployment { + deployment := sdkapps.AppDeployment{ + Mode: sdkapps.AppDeploymentModeSnapshot, + SourceCodePath: sourcePath, + } + + if gitSource != nil { + deployment.GitSource = gitSource + } + + if config != nil { + if len(config.Command) > 0 { + deployment.Command = config.Command + } + + if len(config.Env) > 0 { + deployment.EnvVars = make([]sdkapps.EnvVar, len(config.Env)) + for i, env := range config.Env { + deployment.EnvVars[i] = sdkapps.EnvVar{ + Name: env.Name, + Value: env.Value, + ValueFrom: env.ValueFrom, + } + } + } + } + + return deployment +} + +// WaitForDeploymentToComplete waits for active and pending deployments on an app to finish. +func WaitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *sdkapps.App) error { + if app.ActiveDeployment != nil && + app.ActiveDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the active deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Active deployment is completed!") + } + + if app.PendingDeployment != nil && + app.PendingDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the pending deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Pending deployment is completed!") + } + + return nil +} + +// Deploy deploys the app using the provided deployment request. +// If another deployment is in progress, it waits for it to complete and retries. +func Deploy(ctx context.Context, w *databricks.WorkspaceClient, appName string, deployment sdkapps.AppDeployment) error { + wait, err := w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + existingApp, getErr := w.Apps.Get(ctx, sdkapps.GetAppRequest{Name: appName}) + if getErr != nil { + return fmt.Errorf("failed to get app %s: %w", appName, getErr) + } + + if waitErr := WaitForDeploymentToComplete(ctx, w, existingApp); waitErr != nil { + return waitErr + } + + wait, err = w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + return err + } + } + + _, err = wait.OnProgress(func(ad *sdkapps.AppDeployment) { + if ad.Status == nil { + return + } + logProgress(ctx, ad.Status.Message) + }).Get() + return err +} diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go new file mode 100644 index 0000000000..f2b3ef317c --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -0,0 +1,48 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +type validateLifecycleStarted struct { + engine engine.EngineType +} + +// ValidateLifecycleStarted returns a mutator that validates lifecycle.started +// is only used on supported resource types (apps, clusters, sql_warehouses). +// lifecycle.started is only supported in direct deployment mode. +func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { + return &validateLifecycleStarted{engine: e} +} + +func (m *validateLifecycleStarted) Name() string { + return "ValidateLifecycleStarted" +} + +func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + for _, group := range b.Config.Resources.AllResources() { + for _, resource := range group.Resources { + lws, ok := resource.GetLifecycle().(resources.LifecycleWithStarted) + if !ok || lws.Started == nil || !*lws.Started { + continue + } + + // lifecycle.started is a direct-mode-only feature. + if !m.engine.IsDirect() { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.started is only supported in direct deployment mode", + }) + } + } + } + + return diags +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index aaf4687a84..977c1b04ea 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -53,6 +53,9 @@ type ConfigResource interface { // InitializeURL initializes the URL field of the resource. InitializeURL(baseURL url.URL) + + // GetLifecycle returns the lifecycle settings for the resource. + GetLifecycle() resources.ILifecycle } // ResourceGroup represents a group of resources of the same type. diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go index e48f6e7dae..26faa7d3a5 100644 --- a/bundle/config/resources/apps.go +++ b/bundle/config/resources/apps.go @@ -37,6 +37,9 @@ type App struct { apps.App // nolint App struct also defines Id and URL field with the same json tag "id" and "url" // Note: apps.App already includes GitRepository field from the SDK + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + // SourceCodePath is a required field used by DABs to point to Databricks app source code // on local disk and to the corresponding workspace path during app deployment. SourceCodePath string `json:"source_code_path,omitempty"` @@ -53,6 +56,11 @@ type App struct { Permissions []AppPermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (a *App) GetLifecycle() ILifecycle { + return a.Lifecycle +} + func (a *App) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, a) } diff --git a/bundle/config/resources/base.go b/bundle/config/resources/base.go index 792db28972..5af311304b 100644 --- a/bundle/config/resources/base.go +++ b/bundle/config/resources/base.go @@ -7,3 +7,8 @@ type BaseResource struct { URL string `json:"url,omitempty" bundle:"internal"` Lifecycle Lifecycle `json:"lifecycle,omitempty"` } + +// GetLifecycle returns the lifecycle settings for the resource. +func (b *BaseResource) GetLifecycle() ILifecycle { + return b.Lifecycle +} diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index c549ac4a6b..1b1eb1a323 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -14,9 +14,17 @@ type Cluster struct { BaseResource compute.ClusterSpec + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + Permissions []ClusterPermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (s *Cluster) GetLifecycle() ILifecycle { + return s.Lifecycle +} + func (s *Cluster) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) } diff --git a/bundle/config/resources/external_location.go b/bundle/config/resources/external_location.go index 29cb5469d2..36b4e37db7 100644 --- a/bundle/config/resources/external_location.go +++ b/bundle/config/resources/external_location.go @@ -62,6 +62,11 @@ func (e *ExternalLocation) GetName() string { return e.Name } +// GetLifecycle returns the lifecycle settings for the resource. +func (e *ExternalLocation) GetLifecycle() ILifecycle { + return e.Lifecycle +} + func (e *ExternalLocation) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, e) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index c3de7ce8ea..9805199c74 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -1,8 +1,33 @@ package resources -// Lifecycle is a struct that contains the lifecycle settings for a resource. -// It controls the behavior of the resource when it is deployed or destroyed. +// ILifecycle is implemented by Lifecycle and LifecycleWithStarted. +type ILifecycle interface { + HasPreventDestroy() bool +} + +// Lifecycle contains base lifecycle settings supported by all resources. type Lifecycle struct { // Lifecycle setting to prevent the resource from being destroyed. PreventDestroy bool `json:"prevent_destroy,omitempty"` } + +// HasPreventDestroy returns true if prevent_destroy is set. +func (l Lifecycle) HasPreventDestroy() bool { + return l.PreventDestroy +} + +// LifecycleWithStarted contains lifecycle settings for resources that support lifecycle.started. +// It is used by apps, clusters, and sql_warehouses. +type LifecycleWithStarted struct { + // Lifecycle setting to prevent the resource from being destroyed. + PreventDestroy bool `json:"prevent_destroy,omitempty"` + + // If set to true, the resource will be deployed in started mode. + // Supported only for apps, clusters, and sql_warehouses. + Started *bool `json:"started,omitempty"` +} + +// HasPreventDestroy returns true if prevent_destroy is set. +func (l LifecycleWithStarted) HasPreventDestroy() bool { + return l.PreventDestroy +} diff --git a/bundle/config/resources/sql_warehouses.go b/bundle/config/resources/sql_warehouses.go index bed567b805..0251ac0532 100644 --- a/bundle/config/resources/sql_warehouses.go +++ b/bundle/config/resources/sql_warehouses.go @@ -14,9 +14,17 @@ type SqlWarehouse struct { BaseResource sql.CreateWarehouseRequest + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + Permissions []SqlWarehousePermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (sw *SqlWarehouse) GetLifecycle() ILifecycle { + return sw.Lifecycle +} + func (sw *SqlWarehouse) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, sw) } diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go index b25d403766..87095eb0aa 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app.go +++ b/bundle/deploy/terraform/tfdyn/convert_app.go @@ -38,7 +38,7 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out return err } - // We always set no_compute to true as it allows DABs not to wait for app compute to be started when app is created. + // Always skip compute during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_compute", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go index e53b22a38d..3a2439014a 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -29,7 +29,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, return err } - // We always set no_wait as it allows DABs not to wait for cluster to be started. + // Always skip wait during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_wait", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/lifecycle.go b/bundle/deploy/terraform/tfdyn/lifecycle.go index 1600cdef2d..669aa83040 100644 --- a/bundle/deploy/terraform/tfdyn/lifecycle.go +++ b/bundle/deploy/terraform/tfdyn/lifecycle.go @@ -11,7 +11,19 @@ func convertLifecycle(ctx context.Context, vout, vLifecycle dyn.Value) (dyn.Valu return vout, nil } - vout, err := dyn.Set(vout, "lifecycle", vLifecycle) + // Strip lifecycle.started: it is a DABs-only field not understood by Terraform. + var err error + vLifecycle, err = dyn.DropKeys(vLifecycle, []string{"started"}) + if err != nil { + return dyn.InvalidValue, err + } + + // If only lifecycle.started was set (now empty), skip setting the lifecycle block. + if m, ok := vLifecycle.AsMap(); ok && m.Len() == 0 { + return vout, nil + } + + vout, err = dyn.Set(vout, "lifecycle", vLifecycle) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index e753079dac..ff6843cb50 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -7,13 +7,26 @@ import ( "strings" "time" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/apps" ) +// AppState is the state type for App resources. It extends apps.App with fields +// needed for app deployments (Apps.Deploy) that are not part of the remote state. +type AppState struct { + apps.App + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Started bool `json:"started,omitempty"` +} + type ResourceApp struct { client *databricks.WorkspaceClient } @@ -22,18 +35,41 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { return &ResourceApp{client: client} } -func (*ResourceApp) PrepareState(input *resources.App) *apps.App { - return &input.App +func (*ResourceApp) PrepareState(input *resources.App) *AppState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &AppState{ + App: input.App, + SourceCodePath: input.SourceCodePath, + Config: input.Config, + GitSource: input.GitSource, + Started: started, + } +} + +// RemapState maps the remote apps.App to AppState for diff comparison. +// Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, +// so they default to zero values, which prevents false drift detection. +func (*ResourceApp) RemapState(remote *apps.App) *AppState { + return &AppState{ + App: *remote, + SourceCodePath: "", + Config: nil, + GitSource: nil, + Started: false, + } } func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) { return r.client.Apps.GetByName(ctx, id) } -func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, *apps.App, error) { +func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { + // With lifecycle.started=true, start the app compute (no_compute=false). + // Otherwise, skip compute startup during creation. + noCompute := !config.Started request := apps.CreateAppRequest{ - App: *config, - NoCompute: true, + App: config.App, + NoCompute: noCompute, ForceSendFields: nil, } @@ -68,11 +104,11 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, * return app.Name, nil, nil } -func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, changes Changes) (*apps.App, error) { +func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, changes Changes) (*apps.App, error) { updateMask := strings.Join(collectUpdatePathsWithPrefix(changes, ""), ",") request := apps.AsyncUpdateAppRequest{ - App: config, + App: &config.App, AppName: id, UpdateMask: updateMask, } @@ -89,15 +125,64 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) } + + // With lifecycle.started=true, ensure the app compute is running and deploy the latest code. + if config.Started { + // Start compute if it is stopped (mirrors bundle run behavior). + app, err := r.client.Apps.GetByName(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get app %s: %w", id, err) + } + if isComputeStopped(app) { + startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) + if err != nil { + return nil, fmt.Errorf("failed to start app %s: %w", id, err) + } + startedApp, err := startWaiter.Get() + if err != nil { + return nil, fmt.Errorf("failed to wait for app %s to start: %w", id, err) + } + if err := appdeploy.WaitForDeploymentToComplete(ctx, r.client, startedApp); err != nil { + return nil, err + } + } + deployment := appdeploy.BuildDeployment(config.SourceCodePath, config.Config, config.GitSource) + if err := appdeploy.Deploy(ctx, r.client, id, deployment); err != nil { + return nil, err + } + } + return nil, nil } +// localOnlyFields are AppState fields that have no counterpart in the remote state. +// They must not appear in the App update_mask. +var localOnlyFields = map[string]bool{ + "source_code_path": true, + "config": true, + "git_source": true, + "started": true, +} + +func (*ResourceApp) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *apps.App) error { + if change.Action == deployplan.Update && localOnlyFields[p.Prefix(1).String()] { + change.Action = deployplan.Skip + } + return nil +} + +func isComputeStopped(app *apps.App) bool { + return app.ComputeStatus == nil || + app.ComputeStatus.State == apps.ComputeStateStopped || + app.ComputeStatus.State == apps.ComputeStateError +} + func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { _, err := r.client.Apps.DeleteByName(ctx, id) return err } -func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *apps.App) (*apps.App, error) { +func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*apps.App, error) { return r.waitForApp(ctx, r.client, config.Name) } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index e0eeed5b77..ad9ca01e8a 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -57,7 +57,7 @@ func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) @@ -113,7 +113,7 @@ func TestAppDoCreate_RetriesWhenGetReturnsNotFound(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index a8f78d12f9..2eea7d517d 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -16,6 +16,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) +type ClusterState struct { + compute.ClusterSpec + Started bool `json:"started,omitempty"` +} + type ResourceCluster struct { client *databricks.WorkspaceClient } @@ -26,11 +31,15 @@ func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { } } -func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.ClusterSpec { - return &input.ClusterSpec +func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &ClusterState{ + ClusterSpec: input.ClusterSpec, + Started: started, + } } -func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { +func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterState { spec := &compute.ClusterSpec{ ApplyPolicyDefaultValues: false, Autoscale: input.Autoscale, @@ -71,27 +80,35 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.Clu if input.Spec != nil { spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues } - return spec + return &ClusterState{ClusterSpec: *spec, Started: false} } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*compute.ClusterDetails, error) { return r.client.Clusters.GetByClusterId(ctx, id) } -func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterSpec) (string, *compute.ClusterDetails, error) { - wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(config)) +func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (string, *compute.ClusterDetails, error) { + wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(&config.ClusterSpec)) if err != nil { return "", nil, err } + // With lifecycle.started=true, wait for the cluster to reach the running state. + if config.Started { + details, err := wait.Get() + if err != nil { + return "", nil, err + } + return details.ClusterId, details, nil + } return wait.ClusterId, nil, nil } -func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec, _ Changes) (*compute.ClusterDetails, error) { +func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, _ Changes) (*compute.ClusterDetails, error) { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 timeout := 15 * time.Minute _, err := retries.Poll(ctx, timeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { - wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, config)) + wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, &config.ClusterSpec)) if err == nil { return wait, nil } @@ -107,7 +124,7 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compu return nil, err } -func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *compute.ClusterSpec) error { +func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *ClusterState) error { _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ ClusterId: id, NumWorkers: config.NumWorkers, @@ -129,6 +146,10 @@ func (r *ResourceCluster) OverrideChangeDesc(ctx context.Context, p *structpath. path := p.Prefix(1).String() switch path { + case "started": + // started is lifecycle metadata, not an actual cluster property. + change.Action = deployplan.Skip + case "data_security_mode": // We do change skip here in the same way TF provider does suppress diff if the alias is used. // https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L109-L117 diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 5d9d7793b7..066f63e184 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -4,12 +4,19 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/sql" ) +type SqlWarehouseState struct { + sql.CreateWarehouseRequest + Started bool `json:"started,omitempty"` +} + type ResourceSqlWarehouse struct { client *databricks.WorkspaceClient } @@ -20,12 +27,16 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq } // PrepareState converts bundle config to the SDK type. -func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *sql.CreateWarehouseRequest { - return &input.CreateWarehouseRequest +func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &SqlWarehouseState{ + CreateWarehouseRequest: input.CreateWarehouseRequest, + Started: started, + } } -func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sql.CreateWarehouseRequest { - return &sql.CreateWarehouseRequest{ +func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *SqlWarehouseState { + return &SqlWarehouseState{Started: false, CreateWarehouseRequest: sql.CreateWarehouseRequest{ AutoStopMins: warehouse.AutoStopMins, Channel: warehouse.Channel, ClusterSize: warehouse.ClusterSize, @@ -40,7 +51,7 @@ func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sq Tags: warehouse.Tags, WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), - } + }} } // DoRead reads the warehouse by id. @@ -49,16 +60,24 @@ func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*sql.GetW } // DoCreate creates the warehouse and returns its id. -func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateWarehouseRequest) (string, *sql.GetWarehouseResponse, error) { - waiter, err := r.client.Warehouses.Create(ctx, *config) +func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehouseState) (string, *sql.GetWarehouseResponse, error) { + waiter, err := r.client.Warehouses.Create(ctx, config.CreateWarehouseRequest) if err != nil { return "", nil, err } + // With lifecycle.started=true, wait for the warehouse to reach the running state. + if config.Started { + warehouse, err := waiter.Get() + if err != nil { + return "", nil, err + } + return warehouse.Id, warehouse, nil + } return waiter.Id, nil, nil } // DoUpdate updates the warehouse in place. -func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ Changes) (*sql.GetWarehouseResponse, error) { +func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, _ Changes) (*sql.GetWarehouseResponse, error) { request := sql.EditWarehouseRequest{ AutoStopMins: config.AutoStopMins, Channel: config.Channel, @@ -92,3 +111,11 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string) error { return r.client.Warehouses.DeleteById(ctx, oldID) } + +func (*ResourceSqlWarehouse) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *sql.GetWarehouseResponse) error { + if change.Action == deployplan.Update && p.Prefix(1).String() == "started" { + // started is lifecycle metadata, not an actual warehouse property. + change.Action = deployplan.Skip + } + return nil +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d061d4d0da..b4436b784e 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,8 +16,16 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ + // source_code_path, config, git_source, and started are bundle-specific deployment fields not present in the remote App state. + "apps": { + "config", + "git_source", + "source_code_path", + "started", + }, "clusters": { "apply_policy_default_values", + "started", }, "external_locations": { "skip_validation", @@ -33,6 +41,9 @@ var knownMissingInRemoteType = map[string][]string{ "route_optimized", "tags", }, + "sql_warehouses": { + "started", + }, "quality_monitors": { "skip_builtin_dashboard", "warehouse_id", @@ -91,11 +102,6 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, - "apps": { - "config", - "source_code_path", - "git_source", - }, "dashboards": { "file_path", }, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 2eb01cd85a..86a799fe00 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -700,6 +700,13 @@ github.com/databricks/cli/bundle/config/resources.Lifecycle: "prevent_destroy": "description": |- Lifecycle setting to prevent the resource from being destroyed. +github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted: + "prevent_destroy": + "description": |- + Lifecycle setting to prevent the resource from being destroyed. + "started": + "description": |- + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermission: "group_name": "description": |- diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 6e43f42291..fd692c9614 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" @@ -13,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt" - "github.com/databricks/cli/libs/dyn" ) // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". @@ -25,6 +25,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), + mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) } @@ -32,32 +33,27 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { - root := b.Config.Value() var errs []error for _, action := range actions { if action.ActionType != deployplan.Recreate && action.ActionType != deployplan.Delete { continue } - path, err := dyn.NewPathFromString(action.ResourceKey) - if err != nil { - return fmt.Errorf("failed to parse %q", action.ResourceKey) - } - - path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) - - // If there is no prevent_destroy, skip - preventDestroyV, err := dyn.GetByPath(root, path) - if err != nil { + // ResourceKey format: "resources.{type}.{key}" + parts := strings.SplitN(action.ResourceKey, ".", 3) + if len(parts) != 3 || parts[0] != "resources" { continue } - - preventDestroy, ok := preventDestroyV.AsBool() - if !ok { - return fmt.Errorf("internal error: prevent_destroy is not a boolean for %s", action.ResourceKey) - } - if preventDestroy { - errs = append(errs, fmt.Errorf("%s has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for %s", action.ResourceKey, action.ResourceKey)) + resourceType, resourceKey := parts[1], parts[2] + + for _, group := range b.Config.Resources.AllResources() { + if group.Description.PluralName != resourceType { + continue + } + if r, ok := group.Resources[resourceKey]; ok && r.GetLifecycle().HasPreventDestroy() { + errs = append(errs, fmt.Errorf("%s has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for %s", action.ResourceKey, action.ResourceKey)) + } + break } } diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 28b19d0e1e..3873f19f8c 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -1,65 +1,94 @@ package phases import ( - "context" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) -func TestCheckPreventDestroyForAllResources(t *testing.T) { - supportedResources := config.SupportedResources() +func TestCheckPreventDestroyForJob(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_resource": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{PreventDestroy: true}, + }, + JobSettings: jobs.JobSettings{}, + }, + }, + }, + }, + } + + actions := []deployplan.Action{ + { + ResourceKey: "resources.jobs.test_resource", + ActionType: deployplan.Recreate, + }, + } + + err := checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.jobs.test_resource has lifecycle.prevent_destroy set") + require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") + require.Contains(t, err.Error(), "disable lifecycle.prevent_destroy for resources.jobs.test_resource") +} - for resourceType := range supportedResources { - t.Run(resourceType, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", +func TestCheckPreventDestroyForApp(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "test_resource": { + Lifecycle: resources.LifecycleWithStarted{PreventDestroy: true}, }, - Resources: config.Resources{}, }, - } + }, + }, + } + + actions := []deployplan.Action{ + { + ResourceKey: "resources.apps.test_resource", + ActionType: deployplan.Delete, + }, + } - ctx := t.Context() - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - // Use Mutate to set the configuration dynamically - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - // Set the resource with lifecycle.prevent_destroy = true - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - resourceType: dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) - require.NoError(t, err) - }) + err := checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.apps.test_resource has lifecycle.prevent_destroy set") +} - actions := []deployplan.Action{ - { - ResourceKey: "resources." + resourceType + ".test_resource", - ActionType: deployplan.Recreate, +func TestCheckPreventDestroyNoError(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_resource": { + JobSettings: jobs.JobSettings{}, + }, }, - } + }, + }, + } - err := checkForPreventDestroy(b, actions) - require.Error(t, err) - require.Contains(t, err.Error(), "resources."+resourceType+".test_resource has lifecycle.prevent_destroy set") - require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") - require.Contains(t, err.Error(), "disable lifecycle.prevent_destroy for resources."+resourceType+".test_resource") - }) + actions := []deployplan.Action{ + { + ResourceKey: "resources.jobs.test_resource", + ActionType: deployplan.Recreate, + }, } + + err := checkForPreventDestroy(b, actions) + require.NoError(t, err) } func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { @@ -71,9 +100,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test_job": { - JobSettings: jobs.JobSettings{ - Tasks: []jobs.Task{}, - }, + JobSettings: jobs.JobSettings{}, }, }, Apps: map[string]*resources.App{ @@ -81,11 +108,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { App: apps.App{ Name: "Test App", }, - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - PreventDestroy: true, - }, - }, + Lifecycle: resources.LifecycleWithStarted{PreventDestroy: true}, }, }, }, @@ -103,10 +126,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { }, } - ctx := t.Context() - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := checkForPreventDestroy(b, actions) - require.Error(t, err) - require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") - }) + err := checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") } diff --git a/bundle/run/app.go b/bundle/run/app.go index 1dd4484093..c3a6497f1d 100644 --- a/bundle/run/app.go +++ b/bundle/run/app.go @@ -7,10 +7,10 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -124,7 +124,7 @@ func (a *appRunner) start(ctx context.Context) error { // active and pending deployments fields (if any). If there are active or pending deployments, // we need to wait for them to complete before we can do the new deployment. // Otherwise, the new deployment will fail. - err = waitForDeploymentToComplete(ctx, w, startedApp) + err = appdeploy.WaitForDeploymentToComplete(ctx, w, startedApp) if err != nil { return err } @@ -133,108 +133,10 @@ func (a *appRunner) start(ctx context.Context) error { return nil } -func waitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *apps.App) error { - // We first wait for the active deployment to complete. - if app.ActiveDeployment != nil && - app.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the active deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Active deployment is completed!") - } - - // Then, we wait for the pending deployment to complete. - if app.PendingDeployment != nil && - app.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the pending deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Pending deployment is completed!") - } - - return nil -} - -// buildAppDeployment creates an AppDeployment struct with inline config if provided -func (a *appRunner) buildAppDeployment() apps.AppDeployment { - deployment := apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: a.app.SourceCodePath, - } - - // Add git source if provided - if a.app.GitSource != nil { - deployment.GitSource = a.app.GitSource - } - - // Add inline config if provided - if a.app.Config != nil { - if len(a.app.Config.Command) > 0 { - deployment.Command = a.app.Config.Command - } - - if len(a.app.Config.Env) > 0 { - deployment.EnvVars = make([]apps.EnvVar, len(a.app.Config.Env)) - for i, env := range a.app.Config.Env { - deployment.EnvVars[i] = apps.EnvVar{ - Name: env.Name, - Value: env.Value, - ValueFrom: env.ValueFrom, - } - } - } - } - - return deployment -} - func (a *appRunner) deploy(ctx context.Context) error { - app := a.app - b := a.bundle - w := b.WorkspaceClient() - - wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - // If deploy returns an error, then there's an active deployment in progress, wait for it to complete. - // For this we first need to get an app and its acrive and pending deployments and then wait for them. - if err != nil { - app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name}) - if err != nil { - return fmt.Errorf("failed to get app %s: %w", app.Name, err) - } - - err = waitForDeploymentToComplete(ctx, w, app) - if err != nil { - return err - } - - // Now we can try to deploy the app again - wait, err = w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - if err != nil { - return err - } - } - - _, err = wait.OnProgress(func(ad *apps.AppDeployment) { - if ad.Status == nil { - return - } - logProgress(ctx, ad.Status.Message) - }).Get() - if err != nil { - return err - } - - return nil + w := a.bundle.WorkspaceClient() + deployment := appdeploy.BuildDeployment(a.app.SourceCodePath, a.app.Config, a.app.GitSource) + return appdeploy.Deploy(ctx, w, a.app.Name, deployment) } func (a *appRunner) Cancel(ctx context.Context) error { diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 112fcea28d..1c05fa63eb 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" @@ -294,11 +295,7 @@ func TestBuildAppDeploymentWithValueFrom(t *testing.T) { }, } - runner := &appRunner{ - app: app, - } - - deployment := runner.buildAppDeployment() + deployment := appdeploy.BuildDeployment(app.SourceCodePath, app.Config, app.GitSource) require.Equal(t, apps.AppDeploymentModeSnapshot, deployment.Mode) require.Equal(t, "/path/to/app", deployment.SourceCodePath) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 217e688796..29a318f2ee 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -197,7 +197,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "name": { "description": "The name of the app. The name must contain only lowercase alphanumeric characters and hyphens.\nIt must be unique within the workspace.", @@ -462,7 +462,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "node_type_id": { "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.", @@ -1119,6 +1119,28 @@ } ] }, + "resources.LifecycleWithStarted": { + "oneOf": [ + { + "type": "object", + "properties": { + "prevent_destroy": { + "description": "Lifecycle setting to prevent the resource from being destroyed.", + "$ref": "#/$defs/bool" + }, + "started": { + "description": "Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.MlflowExperiment": { "oneOf": [ { @@ -2087,7 +2109,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "max_num_clusters": { "description": "Maximum number of clusters that the autoscaler will create to handle\nconcurrent queries.\n\nSupported values:\n- Must be \u003e= min_num_clusters\n- Must be \u003c= 40.\n\nDefaults to min_clusters if unset.", diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 8abd4cd416..8d7ac8bfc7 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -89,6 +89,79 @@ func (s *FakeWorkspace) AppsGetUpdate(_ Request, name string) Response { } } +func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + var deployment apps.AppDeployment + if err := json.Unmarshal(req.Body, &deployment); err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + + deployment.DeploymentId = "deploy-1" + deployment.Status = &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + } + + return Response{Body: deployment} +} + +func (s *FakeWorkspace) AppsGetDeployment(_ Request, name, deploymentID string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: apps.AppDeployment{ + DeploymentId: deploymentID, + Status: &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + }, + }} +} + +func (s *FakeWorkspace) AppsStart(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateActive, + Message: "App compute is active.", + } + s.Apps[name] = app + + return Response{Body: app} +} + +func (s *FakeWorkspace) AppsStop(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateStopped, + Message: "App compute is stopped.", + } + s.Apps[name] = app + + return Response{Body: app} +} + func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { var app apps.App diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9e30cb5f0c..b2a95b1902 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -379,6 +379,22 @@ func AddDefaultHandlers(server *Server) { // Apps: + server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req Request) any { + return req.Workspace.AppsCreateDeployment(req, req.Vars["name"]) + }) + + server.Handle("GET", "/api/2.0/apps/{name}/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.AppsGetDeployment(req, req.Vars["name"], req.Vars["deployment_id"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req Request) any { + return req.Workspace.AppsStart(req, req.Vars["name"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/stop", func(req Request) any { + return req.Workspace.AppsStop(req, req.Vars["name"]) + }) + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req Request) any { return req.Workspace.AppsCreateUpdate(req, req.Vars["name"]) })