acc: run bundle/deploy/mlops-stacks locally

acceptance/bundle/deploy/mlops-stacks/script

@@ -20,7 +20,11 @@ cleanup() {
 }
 trap cleanup EXIT
 
-trace $CLI bundle init mlops-stacks --config-file config.json
+if [ -z "$CLOUD_ENV" ]; then
+  trace $CLI bundle init "$TESTDIR/template" --config-file config.json
+else
+  trace $CLI bundle init mlops-stacks --config-file config.json
+fi
 trace cat test_repo_mlops_stacks/README.md | head -n 4
 
 cd "test_repo_mlops_stacks/project_name_${UNIQUE_NAME}" || exit 1

acceptance/bundle/deploy/mlops-stacks/template.REVISION

@@ -0,0 +1 @@
+1c87ae24d6c06050adc51185a5022ea58a8961f2

acceptance/bundle/deploy/mlops-stacks/template/databricks_template_schema.json

@@ -0,0 +1,360 @@
+{
+  "welcome_message": "Welcome to MLOps Stacks. For detailed information on project generation, see the README at https://github.com/databricks/mlops-stacks/blob/main/README.md.",
+  "min_databricks_cli_version": "v0.236.0",
+  "properties": {
+    "input_setup_cicd_and_project": {
+      "order": 1,
+      "type": "string",
+      "description": "{{if false}}\n\nERROR: This template is not supported by your current Databricks CLI version.\nPlease hit control-C and go to https://docs.databricks.com/en/dev-tools/cli/install.html for instructions on upgrading the CLI to the minimum version supported by MLOps Stacks.\n\n\n{{end}}\nSelect if both CI/CD and the Project should be set up, or only one of them.\nYou can always set up the other later by running initialization again",
+      "default": "CICD_and_Project",
+      "enum": ["CICD_and_Project", "Project_Only", "CICD_Only"]
+    },
+    "input_project_name": {
+      "order": 2,
+      "type": "string",
+      "default": "my_mlops_project",
+      "description": "\nProject Name. Default",
+      "pattern": "^[^ .\\\\/]{3,}$",
+      "pattern_match_failure_message": "Project name must be at least 3 characters long and cannot contain the following characters: \"\\\", \"/\", \" \" and \".\".",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_root_dir": {
+      "order": 3,
+      "type": "string",
+      "default": "{{ .input_project_name }}",
+      "description": "\nRoot directory name.\nFor monorepos, name of the root directory that contains all the projects.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_and_Project"
+          }
+        }
+      }
+    },
+    "input_cloud": {
+      "order": 4,
+      "type": "string",
+      "description": "\nSelect cloud",
+      "default": "azure",
+      "enum": ["azure", "aws", "gcp"]
+    },
+    "input_cicd_platform": {
+      "order": 5,
+      "type": "string",
+      "description": "\nSelect CICD platform",
+      "default": "github_actions",
+      "enum": ["github_actions", "github_actions_for_github_enterprise_servers", "azure_devops", "gitlab"],
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_databricks_staging_workspace_host": {
+      "order": 6,
+      "type": "string",
+      "default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-staging-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-staging-workspace.gcp.databricks.com{{end}}",
+      "description": "\nURL of staging Databricks workspace,\nIt will run PR CI and preview changes before they're deployed to production.\nDefault",
+      "pattern": "^(https.*)?$",
+      "pattern_match_failure_message": "Databricks staging workspace host URLs must start with https. Got invalid workspace host.",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_databricks_prod_workspace_host": {
+      "order": 7,
+      "type": "string",
+      "default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-prod-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-prod-workspace.gcp.databricks.com{{end}}",
+      "description": "\nURL of production Databricks workspace.\nDefault",
+      "pattern": "^(https.*)?$",
+      "pattern_match_failure_message": "Databricks production workspace host URLs must start with https. Got invalid workspace host.",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_default_branch": {
+      "order": 8,
+      "type": "string",
+      "default": "main",
+      "description": "\nName of the default branch,\nStaging resources are deployed from this branch and stages the latest ML code.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_release_branch": {
+      "order": 9,
+      "type": "string",
+      "default": "release",
+      "description": "\nName of the release branch.\nThe training and other production jobs pull ML code from this branch.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_read_user_group": {
+      "order": 10,
+      "type": "string",
+      "default": "users",
+      "description": "\nUser group name to give READ permissions to for project resources\n(ML jobs, integration test job runs, and machine learning resources).\nA group with this name must exist in both the staging and prod workspaces.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_include_models_in_unity_catalog": {
+      "order": 11,
+      "type": "string",
+      "description": "\nWhether to use the Model Registry with Unity Catalog",
+      "default": "no",
+      "enum": ["yes", "no"]
+    },
+    "input_staging_catalog_name": {
+      "order": 12,
+      "type": "string",
+      "description": "\nName of the catalog in Unity Catalog that will host the staging UC resources. \nThis catalog must already exist and service principals must have access to it.\nDefault",
+      "default": "staging",
+      "skip_prompt_if": {
+        "anyOf": [
+          {
+            "properties": {
+              "input_include_models_in_unity_catalog": {
+                "const": "no"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "Project_Only"
+              }
+            }
+          }
+        ]
+      }
+    },
+    "input_prod_catalog_name": {
+      "order": 13,
+      "type": "string",
+      "description": "\nName of the catalog in Unity Catalog that will host the production UC resources.\nThis catalog must already exist and service principals must have access to it.\nDefault",
+      "default": "prod",
+      "skip_prompt_if": {
+        "anyOf": [
+          {
+            "properties": {
+              "input_include_models_in_unity_catalog": {
+                "const": "no"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "Project_Only"
+              }
+            }
+          }
+        ]
+      }
+    },
+    "input_test_catalog_name": {
+      "order": 14,
+      "type": "string",
+      "description": "\nName of the catalog in Unity Catalog that will be used for integration tests.\nThis catalog must already exist and service principals must have access to it.\nDefault",
+      "default": "test",
+      "skip_prompt_if": {
+        "anyOf": [
+          {
+            "properties": {
+              "input_include_models_in_unity_catalog": {
+                "const": "no"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "Project_Only"
+              }
+            }
+          }
+        ]
+      }
+    },
+    "input_schema_name": {
+      "order": 15,
+      "type": "string",
+      "description": "\nName of schema to use when registering a model in Unity Catalog.\nThis schema must already exist and service principals must have access.\nWe recommend using the project name.\nDefault",
+      "default": "{{if (eq .input_include_models_in_unity_catalog `no`)}}schema{{else}}{{ .input_project_name }}{{end}}",
+      "pattern": "^[^ .\\-\\/]*$",
+      "pattern_match_failure_message": "Valid schema names cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
+      "skip_prompt_if": {
+        "anyOf":[
+          {
+            "properties": {
+              "input_include_models_in_unity_catalog": {
+                "const": "no"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "CICD_Only"
+              }
+            }
+          }
+        ]
+      }
+    },
+    "input_unity_catalog_read_user_group": {
+      "order": 16,
+      "type": "string",
+      "default": "account users",
+      "description": "\nUser group name to give EXECUTE privileges to models in Unity Catalog (UC).\nIt must exist in UC with access granted to the staging and prod workspaces.\nDefault",
+      "skip_prompt_if": {
+        "anyOf":[
+          {
+            "properties": {
+              "input_include_models_in_unity_catalog": {
+                "const": "no"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "CICD_Only"
+              }
+            }
+          }
+        ]
+      }
+    },
+    "input_inference_table_name": {
+      "order": 17,
+      "type": "string",
+      "description": "\nFully qualified name of inference table to attach monitoring to.\nThis table must already exist and service principals must have access.",
+      "default": "dev.{{ .input_project_name }}.predictions",
+      "pattern": "^[^ .\\-\\/]+(\\.[^ .\\-\\/]+){2}$",
+      "pattern_match_failure_message": "Fully qualified Unity Catalog table names must have catalog, schema, and table separated by \".\" and each cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
+      "skip_prompt_if": {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "CICD_Only"
+              }
+            }
+      }
+    },
+    "input_include_feature_store": {
+      "order": 18,
+      "type": "string",
+      "description": "\nWhether to include Feature Store",
+      "default": "no",
+      "enum": ["no", "yes"],
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_include_mlflow_recipes": {
+      "order": 19,
+      "type": "string",
+      "description": "\nWhether to include MLflow Recipes",
+      "default": "no",
+      "enum": ["no", "yes"],
+      "skip_prompt_if": {
+        "anyOf":[
+          {
+            "properties": {
+              "input_include_models_in_unity_catalog": {
+                "const": "yes"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_include_feature_store": {
+                "const": "yes"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "CICD_Only"
+              }
+            }
+          }
+        ]
+      }
+    },      
+    "input_docker_image": {
+      "order": 20,
+      "type": "string",
+      "description": "\nDocker image for the execution of Gitlab pipelines",
+      "default": "databricksfieldeng/mlopsstacks:latest",
+      "skip_prompt_if": {
+        "anyOf":[
+          {
+            "properties": {
+              "input_cicd_platform": {
+                "const": "github_actions"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_cicd_platform": {
+                "const": "github_actions_for_github_enterprise_servers"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_cicd_platform": {
+                "const": "azure_devops"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "Project_Only"
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "success_message" : "\n*** Your MLOps Stack has been created in the '{{.input_root_dir}}{{if not (eq .input_setup_cicd_and_project `CICD_Only`) }}/{{.input_project_name}}{{end}}' directory! ***\n\nPlease refer to the README.md for further instructions on getting started."
+}