databricks.txt

# Databricks Connect: https://docs.databricks.com/dev-tools/databricks-connect.html
config = Config(profile = "my_profile")
spark = DatabricksSession.builder.sdkConfig(config).getOrCreate()

# Make sure ~/.databrickscfg have the profile like so
[my_profile]
host = https://adb-12345678.20.databricks.net
token = dapi12345678
cluster_id = 0134-99999-1mkl20wer

# List all widgets in the notebook: https://stackoverflow.com/a/71971154
my_widgets = dbutils.notebook.entry_point.getCurrentBindings()
{key: my_widgets[key] for key in my_widgets}

# Show all mount points 
display(dbutils.fs.mounts())

# First time authentication
databricks auth login --host [workspace_url]

# Job export import
# ============================================================
# NOTE: might want to pause the job schedule first before exporting
JOB_ID=12345 SOURCE_PROFILE=profile_1 TARGET_PROFILE=profile_2

# Export job as "databricks_job.json". Require jq to get "settings" field in JSON
# By default set the schedule to PAUSED. Please start it manually after finish exporting.
databricks jobs --profile $SOURCE_PROFILE get $JOB_ID -o json | jq '.settings | walk(if type == "object" and has("pause_status") then .pause_status = "PAUSED" else . end)' > databricks_job.json

# Create job from exported json file
# May want to pause the job first: "pause_status": "PAUSED"
databricks jobs --profile $TARGET_PROFILE create --json @databricks_job.json
# ============================================================