databricks.yml

# yaml-language-server: $schema=bundle_config_schema.json
bundle:
  name: gtfs-exo-bundle

workspace:
  host: https://adb-7865087530835937.17.azuredatabricks.net/

artifacts:
  default:
    type: whl
    build: poetry build
    path: exo-ingestion-gtfs

resources:
  jobs:
    exo-ingestion-gtfs:
      name: exo-ingestion-gtfs-${bundle.target}
      tasks:
        - task_key: exo-ingestion-gtfs-task
          existing_cluster_id: 1130-163615-1mtvpjso
          python_wheel_task:
            entry_point: ingest
            package_name: exo_ingestion_gtfs
            parameters:
              - --env
              - ${bundle.target}
          libraries:
            - whl: exo-ingestion-gtfs/dist/*.whl

  pipelines:
    exo-refine-gtfs:
      name: exo-refine-gtfs-${bundle.target}
      catalog: exo_${bundle.target}
      target: gtfs_refined
      continuous: false
      channel: CURRENT
      photon: false
      libraries:
        - notebook:
            path: ./exo-refined-gtfs/exo_gtfs_refine_deltalivetable-${bundle.target}.ipynb
      edition: CORE
      clusters:
        - label: "default"
          num_workers: 1
          node_type_id: "Standard_F4"
          driver_node_type_id: "Standard_F4"

targets:
  dev:
    default: true
    mode: development
  qa:
    mode: development
    resources:
      jobs:
        exo-ingestion-gtfs:
          tasks:
          - task_key: exo-ingestion-gtfs-task
            description: exo gtfs en qa
  prod:
    resources:
      pipelines:
        exo-refine-gtfs:
          name: exo-refine-gtfs-${bundle.target}
          clusters:
            - label: "default"
              num_workers: 4
    workspace:
      root_path: /exo-gtfs
    git:
      branch: main