spark_config:
  # choice for spark runtime. Currently support: azure_synapse, databricks
  # The `databricks` configs will be ignored if `azure_synapse` is set and vice versa.
  spark_cluster: "azure_synapse"
  # configure number of parts for the spark output for feature generation job
  spark_result_output_parts: "1"

  databricks:
    # workspace instance
    workspace_instance_url: 'https://adb-6885802458123232.12.azuredatabricks.net/'
    # config string including run time information, spark version, machine size, etc.
    # the config follows the format in the databricks documentation: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/2.0/jobs#--request-structure-6
    # The fields marked as "FEATHR_FILL_IN" will be managed by Feathr. Other parameters can be customizable. For example, you can customize the node type, spark version, number of workers, instance pools, timeout, etc.
    config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","node_type_id":"Standard_D3_v2","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"}},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}'
    # workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here
    work_dir: "dbfs:/feathr_getting_started"
    # This is the location of the runtime jar for Spark job submission. If you have compiled the runtime yourself, you need to specify this location.
    # Or use https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-LATEST.jar so you don't have to compile the runtime yourself
    # Local path, path starting with `http(s)://` or `dbfs://` are supported. If not specified, the latest jar from Maven would be used
    feathr_runtime_location: "https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-LATEST.jar"

2022-09-26 00:10:42.789 | ERROR    | feathr.spark_provider._databricks_submission:wait_for_completion:210 - Feathr job has failed. Please visit this page to view error message: https://adb-1996253548709298.18.azuredatabricks.net/?o=1996253548709298#job/464256084943565/run/253311
2022-09-26 00:10:42.789 | ERROR    | feathr.spark_provider._databricks_submission:wait_for_completion:212 - Error Code: Run result unavailable: job failed with error message
 Library installation failed for library due to user error for maven {
  coordinates: "com.linkedin.feathr:feathr_2.12:0.8.0"
}
. Error messages:
Library installation attempted on the driver node of cluster 0926-000613-mqjvrxxu and failed. Please refer to the following error message to fix the library or contact Databricks support. Error Code: DRIVER_LIBRARY_INSTALLATION_FAILURE. Error Message: Library resolution failed. Cause: java.lang.RuntimeException: commons-logging:commons-logging download failed.