microsoft · j-so · Jun 26, 2020 · May 13, 2020 · May 14, 2020 · May 14, 2020
diff --git a/.pipelines/diabetes_regression-cd-deploy.yml b/.pipelines/diabetes_regression-cd-deploy.yml
@@ -0,0 +1,157 @@
+# Continuous Integration (CI) pipeline that orchestrates the deployment of the diabetes_regression model.
+
+# Runtime parameters to select artifacts
+parameters:
+- name : artifactBuildId
+  displayName: Model Train CI Build ID. Default is 'latest'.
+  type: string
+  default: latest
+
+pr: none
+
+# Trigger this pipeline on model-train pipeline completion
+trigger: none
+resources:
+  containers:
+  - container: mlops
+    image: mcr.microsoft.com/mlops/python:latest
+  pipelines:
+  - pipeline: model-train-ci
+    source: Model-Train-Register-CI # Name of the triggering pipeline
+    trigger:
+      branches:
+        include:
+        - master
+
+variables:
+- template: diabetes_regression-variables-template.yml
+- group: devopsforai-aml-vg
+
+stages:
+- stage: 'Deploy_ACI'
+  displayName: 'Deploy to ACI'
+  condition: variables['ACI_DEPLOYMENT_NAME']
+  jobs:
+  - job: "Deploy_ACI"
+    displayName: "Deploy to ACI"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - download: none
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+    - task: AzureCLI@1
+      displayName: 'Install AzureML CLI'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
+        inlineScript: 'az extension add -n azure-cli-ml'
+    - task: AzureCLI@1
+      displayName: "Deploy to ACI (CLI)"
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring
+        inlineScript: |
+          az ml model deploy --name $(ACI_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(MODEL_VERSION)' \
+          --ic inference_config.yml \
+          --dc deployment_config_aci.yml \
+          -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \
+          --overwrite -v
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_AKS'
+  displayName: 'Deploy to AKS'
+  dependsOn: Deploy_ACI
+  condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_AKS"
+    displayName: "Deploy to AKS"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+    - task: AzureCLI@1
+      displayName: 'Install AzureML CLI'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
+        inlineScript: 'az extension add -n azure-cli-ml'
+    - task: AzureCLI@1
+      displayName: "Deploy to AKS (CLI)"
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring
+        inlineScript: |
+          az ml model deploy --name $(AKS_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(MODEL_VERSION)' \
+          --compute-target $(AKS_COMPUTE_NAME) \
+          --ic inference_config.yml \
+          --dc deployment_config_aks.yml \
+          -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \
+          --overwrite -v
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_Webapp'
+  displayName: 'Deploy to Webapp'
+  condition: variables['WEBAPP_DEPLOYMENT_NAME']
+  jobs:
+  - job: "Deploy_Webapp"
+    displayName: "Package and deploy model"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - template: diabetes_regression-get-model-id-artifact-template.yml
+      parameters:
+        projectId: '$(resources.pipeline.model-train-ci.projectID)'
+        pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)'
+        artifactBuildId: ${{ parameters.artifactBuildId }}
+    - template: diabetes_regression-package-model-template.yml
+      parameters:
+        modelId: $(MODEL_NAME):$(MODEL_VERSION)
+        scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/score.py'
+        condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml'
+    - script: echo $(IMAGE_LOCATION) >image_location.txt
+      displayName: "Write image location file"
+    - task: AzureWebAppContainer@1
+      name: WebAppDeploy
+      displayName: 'Azure Web App on Container Deploy'
+      inputs:
+        azureSubscription: '$(AZURE_RM_SVC_CONNECTION)'
+        appName: '$(WEBAPP_DEPLOYMENT_NAME)'
+        resourceGroupName: '$(RESOURCE_GROUP)'
+        imageName: '$(IMAGE_LOCATION)'
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"
diff --git a/.pipelines/diabetes_regression-ci-train-register.yml b/.pipelines/diabetes_regression-ci-train-register.yml
@@ -0,0 +1,97 @@
+# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model.
+
+resources:
+  containers:
+  - container: mlops
+    image: mcr.microsoft.com/mlops/python:latest
+
+pr: none
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - diabetes_regression/
+    - ml_service/pipelines/diabetes_regression_build_train_pipeline.py
+    - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py
+    - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py
+
+variables:
+- template: diabetes_regression-variables-template.yml
+- group: devopsforai-aml-vg
+
+pool:
+  vmImage: ubuntu-latest
+
+stages:
+- stage: 'Model_CI'
+  displayName: 'Model CI'
+  jobs:
+  - job: "Model_CI_Pipeline"
+    displayName: "Model CI Pipeline"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - template: code-quality-template.yml
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          # Invoke the Python building and publishing a training pipeline
+          python -m ml_service.pipelines.diabetes_regression_build_train_pipeline
+      displayName: 'Publish Azure Machine Learning Pipeline'
+
+- stage: 'Trigger_AML_Pipeline'
+  displayName: 'Train and evaluate model'
+  condition: succeeded()
+  variables:
+    BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)'
+  jobs:
+  - job: "Get_Pipeline_ID"
+    condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true'))
+    displayName: "Get Pipeline ID for execution"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
+          # Set AMLPIPELINEID variable for next AML Pipeline task in next job
+          AMLPIPELINEID="$(cat pipeline_id.txt)"
+          echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID"
+      name: 'getpipelineid'
+      displayName: 'Get Pipeline ID'
+  - job: "Run_ML_Pipeline"
+    dependsOn: "Get_Pipeline_ID"
+    displayName: "Trigger ML Training Pipeline"
+    timeoutInMinutes: 0
+    pool: server
+    variables:
+      AMLPIPELINE_ID: $[ dependencies.Get_Pipeline_ID.outputs['getpipelineid.AMLPIPELINEID'] ]
+    steps:
+    - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0
+      displayName: 'Invoke ML pipeline'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        PipelineId: '$(AMLPIPELINE_ID)'
+        ExperimentName: '$(EXPERIMENT_NAME)'
+        PipelineParameters: '"ParameterAssignments": {"model_name": "$(MODEL_NAME)"}, "tags": {"BuildId": "$(Build.BuildId)", "BuildUri": "$(BUILD_URI)"}, "StepTags": {"BuildId": "$(Build.BuildId)", "BuildUri": "$(BUILD_URI)"}'
+  - job: "Training_Run_Report"
+    dependsOn: "Run_ML_Pipeline"
+    condition: always()
+    displayName: "Publish artifact if new model was registered"
+    container: mlops
+    timeoutInMinutes: 0
+    steps:
+    - template: diabetes_regression-publish-model-artifact-template.yml
diff --git a/.pipelines/diabetes_regression-get-model-id-artifact-template.yml b/.pipelines/diabetes_regression-get-model-id-artifact-template.yml
@@ -0,0 +1,45 @@
+parameters:
+- name: projectId
+  type: string
+  default: ''
+- name: pipelineId
+  type: string
+  default: ''
+- name: artifactBuildId
+  type: string
+  default: latest
+
+steps:
+  - download: none
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Pipeline Artifacts
+    inputs:
+      source: 'specific'
+      project: '${{ parameters.projectId }}'
+      pipeline: '${{ parameters.pipelineId }}'
+      preferTriggeringPipeline: true
+      ${{ if eq(parameters.artifactBuildId, 'latest') }}:
+        buildVersionToDownload: 'latestFromBranch'
+      ${{ if ne(parameters.artifactBuildId, 'latest') }}:
+        buildVersionToDownload: 'specific'
+        runId: '${{ parameters.artifactBuildId }}'
+      runBranch: '$(Build.SourceBranch)'
+      path: $(Build.SourcesDirectory)/bin
+  - task: Bash@3
+    displayName: Parse Json for Model Name and Version
+    inputs:
+      targetType: 'inline'
+      script: |
+        # Print JSON
+        cat $(Build.SourcesDirectory)/bin/model/model.json | jq '.'
+
+        # Set model name and version variables
+        MODEL_NAME=$(jq -r '.name' <$(Build.SourcesDirectory)/bin/model/model.json)
+        MODEL_VERSION=$(jq -r '.version' <$(Build.SourcesDirectory)/bin/model/model.json)
+
+        echo "Model Name: $MODEL_NAME"
+        echo "Model Version: $MODEL_VERSION"
+
+        # Set environment variables
+        echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION"
+        echo "##vso[task.setvariable variable=MODEL_NAME]$MODEL_NAME"
diff --git a/.pipelines/diabetes_regression-package-model-template.yml b/.pipelines/diabetes_regression-package-model-template.yml
@@ -0,0 +1,39 @@
+# Pipeline template that creates a model package and adds the package location to the environment for subsequent tasks to use.
+parameters:
+- name: modelId
+  type: string
+  default: ''
+- name: scoringScriptPath
+  type: string
+  default: ''
+- name: condaFilePath
+  type: string
+  default: ''
+
+steps:
+  - task: AzureCLI@1
+    displayName: 'Install AzureML CLI'
+    inputs:
+      azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+      scriptLocation: inlineScript
+      workingDirectory: $(Build.SourcesDirectory)
+      inlineScript: 'az extension add -n azure-cli-ml'
+  - task: AzureCLI@1
+    displayName: 'Create model package and set IMAGE_LOCATION variable'
+    inputs:
+      azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+      scriptLocation: inlineScript
+      inlineScript: |
+        set -e # fail on error
+
+        # Create model package using CLI
+        IMAGE_LOCATION=$(\
+        az ml model package --workspace-name $(WORKSPACE_NAME) -g $(RESOURCE_GROUP) \
+        --model '${{ parameters.modelId }}' \
+        --entry-script '${{ parameters.scoringScriptPath }}' \
+        --cf '${{ parameters.condaFilePath }}' \
+        --rt python --query 'location' -o tsv)
+
+        # Set environment variable
+        echo $IMAGE_LOCATION
+        echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
diff --git a/.pipelines/diabetes_regression-publish-model-artifact-template.yml b/.pipelines/diabetes_regression-publish-model-artifact-template.yml
@@ -0,0 +1,29 @@
+# Pipeline template to check if a model was registered for the build and publishes an artifact with the model JSON
+steps:
+- task: AzureCLI@1
+  displayName: 'Install AzureML CLI'
+  inputs:
+    azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+    scriptLocation: inlineScript
+    workingDirectory: $(Build.SourcesDirectory)
+    inlineScript: 'az extension add -n azure-cli-ml'
+- task: AzureCLI@1
+  inputs:
+    azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+    scriptLocation: inlineScript
+    workingDirectory: $(Build.SourcesDirectory)
+    inlineScript: |
+      set -e # fail on error
+
+      # Get the model using the build ID tag
+      FOUND_MODEL=$(az ml model list -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) --tag BuildId=$(modelbuildid) --query '[0]')
+
+      # If the variable is empty, print and fail
+      [[ -z "$FOUND_MODEL" ]] && { echo "Model was not registered for this run." ; exit 1; }
+
+      # Write to a file
+      echo $FOUND_MODEL >model.json
+  name: 'getversion'
+  displayName: "Determine if evaluation succeeded and new model is registered (CLI)"
+- publish: model.json
+  artifact: model
diff --git a/diabetes_regression/ci_dependencies.yml b/diabetes_regression/ci_dependencies.yml
@@ -12,6 +12,7 @@ dependencies:
   - r=3.6.0
   - r-essentials=3.6.0
 
+  - conda-forge::jq
   - pip=20.0.*
 
   - pip:

diff --git a/docs/images/model-deploy-configure.png b/docs/images/model-deploy-configure.png
diff --git a/docs/images/model-deploy-result.png b/docs/images/model-deploy-result.png
diff --git a/docs/images/model-train-register-artifacts.png b/docs/images/model-train-register-artifacts.png
diff --git a/docs/images/model-train-register.png b/docs/images/model-train-register.png
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,6 +12,7 @@ dependencies: @@
       - r=3.6.0
       - r-essentials=3.6.0
+      - conda-forge::jq
       - pip=20.0.*
       - pip:
@@ Expand Down @@