uc-cdis · paulineribeyre · Sep 14, 2021 · Jul 14, 2021
diff --git a/azure-devops-pipeline.yaml b/azure-devops-pipeline.yaml
@@ -0,0 +1,147 @@
+trigger:
+  branches:
+    include:
+      - master
+variables:
+  - name: SP_CLIENT_ID
+    value: "MY_AZ_SP_CLIENT_ID"
+  - name: SP_CLIENT_PASS
+    value: "MY_AZ_SP_CLIENT_PASS"
+  - name: TENANT_ID
+    value: "MY_AZ_TENANT_ID"
+  - name: ACR_NAME
+    value: "myacrname"
+  - name: LOCAL_TEST_POSTGRESQL_USERNAME
+    value: test
+  - name: LOCAL_TEST_POSTGRESQL_PASSWORD
+    value: test
+  - name: LOCAL_POSTGRESQL_USERNAME
+    value: postgres
+  - name: LOCAL_POSTGRESQL_PASSWORD
+    value: test
+  - name: LOCAL_POSTGRESQL_PORT
+    value: 5433
+stages:
+  - stage: build
+    jobs:
+      - job: run_build_push_acr
+        pool:
+          vmImage: ubuntu-latest
+        steps:
+          - script: |-
+              sudo apt-get update
+              sudo apt-get install python3
+            displayName: Install Python 3
+
+          - script: |-
+              python3 -m venv py38-venv
+            displayName: Create Python Virtual Environment
+
+          - script: |-
+              # install psql
+              sudo apt install postgresql-client-common
+              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+              wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+              sudo apt-get update
+              sudo apt-get install postgresql-12
+              sudo pg_ctlcluster 12 main start
+
+              # wait for psql to be ready
+              tail /var/log/postgresql/postgresql-12-main.log | sed '/^database system is ready to accept connections$/ q'
+
+              # run psql scripts to initialize db
+              curDir=$(pwd)
+              ls "${curDir}/deployment/scripts/postgresql/postgresql_test_init.sql"
+              sudo -u postgres psql -f "${curDir}/deployment/scripts/postgresql/postgresql_test_init.sql" -p "$(LOCAL_POSTGRESQL_PORT)"
+
+              # setup root user
+              sudo -u postgres psql -c "ALTER USER $(LOCAL_POSTGRESQL_USERNAME) WITH PASSWORD '$(LOCAL_POSTGRESQL_PASSWORD)';" -p "$(LOCAL_POSTGRESQL_PORT)"
+            displayName: Setup Local Postgresql for Testing
+
+          - script: |-
+              . py38-venv/bin/activate
+              python3 -m pip install --upgrade pip
+
+              export CRYPTOGRAPHY_DONT_BUILD_RUST=1
+              python3 -m pip install -r requirements.txt
+              python3 setup.py develop
+              python3 -m pip install -r dev-requirements.txt
+
+              # if you make changes in the /bin scripts then make sure to re-install
+              python3 setup.py build
+              python3 setup.py install
+            displayName: Install Sheepdog Dependencies
+
+          - script: |-
+              . py38-venv/bin/activate
+
+              python3 bin/setup_test_database.py --database "sheepdog_automated_test" --root_user $(LOCAL_POSTGRESQL_USERNAME) --root_password $(LOCAL_POSTGRESQL_PASSWORD) --user $(LOCAL_TEST_POSTGRESQL_USERNAME) --password $(LOCAL_TEST_POSTGRESQL_PASSWORD) --port $(LOCAL_POSTGRESQL_PORT)
+              mkdir -p tests/integration/resources/keys; cd tests/integration/resources/keys; openssl genrsa -out test_private_key.pem 2048; openssl rsa -in test_private_key.pem -pubout -out test_public_key.pem; cd -
+            displayName: Setup Test Environment
+
+          - script: |-
+              . py38-venv/bin/activate
+              py.test -vv --cov=sheepdog --cov-report xml --junitxml="test-results-datadict.xml" tests/integration/datadict
+              py.test -vv --cov=sheepdog --cov-report xml --cov-append --junitxml="test-results-datadictwithobjid.xml" tests/integration/datadictwithobjid
+              py.test -vv --cov=sheepdog --cov-report xml --cov-append --junitxml="test-results-unit.xml" tests/unit
+            displayName: Run Sheepdog Test Suite
+            env:
+              PGPORT: $(LOCAL_POSTGRESQL_PORT)
+
+          - script: |-
+              . py38-venv/bin/activate
+              python3 -m pip install junitparser
+
+              # Use script to merge together test results
+              # https://pypi.org/project/junitparser/
+              eval $(python 2> /dev/null <<EOM
+              from junitparser import JUnitXml
+
+              xml_data_dict = JUnitXml.fromfile('test-results-datadict.xml')
+              xml_data_dict_with_objid =JUnitXml.fromfile('test-results-datadictwithobjid.xml')
+              xml_unit = JUnitXml.fromfile('test-results-unit.xml')
+
+              xml_combined = xml_data_dict + xml_data_dict_with_objid + xml_unit
+
+              xml_combined.write('test-results.xml')
+              EOM
+              )
+            displayName: Combine Test Results
+
+          - task: PublishTestResults@2
+            condition: succeededOrFailed()
+            inputs:
+              testResultsFiles: '**/test-*.xml'
+              testRunTitle: 'Publish test results for Python $(python.version)'
+
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+              reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
+
+          - script: |-
+              set -e
+              echo "az login --service-principal --username $(SP_CLIENT_ID) --password $(SP_CLIENT_PASS) --tenant $(TENANT_ID)"
+              az login --service-principal --username "$(SP_CLIENT_ID)" --password "$(SP_CLIENT_PASS)" --tenant "$(TENANT_ID)"
+            displayName: Azure Login
+
+          - script: |-
+              set -e
+              echo "PWD:"
+              pwd
+              ls -a
+              export BUILD_REPO_NAME=$(echo $(Build.Repository.Name) | tr '[:upper:]' '[:lower:]')
+              export IMAGE_TAG=$(echo $(Build.SourceBranchName) | tr / - | tr . - | tr _ - )-$(Build.BuildNumber)
+              export IMAGE_NAME=$BUILD_REPO_NAME:$IMAGE_TAG
+              echo "Image Name: $IMAGE_NAME"
+              ACR_BUILD_COMMAND="az acr build -r $(ACR_NAME) --image $IMAGE_NAME ."
+
+              cd ./
+              echo "PWD:"
+              pwd
+              ls -a
+              echo "ACR BUILD COMMAND: $ACR_BUILD_COMMAND"
+              $ACR_BUILD_COMMAND
+            displayName: ACR Build and Publish
+
diff --git a/deployment/scripts/postgresql/postgresql_test_init.sql b/deployment/scripts/postgresql/postgresql_test_init.sql
@@ -1 +1,6 @@
+/*
+Set up PostgreSQL for testing
+See example usage in ADO pipeline: (under ../../../azure-devops-pipeline.yaml)
+*/
+
 CREATE DATABASE sheepdog_automated_test;
diff --git a/docs/azure_devops_pipeline.md b/docs/azure_devops_pipeline.md
@@ -0,0 +1,126 @@
+# Azure DevOps Build Pipeline
+
+The purpose of this [Azure DevOps Pipeline](../azure-devops-pipeline.yaml) is to build `sheepdog`, run a test suite, and then push the `sheepdog` container into an [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal).
+
+## Getting Started
+
+If you don't already have access, you can use the free sign up with [Azure Devops](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/pipelines-sign-up?view=azure-devops).
+
+You can also import the [pipeline](../azure-devops-pipeline.yaml), see these [doc notes](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/clone-import-pipeline?view=azure-devops&tabs=yaml#export-and-import-a-pipeline) as a guide.
+
+### Setup Azure Container Registry
+
+[Create a Service Principal](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli#password-based-authentication) in your Azure Subscription using [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli).
+
+First, log into `az` cli:
+
+```bash
+az login
+az account set -s <Subscription ID>
+```
+
+You can create a **service principal** in Azure AD:
+
+```bash
+spObject=$(az ad sp create-for-rbac --name ServicePrincipalName)
+
+# this can be used for the SP_CLIENT_ID
+spClientId=$(echo $spObject | jq -r ".appId")
+
+# this can be used for the SP_CLIENT_PASSWORD
+spPassword=$(echo $spObject | jq -r ".password")
+
+# this can be used for the TENANT_ID
+spTenantId=$(echo $spObject | jq -r ".tenant")
+```
+
+> You will need to have appropriate permissions in the AAD directory.  If you don't have access, please work with your Azure Subscription administrator to obtain a Service Principal.
+
+You can also create an **Azure Container Registry** using [azure cli](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-azure-cli) or the [portal](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal).
+
+You can use the following `az` cli commands in `bash` for reference:
+
+```bash
+az group create --name myResourceGroup --location eastus
+az acr create --resource-group myResourceGroup --name myContainerRegistry --sku Basic
+```
+
+Also, make sure that the **Service Principal** has rights to the [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-roles?tabs=azure-cli) to **acrPull** and **acrPush**.
+
+```bash
+acrResourceId="$(az acr show -n myContainerRegistry -g myResourceGroup --query "id" -o tsv)"
+
+az role assignment create --assignee $spClientId --role 'AcrPull' --scope $acrResourceId
+
+az role assignment create --assignee $spClientId --role 'AcrPush' --scope $acrResourceId
+```
+
+To verify if the pipeline context will have access to ACR, you can login.
+
+> Note, this is an approach for dev / test, but in a production scenario, it is more likely that your SP Credentials used in the Azure DevOps Pipeline would be populated as secrets through variables or Variable Groups.
+
+```bash
+az login --service-principal --username "$spClientId" --password "$spPassword" --tenant "$spTenantId"
+
+az acr login --name myContainerRegistry
+```
+
+You can also verify that this service principal will have `ACRPush` and `ACRPull` permission with ACR, which you can check how the [getting started with docker guide](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-docker-cli?tabs=azure-cli) for more details.
+
+First, pull and tag an image:
+
+```bash
+docker pull mcr.microsoft.com/oss/nginx/nginx:1.15.5-alpine
+
+docker tag mcr.microsoft.com/oss/nginx/nginx:1.15.5-alpine mycontainerregistry.azurecr.io/samples/nginx
+```
+
+> Note that the ACR names will default to **lowercase** for the `fqdn`, so make sure that when you're tagging images to use **lowercase** for the ACR name.
+
+Check that you can push an image to ACR:
+
+```bash
+docker push mycontainerregistry.azurecr.io/samples/nginx
+```
+
+Check that you can pull an image from ACR:
+
+```bash
+docker pull mycontainerregistry.azurecr.io/samples/nginx
+```
+
+You can also list out the images in the ACR with `az` cli:
+
+```bash
+az acr repository list --name mycontainerregistry
+```
+
+## Configuring the Pipeline
+
+You can set the variables on your **Azure DevOps pipeline**.
+
+First, make sure you have already [imported your Azure DevOps Pipeline](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/clone-import-pipeline?view=azure-devops&tabs=yaml#export-and-import-a-pipeline).
+
+Click on the pipeline and then click edit, which will let you update the variables in the Azure DevOps pipeline:
+
+![Click on Variables](azure_devops_pipeline_config_1.png)
+
+Variable Name | Description  
+------ | ------
+SP_CLIENT_ID | This is your Service Principal Client ID.
+SP_CLIENT_PASS | This is your Service Principal Password.  You can override this value when running the Azure DevOps pipeline.
+TENANT_ID | This is the Azure AD tenant ID where the SP and the ACR reside.
+ACR_NAME | This is the Azure Container Registry name.  Note, it is not the **FQDN** (e.g. `myacrname` instead of `myacrname.azurecr.io`).
+LOCAL_TEST_POSTGRESQL_USERNAME | Automated Test Username for the local `PostgreSQL` in the pipeline for running the `sheepdog` test suite.  Default value is `test`.
+LOCAL_TEST_POSTGRESQL_PASSWORD | Automated Test Password for the local `PostgreSQL` in the pipeline for running the `sheepdog` test suite.  Default value is `test`.
+LOCAL_POSTGRESQL_USERNAME | Username in `PostgreSQL`, and used to setup PostgreSQL schema in the pipeline for running the `sheepdog` test suite.  Default value is `postgres`.
+LOCAL_POSTGRESQL_PASSWORD | Password for the user in `PostgreSQL`  used to setup PostgreSQL schema in the pipeline for running the `sheepdog` test suite.  Default value is `test`.
+LOCAL_POSTGRESQL_PORT | This is the Local PostgreSQL Port number.  The default port for a `PostgreSQL` server is `5432`, but you can change this to another port in case this port is already in use on the host. For example you can use `5433`.
+
+After updating the variables, be sure to click **save**:
+
+![Save updated variables](azure_devops_pipeline_config_2.png)
+
+You can run the pipeline to validate the `sheepdog` build and push to ACR.
+
+![Run the pipeline](azure_devops_pipeline_config_3.png)
diff --git a/docs/azure_devops_pipeline_config_1.png b/docs/azure_devops_pipeline_config_1.png
diff --git a/docs/azure_devops_pipeline_config_2.png b/docs/azure_devops_pipeline_config_2.png
diff --git a/docs/azure_devops_pipeline_config_3.png b/docs/azure_devops_pipeline_config_3.png