#!/bin/bash
MG_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd "../must-gather" && pwd )"

function must_gather_help() {
  [[ -n "$1" ]] && echo_warning "$1"
  reset_colors
  cat << EOM
Usage:
  mas must-gather [options]
Where ${COLOR_YELLOW}specified${TEXT_RESET} each option may also be defined by setting the appropriate environment variable.

Destination:
  -d, --directory ${COLOR_YELLOW}MG_DIR${TEXT_RESET}     Directory where the must-gather will be saved, defaults to /tmp/must-gather
  -k, --keep-files           Do not delete individual files after creating the must-gather compressed tar archive

General Controls:
  --summary-only             Perform a much faster must-gather that only gathers high level summary information
  --no-logs                  Skip collection of pod logs, greatly speeds up must-gather collection time when pod logs are not required
  --secret-data              Include secrets content in the must-gather

MAS Content Controls:
  --mas-instance-ids         Limit must-gather to a list of MAS instance IDs (comma-seperated list)
  --mas-app-ids              Limit must-gather to a subset of MAS namespaces (comma-seperated list)

AI Service Content Controls:
  --aiservice-instance-ids   Limit must-gather to a list of AI Service instance IDs (comma-seperated list)
  --aiservice-tenant-ids     Limit must-gather to a list of AI Service tenant IDs (comma-seperated list)

Disable Collectors:
  --no-ocp                   Disable must-gather for the OCP cluster itself
  --no-dependencies          Disable must-gather for in-cluster dependencies (Db2, Cloud Pak for Data, Cloud Pak Foundational Services, Mongo)
  --no-sls                   Disable must-gather for IBM Suite License Service
  --no-mas-quick-summary     Disable MAS quick summary reports

Additional Collectors:
  --extra-namespaces         Enable must-gather in custom namespaces (comma-seperated list)

Artifactory Upload:
  --artifactory-token ${COLOR_YELLOW}ARTIFACTORY_TOKEN${TEXT_RESET}              Provide a token for Artifactory to automatically upload the file to ARTIFACTORY_UPLOAD_DIR
  --artifactory-upload-dir ${COLOR_YELLOW}ARTIFACTORY_UPLOAD_DIR${TEXT_RESET}    Working URL to the root directory in Artifactory where the must-gather file should be uploaded

Other Options:
  -h, --help    Show this help message
EOM
  [[ -n "$1" ]] && exit 1 || exit 0
}

function startTimer() {
  TIMER_START=`date +%s`
}
function endTimer() {
  TIMER_END=`date +%s`
  echo
  echo_green "Collection for $1 completed in $(expr $TIMER_END - $TIMER_START) seconds"
}

function genericMustGather() {
  echo
  $MG_SCRIPT_DIR/mg-collect-ibm-custom-resources -n $1 -d $OUTPUT_DIR $POD_LOGS_FLAG $NO_DETAIL_FLAG
  echo
  echo_h4 "Collect Standard Kubernetes Resources"
  ADDITIONAL_RESOURCES=$(echo "$2" | tr "," " ")
  for RESOURCE in $ADDITIONAL_RESOURCES configmaps services routes deployments jobs pvc operatorconditions clusterserviceversions installplans subscriptions serviceaccounts roles rolebindings certificates statefulsets
  do
    $MG_SCRIPT_DIR/mg-collect-resources -n $1 -r $RESOURCE -d $OUTPUT_DIR/resources $NO_DETAIL_FLAG
  done
  $MG_SCRIPT_DIR/mg-collect-pods -n $1 -d $OUTPUT_DIR/resources $POD_LOGS_FLAG $NO_DETAIL_FLAG
  $MG_SCRIPT_DIR/mg-collect-secrets -n $1 -r $RESOURCE -d $OUTPUT_DIR/resources -s $SECRET_DATA
}

function mustgather() {
  MG_COMMAND=$(echo $@)
  # Take the first parameter off (it will be "must-gather")
  shift

  # Set defaults
  SUMMARY_ONLY=false
  NO_DETAIL_FLAG=""
  POD_LOGS=true
  POD_LOGS_FLAG="-p"
  SECRET_DATA=false
  OCP_REPORT=true
  DEPENDENCY_REPORT=true
  SLS_REPORT=true
  MAS_QUICK_SUMMARY=true
  MAS_APP_IDS="core,add,assist,iot,monitor,manage,optimizer,predict,visualinspection,pipelines,facilities"
  EXTRA_NAMESPACES=""


  # If a /must-gather directory exists, default to that as the output directory, otherwise
  # the default is /tmp/must-gather
  MG_DIR=/tmp/must-gather
  if [[ -d "/must-gather" ]]; then
    MG_DIR=/must-gather
  fi

  # Process command flags
  while [[ $# -gt 0 ]]
  do
    key="$1"
    shift
    case $key in
    -d|--directory)
      MG_DIR=$1; shift
      ;;

    # Summary only mode will omit all the generic Kubernetes resources from the must-gather (pods, deployments, secrets, configmaps etc)
    # It results in an incredibly fast gather, but the result is of limited use
    --summary-only)
      SUMMARY_ONLY=true
      NO_DETAIL_FLAG="--no-detail"
      ;;

    # Omitting the logs from the must-gather can significantly reduce the time to generate and size of the must-gather file, in many cases
    # it will also result in key information being missing from the must-gather collection
    --no-logs|--no-pod-logs)
      POD_LOGS=false
      POD_LOGS_FLAG=""
      ;;

    # Restict the gather to processing a specific list of MAS instance IDs, generally discouraged as problems in one MAS instance may be
    # caused by anything on the cluster, including other MAS instances
    --mas-instance-ids)
      MAS_INSTANCE_IDS=$1; shift
      ;;

    # Limit the must-gather to only specific MAS applications, again highly discouraged as a problem in one application may be caused by something
    # in another application and without the full picture we may not be able to diagnose the problem
    --mas-app-ids)
      MAS_APP_IDS=$1; shift
      ;;

    # Restict the gather to processing a specific list of AI Service instance IDs, generally discouraged as problems in one instance may be
    # caused by anything on the cluster, including other instances
    --aiservice-instance-ids)
      AISERVICE_INSTANCE_IDS=$1; shift
      ;;

    # Limit the must-gather to only specific AI Service tenants,
    --aiservice-tenant-ids)
      AISERVICE_TENANT_IDS=$1; shift
      ;;

    # Provide a list of additional namespaces where generic Kubernetes must-gather will be collected from
    --extra-namespaces)
      EXTRA_NAMESPACES=$1; shift
      ;;

    # By default the contents of secrets are not included in the must-gather, in most cases you will not need to use this option.
    --secret-data)
      SECRET_DATA=true
      ;;

    # Disable parts of the must-gather process
    --no-ocp)
      OCP_REPORT=false
      ;;
    --no-dependencies)
      DEPENDENCY_REPORT=false
      ;;
    --no-sls)
      SLS_REPORT=false
      ;;
    --no-mas-quick-summary)
      MAS_QUICK_SUMMARY=false
      ;;

    # Automatic upload of the must-gather to Artifactory (primarily used in MAS FVT)
    --artifactory-token)
      ARTIFACTORY_TOKEN=$1; shift
      ;;
    --artifactory-upload-dir)
      ARTIFACTORY_UPLOAD_DIR=$1; shift
      ;;

    # Retain the source files after generating the tgz (for easy local analysis of the must-gather)
    -k|--keep-files)
      KEEP_FILES=true;
      ;;

    -h|--help)
      must_gather_help
      exit 0
      ;;
    *)
      # unknown option
      echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${TEXT_RESET}\n"
      must_gather_help
      exit 1
      ;;
    esac
  done

  TIMESTAMP=$(date +"%Y%m%d-%H%M%S")

  # All the output is generated here
  OUTPUT_DIR=${MG_DIR}/${TIMESTAMP}
  mkdir -p $OUTPUT_DIR

  if [[ "$?" != "0" ]]; then
    echo_warning "Unable to create must-gather output directory '${MG_DIR}/${TIMESTAMP}'"
    exit 1
  fi

  # The final destination for the must-gather is a tgz
  OUTPUT_FILENAME=must-gather-${TIMESTAMP}.tgz
  OUTPUT_FILE=${MG_DIR}/${OUTPUT_FILENAME}

  LOG_FILE=${OUTPUT_DIR}/must-gather.log
  exec > >(tee ${LOG_FILE}) 2>&1

  echo "Command executed: $MG_COMMAND"
  echo "Must gather generated with mas cli version: $VERSION"
  echo "Must gather will be saved to: $OUTPUT_FILE"
  echo "For help reviewing the content of the must gather refer to:"
  echo "https://www.ibm.com/support/pages/how-review-maximo-application-suite-must-gather"

  if [ "$(oc whoami 2>/dev/null)" == "" ] ; then 
    echo_warning "You must be logged in to the server as a cluster administrator before running the must-gather command"
    exit 1
  elif [ "$(oc get clusterrolebindings 2>&1 | grep forbidden)" != "" ] ; then 
    echo_warning "Your user does not appear to be a cluster administrator, you must be logged in to the server as a cluster administrator before running the must-gather command"
    exit 1
  fi

  # ---------------------------------------------------------------------------
  # Generate OCP Report
  # ---------------------------------------------------------------------------
  echo_h2 "OpenShift Container Platform"
  if [[ "$OCP_REPORT" == "true" ]]; then
    echo_h3 "Cluster Resources"
    startTimer

    for RESOURCE in storageclasses clusterversions objectbucket objectbucketclaim objectstoragecfg
    do
      $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources
    done

    for RESOURCE in nodes
    do
      $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources --describe
    done

    # collection for airgap environment
    if [[ $(oc get imagecontentsourcepolicy,imagedigestmirrorset,imagetagmirrorset  2> /dev/null) != "" ]]; then 
    
      for RESOURCE in imagecontentsourcepolicy imagedigestmirrorset imagetagmirrorset machineconfig machineconfigpool
      do
        $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources
      done

      $MG_SCRIPT_DIR/mg-collect-node-files -d $OUTPUT_DIR/resources -f /host/etc/containers/registries.conf
    fi


    

    for RESOURCE in namespaces packagemanifests clusterroles clusterrolebindings
    do
      $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources --no-detail
    done

    echo_h3 "OpenShift Marketplace"
    for RESOURCE in catalogsources jobs
    do
      $MG_SCRIPT_DIR/mg-collect-resources -n openshift-marketplace -r $RESOURCE -d $OUTPUT_DIR/resources
    done

    echo_h3 "Kubernetes Operators"
    for RESOURCE in subscriptions installplans operatorconditions
    do
      $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources -a
    done
    endTimer "OCP"
  else
    echo_warning "OpenShift Container Platform must-gather disabled"
  fi

  # ---------------------------------------------------------------------------
  # MAS Dependencies
  # ---------------------------------------------------------------------------
  echo_h2 "In-Cluster Dependencies"
  if [[ "$DEPENDENCY_REPORT" == "true" ]]; then
    startTimer
    echo_h3 "IBM CloudPak Foundation Services"
    $MG_SCRIPT_DIR/mg-summary-ibm-common-services &> ${OUTPUT_DIR}/ibm-common-services.txt
    genericMustGather ibm-common-services

    echo_h3 "IBM CloudPak for Data"
    NAMESPACE_LOOKUP=$(oc get namespace ibm-cpd-operators --ignore-not-found)
    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      $MG_SCRIPT_DIR/mg-summary-cp4d &> ${OUTPUT_DIR}/cp4d.txt
      genericMustGather ibm-cpd
      genericMustGather ibm-cpd-operators
    else
      echo_highlight "Unable to find ibm-cpd-operators namespace"
    fi

    echo_h3 "IBM Db2 Universal Operator"
    if [[ "$MAS_INSTANCE_IDS" != "" ]]; then
      # using the jdbccfg determine what db2 service is used, search for that service to determine the db2u namespace
      DB2U_NAMESPACES=$(for JDBC_URL in $(oc get jdbccfg -n mas-${MAS_INSTANCE_IDS}-core -o jsonpath='{.items[*].spec.config.url}'); do echo $JDBC_URL| cut -d"/" -f3| cut -d"." -f2; done | sort | uniq )
    else
      DB2U_NAMESPACES=$(oc get db2ucluster -A --ignore-not-found -o jsonpath='{.items[*].metadata.namespace}' | tr " " "\n"  |sort  | uniq)
    fi
    for DB2U_NAMESPACE in $DB2U_NAMESPACES
    do
      $MG_SCRIPT_DIR/mg-summary-db2u &> ${OUTPUT_DIR}/db2u.txt
      genericMustGather $DB2U_NAMESPACE
    done

    echo_h3 "IBM Data Reporter Operator"
    DRO_NAMESPACE_LOOKUP=$(oc get DataReporterConfig -A --ignore-not-found -o jsonpath='{.items[*].metadata.namespace}')
    if [[ "$DRO_NAMESPACE_LOOKUP" != "" ]]; then
      $MG_SCRIPT_DIR/mg-summary-dro &> ${OUTPUT_DIR}/dro.txt
      genericMustGather $DRO_NAMESPACE_LOOKUP DataReporterConfig,MarketplaceConfig,MeterReport,MeterBase,RazeeDeployment,MeterDefinition
    else
      echo_highlight "Unable to find namespace for DRO"
    fi 

    echo_h3 "Red Hat Certificate Manager"
    NAMESPACE_LOOKUP=$(oc get namespace cert-manager-operator --ignore-not-found)
    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      $MG_SCRIPT_DIR/mg-summary-cert-manager &> ${OUTPUT_DIR}/cert-manager.txt
      genericMustGather cert-manager-operator CertificateRequest,Certificate,Challenge,ClusterIssuer,Issuer,Order,CertManager
    else
      echo_highlight "Unable to find cert-manager-operator namespace"
    fi

    NAMESPACE_LOOKUP=$(oc get namespace cert-manager --ignore-not-found)
    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      genericMustGather cert-manager
    else
      echo_highlight "Unable to find cert-manager namespace"
    fi

    NAMESPACE_LOOKUP=$(oc get namespace openshift-storage --ignore-not-found)
    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      genericMustGather openshift-storage StorageCluster
    else
      echo_highlight "Unable to find openshift-storage namespace"
    fi

    echo_h3 "Kafka"
    KAFKA_NAMESPACES=$(oc get Kafka -A --ignore-not-found -o jsonpath='{.items[*].metadata.namespace}')
    for KAFKA_NAMESPACE in $KAFKA_NAMESPACES
    do
      $MG_SCRIPT_DIR/mg-summary-kafka &> ${OUTPUT_DIR}/kafka.txt
      genericMustGather $KAFKA_NAMESPACE Kafka,KafkaUser
    done

    echo_h3 "Grafana"
    GRAFANA_NAMESPACES=$(oc get Grafana -A --ignore-not-found -o jsonpath='{.items[*].metadata.namespace}')
    for GRAFANA_NAMESPACE in $GRAFANA_NAMESPACES
    do
      $MG_SCRIPT_DIR/mg-summary-grafana &> ${OUTPUT_DIR}/kafka.txt
      genericMustGather $GRAFANA_NAMESPACE Grafana,GrafanaDatasource
    done

    MONGOCE_NAMESPACES=$(oc get mongodbcommunity --all-namespaces --ignore-not-found -o jsonpath='{.items[*].metadata.namespace}')
    for MONGOCE_NAMESPACE in $MONGOCE_NAMESPACES
    do
      echo_h3 "MongoCE: ${MONGOCE_NAMESPACE}"
      $MG_SCRIPT_DIR/mg-summary-mongoce $MONGOCE_NAMESPACE &> ${OUTPUT_DIR}/mongoce-${MONGOCE_NAMESPACE}.txt
      genericMustGather $MONGOCE_NAMESPACE mongodbcommunity
    done
    endTimer "in-cluster dependencies"
  else
    echo_warning "Maximo Application Suite dependency must-gather disabled"
  fi

  # ---------------------------------------------------------------------------
  # SLS Must-Gather
  # ---------------------------------------------------------------------------
  echo_h2 "Suite License Service"
  if [[ "$SLS_REPORT" == "true" ]]; then
    startTimer
    if [[ "$MAS_INSTANCE_IDS" != "" ]]; then
      # using the slscfg determine what sls service is used, search for that service to determine the sls namespace
      SLS_NAMESPACES=$(for SLS_URL in $(oc get slscfg -n mas-${MAS_INSTANCE_IDS}-core -o jsonpath='{.items[*].spec.config.url}'); do SLS_ROUTE=$(echo $SLS_URL| cut -d"/" -f3) ; oc get route -A| grep $SLS_ROUTE | cut -d" " -f1; done)
    else
      SLS_NAMESPACES=$(oc get LicenseService --all-namespaces --ignore-not-found -o jsonpath='{.items[*].metadata.namespace}')
    fi
    
    echo ""
    for SLS_NAMESPACE in $SLS_NAMESPACES
    do
      # assume that there is only one LicenseService in the namespace
      SLS_NAME=$(oc get LicenseService -n ${SLS_NAMESPACE} --ignore-not-found -o jsonpath='{.items[0].metadata.name}')
      echo_h3 "Namespace: ${SLS_NAMESPACE}"

      # SLS-specific must-gather
      $MG_SCRIPT_DIR/mg-summary-sls $SLS_NAMESPACE &> ${OUTPUT_DIR}/${SLS_NAMESPACE}.txt
      genericMustGather $SLS_NAMESPACE
    done
    endTimer "SLS"
  else
    echo_warning "IBM Suite License Service must-gather disabled"
  fi


  # ---------------------------------------------------------------------------
  # MAS Must-Gather
  # ---------------------------------------------------------------------------
  if [[ -z $MAS_INSTANCE_IDS ]]; then
    # Find MAS instances
    MAS_INSTANCE_IDS=$(oc get suite --all-namespaces --ignore-not-found -o jsonpath='{.items[*].metadata.name}')
    if [[ -z $MAS_INSTANCE_IDS ]]; then
      # if there is no suite CR, search for mas-xxx-pipelines namespace to collect from that namespace
      echo_warning "Suite CR not found, searching mas-xxx-pipelines project to get mas instance id"
      MAS_INSTANCE_IDS=$(oc get project -oname | grep -o  mas-.*-pipelines | cut -d"-" -f 2)
    fi
  fi



  echo ""
  for MAS_INSTANCE_ID in $MAS_INSTANCE_IDS
  do
    echo_h2 "Maximo Application Suite: ${MAS_INSTANCE_ID}"
    startTimer
    for MAS_APP_ID in $(echo $MAS_APP_IDS | tr "," " ")
    do
      MAS_APP_NAMESPACE=mas-${MAS_INSTANCE_ID}-${MAS_APP_ID}
      NAMESPACE_LOOKUP=$(oc get namespace $MAS_APP_NAMESPACE --ignore-not-found)

      if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
        echo_h3 "Namespace: ${MAS_APP_NAMESPACE}"
        echo
        # MAS-specific must-gather
        $MG_SCRIPT_DIR/mg-summary-mas-${MAS_APP_ID} $MAS_APP_NAMESPACE &> ${OUTPUT_DIR}/mas-${MAS_INSTANCE_ID}-${MAS_APP_ID}.txt
        $MG_SCRIPT_DIR/mg-collect-mas-${MAS_APP_ID} $MAS_APP_NAMESPACE $OUTPUT_DIR

        genericMustGather $MAS_APP_NAMESPACE
      fi
    done

    if [[ "$MAS_QUICK_SUMMARY" == "true" ]]; then

      MAS_QUICK_SUMMARY_DIR_PATH=${OUTPUT_DIR}/mas-quick-summary
      mkdir -p $MAS_QUICK_SUMMARY_DIR_PATH

      # Generate quick summary report that helps troubleshoot user synchronization problems
      MAS_QUICK_SUMMARY_FILE_PATH=${MAS_QUICK_SUMMARY_DIR_PATH}/${MAS_INSTANCE_ID}.txt
      echo_h3 "MAS quick summary: ${MAS_INSTANCE_ID}"
      echo
      $MG_SCRIPT_DIR/mg-quick-summary-mas $MAS_INSTANCE_ID $MAS_QUICK_SUMMARY_FILE_PATH
      echo "MAS quick summary generated"
      echo
    fi
    
    endTimer "${MAS_INSTANCE_ID}:${MAS_APP_ID}"
  done

  # ---------------------------------------------------------------------------
  # AI Service Dependencies
  # ---------------------------------------------------------------------------
  echo_h2 "In-Cluster AI Service Dependencies"
  if [[ "$DEPENDENCY_REPORT" == "true" ]]; then
    startTimer
    
    echo_h3 "Open Data Hub"
    ODH_NAMESPACE_LOOKUP=$(oc get namespace opendatahub --ignore-not-found)
    if [[ "$ODH_NAMESPACE_LOOKUP" != "" ]]; then
      $MG_SCRIPT_DIR/mg-summary-opendatahub &> ${OUTPUT_DIR}/opendatahub.txt
      genericMustGather opendatahub Auth,DataScienceCluster,DataSciencePipelines,DataSciencePipelinesApplication,DSCInitialization,Kserve,ModelController
    else
      echo_highlight "Unable to find namespace for opendatahub"
    fi

    echo_h3 "Minio"
    MINIO_DEPLOYMENT_LOOKUP=$(oc get deployment -A | grep minio)
    if [[ "$MINIO_DEPLOYMENT_LOOKUP" != "" ]]; then
      genericMustGather minio
    else
      echo_highlight "Unable to find deployment for minio"
    fi

    endTimer "in-cluster AI Service dependencies"
  else
    echo_warning "Maximo AI Service dependency must-gather disabled"
  fi

  # ---------------------------------------------------------------------------
  # AI Service Must-Gather
  # ---------------------------------------------------------------------------
  if [[ -z $AISERVICE_INSTANCE_IDS ]]; then
    # Find AI Service instances
    AISERVICE_INSTANCE_IDS=$(oc get aiserviceapp --all-namespaces --ignore-not-found -o jsonpath='{.items[*].metadata.name}')
  fi

  echo ""
  for AISERVICE_INSTANCE_ID in $AISERVICE_INSTANCE_IDS
  do
    echo_h2 "AI Service instance: ${AISERVICE_INSTANCE_ID}"
    startTimer
    AISERVICE_NAMESPACE=aiservice-${AISERVICE_INSTANCE_ID}
    NAMESPACE_LOOKUP=$(oc get namespace $AISERVICE_NAMESPACE --ignore-not-found)

    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      echo_h3 "Namespace: ${AISERVICE_NAMESPACE}"
      echo
      
      $MG_SCRIPT_DIR/mg-summary-aiservice $AISERVICE_NAMESPACE &> ${OUTPUT_DIR}/${AISERVICE_NAMESPACE}.txt
      $MG_SCRIPT_DIR/mg-collect-aiservice $AISERVICE_NAMESPACE $OUTPUT_DIR

      genericMustGather $AISERVICE_NAMESPACE pipelinerun   # Including model training logs
    fi

    
    AISERVICE_PIPELINE_NAMESPACE=aiservice-${AISERVICE_INSTANCE_ID}-pipelines
    NAMESPACE_LOOKUP=$(oc get namespace $AISERVICE_PIPELINE_NAMESPACE --ignore-not-found)

    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      echo_h3 "Namespace: ${AISERVICE_PIPELINE_NAMESPACE}"
      echo

      # Reuse script from mas
      $MG_SCRIPT_DIR/mg-summary-mas-pipelines $AISERVICE_PIPELINE_NAMESPACE &> ${OUTPUT_DIR}/${AISERVICE_PIPELINE_NAMESPACE}.txt
      $MG_SCRIPT_DIR/mg-collect-mas-pipelines $AISERVICE_PIPELINE_NAMESPACE $OUTPUT_DIR
      
      genericMustGather $AISERVICE_PIPELINE_NAMESPACE
    fi

    endTimer "${AISERVICE_INSTANCE_ID}"
  done


  # ---------------------------------------------------------------------------
  # AI Service Tenant must-gather
  # ---------------------------------------------------------------------------
  if [[ -z $AISERVICE_TENANT_IDS ]]; then
    # Find AI Service instances
    AISERVICE_TENANT_IDS=$(oc get aiservicetenant --all-namespaces --ignore-not-found -o jsonpath='{.items[*].metadata.name}')
  fi

  echo ""
  for AISERVICE_TENANT_ID in $AISERVICE_TENANT_IDS
  do
    echo_h2 "AI Service Tenant: ${AISERVICE_TENANT_ID}"
    startTimer
    NAMESPACE_LOOKUP=$(oc get namespace $AISERVICE_TENANT_ID --ignore-not-found)

    if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
      echo_h3 "Namespace: ${AISERVICE_TENANT_ID}"
      echo
      
      genericMustGather $AISERVICE_TENANT_ID InferenceService
    fi
    endTimer "${AISERVICE_TENANT_ID}"
  done


  # ---------------------------------------------------------------------------
  # Cluster Pipeline Namespace (update pipeline)
  # ---------------------------------------------------------------------------
  NAMESPACE_LOOKUP=$(oc get namespace mas-pipelines --ignore-not-found)
  if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
    echo_h2 "Maximo Application Suite: Cluster"
    startTimer
    echo_h3 "Namespace: mas-pipelines"
    echo
    # MAS-specific must-gather
    $MG_SCRIPT_DIR/mg-summary-mas-pipelines mas-pipelines &> ${OUTPUT_DIR}/mas-pipelines.txt
    $MG_SCRIPT_DIR/mg-collect-mas-pipelines mas-pipelines $OUTPUT_DIR

    genericMustGather mas-pipelines
    endTimer "mas-pipelines"
  fi


  # ---------------------------------------------------------------------------
  # Argo Applications and ApplicationSets
  # ---------------------------------------------------------------------------
  NAMESPACE_LOOKUP=$(oc get namespace openshift-gitops --ignore-not-found)
  if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
    echo_h2 "Argo resources"
    startTimer
    echo_h3 "Namespace: openshift-gitops"
    echo
    # Argo-specific must-gather
    $MG_SCRIPT_DIR/mg-summary-argo openshift-gitops &> ${OUTPUT_DIR}/argo.txt
    $MG_SCRIPT_DIR/mg-collect-argo openshift-gitops $OUTPUT_DIR

    genericMustGather openshift-gitops
    endTimer "openshift-gitops"
  fi


  # ---------------------------------------------------------------------------
  # Extra Namespaces Must-Gather
  # ---------------------------------------------------------------------------
  if [[ "$EXTRA_NAMESPACES" != "" ]] && [[ "$SUMMARY_ONLY" == false ]]; then
    echo_h2 "Extra Namespaces"
    for EXTRA_NAMESPACE in $(echo $EXTRA_NAMESPACES | tr "," " ")
    do
      echo_h3 "Namespace: ${EXTRA_NAMESPACE}"
      startTimer
      NAMESPACE_LOOKUP=$(oc get namespace $EXTRA_NAMESPACE --ignore-not-found)
      if [[ "$NAMESPACE_LOOKUP" != "" ]]; then
        genericMustGather $EXTRA_NAMESPACE
      else
        echo_highlight "Unable to find $EXTRA_NAMESPACE namespace"
      fi
      endTimer "namespace $EXTRA_NAMESPACE"
    done
  fi

  


  # ---------------------------------------------------------------------------
  # Generate Summary Report (we can't generate a report for partial must-gather)
  # ---------------------------------------------------------------------------
  if [[ "$OCP_REPORT" == "true" ]]; then
    python3 $MG_SCRIPT_DIR/summarizer/mg-print-summary.py $OUTPUT_DIR >  $OUTPUT_DIR/summary.txt
  fi

  tar -czf $OUTPUT_FILE -C $MG_DIR $TIMESTAMP


  # ---------------------------------------------------------------------------
  # Artifactory Upload
  # ---------------------------------------------------------------------------
  if [[ -n "$ARTIFACTORY_TOKEN" && -n "$ARTIFACTORY_UPLOAD_DIR" ]]; then
    set -e
    ARTIFACTORY_AUTH_HEADER="Authorization:Bearer $ARTIFACTORY_TOKEN"
    TARGET_URL="${ARTIFACTORY_UPLOAD_DIR}/${OUTPUT_FILENAME}"

    MD5_VALUE="`md5sum "$OUTPUT_FILE"`"
    MD5_VALUE="${MD5_VALUE:0:32}"

    SHA1_VALUE="`sha1sum "$OUTPUT_FILE"`"
    SHA1_VALUE="${SHA1_VALUE:0:40}"

    echo "Uploading $OUTPUT_FILE to $TARGET_URL"
    curl -H "$ARTIFACTORY_AUTH_HEADER"  -H "X-Checksum-Md5: $MD5_VALUE" -H "X-Checksum-Sha1: $SHA1_VALUE" -T $OUTPUT_FILE $TARGET_URL
    set +e
  fi

  # ---------------------------------------------------------------------------
  # Cleanup
  # ---------------------------------------------------------------------------
  if [[ "$KEEP_FILES" != "true" ]]; then
    rm -rf "$OUTPUT_DIR"
  fi

  echo
  echo_green "Must gather successfully saved to: $OUTPUT_FILE"
  echo

  true
}
