From c27a5d6cc8bf9113faf94b54db87b5a4bfb814dc Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Fri, 8 May 2020 10:08:51 -0700 Subject: [PATCH 01/10] checkpointing --- .../bikesw_training/bw_hptune_standalone.py | 257 ++++++++++++++++++ .../bikesw_training/kchief_deployment.yaml | 42 +++ .../bikesw_training/kchief_service.yaml | 16 ++ .../bikesw_training/ktuner_deployment.yaml | 38 +++ .../containers/bikesw_training/Dockerfile | 23 ++ .../containers/bikesw_training/build.sh | 31 +++ 6 files changed, 407 insertions(+) create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile create mode 100755 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py new file mode 100644 index 0000000..2dcc150 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py @@ -0,0 +1,257 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Adapted in part from: +# https://github.com/GoogleCloudPlatform/data-science-on-gcp/blob/master/09_cloudml/flights_model_tf2.ipynb +# by Valliappa Lakshmanan. (See that repo for more info about the accompanying book, +# "Data Science on the Google Cloud Platform", from O'Reilly.) + + +import argparse +import logging +import os, json, math, time, shutil +import numpy as np + +# import pathlib2 +import tensorflow as tf +from kerastuner.tuners import RandomSearch, Hyperband + + +DEVELOP_MODE = False +NBUCKETS = 5 # for embeddings +NUM_EXAMPLES = 1000*1000 * 20 # assume 20 million examples +# DNN_HIDDEN_UNITS = '128,64,32' + +CSV_COLUMNS = ('duration,end_station_id,bike_id,ts,day_of_week,start_station_id' + + ',start_latitude,start_longitude,end_latitude,end_longitude' + + ',euclidean,loc_cross,prcp,max,min,temp,dewp').split(',') +LABEL_COLUMN = 'duration' +DEFAULTS = [[0.0],['na'],['na'],[0.0],['na'],['na'], + [0.0],[0.0],[0.0],[0.0], + [0.0],['na'],[0.0],[0.0],[0.0],[0.0], [0.0]] + +STRATEGY = tf.distribute.MirroredStrategy() +# TRAIN_BATCH_SIZE = 256 +TRAIN_BATCH_SIZE = 256 * STRATEGY.num_replicas_in_sync + + +def load_dataset(pattern, batch_size=1): + return tf.data.experimental.make_csv_dataset(pattern, batch_size, CSV_COLUMNS, DEFAULTS) + +def features_and_labels(features): + label = features.pop('duration') # this is what we will train for + features.pop('bike_id') + return features, label + +# def parse_fn(filename): +# return tf.data.Dataset.range(10) + + +def read_dataset(pattern, batch_size, mode=tf.estimator.ModeKeys.TRAIN, truncate=None): + dataset = load_dataset(pattern, batch_size) + dataset = dataset.map(features_and_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) + if mode == tf.estimator.ModeKeys.TRAIN: + dataset = dataset.repeat().shuffle(batch_size*10) + # dataset = dataset.repeat() + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + if truncate is not None: + dataset = dataset.take(truncate) + return dataset + + +# Build a wide-and-deep model. +def wide_and_deep_classifier(inputs, linear_feature_columns, dnn_feature_columns, + num_hidden_layers, dnn_hidden_units1, learning_rate): + deep = tf.keras.layers.DenseFeatures(dnn_feature_columns, name='deep_inputs')(inputs) + # layers = [int(x) for x in dnn_hidden_units.split(',')] + layers = [dnn_hidden_units1] + if num_hidden_layers > 1: + layers += [int(dnn_hidden_units1/(x*2)) for x in range(1, num_hidden_layers)] + # layers = [dnn_hidden_units1, dnn_hidden_units1/2, dnn_hidden_units1/4] # using hp tuning val, but hardwired to 3 layers currently. + for layerno, numnodes in enumerate(layers): + deep = tf.keras.layers.Dense(numnodes, activation='relu', name='dnn_{}'.format(layerno+1))(deep) + wide = tf.keras.layers.DenseFeatures(linear_feature_columns, name='wide_inputs')(inputs) + both = tf.keras.layers.concatenate([deep, wide], name='both') + output = tf.keras.layers.Dense(1, name='dur')(both) + model = tf.keras.Model(inputs, output) + optimizer = tf.keras.optimizers.RMSprop(learning_rate) + model.compile(loss='mse', optimizer=optimizer, + metrics=['mse', 'mae', tf.keras.metrics.RootMeanSquaredError()]) + return model + + +def create_model(hp): + + # duration,end_station_id,bike_id,ts,day_of_week,start_station_id,start_latitude,start_longitude,end_latitude,end_longitude, + # euclidean,loc_cross,prcp,max,min,temp,dewp + + real = { + colname : tf.feature_column.numeric_column(colname) + for colname in + # ('ts,start_latitude,start_longitude,end_latitude,end_longitude,euclidean,prcp,max,min,temp,dewp').split(',') + ('ts,euclidean,prcp,max,min,temp,dewp').split(',') + } + sparse = { + 'day_of_week': tf.feature_column.categorical_column_with_vocabulary_list('day_of_week', + vocabulary_list='1,2,3,4,5,6,7'.split(',')), + # 'end_station_id' : tf.feature_column.categorical_column_with_hash_bucket('end_station_id', hash_bucket_size=800), + # 'start_station_id' : tf.feature_column.categorical_column_with_hash_bucket('start_station_id', hash_bucket_size=800), + 'loc_cross' : tf.feature_column.categorical_column_with_hash_bucket('loc_cross', hash_bucket_size=21000), + # 'bike_id' : tf.feature_column.categorical_column_with_hash_bucket('bike_id', hash_bucket_size=14000) + } + + inputs = { + colname : tf.keras.layers.Input(name=colname, shape=(), dtype='float32') + for colname in real.keys() + } + inputs.update({ + colname : tf.keras.layers.Input(name=colname, shape=(), dtype='string') + for colname in sparse.keys() + }) + + # embed all the sparse columns + embed = { + 'embed_{}'.format(colname) : tf.feature_column.embedding_column(col, 10) + for colname, col in sparse.items() + } + real.update(embed) + + # one-hot encode the sparse columns + sparse = { + colname : tf.feature_column.indicator_column(col) + for colname, col in sparse.items() + } + + if DEVELOP_MODE: + print(sparse.keys()) + print(real.keys()) + + model = None + print('num replicas...') + print(STRATEGY.num_replicas_in_sync) + + # with STRATEGY.scope(): # hmmm + model = wide_and_deep_classifier( + inputs, + linear_feature_columns = sparse.values(), + dnn_feature_columns = real.values(), + # dnn_hidden_units = DNN_HIDDEN_UNITS, + num_hidden_layers = hp.Int('num_hidden_layers', 2,3, step=1, default=3), + dnn_hidden_units1 = hp.Int('hidden_size', 32, 128, step=32, default=96), + learning_rate=hp.Choice('learning_rate', + values=[1e-2, 1e-3, 1e-4]) + ) + + model.summary() + return model + +def main(): + + logging.getLogger().setLevel(logging.INFO) + parser = argparse.ArgumentParser(description='ML Trainer') + parser.add_argument( + '--epochs', type=int, + default=1) + parser.add_argument( + '--steps-per-epoch', type=int, + default=-1) # if set to -1, don't override the normal calcs for this + parser.add_argument( + '--tuner-proj', + required=True) + parser.add_argument( + '--tuner-dir', + required=True) + parser.add_argument( + '--data-dir', + default='gs://aju-dev-demos-codelabs/bikes_weather/') + + + args = parser.parse_args() + logging.info("Tensorflow version " + tf.__version__) + + TRAIN_DATA_PATTERN = args.data_dir + "train*" + EVAL_DATA_PATTERN = args.data_dir + "test*" + OUTPUT_DIR='{}/bwmodel/trained_model'.format(args.tuner_dir) + logging.info('Writing trained model to {}'.format(OUTPUT_DIR)) + # learning_rate = 0.001 + + train_batch_size = TRAIN_BATCH_SIZE + eval_batch_size = 1000 + if args.steps_per_epoch == -1: # calc based on dataset size + steps_per_epoch = NUM_EXAMPLES // train_batch_size + else: + steps_per_epoch = args.steps_per_epoch + logging.info('using {} steps per epoch'.format(steps_per_epoch)) + + logging.info('using train batch size %s', train_batch_size) + train_dataset = read_dataset(TRAIN_DATA_PATTERN, train_batch_size) + eval_dataset = read_dataset(EVAL_DATA_PATTERN, eval_batch_size, tf.estimator.ModeKeys.EVAL, + eval_batch_size * 100 * STRATEGY.num_replicas_in_sync + ) + + tuner = RandomSearch( + # tuner = Hyperband( + create_model, + objective='val_mae', + # max_epochs=10, + # hyperband_iterations=2, + max_trials=25, + # distribution_strategy=tf.distribute.MirroredStrategy(), + executions_per_trial=1, + directory=args.tuner_dir, + project_name=args.tuner_proj + ) + + tuner.search_space_summary() + + checkpoint_path = '{}/checkpoints/bikes_weather.cpt'.format(OUTPUT_DIR) + logging.info("checkpoint path: %s", checkpoint_path) + cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, + save_weights_only=True, + verbose=1) + tb_callback = tf.keras.callbacks.TensorBoard(log_dir='{}/logs'.format(OUTPUT_DIR), + update_freq=10000) + + logging.info("hp tuning model....") + tuner.search(train_dataset, + validation_data=eval_dataset, + validation_steps=eval_batch_size, + epochs=args.epochs, + steps_per_epoch=steps_per_epoch, + # callbacks=[cp_callback # , tb_callback + # ] + ) + best_hyperparameters = tuner.get_best_hyperparameters(1)[0] + logging.info('best hyperparameters: {}, {}'.format(best_hyperparameters, + best_hyperparameters.values)) + best_model = tuner.get_best_models(1)[0] + logging.info('best model: {}'.format(best_model)) + + ts = str(int(time.time())) + export_dir = '{}/export/bikesw/{}'.format(OUTPUT_DIR, ts) + logging.info('Exporting to {}'.format(export_dir)) + + try: + logging.info("exporting model....") + tf.saved_model.save(best_model, export_dir) + except Exception as e: # retry once if error + logging.warning(e) + logging.info("retrying...") + time.sleep(10) + logging.info("again ... exporting model....") + tf.saved_model.save(best_model, export_dir) + + +if __name__ == "__main__": + main() diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml new file mode 100644 index 0000000..c372793 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: ktuner-chief + name: kchief-dep + namespace: default +spec: + replicas: 1 + template: + metadata: + labels: + app: ktuner-chief + version: v1 + spec: + containers: + - args: + - --epochs=2 + - --tuner-dir=gs://aju-pipelines/hptest2 + - --tuner-proj=p3 + image: gcr.io/aju-vtests2/ml-pipeline-bikes-tuner + env: + - name: KERASTUNER_TUNER_ID + value: chief + # - name: KERASTUNER_ORACLE_IP + # value: 10.60.2.37 + - name: KERASTUNER_ORACLE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KERASTUNER_ORACLE_PORT + value: "9000" + imagePullPolicy: Always + name: ktuner-chief + ports: + - name: tuner-port + containerPort: 9000 + resources: + limits: + cpu: 1 + memory: 2Gi diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml new file mode 100644 index 0000000..2f8869e --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: ktuner-chief + name: ktuner-chief + namespace: default +spec: + ports: + - name: grpc + port: 9000 + targetPort: 9000 + selector: + app: ktuner-chief + type: ClusterIP diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml new file mode 100644 index 0000000..924c00b --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: ktuner-tuner + name: ktuner-dep + namespace: default +spec: + replicas: 2 + template: + metadata: + labels: + app: ktuner-tuner + version: v1 + spec: + containers: + - args: + - --epochs=2 + - --tuner-dir=gs://aju-pipelines/hptest2 + - --tuner-proj=p3 + image: gcr.io/aju-vtests2/ml-pipeline-bikes-tuner + env: + - name: KERASTUNER_TUNER_ID + value: tuner1 + - name: KERASTUNER_ORACLE_IP + value: ktuner-chief + - name: KERASTUNER_ORACLE_PORT + value: "9000" + imagePullPolicy: Always + name: kktuner-tuner + ports: + - name: tuner-port + containerPort: 9000 + resources: + limits: + nvidia.com/gpu: 1 + memory: 16Gi diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile new file mode 100644 index 0000000..bc8cf72 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM tensorflow/tensorflow:2.1.0-gpu-py3 + +RUN pip install --upgrade pip +RUN pip install keras-tuner + + +ADD build /ml + +ENTRYPOINT ["python", "/ml/bw_hptune_standalone.py"] diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh new file mode 100755 index 0000000..990c2f7 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +if [ -z "$1" ] + then + PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)") +else + PROJECT_ID=$1 +fi + +mkdir -p ./build +rsync -arvp "../../bikesw_training/"/ ./build/ + +docker build -t ml-pipeline-bikes-tuner . +rm -rf ./build + +docker tag ml-pipeline-bikes-tuner gcr.io/${PROJECT_ID}/ml-pipeline-bikes-tuner +docker push gcr.io/${PROJECT_ID}/ml-pipeline-bikes-tuner From 093f870bb655c496f046fd610e6d1f30bc0026b5 Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Fri, 8 May 2020 15:12:17 -0700 Subject: [PATCH 02/10] checkpointing --- .../bikesw_training/bw_hptune_standalone.py | 14 +++++--- ...ment.yaml => kchief_deployment_templ.yaml} | 34 ++++++++++++++----- .../bikesw_training/kchief_service.yaml | 16 --------- ...ent.yaml => ktuners_deployment_templ.yaml} | 28 ++++++++------- 4 files changed, 50 insertions(+), 42 deletions(-) rename ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/{kchief_deployment.yaml => kchief_deployment_templ.yaml} (59%) delete mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml rename ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/{ktuner_deployment.yaml => ktuners_deployment_templ.yaml} (59%) diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py index 2dcc150..d185aa5 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py @@ -171,7 +171,13 @@ def main(): required=True) parser.add_argument( '--tuner-dir', - required=True) + required=True) + parser.add_argument( + '--executions-per-trial', type=int, + default=1) + parser.add_argument( + '--max-trials', type=int, + default=20) parser.add_argument( '--data-dir', default='gs://aju-dev-demos-codelabs/bikes_weather/') @@ -182,7 +188,7 @@ def main(): TRAIN_DATA_PATTERN = args.data_dir + "train*" EVAL_DATA_PATTERN = args.data_dir + "test*" - OUTPUT_DIR='{}/bwmodel/trained_model'.format(args.tuner_dir) + OUTPUT_DIR='{}/{}/bwmodel/trained_model'.format(args.tuner_dir, args.tuner_proj) logging.info('Writing trained model to {}'.format(OUTPUT_DIR)) # learning_rate = 0.001 @@ -206,9 +212,9 @@ def main(): objective='val_mae', # max_epochs=10, # hyperband_iterations=2, - max_trials=25, + max_trials=args.max_trials, # distribution_strategy=tf.distribute.MirroredStrategy(), - executions_per_trial=1, + executions_per_trial=args.executions_per_trial, directory=args.tuner_dir, project_name=args.tuner_proj ) diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml similarity index 59% rename from ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml rename to ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml index c372793..6e47142 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment.yaml +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml @@ -1,13 +1,29 @@ --- -apiVersion: extensions/v1beta1 -kind: Deployment + apiVersion: v1 + kind: Service + metadata: + labels: + app: ktuner-chief + name: ktuner-chief + namespace: NAMESPACE + spec: + ports: + - name: grpc + port: 9000 + targetPort: 9000 + selector: + app: ktuner-chief + type: ClusterIP +--- +apiVersion: batch/v1 +kind: Job metadata: labels: app: ktuner-chief name: kchief-dep - namespace: default + namespace: NAMESPACE spec: - replicas: 1 + # replicas: 1 template: metadata: labels: @@ -16,15 +32,14 @@ spec: spec: containers: - args: - - --epochs=2 - - --tuner-dir=gs://aju-pipelines/hptest2 - - --tuner-proj=p3 + - --epochs=EPOCHS + - --tuner-dir=TUNER_DIR + - --tuner-proj=TUNER_PROJ + - --max-trials=MAX_TRIALS image: gcr.io/aju-vtests2/ml-pipeline-bikes-tuner env: - name: KERASTUNER_TUNER_ID value: chief - # - name: KERASTUNER_ORACLE_IP - # value: 10.60.2.37 - name: KERASTUNER_ORACLE_IP valueFrom: fieldRef: @@ -40,3 +55,4 @@ spec: limits: cpu: 1 memory: 2Gi + restartPolicy: Never diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml deleted file mode 100644 index 2f8869e..0000000 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_service.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: ktuner-chief - name: ktuner-chief - namespace: default -spec: - ports: - - name: grpc - port: 9000 - targetPort: 9000 - selector: - app: ktuner-chief - type: ClusterIP diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml similarity index 59% rename from ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml rename to ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml index 924c00b..dffa009 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuner_deployment.yaml +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml @@ -1,13 +1,13 @@ --- -apiVersion: extensions/v1beta1 -kind: Deployment +apiVersion: batch/v1 +kind: Job metadata: labels: app: ktuner-tuner - name: ktuner-dep - namespace: default + name: KTUNER_NAME + namespace: NAMESPACE spec: - replicas: 2 + # replicas: 1 template: metadata: labels: @@ -16,13 +16,14 @@ spec: spec: containers: - args: - - --epochs=2 - - --tuner-dir=gs://aju-pipelines/hptest2 - - --tuner-proj=p3 + - --epochs=EPOCHS + - --tuner-dir=TUNER_DIR + - --tuner-proj=TUNER_PROJ + - --max-trials=MAX_TRIALS image: gcr.io/aju-vtests2/ml-pipeline-bikes-tuner env: - name: KERASTUNER_TUNER_ID - value: tuner1 + value: KTUNER_ID - name: KERASTUNER_ORACLE_IP value: ktuner-chief - name: KERASTUNER_ORACLE_PORT @@ -32,7 +33,8 @@ spec: ports: - name: tuner-port containerPort: 9000 - resources: - limits: - nvidia.com/gpu: 1 - memory: 16Gi + # resources: + # limits: + # nvidia.com/gpu: 1 + # memory: 20Gi + restartPolicy: Never From ba4551c4b3ec7b639943b38ec0573b0f58d1db33 Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Fri, 17 Jul 2020 14:04:12 -0700 Subject: [PATCH 03/10] checkpointing... + initial v. of the pipeline --- .../bikesw_training/deploy_tuner.py | 112 ++++++++++++++++++ .../kchief_deployment_templ.yaml | 4 +- .../ktuners_deployment_templ.yaml | 4 +- .../example_pipelines/bw_ktune.py | 66 +++++++++++ 4 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py create mode 100644 ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py new file mode 100644 index 0000000..7d6e077 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py @@ -0,0 +1,112 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import os +import time +import logging +import subprocess +# import requests + + +def main(): + parser = argparse.ArgumentParser(description='Keras distributed tuner') + parser.add_argument( + '--epochs', type=int, required=True) + parser.add_argument( + '--num-tuners', type=int, required=True) + parser.add_argument( + '--tuner-dir', required=True) + parser.add_argument( + '--tuner-proj', required=True) + parser.add_argument( + '--max-trials', type=int, required=True) + parser.add_argument( + '--namespace', default='default') + + args = parser.parse_args() + + logging.getLogger().setLevel(logging.INFO) + args_dict = vars(args) + # # Get cluster name and zone from metadata + # metadata_server = "http://metadata/computeMetadata/v1/instance/" + # metadata_flavor = {'Metadata-Flavor' : 'Google'} + # cluster = requests.get(metadata_server + "attributes/cluster-name", + # headers=metadata_flavor).text + # zone = requests.get(metadata_server + "zone", + # headers=metadata_flavor).text.split('/')[-1] + + logging.info('Generating tuner deployment templates.') + ts = int(time.time()) + KTUNER_CHIEF = 'ktuner{}-chief'.format(ts) + logging.info('KTUNER_CHIEF: {}'.format(KTUNER_CHIEF)) + KTUNER_DEP_PREFIX = 'ktuner{}-dep'.format(ts) + logging.info('KTUNER_DEP_PREFIX: {}'.format(KTUNER_DEP_PREFIX)) + + template_file = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'kchief_deployment_templ.yaml') + chief_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'kchief_dep.yaml') + + with open(template_file, 'r') as f: + with open(chief_file_path, "w") as target: + data = f.read() + changed = data.replace('EPOCHS', str(args.epochs)).replace( + 'TUNER_DIR', args.tuner_dir).replace('NAMESPACE', args.namespace).replace( + 'TUNER_PROJ', args.tuner_proj).replace('MAX_TRIALS', str(args.max_trials)).replace( + 'KTUNER_CHIEF', KTUNER_CHIEF) + target.write(changed) + + tuner_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ktuners_dep.yaml') + logging.info("tuner file path: %s", tuner_file_path) + if os.path.exists(tuner_file_path): + os.remove(tuner_file_path) + template_file = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'ktuners_deployment_templ.yaml') + logging.info("num tuners: %s", args.num_tuners) + with open(template_file, 'r') as f: + with open(tuner_file_path, "a") as target: + data = f.read() + for i in range(args.num_tuners): + changed = data.replace('EPOCHS', str(args.epochs)).replace( + 'TUNER_DIR', args.tuner_dir).replace('NAMESPACE', args.namespace).replace( + 'TUNER_PROJ', args.tuner_proj).replace('KTUNER_CHIEF', KTUNER_CHIEF).replace( + 'MAX_TRIALS', str(args.max_trials)) + changed = changed.replace( + 'KTUNER_DEP_NAME', KTUNER_DEP_PREFIX +'{}'.format(i)).replace( + 'KTUNER_ID', 'tuner{}'.format(i)) + target.write(changed) + + logging.info('deploying chief...') + subprocess.call(['kubectl', 'apply', '-f', chief_file_path]) + logging.info('pausing before tuner worker deployment...') + time.sleep(120) + logging.info('deploying tuners...') + subprocess.call(['kubectl', 'apply', '-f', tuner_file_path]) + logging.info('finished deployments.') + + logging.info('pausing before start the wait for job completion...') + time.sleep(180) + # wait for all the tuner workers to complete + for i in range(args.num_tuners): # hmm... + logging.info('waiting for completion of tuner %s...', i) + # negative timeout value --> one week + subprocess.call(['kubectl', 'wait', '--for=condition=complete', '--timeout=-1m', 'job/{}{}'.format(KTUNER_DEP_PREFIX, i)]) + + +if __name__ == "__main__": + main() + +# python deploy_tuner.py --epochs 2 --num-tuners 3 --tuner-dir gs://aju-pipelines/hptest1 --tuner-proj p2 --max-trials 8 +# kubectl create clusterrolebinding sa-admin --clusterrole=cluster-admin --serviceaccount=kubeflow:pipeline-runner diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml index 6e47142..465b80f 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/kchief_deployment_templ.yaml @@ -4,7 +4,7 @@ metadata: labels: app: ktuner-chief - name: ktuner-chief + name: KTUNER_CHIEF namespace: NAMESPACE spec: ports: @@ -20,7 +20,7 @@ kind: Job metadata: labels: app: ktuner-chief - name: kchief-dep + name: KTUNER_CHIEF-dep namespace: NAMESPACE spec: # replicas: 1 diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml index dffa009..93347e9 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml @@ -4,7 +4,7 @@ kind: Job metadata: labels: app: ktuner-tuner - name: KTUNER_NAME + name: KTUNER_DEP_NAME namespace: NAMESPACE spec: # replicas: 1 @@ -25,7 +25,7 @@ spec: - name: KERASTUNER_TUNER_ID value: KTUNER_ID - name: KERASTUNER_ORACLE_IP - value: ktuner-chief + value: KTUNER_CHIEF - name: KERASTUNER_ORACLE_PORT value: "9000" imagePullPolicy: Always diff --git a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py new file mode 100644 index 0000000..2432b84 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py @@ -0,0 +1,66 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl +import kfp.gcp as gcp +import kfp.components as comp +from kfp.dsl.types import GCSPath, String + +# train_op = comp.load_component_from_url( +# 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/train_component.yaml' # pylint: disable=line-too-long +# ) +# serve_op = comp.load_component_from_url( +# 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/serve_component.yaml' # pylint: disable=line-too-long +# ) + + +@dsl.pipeline( + name='bikes_weather_keras_tuner', + description='Model bike rental duration given weather, use Keras Tuner' +) +def bikes_weather_hptune( #pylint: disable=unused-argument + epochs: int = 2, + num_tuners: int = 3, + tuner_dir: str = 'gs://aju-pipelines/hptest2', + tuner_proj: str = 'p1', + max_trials:int = 8 + ): + + hptune = dsl.ContainerOp( + name='ktune', + image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v3', + arguments=['--epochs', epochs, '--num-tuners', num_tuners, '--tuner-dir', tuner_dir, + '--tuner-proj', tuner_proj, '--max-trials', max_trials + ] + ) + + # train = train_op( + # data_dir=data_dir, + # workdir='%s/%s' % (working_dir, dsl.RUN_ID_PLACEHOLDER), + # epochs=epochs, steps_per_epoch=steps_per_epoch, + # load_checkpoint=load_checkpoint + # ).apply(gcp.use_gcp_secret('user-gcp-sa')) + + + # serve = serve_op( + # model_path=train.outputs['train_output_path'], + # model_name='bikesw' + # ).apply(gcp.use_gcp_secret('user-gcp-sa')) + + # train.set_gpu_limit(1) + +if __name__ == '__main__': + import kfp.compiler as compiler + compiler.Compiler().compile(bikes_weather_hptune, __file__ + '.tar.gz') From 3721704cb94f06499087ed57c5b20355cb5e5077 Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Mon, 20 Jul 2020 17:51:28 -0700 Subject: [PATCH 04/10] checkpointing --- .../bikesw_training/deploy_tuner.py | 86 ++++++++++++------- .../ktuners_deployment_templ.yaml | 6 +- .../containers/bikesw_training/Dockerfile | 6 +- .../containers/bikesw_training/build.sh | 8 +- .../bikesw_training_hptune/Dockerfile | 23 +++++ .../bikesw_training_hptune/build.sh | 31 +++++++ .../containers/deploy_jobs/Dockerfile | 48 +++++++++++ .../containers/deploy_jobs/build.sh | 32 +++++++ .../example_pipelines/bw_ktune.py | 37 ++++++-- 9 files changed, 226 insertions(+), 51 deletions(-) create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/Dockerfile create mode 100755 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/build.sh create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/Dockerfile create mode 100755 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/build.sh diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py index 7d6e077..f7157b9 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py @@ -14,12 +14,16 @@ import argparse -import os -import time +import json import logging +import os import subprocess -# import requests +import time +from google.cloud import storage + + +OUTPUT_PATH = '/tmp/hps.json' def main(): parser = argparse.ArgumentParser(description='Keras distributed tuner') @@ -27,6 +31,8 @@ def main(): '--epochs', type=int, required=True) parser.add_argument( '--num-tuners', type=int, required=True) + parser.add_argument( + '--bucket-name', required=True) parser.add_argument( '--tuner-dir', required=True) parser.add_argument( @@ -35,18 +41,14 @@ def main(): '--max-trials', type=int, required=True) parser.add_argument( '--namespace', default='default') - - args = parser.parse_args() + parser.add_argument('--deploy', default=False, action='store_true') + parser.add_argument('--no-deploy', dest='deploy', action='store_false') + args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) args_dict = vars(args) - # # Get cluster name and zone from metadata - # metadata_server = "http://metadata/computeMetadata/v1/instance/" - # metadata_flavor = {'Metadata-Flavor' : 'Google'} - # cluster = requests.get(metadata_server + "attributes/cluster-name", - # headers=metadata_flavor).text - # zone = requests.get(metadata_server + "zone", - # headers=metadata_flavor).text.split('/')[-1] + tuner_path = 'gs://{}/{}'.format(args.bucket_name, args.tuner_dir) + logging.info('tuner path: %s', tuner_path) logging.info('Generating tuner deployment templates.') ts = int(time.time()) @@ -63,7 +65,7 @@ def main(): with open(chief_file_path, "w") as target: data = f.read() changed = data.replace('EPOCHS', str(args.epochs)).replace( - 'TUNER_DIR', args.tuner_dir).replace('NAMESPACE', args.namespace).replace( + 'TUNER_DIR', tuner_path).replace('NAMESPACE', args.namespace).replace( 'TUNER_PROJ', args.tuner_proj).replace('MAX_TRIALS', str(args.max_trials)).replace( 'KTUNER_CHIEF', KTUNER_CHIEF) target.write(changed) @@ -80,33 +82,53 @@ def main(): data = f.read() for i in range(args.num_tuners): changed = data.replace('EPOCHS', str(args.epochs)).replace( - 'TUNER_DIR', args.tuner_dir).replace('NAMESPACE', args.namespace).replace( + 'TUNER_DIR', tuner_path).replace('NAMESPACE', args.namespace).replace( 'TUNER_PROJ', args.tuner_proj).replace('KTUNER_CHIEF', KTUNER_CHIEF).replace( - 'MAX_TRIALS', str(args.max_trials)) + 'MAX_TRIALS', str(args.max_trials)) changed = changed.replace( 'KTUNER_DEP_NAME', KTUNER_DEP_PREFIX +'{}'.format(i)).replace( 'KTUNER_ID', 'tuner{}'.format(i)) target.write(changed) - logging.info('deploying chief...') - subprocess.call(['kubectl', 'apply', '-f', chief_file_path]) - logging.info('pausing before tuner worker deployment...') - time.sleep(120) - logging.info('deploying tuners...') - subprocess.call(['kubectl', 'apply', '-f', tuner_file_path]) - logging.info('finished deployments.') - - logging.info('pausing before start the wait for job completion...') - time.sleep(180) - # wait for all the tuner workers to complete - for i in range(args.num_tuners): # hmm... - logging.info('waiting for completion of tuner %s...', i) - # negative timeout value --> one week - subprocess.call(['kubectl', 'wait', '--for=condition=complete', '--timeout=-1m', 'job/{}{}'.format(KTUNER_DEP_PREFIX, i)]) - + if args.deploy: + logging.info('deploying chief...') + subprocess.call(['kubectl', 'apply', '-f', chief_file_path]) + logging.info('pausing before tuner worker deployment...') + time.sleep(120) + logging.info('deploying tuners...') + subprocess.call(['kubectl', 'apply', '-f', tuner_file_path]) + logging.info('finished deployments.') + + logging.info('pausing 5 mins before starting the wait for job completion...') + time.sleep(300) + # wait for all the tuner workers to complete + for i in range(args.num_tuners): # hmm... + logging.info('waiting for completion of tuner %s...', i) + # negative timeout value --> one week + subprocess.call(['kubectl', '-n={}'.format(args.namespace), 'wait', + '--for=condition=complete', '--timeout=-1m', 'job/{}{}'.format(KTUNER_DEP_PREFIX, i)]) + + # parse the final oracle.json file to get the best params + # (is there a more preferred way to do this?) + + client = storage.Client() + bucket = client.get_bucket(args.bucket_name) + blob = bucket.get_blob('{}/{}/oracle.json'.format(args.tuner_dir, args.tuner_proj)) + + oracle_json_str = blob.download_as_string() + logging.info('got oracle info: %s', oracle_json_str) + oracle_json = json.loads(oracle_json_str) + logging.info('oracle json: %s', oracle_json) + o_values = oracle_json['hyperparameters']['values'] + hp_values_str = json.dumps(o_values) + logging.info('oracle values: %s', hp_values_str) + + with open(OUTPUT_PATH, 'w') as f: + f.write(hp_values_str) if __name__ == "__main__": main() -# python deploy_tuner.py --epochs 2 --num-tuners 3 --tuner-dir gs://aju-pipelines/hptest1 --tuner-proj p2 --max-trials 8 + # kubectl create clusterrolebinding sa-admin --clusterrole=cluster-admin --serviceaccount=kubeflow:pipeline-runner +# kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded.yaml diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml index 93347e9..0a9400a 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/ktuners_deployment_templ.yaml @@ -33,8 +33,6 @@ spec: ports: - name: tuner-port containerPort: 9000 - # resources: - # limits: - # nvidia.com/gpu: 1 - # memory: 20Gi + resources: + limits: {nvidia.com/gpu: 1} restartPolicy: Never diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile index bc8cf72..06919ef 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/Dockerfile @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM tensorflow/tensorflow:2.1.0-gpu-py3 +FROM tensorflow/tensorflow:2.0.0-gpu-py3 RUN pip install --upgrade pip -RUN pip install keras-tuner +RUN pip install pathlib2 ADD build /ml -ENTRYPOINT ["python", "/ml/bw_hptune_standalone.py"] +ENTRYPOINT ["python", "/ml/bikes_weather_limited.py"] diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh index 990c2f7..27e7cfd 100755 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh @@ -22,10 +22,10 @@ else fi mkdir -p ./build -rsync -arvp "../../bikesw_training/"/ ./build/ +rsync -arvp "../../bikesw_training"/ ./build/ -docker build -t ml-pipeline-bikes-tuner . +docker build -t ml-pipeline-bikes-train . rm -rf ./build -docker tag ml-pipeline-bikes-tuner gcr.io/${PROJECT_ID}/ml-pipeline-bikes-tuner -docker push gcr.io/${PROJECT_ID}/ml-pipeline-bikes-tuner +docker tag ml-pipeline-bikes-train gcr.io/${PROJECT_ID}/ml-pipeline-bikes-train +docker push gcr.io/${PROJECT_ID}/ml-pipeline-bikes-train diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/Dockerfile b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/Dockerfile new file mode 100644 index 0000000..bc8cf72 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM tensorflow/tensorflow:2.1.0-gpu-py3 + +RUN pip install --upgrade pip +RUN pip install keras-tuner + + +ADD build /ml + +ENTRYPOINT ["python", "/ml/bw_hptune_standalone.py"] diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/build.sh b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/build.sh new file mode 100755 index 0000000..990c2f7 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training_hptune/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +if [ -z "$1" ] + then + PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)") +else + PROJECT_ID=$1 +fi + +mkdir -p ./build +rsync -arvp "../../bikesw_training/"/ ./build/ + +docker build -t ml-pipeline-bikes-tuner . +rm -rf ./build + +docker tag ml-pipeline-bikes-tuner gcr.io/${PROJECT_ID}/ml-pipeline-bikes-tuner +docker push gcr.io/${PROJECT_ID}/ml-pipeline-bikes-tuner diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/Dockerfile b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/Dockerfile new file mode 100644 index 0000000..21f5d86 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/Dockerfile @@ -0,0 +1,48 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 + +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev wget unzip \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 install --upgrade pip + +# RUN apt-get install -y wget unzip git + +RUN pip install --upgrade pip +RUN pip install urllib3 certifi retrying +RUN pip install google-cloud-storage + + +RUN wget -nv https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip && \ + unzip -qq google-cloud-sdk.zip -d tools && \ + rm google-cloud-sdk.zip && \ + tools/google-cloud-sdk/install.sh --usage-reporting=false \ + --path-update=false --bash-completion=false \ + --disable-installation-options && \ + tools/google-cloud-sdk/bin/gcloud -q components update \ + gcloud core gsutil && \ + tools/google-cloud-sdk/bin/gcloud -q components install kubectl && \ + tools/google-cloud-sdk/bin/gcloud config set component_manager/disable_update_check true && \ + touch /tools/google-cloud-sdk/lib/third_party/google.py + + +ENV PATH $PATH:/tools/google-cloud-sdk/bin + +ADD build /ml + +ENTRYPOINT ["python", "/ml/deploy_tuner.py"] + diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/build.sh b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/build.sh new file mode 100755 index 0000000..9be1a32 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/deploy_jobs/build.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +if [ -z "$1" ] + then + PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)") +else + PROJECT_ID=$1 +fi + +mkdir -p ./build +rsync -arvp "../../bikesw_training/"/ ./build/ + +docker build -t ml-pipeline-bikes-dep . +rm -rf ./build + +docker tag ml-pipeline-bikes-dep gcr.io/${PROJECT_ID}/ml-pipeline-bikes-dep +docker push gcr.io/${PROJECT_ID}/ml-pipeline-bikes-dep + diff --git a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py index 2432b84..419011a 100644 --- a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py +++ b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import time import kfp.dsl as dsl import kfp.gcp as gcp @@ -31,20 +32,40 @@ description='Model bike rental duration given weather, use Keras Tuner' ) def bikes_weather_hptune( #pylint: disable=unused-argument - epochs: int = 2, + tune_epochs: int = 2, + train_epochs: int = 3, num_tuners: int = 3, - tuner_dir: str = 'gs://aju-pipelines/hptest2', + bucket_name: str = 'aju-pipelines', + tuner_dir_prefix: str = 'hptest', tuner_proj: str = 'p1', - max_trials:int = 8 + max_trials: int = 32, + working_dir: str = 'gs://YOUR_GCS_DIR_HERE', + data_dir: str = 'gs://aju-dev-demos-codelabs/bikes_weather/', + steps_per_epoch: int = -1 , # if -1, don't override normal calcs based on dataset size + # load_checkpoint: str = '' ): hptune = dsl.ContainerOp( name='ktune', - image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v3', - arguments=['--epochs', epochs, '--num-tuners', num_tuners, '--tuner-dir', tuner_dir, - '--tuner-proj', tuner_proj, '--max-trials', max_trials - ] + image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v2', + arguments=['--epochs', tune_epochs, '--num-tuners', num_tuners, + '--tuner-dir', '{}_{}'.format(tuner_dir_prefix, int(time.time())), + '--tuner-proj', tuner_proj, '--bucket-name', bucket_name, '--max-trials', max_trials, + '--deploy' + ], + file_outputs={'hps': '/tmp/hps.json'}, ) + train = dsl.ContainerOp( + name='train', + image='gcr.io/aju-vtests2/ml-pipeline-bikes-train:v2', + arguments=[ + '--data-dir', data_dir, '--steps-per-epoch', steps_per_epoch, + '--workdir', '%s/%s' % (working_dir, dsl.RUN_ID_PLACEHOLDER), + # '--load-checkpoint', load_checkpoint, + '--epochs', train_epochs, '--hptune-results', hptune.outputs['hps'] + ], + file_outputs={'train_output_path': '/tmp/train_output_path.txt'}, + ) # train = train_op( # data_dir=data_dir, @@ -59,7 +80,7 @@ def bikes_weather_hptune( #pylint: disable=unused-argument # model_name='bikesw' # ).apply(gcp.use_gcp_secret('user-gcp-sa')) - # train.set_gpu_limit(1) + train.set_gpu_limit(1) if __name__ == '__main__': import kfp.compiler as compiler From a39e9439075961e1fd24b2a4f0c4ec54f7cfcba6 Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Tue, 28 Jul 2020 09:11:44 -0700 Subject: [PATCH 05/10] checkpointing --- .../bikesw_training/deploy_tuner.py | 11 +++++++--- .../containers/bikesw_training/build.sh | 6 +++--- .../example_pipelines/bw_ktune.py | 20 +++++++++---------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py index f7157b9..feeacd4 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/deploy_tuner.py @@ -99,10 +99,15 @@ def main(): subprocess.call(['kubectl', 'apply', '-f', tuner_file_path]) logging.info('finished deployments.') - logging.info('pausing 5 mins before starting the wait for job completion...') - time.sleep(300) + # wait for the tuner pods to be ready... if we're autoscaling the GPU pool, + # this might take a while. + for i in range(args.num_tuners): + logging.info('waiting for tuner %s pod to be ready...', i) + subprocess.call(['kubectl', '-n={}'.format(args.namespace), 'wait', 'pod', + '--for=condition=ready', '--timeout=15m', '-l=job-name={}{}'.format(KTUNER_DEP_PREFIX, i)]) + # wait for all the tuner workers to complete - for i in range(args.num_tuners): # hmm... + for i in range(args.num_tuners): logging.info('waiting for completion of tuner %s...', i) # negative timeout value --> one week subprocess.call(['kubectl', '-n={}'.format(args.namespace), 'wait', diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh index 27e7cfd..db5a7c6 100755 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/bikesw_training/build.sh @@ -24,8 +24,8 @@ fi mkdir -p ./build rsync -arvp "../../bikesw_training"/ ./build/ -docker build -t ml-pipeline-bikes-train . +docker build -t ml-pl-bikes-train . rm -rf ./build -docker tag ml-pipeline-bikes-train gcr.io/${PROJECT_ID}/ml-pipeline-bikes-train -docker push gcr.io/${PROJECT_ID}/ml-pipeline-bikes-train +docker tag ml-pl-bikes-train gcr.io/${PROJECT_ID}/ml-pl-bikes-train +docker push gcr.io/${PROJECT_ID}/ml-pl-bikes-train diff --git a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py index 419011a..1437bc6 100644 --- a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py +++ b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py @@ -22,9 +22,9 @@ # train_op = comp.load_component_from_url( # 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/train_component.yaml' # pylint: disable=line-too-long # ) -# serve_op = comp.load_component_from_url( -# 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/serve_component.yaml' # pylint: disable=line-too-long -# ) +serve_op = comp.load_component_from_url( + 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/serve_component.yaml' # pylint: disable=line-too-long + ) @dsl.pipeline( @@ -47,7 +47,7 @@ def bikes_weather_hptune( #pylint: disable=unused-argument hptune = dsl.ContainerOp( name='ktune', - image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v2', + image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v5', arguments=['--epochs', tune_epochs, '--num-tuners', num_tuners, '--tuner-dir', '{}_{}'.format(tuner_dir_prefix, int(time.time())), '--tuner-proj', tuner_proj, '--bucket-name', bucket_name, '--max-trials', max_trials, @@ -57,7 +57,7 @@ def bikes_weather_hptune( #pylint: disable=unused-argument ) train = dsl.ContainerOp( name='train', - image='gcr.io/aju-vtests2/ml-pipeline-bikes-train:v2', + image='gcr.io/aju-vtests2/ml-pl-bikes-train:v1', arguments=[ '--data-dir', data_dir, '--steps-per-epoch', steps_per_epoch, '--workdir', '%s/%s' % (working_dir, dsl.RUN_ID_PLACEHOLDER), @@ -75,12 +75,12 @@ def bikes_weather_hptune( #pylint: disable=unused-argument # ).apply(gcp.use_gcp_secret('user-gcp-sa')) - # serve = serve_op( - # model_path=train.outputs['train_output_path'], - # model_name='bikesw' - # ).apply(gcp.use_gcp_secret('user-gcp-sa')) + serve = serve_op( + model_path=train.outputs['train_output_path'], + model_name='bikesw' + ) - train.set_gpu_limit(1) + train.set_gpu_limit(2) if __name__ == '__main__': import kfp.compiler as compiler From b6400e0f629686a54e78b579b99a00fcb822e604 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 12 May 2020 15:34:09 -0700 Subject: [PATCH 06/10] CAIPP remote deployment (#65) * CAIP PLs remote deployment example * CAIP PLs remote deployment example --- .../functions/hosted_kfp_gcf.ipynb | 364 ++++++++++++++++++ ml/notebook_examples/functions/main.py | 63 +++ .../functions/requirements.txt | 1 + 3 files changed, 428 insertions(+) create mode 100644 ml/notebook_examples/functions/hosted_kfp_gcf.ipynb create mode 100644 ml/notebook_examples/functions/main.py create mode 100644 ml/notebook_examples/functions/requirements.txt diff --git a/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb b/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb new file mode 100644 index 0000000..7155253 --- /dev/null +++ b/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb @@ -0,0 +1,364 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using Google Cloud Functions to support event-based triggering of Cloud AI Platform Pipelines\n", + "\n", + "This example shows how you can run a [Cloud AI Platform Pipeline](https://cloud.google.com/blog/products/ai-machine-learning/introducing-cloud-ai-platform-pipelines) from a [Google Cloud Function](https://cloud.google.com/functions/docs/), thus providing a way for Pipeline runs to be triggered by events (in the interim before this is supported by Pipelines itself). \n", + "\n", + "In this example, the function is triggered by the addition of or update to a file in a [Google Cloud Storage](https://cloud.google.com/storage/) (GCS) bucket, but Cloud Functions can have other triggers too (including [Pub/Sub](https://cloud.google.com/pubsub/docs/)-based triggers).\n", + "\n", + "The example is Google Cloud Platform (GCP)-specific, and requires a [Cloud AI Platform Pipelines](https://cloud.google.com/ai-platform/pipelines/docs) installation using Pipelines version >= 0.4.\n", + "\n", + "(If you are instead interested in how to do this with a Kubeflow-based pipelines installation, see [this notebook](https://github.com/amygdala/kubeflow-examples/blob/cookbook/cookbook/pipelines/notebooks/gcf_kfp_trigger.ipynb)).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "### Create a Cloud AI Platform Pipelines installation\n", + "\n", + "Follow the instructions in the [documentation](https://cloud.google.com/ai-platform/pipelines/docs) to create a Cloud AI Platform Pipelines installation. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Identify (or create) a Cloud Storage bucket to use for the example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Before executing the next cell**, edit it to **set the `TRIGGER_BUCKET` environment variable** to a Google Cloud Storage bucket ([create a bucket first](https://console.cloud.google.com/storage/browser) if necessary). Do *not* include the `gs://` prefix in the bucket name.\n", + "\n", + "We'll deploy the GCF function so that it will trigger on new and updated files (blobs) in this bucket." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%env TRIGGER_BUCKET=REPLACE_WITH_YOUR_GCS_BUCKET_NAME" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Give Cloud Function's service account the necessary access\n", + "\n", + "First, make sure the Cloud Function API [is enabled](https://console.cloud.google.com/apis/library/cloudfunctions.googleapis.com?q=functions).\n", + "\n", + "Functions uses the project's 'appspot'acccount for its service account. It will have the form: \n", + "` PROJECT_ID@appspot.gserviceaccount.com`. (This is also the App Engine service account).\n", + "\n", + "- Go to your project's [IAM - Service Account page](https://console.cloud.google.com/iam-admin/serviceaccounts).\n", + "- Find the ` PROJECT_ID@appspot.gserviceaccount.com` account and copy its email address.\n", + "- Find the project's Compute Engine (GCE) default service account (this is the default account used for the Pipelines installation). It will have a form like this: `PROJECT_NUMBER@developer.gserviceaccount.com`.\n", + " Click the checkbox next to the GCE service account, and in the 'INFO PANEL' to the right, click **ADD MEMBER**. Add the Functions service account (`PROJECT_ID@appspot.gserviceaccount.com`) as a **Project Viewer** of the GCE service account. \n", + " \n", + "![Add the Functions service account as a project viewer of the GCE service account](https://storage.googleapis.com/amy-jo/images/kfp-deploy/hosted_kfp_setup1.png) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, configure your `TRIGGER_BUCKET` to allow the Functions service account access to that bucket. \n", + "\n", + "- Navigate in the console to your list of buckets in the [Storage Browser](https://console.cloud.google.com/storage/browser).\n", + "- Click the checkbox next to the `TRIGGER_BUCKET`. In the 'INFO PANEL' to the right, click **ADD MEMBER**. Add the service account (`PROJECT_ID@appspot.gserviceaccount.com`) with `Storage Object Admin` permissions. (While not tested, giving both Object view and create permissions should also suffice).\n", + "\n", + "![add the app engine service account to the trigger bucket with view and edit permissions](https://storage.googleapis.com/amy-jo/images/kfp-deploy/hosted_kfp_setup2.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a simple GCF function to test your configuration\n", + "\n", + "First we'll generate and deploy a simple GCF function, to test that the basics are properly configured. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "mkdir -p functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll first create a `requirements.txt` file, to indicate what packages the GCF code requires to be installed. (We won't actually need `kfp` for this first 'sanity check' version of a GCF function, but we'll need it below for the second function we'll create, that deploys a pipeline)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile functions/requirements.txt\n", + "kfp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we'll create a simple GCF function in the `functions/main.py` file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile functions/main.py\n", + "import logging\n", + "\n", + "def gcs_test(data, context):\n", + " \"\"\"Background Cloud Function to be triggered by Cloud Storage.\n", + " This generic function logs relevant data when a file is changed.\n", + "\n", + " Args:\n", + " data (dict): The Cloud Functions event payload.\n", + " context (google.cloud.functions.Context): Metadata of triggering event.\n", + " Returns:\n", + " None; the output is written to Stackdriver Logging\n", + " \"\"\"\n", + "\n", + " logging.info('Event ID: {}'.format(context.event_id))\n", + " logging.info('Event type: {}'.format(context.event_type))\n", + " logging.info('Data: {}'.format(data))\n", + " logging.info('Bucket: {}'.format(data['bucket']))\n", + " logging.info('File: {}'.format(data['name']))\n", + " file_uri = 'gs://%s/%s' % (data['bucket'], data['name'])\n", + " logging.info('Using file uri: %s', file_uri)\n", + "\n", + " logging.info('Metageneration: {}'.format(data['metageneration']))\n", + " logging.info('Created: {}'.format(data['timeCreated']))\n", + " logging.info('Updated: {}'.format(data['updated']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Deploy the GCF function as follows. (You'll need to **wait a moment or two for output of the deployment to display in the notebook**). You can also run this command from a notebook terminal window in the `functions` subdirectory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "cd functions\n", + "gcloud functions deploy gcs_test --runtime python37 --trigger-resource ${TRIGGER_BUCKET} --trigger-event google.storage.object.finalize" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After you've deployed, test your deployment by adding a file to the specified `TRIGGER_BUCKET`. You can do this easily by visiting the **Storage** panel in the Cloud Console, clicking on the bucket in the list, and then clicking on **Upload files** in the bucket details view.\n", + "\n", + "Then, check in the logs viewer panel (https://console.cloud.google.com/logs/viewer) to confirm that the GCF function was triggered and ran correctly. You can select 'Cloud Function' in the first pulldown menu to filter on just those log entries." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy a Pipeline from a GCF function\n", + "\n", + "Next, we'll create a GCF function that deploys an AI Platform Pipeline when triggered. First, preserve your existing main.py in a backup file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "cd functions\n", + "mv main.py main.py.bak" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, **before executing the next cell**, **edit the `HOST` variable** in the code below. You'll replace `` with the correct value for your installation.\n", + "\n", + "To find this URL, visit the [Pipelines panel](https://console.cloud.google.com/ai-platform/pipelines/) in the Cloud Console. \n", + "From here, you can find the URL by clicking on the **SETTINGS** link for the Pipelines installation you want to use, and copying the 'host' string displayed in the client example code (prepend `https://` to that string in the code below). \n", + "You can alternately click on **OPEN PIPELINES DASHBOARD** for the Pipelines installation, and copy that URL, removing the `/#/pipelines` suffix." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile functions/main.py\n", + "import logging\n", + "import datetime\n", + "import logging\n", + "import time\n", + " \n", + "import kfp\n", + "import kfp.compiler as compiler\n", + "import kfp.dsl as dsl\n", + " \n", + "import requests\n", + " \n", + "# TODO: replace with your Pipelines endpoint URL\n", + "HOST = 'https://.pipelines.googleusercontent.com'\n", + "\n", + "@dsl.pipeline(\n", + " name='Sequential',\n", + " description='A pipeline with two sequential steps.'\n", + ")\n", + "def sequential_pipeline(filename='gs://ml-pipeline-playground/shakespeare1.txt'):\n", + " \"\"\"A pipeline with two sequential steps.\"\"\"\n", + " op1 = dsl.ContainerOp(\n", + " name='filechange',\n", + " image='library/bash:4.4.23',\n", + " command=['sh', '-c'],\n", + " arguments=['echo \"%s\" > /tmp/results.txt' % filename],\n", + " file_outputs={'newfile': '/tmp/results.txt'})\n", + " op2 = dsl.ContainerOp(\n", + " name='echo',\n", + " image='library/bash:4.4.23',\n", + " command=['sh', '-c'],\n", + " arguments=['echo \"%s\"' % op1.outputs['newfile']]\n", + " )\n", + " \n", + "def get_access_token():\n", + " url = 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token'\n", + " r = requests.get(url, headers={'Metadata-Flavor': 'Google'})\n", + " r.raise_for_status()\n", + " access_token = r.json()['access_token']\n", + " return access_token\n", + " \n", + "def hosted_kfp_test(data, context):\n", + " logging.info('Event ID: {}'.format(context.event_id))\n", + " logging.info('Event type: {}'.format(context.event_type))\n", + " logging.info('Data: {}'.format(data))\n", + " logging.info('Bucket: {}'.format(data['bucket']))\n", + " logging.info('File: {}'.format(data['name']))\n", + " file_uri = 'gs://%s/%s' % (data['bucket'], data['name'])\n", + " logging.info('Using file uri: %s', file_uri)\n", + " \n", + " logging.info('Metageneration: {}'.format(data['metageneration']))\n", + " logging.info('Created: {}'.format(data['timeCreated']))\n", + " logging.info('Updated: {}'.format(data['updated']))\n", + " \n", + " token = get_access_token() \n", + " logging.info('attempting to launch pipeline run.')\n", + " ts = int(datetime.datetime.utcnow().timestamp() * 100000)\n", + " client = kfp.Client(host=HOST, existing_token=token)\n", + " compiler.Compiler().compile(sequential_pipeline, '/tmp/sequential.tar.gz')\n", + " exp = client.create_experiment(name='gcstriggered') # this is a 'get or create' op\n", + " res = client.run_pipeline(exp.id, 'sequential_' + str(ts), '/tmp/sequential.tar.gz',\n", + " params={'filename': file_uri})\n", + " logging.info(res)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, deploy the new GCF function. As before, **it will take a moment or two for the results of the deployment to display in the notebook**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "cd functions\n", + "gcloud functions deploy hosted_kfp_test --runtime python37 --trigger-resource ${TRIGGER_BUCKET} --trigger-event google.storage.object.finalize" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Add another file to your `TRIGGER_BUCKET`. This time you should see both GCF functions triggered. The `hosted_kfp_test` function will deploy the pipeline. You'll be able to see it running at your Pipeline installation's endpoint, `https://.pipelines.googleusercontent.com/#/pipelines`, under the given Pipelines Experiment (`gcstriggered` as default)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "------------------------------------------\n", + "Copyright 2020, Google, LLC.\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "you may not use this file except in compliance with the License.\n", + "You may obtain a copy of the License at\n", + "\n", + " http://www.apache.org/licenses/LICENSE-2.0\n", + "\n", + "Unless required by applicable law or agreed to in writing, software\n", + "distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "See the License for the specific language governing permissions and\n", + "limitations under the License." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ml/notebook_examples/functions/main.py b/ml/notebook_examples/functions/main.py new file mode 100644 index 0000000..7b4e0e5 --- /dev/null +++ b/ml/notebook_examples/functions/main.py @@ -0,0 +1,63 @@ +import logging +import datetime +import logging +import time + +import kfp +import kfp.compiler as compiler +import kfp.dsl as dsl + +import requests + +# TODO: replace yours +# HOST = 'https://.pipelines.googleusercontent.com' +HOST = 'https://7c7f7f3e3d11e1d4-dot-us-central2.pipelines.googleusercontent.com' + +@dsl.pipeline( + name='Sequential', + description='A pipeline with two sequential steps.' +) +def sequential_pipeline(filename='gs://ml-pipeline-playground/shakespeare1.txt'): + """A pipeline with two sequential steps.""" + op1 = dsl.ContainerOp( + name='filechange', + image='library/bash:4.4.23', + command=['sh', '-c'], + arguments=['echo "%s" > /tmp/results.txt' % filename], + file_outputs={'newfile': '/tmp/results.txt'}) + op2 = dsl.ContainerOp( + name='echo', + image='library/bash:4.4.23', + command=['sh', '-c'], + arguments=['echo "%s"' % op1.outputs['newfile']] + ) + +def get_access_token(): + url = 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token' + r = requests.get(url, headers={'Metadata-Flavor': 'Google'}) + r.raise_for_status() + access_token = r.json()['access_token'] + return access_token + +def hosted_kfp_test(data, context): + logging.info('Event ID: {}'.format(context.event_id)) + logging.info('Event type: {}'.format(context.event_type)) + logging.info('Data: {}'.format(data)) + logging.info('Bucket: {}'.format(data['bucket'])) + logging.info('File: {}'.format(data['name'])) + file_uri = 'gs://%s/%s' % (data['bucket'], data['name']) + logging.info('Using file uri: %s', file_uri) + + logging.info('Metageneration: {}'.format(data['metageneration'])) + logging.info('Created: {}'.format(data['timeCreated'])) + logging.info('Updated: {}'.format(data['updated'])) + + token = get_access_token() + logging.info('attempting to launch pipeline run.') + ts = int(datetime.datetime.utcnow().timestamp() * 100000) + client = kfp.Client(host=HOST, existing_token=token) + compiler.Compiler().compile(sequential_pipeline, '/tmp/sequential.tar.gz') + exp = client.create_experiment(name='gcstriggered') # this is a 'get or create' op + res = client.run_pipeline(exp.id, 'sequential_' + str(ts), '/tmp/sequential.tar.gz', + params={'filename': file_uri}) + logging.info(res) diff --git a/ml/notebook_examples/functions/requirements.txt b/ml/notebook_examples/functions/requirements.txt new file mode 100644 index 0000000..1235b39 --- /dev/null +++ b/ml/notebook_examples/functions/requirements.txt @@ -0,0 +1 @@ +kfp From 62f301177b7f96d0fc6aa576ec0216c2b345d8a0 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 12 May 2020 16:02:36 -0700 Subject: [PATCH 07/10] typo fix (#66) --- ml/notebook_examples/functions/hosted_kfp_gcf.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb b/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb index 7155253..b1bb15e 100644 --- a/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb +++ b/ml/notebook_examples/functions/hosted_kfp_gcf.ipynb @@ -59,8 +59,8 @@ "\n", "First, make sure the Cloud Function API [is enabled](https://console.cloud.google.com/apis/library/cloudfunctions.googleapis.com?q=functions).\n", "\n", - "Functions uses the project's 'appspot'acccount for its service account. It will have the form: \n", - "` PROJECT_ID@appspot.gserviceaccount.com`. (This is also the App Engine service account).\n", + "Cloud Functions uses the project's 'appspot' acccount for its service account. It will have the form: \n", + "`PROJECT_ID@appspot.gserviceaccount.com`. (This is also the project's App Engine service account).\n", "\n", "- Go to your project's [IAM - Service Account page](https://console.cloud.google.com/iam-admin/serviceaccounts).\n", "- Find the ` PROJECT_ID@appspot.gserviceaccount.com` account and copy its email address.\n", From e48b79074967b160a2d4ec2970fd2e719485876b Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 15 May 2020 10:34:24 -0700 Subject: [PATCH 08/10] add simple caipp connection example notebook (#67) --- .../caipp/caipp_connect.ipynb | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 ml/notebook_examples/caipp/caipp_connect.ipynb diff --git a/ml/notebook_examples/caipp/caipp_connect.ipynb b/ml/notebook_examples/caipp/caipp_connect.ipynb new file mode 100644 index 0000000..87ec2d3 --- /dev/null +++ b/ml/notebook_examples/caipp/caipp_connect.ipynb @@ -0,0 +1,190 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Remote deployment of a Cloud AI Platform Pipeline\n", + "\n", + "This example requires that you have a Cloud AI Platform Pipelines installation up and running.\n", + "The easiest way to run this example locally is to have `gcloud` [installed](https://cloud.google.com/sdk/install) and [authorized for your account](https://cloud.google.com/sdk/docs/initializing).\n", + "\n", + "You will also need to have `kfp` installed in your (virtual) environment:\n", + "```sh\n", + "pip install kfp\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, do some imports:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime\n", + "\n", + "import kfp\n", + "import kfp.compiler as compiler\n", + "import kfp.dsl as dsl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define a (very) simple example pipeline to run:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@dsl.pipeline(\n", + " name='Sequential',\n", + " description='A pipeline with two sequential steps.'\n", + ")\n", + "def sequential_pipeline(filename='gs://ml-pipeline-playground/shakespeare1.txt'):\n", + " \"\"\"A pipeline with two sequential steps.\"\"\"\n", + "\n", + " op1 = dsl.ContainerOp(\n", + " name='getfilename',\n", + " image='library/bash:4.4.23',\n", + " command=['sh', '-c'],\n", + " arguments=['echo \"%s\" > /tmp/results.txt' % filename],\n", + " file_outputs={'newfile': '/tmp/results.txt'})\n", + " op2 = dsl.ContainerOp(\n", + " name='echo',\n", + " image='library/bash:4.4.23',\n", + " command=['sh', '-c'],\n", + " arguments=['echo \"%s\"' % op1.outputs['newfile']]\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we'll create an instance of the Kubeflow Pipelines client. For this you need the endpoint of your installation. An easy way to do this is to visit your AI Pipelines dashboard, and click on **SETTINGS**. \n", + "![Click 'Settings' to get information about your installation](https://storage.googleapis.com/amy-jo/images/kfp-deploy/FOubezufx6b.png)\n", + "\n", + "A window will pop up that looks similar to the following:\n", + "![KFP client settings](https://storage.googleapis.com/amy-jo/images/kfp-deploy/9SD6WeYgpTv-2.png)\n", + "\n", + "Copy and paste the code snippet to replace the code below with your endpoint:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import kfp\n", + "# TODO: edit the following for your installation endpoint\n", + "client = kfp.Client(host='.pipelines.googleusercontent.com')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List your installation's existing pipelines to sanity-check the connection. If you haven't yet uploaded any of your own pipelines, you'll just see the example ones listed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client.list_pipelines()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Compile the pipeline, and create a Pipelines `Experiment`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "compiler.Compiler().compile(sequential_pipeline, '/tmp/sequential.tar.gz')\n", + "exp = client.create_experiment(name='sequential')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, run the pipeline:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ts = int(datetime.datetime.utcnow().timestamp() * 100000)\n", + "res = client.run_pipeline(exp.id, 'sequential_' + str(ts), \n", + " '/tmp/sequential.tar.gz',\n", + " )\n", + "print(res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "------------------------------------\n", + "Copyright 2020, Google, LLC.\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "you may not use this file except in compliance with the License.\n", + "You may obtain a copy of the License at\n", + "\n", + " http://www.apache.org/licenses/LICENSE-2.0\n", + "\n", + "Unless required by applicable law or agreed to in writing, software\n", + "distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "See the License for the specific language governing permissions and\n", + "limitations under the License." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 52a8dd3cbd3528a7ae6626d261c9ace8d0243046 Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Mon, 22 Jun 2020 09:26:41 -0700 Subject: [PATCH 09/10] minor cleanup --- ml/automl/tables/kfp_e2e/README.md | 2 +- .../tables/kfp_e2e/tables_pipeline_caip.py | 6 ------ .../kfp_e2e/tables_pipeline_caip.py.tar.gz | Bin 8813 -> 9015 bytes .../tables/kfp_e2e/tables_pipeline_kf.py | 6 ------ .../kfp_e2e/tables_pipeline_kf.py.tar.gz | Bin 8992 -> 9199 bytes 5 files changed, 1 insertion(+), 13 deletions(-) diff --git a/ml/automl/tables/kfp_e2e/README.md b/ml/automl/tables/kfp_e2e/README.md index 4fe4e45..96c815f 100644 --- a/ml/automl/tables/kfp_e2e/README.md +++ b/ml/automl/tables/kfp_e2e/README.md @@ -88,7 +88,7 @@ The uploaded pipeline graph will look similar to this: Click the **+Create Run** button to run the pipeline. You will need to fill in some pipeline parameters. -Specifically, replace `YOUR_PROJECT_HERE` with the name of your project; replace `YOUR_DATASET_NAME` with the name you want to give your new dataset (make it unique, and use letters, numbers and underscores up to 32 characters); and replace `YOUR_BUCKET_NAME` with the name of a [GCS bucket][41]. Do not include the `gs://` prefix— just enter the name. This bucket should be in the [same _region_][42] as that specified by the `gcp_region` parameter. E.g., if you keep the default `us-central1` region, your bucket should also be a _regional_ (not multi-regional) bucket in the `us-central1` region. ++double check that this is necessary.++ +Specifically, replace `YOUR_PROJECT_HERE` with the name of your project; replace `YOUR_DATASET_NAME` with the name you want to give your new dataset (make it unique, and use letters, numbers and underscores up to 32 characters); and replace `YOUR_BUCKET_NAME` with the name of a [GCS bucket][41]. Do not include the `gs://` prefix— just enter the name. This bucket should be in the [same _region_][42] as that specified by the `gcp_region` parameter. E.g., if you keep the default `us-central1` region, your bucket should also be a _regional_ (not multi-regional) bucket in the `us-central1` region. If you want to schedule a recurrent set of runs, you can do that instead. If your data is in [BigQuery][43]— as is the case for this example pipeline— and has a temporal aspect, you could define a _view_ to reflect that, e.g. to return data from a window over the last `N` days or hours. Then, the AutoML pipeline could specify ingestion of data from that view, grabbing an updated data window each time the pipeline is run, and building a new model based on that updated window. diff --git a/ml/automl/tables/kfp_e2e/tables_pipeline_caip.py b/ml/automl/tables/kfp_e2e/tables_pipeline_caip.py index 5953a39..15328f3 100644 --- a/ml/automl/tables/kfp_e2e/tables_pipeline_caip.py +++ b/ml/automl/tables/kfp_e2e/tables_pipeline_caip.py @@ -21,10 +21,6 @@ DEFAULT_SCHEMA = json.dumps({"end_station_id": ["CATEGORY", True], "start_station_id": ["CATEGORY", True], "loc_cross": ["CATEGORY", True], "bike_id": ["CATEGORY", True]}) -# DEFAULT_SCHEMA = json.dumps({"accepted_answer_id": ["CATEGORY", True], "id": ["CATEGORY", True], - # "last_editor_display_name": ["CATEGORY", True], "last_editor_user_id": ["CATEGORY", True], - # "owner_display_name": ["CATEGORY", True], "owner_user_id": ["CATEGORY", True], - # "parent_id": ["CATEGORY", True], "post_type_id": ["CATEGORY", True], "tags": ["CATEGORY", True]}) create_dataset_op = comp.load_component_from_file( @@ -65,11 +61,9 @@ def automl_tables( #pylint: disable=unused-argument model_prefix: str = 'bwmodel', # one of strings: [MAXIMIZE_AU_ROC, MAXIMIZE_AU_PRC, MINIMIZE_LOG_LOSS, MAXIMIZE_RECALL_AT_PRECISION, MAXIMIZE_PRECISION_AT_RECALL, MINIMIZE_RMSE, MINIMIZE_MAE, MINIMIZE_RMSLE] optimization_objective: str = '', # if not set, will use default - # ["title", "body", "answer_count", "comment_count", "creation_date", "favorite_count", "owner_user_id", "score", "view_count"] include_column_spec_names: str = '', exclude_column_spec_names: str = '', bucket_name: str = 'YOUR_BUCKET_NAME', - # thresholds: str = '{"au_prc": 0.9}', thresholds: str = '{"mean_absolute_error": 480}', ): diff --git a/ml/automl/tables/kfp_e2e/tables_pipeline_caip.py.tar.gz b/ml/automl/tables/kfp_e2e/tables_pipeline_caip.py.tar.gz index 1db019944f809e0b05d6e8836f7304f4598d5a77..88c08e0fb666e23381e14edb6732fd11de0acff7 100644 GIT binary patch literal 9015 zcmV-7BgotziwFpW+wfij|8!wuY-Mv_aA|O5Y-w&~Ut?iua4v9pE_7jX0PS6UbK5r3 z@8A9u7-c$=ZzS4I)8182)EhT(d$~*FOyb;4$MNte5t3O`q{mzLgU8u3{`}zJ zyC?f!k^Qee!Y7RrCnDrtDk)d)Gx<0x@82|vy)YOPCt8H7DE!Ux!oAA_$6uX02Rj#D z;Eu_E!sue|hgUmGnmDeLIO82c7Brxdlh9Yrl0xb<377sTai%_vNATJSf-rFsU>%Ri z%}NZk*t$rk6nR-;wAj0tuk6^purISh>|5Vj2SZXpo;|i_;d15qjJut9n1BBX2Yn zp;F`!D*DSy#*vT{R$X!>+NU=pJug_LNel^3k<;8s{RFxEdi?H<{rb)EKaZZB*#9_s zbA+@3G#$QN%vLrwJ)I@C=k|tzdoN^#aD1RiqLf04P|CV8F`C;I~0GfIiG`6p(lbqAYQht@ov4ZCtz21hU&5l0jeUG3ja1QR3m*?s!KkmT2VoIyO8^r_g9K8q4pBhgOsZgHuRp(GPT% zvI^n|T;ezcDS>}HuzDTe$FGYpT=*243hYZ5CSicnGYR9#0V)Q~HE z&T1fJ=0ugBL(xi;LT>6~RxjN|{V5j#tO`2bLXngS0Jevk$0PEL$xGs7BA#4%$vN~I zB@3`<10p0ovqqGp8T~{Q)KXQ@z@##UBop|P3PrH16%U3z!KDXrrl53Lxp+QqRsbbI zHYYI2p!Zx9k!!1)b#-jS@>diw*?2(mKEtOsGgS$dV9Uy)6v0MD|T|D8X3!t z){0}UNPBhE?-N5_RXx!rTl&EPo0Z8C)cFSPg2HALt$6bOCCnMsP@9;7h;w6O#*!gm z3A){SZh;eo&Ab?u#%TqjEE#}Zk)_by7v={{7iI87eDw79KpP&g0i-01Xp%+&Q3f0{ zU`#~=OQ*<4ku4u)bd{788Gnp|$oiSRI(+@&8R+vPTv(w((gaf0UZ}6(?|`b6gcUt# z3CTPaite{K3>1^qj%nmMzV`>Uu_o}(zI6PQ843l@pX2 z0~ThmaES|1YO|kcDrc{(!)TE%L8889Z&1`PCjCRE1hUDI318IJeE!1tk-V*wet%cb z!g5^Kb_DN1PhT7M7=vU|a-p)8nxL&O)HZdQX@&G`bw0^U{`TGL*KdyAzHOIx=0}zC z%^z(ePt&?I&%MIox{Y$kE80AN%`0oj_b11X@0PKE6tZ^6Bz<0n3Qdp|2Pd-ShmITH zYpK+gdR`ctKee-`MT-RJ-I#o%b7ox80K9}>gDf}phrJm{b2Vc=_YicLC>*R+xEhqQ zKRuHyDe*GSjkB8d_h0@ZGnbeMnFg}RNkFn>JYiL2+hFi}30>NDf0w`%m*ieSRu+3w zu}-t^XQfgX;x}THDhow64>x|7zdS36z_3^zq4@IL4?`k#2QC_D0HXt8qJh#;T6Ig- zEPle+!WEF~!59e9OO%|0H5K7>9!R`2xh*v0@mr=V%3N2Dvic|7N)A^vkeNc)T|tL| za+#M|%&n`)lWHXO!YQ=DS>ChkS6)7E=`27qau3q@8|$y^Iro8OnG~+D!sT=$+=sGj zOn+(gzUf4obI|l!c>mZdQHjD4nRs(&mKaLb;Mg2Vmd%i0KaT5tbL(5@P*_XG3^ecF z=KPU&Z4SBnk>?>}!_4q%a5`s?N3nY`LOD4m7xPtC-d6B2!oXpGYttOOmEqS0c$GQ$ zP5yfmY+6rg*28V{*#O&=SgUx0iY8R5#wNsvmHD5bI*%DlR0bOAtUAV@4;Hy2WY+Q+ zYHD0*%JdrBLhlh~3zlIMj*FP#3WmIOU`0JxF%r$h0m_P0GKQ(;DPxhN9E+^rV+JlY zjtra~ad39Tz}XRkvm*x1RxqvwbQS&va7LAiysmO9o3Iy7RlGF=Je=ym;4{294;Gy% zw|qS%ACDPLUtsTw$KO2Qs@PUS-(%u?abieO31(NOs2!aY9!lN`u(?C=l6~0E%@mSOW}JDi>NU9XQs3 zW7`79gg?fh1C^Z1?@{&LtnHh1?m3pVBljC4KEU7u{BOYg#yb^AME?X86m~yjLvGfV zWofJ{@u7&uu#Ssq36^8&1`RbCYW|fPMLDT`J2b){a#j-~KBy!!n+_B6V>M}|9WkkV zGt?bI6$~oh$PB7`x0m;BxS~9cRe_wc4z%*uO8++TDnp88hGp2Zf-6WU1Ro;QeB|WZ zi=}*)&NZx2L7FaMz3d?d$T}V+75Wl8QfHMJMhvq}LB59C# zJ|A%`=a|glTYRp{Iw%R`24g#~6n`>kQJJh%uHHJ3UTKWNV&fUXdrL#VGy4q)3hH_+ zk3)pq^2|4xlYmsDMk+*Mv-$}#kbD<+k|gv`|d^?cNNUdyoI+q z7D8W4E0GWq=hLRKRX3^&A<=#&WJ0TbFoa82et50Otp^wgQC^8w1u)!JzkJLMo$X== za9e#Ef9jY3J}!hwUv|)M9uH6fTb zfv>{kr!WN}a_fD7MNY6hw&%#ikkwI^)m<_G)_G-i>IWB3q`<8FQEOmTkneP`={>=w zyzj*Nj)>6F4l><7V)*&Or5l1ej~^7MV_Nrpxqwaw+I$9}&Fy_UA9a&HKN-Gnj%cg*)#fZ`QOOj*&{7{i z+!4k*^M3uc%z8%!fr=OB4k`*8SlZn1W!ijz!rbQ5zN|^hYOG1`b zFgLw(YNPv@jRVn;vfnr#A1o*2wd->;RO zUA>EVj_SJOEALddpYwVl_a@e=N#ZsWHhdna`}sEK z{CyrL`>CXH5=LOKl!&%c&-Q+4CGfe%TP?r%20q%9WGz(2aRf#?yWjhPZ1V1Wx>W3i96N@}_@=G5(?mRs~T7 zON01Q*%=6`N|$A*l_z`Lsst+<;feJy=b8(n)3$Ck=$UpcmlG2?tkQ0a(BYA`2sVk?s*VxrXDOkE;nm=MF+X%jdpQHejt}+8oeer zYoNPX|DRCy=$B^xk-T084odG_oS7;vdolZ*;V@656N8FZOa9bU@>02<-oD(8rEXc~ zoQ#`<3Y{j?gi&x00!=x?P?2i;XO<9UIId(e!});ymtlB;?S->IEH@?vI(u|)arhII1-J4PH{B+GspQ%xi4T8AL=b?qQ1Y6^U3F%q_&r2dY8O!w{Hw0(#awB``a;rp*L-u*tcT z^x2^yB=Q#Ls8y~p=7TL^D9Mgw=72gn!2WJfGOgDDENG=Yd6m6xz)y2FY`xfE;V{Tr zZlkPtQV*JiV(X&snk_LoeWQXYw^m!-J2nKoWW%L(`>2_1)<%s5E)Q%^UaYbv(8Ro{ z_v4{#-L+cmmbg5+gI1Dp0W>cSEz=ji&ORAyV4DcaHnwIz^RkT5XM+yA{;+~DOko{^ zbdSiJ($F5i58em={qKKo*6`o^pj@8{t~lk9=hBHT-0&(e=Dt2#n_5G-2#k#(UiZ;W zZ_b&8oU)uNs~n`uDUDdD1W-1u+$)&yhJf_FH;4b}nX6ZNxNew1Y0ScqvJt@?KGG;+ zo*y?|uHr#)8vE|;c&45}tK=}JYWJ^!eW4O-(iK2w1rhqNO?E;JxqwHFtLa8VW^_n#{3Ph7&&9uNqQnh)x{UwXk z3T?>1iG2B1G%8fEO5|2p#AX~svS#mFLU$7Dqe!f~%;1UzQMS@C2Da42T{cEu;*@;> za>79eQ-qgl#7UIS_#yx&`Vixgxc}+gxui1Y=h8VeojWWs6y-+4lf~ha*jogS&kiDF zHL)rdCmt?LJb85@BFp~X+%jDd=RE~+GBhSL*KQxZna4O3M_DLq<1Ga+DwjiwLn`D) zTM1q)1bC5up1n!BX9bzVV*h>*$1ulz6oI7G;xj8#W9Y=a#=Mq7OiV?U&9hHZ%nhSc z4<1x>suEUG8J4<9lxPIksEajNEbzMKGjbgr7LT`bG8uJTFY|Z`qs?1LUY&T@Du8D)8PkN6dx&+T}9xkaX`w!sPA4BkE@R4C=J0f z1NKxsPK1rH`Z(E|2-xm01Rj9PZ$+=mLAp0RuU0$~D_Kcr$9oUuu!eHvawe9v(B*p# zL~eaV#2lK!#=8N!cwzzk5Pr17Yb=(MuHWzf2haFt!zVjLm&tU#JAR%9GZb<|i30%v z$ZjH_xc{8(AI-=Lr7vj^T<0g$_xVKv*Rb+k^6lp=b|p)R(Jbt6Vcv`Elrb{BCTKsg z6F2aR19}~_(#gFfhvb3P`>M7RP!L}E8iER&J>d%EGkH=;h zmitIyu{HgK(e2#fq!2c5KFK*DlJqffj*Fo@tc*Y(OM_$3XQHaV+cGL$O)Dugl|%Ju zCasP$QTlGs&jrN&oM7^2yPe}>#Cw0-xb}xx1+9=vCjpBrfw9AgVZy-I$EBKnQDOPz z=c3v>f0}b=TI4CZT#t_hF&H|oJ@v!sKoavH^BAthycgmL`mw~B{fyplSp%%-)2Z*x z>@@PZgHqFpyf~S8?92NlQfy5$CBA-oC0cf%;dGXoT zqHtyiLnG2hbAAYp?ZPDS{CL0zz|dgjH2Vzi^Msx1vsK0|j(hn!W*iEcfXpCB+Fs?5q@OZTFy|GM~&VLET>C*+8b$#c?t{t&R)4 zl@rXuGy1>rZy>37Lfa#=KLTCft`UR9u)qpCCoqYc_(ppARw$a!pJ97uV;oQ~yoKdTj3NZDL5al9*D zgQb^1C4Zm97+~PWhU!cA*jj>Wx1Lwv1Yr{=M$@KmG&!fR)#ZvF!MdOUL$N@8!)mu` z1}NuH4T(fmQhOEbjGm=pjbN>=@f#YTD?i!Mh|zwE)A`){!0$HOTg*Lc6)f^dgly@r zml{Dp%zp5Pw;KGVD2+<@A)|r>r~7B>%WL`Nf&S87g?S#46D;Te<;b-j_5gLVc#Qf0 z{>0uNbTT;DACm73bW^E=yrka25A4V180)kz#2nEY1^HW|TS}pEmS-0e$w1BsI`y z27LCPBIOTH&#B}7@N|#=nTGB)JUaN{XRcv|3?nac*#(pbKfDi~dP|V`(QLw^X0ALp zIiK{t|LfPiJ>bQ;>@oi4s>=mx=nPRGseN`04ae+qY`8@{-rIxLlNw!NTFFQ4!#p$& z>4n*^1c^!6>RnkA8(9|@S&oU!BM7hPbmXj7h5$l2WO6Z!jTBj|q*Uac`XI#n4nW*u z@Zt6X4>t`u?2e=Dj-%buakOQl1lZqd`MslI1-V8S3sU?pxD{K1;+Ct^UE{94enOWk zXtFf1i|s!k(R)`tlZMLm6Sw$SA;<3nZ3~}jEUsb(dly^u_@v~7J1QW6%yF=)NXbrl z-&}!&)R?6@-tsjf_}U)BCXYwvIkgznq9wLCq-iZLTrV2%AF;s%5v~sOY8@dO85={V zN`X`Ld9%i{r`HLM2cnIY=FG>Top*pF6O9IThKFh?yQtuMjSB7#cKz%Qc9p_W^2{a6 zb!)B716riEj<*^8{mG86Z5!?Wse-FNhgk40C&>B>Ismu}?fz37@+-$8eyjt4w-ld! z%fNPX1Y>)yxv6R7s)i(X=Lp~BKwi~cX(&);W_+SUic4m4Wu&(hC7%GJ{ z8R+FKmW7pwMFeW_GOQVKEpk>0{>T_$s0SNIoCgOFXjW*Qopz&FxlwkS>pn3yNM2#_ zw=SxH6+WCRl{ZCaNF~F1W#u22jz@Sdf=96CYN9+c#;OfHdsdXB9DyQnw2W7!RrAsU zWOPpTVKQ}$E5l{vENbs}tKp6KAUCM17^8MWit=$~e9Xc$hqa?CCo(!@IrT?4ZM;M8 z{;cSoB=t}ht4++@C!lYmBu0OGWP1i%apSLl>O zA;2%Wu_`lujv|t0d8>;lRW5I`w#{81Moz|~l~hj0dC5x(A!n1mJkgWyrxEP5A5 z6ELMwL|s&-B!>1`I&#)krj)Vw4KSpMtb-!&3yKtM_$E)#YkUQ|0UoI!R9i%+{%p9X z$-|Q8Zkik2rIrs6(W-Ftbg)(jYi$|UGP+yuh)zkbnE!dBJ1fne@R8a2H67wp<$l)$?F(#Dm2_>fQC2C2$vBJh3AxP51mMhn}`~71sWyKJD#=E`qZ0;1PZbK z6aC1W@qW%TsYQ0Q{w}?j`m?ys5}OziD%@?rl;s7z7Oq;tdzmd-KzhQ&iJWUrbc$0F z%(T?^adOMMDwX~>157{!_Me@=ax7>hy)$0b;Z&dSWDyCqs;`cITgbhkR_1dzJ})dQ zUAedeQgXqx2D3Iqvt(4|wn1LD?70TdldLZ|!Eyt`6*- znl@Ip%QPOP*IcR%XO?i3;1jV?z}OF28E#>fD?0b1z*Qqjy^HBv%eUMkqtFwBD0) z9#>wcDG^m&rCDu>5!GZso~Qd~g~2enWM@5?YA40-@Z}Y|2(dWXCCNqHkgR;rteJzx zzviScQSi_h7{CnFC5pxSh&W8pu_YbU)?sR8kEs=WCbyJXmy(uhvPwm23)4CdMoplFg{05iQ(4N~k#(i|b?5S-hp(CHG{W zv!orig-|M<&7#V(;MrQ%kkyDuCPM+rQfphq=)mE-fWrodZwCx_!0_h{49fse>53f< zTq?a+V4#iA-;7khzRSID{#5Q^R%jC+rs41ZpHf%LUmiFw=AIeF4|Q@=iZ0X zOYD-R_Xj^SI-DYV?~>L;zucGPD_uTTCluwkY;;ajmmM|b`MzfdP(k}VOe4X|brxCY zK*L^C-7F?Hz-Pi!+oZ~u%1h{w##h%yC6^Tq|NJ zjNElsRvq6`R7fSqY%gvTw>(N@UE8JlnFfs=TA9G5xM7C|Gmm08Or_2}>juE5832`% zuAY}76DzoxMPohR3UOMIL#-^U&!}DuR!$K4;=t-zVWcuoh0+lhcbf@?T4<<#p+3~M zE!oSrKJ8=9?vsREneMQvl32p-u<{(OX6_9gQ@RbWx_6AE<}Rv`aSusk-TFuxIZLIV zDp=&bb;mpLjHOr3^sQ?O^$x$!P`}n3>R}wJ4fM6+wQOW{O7#!>shHG)n>~{154|Tg;I#OnWqAv)^pUtxAmcpS5bIq z1C9<1gWL2LZ|Q@~dM2|&SYkCg%y1v{dM%%tG088*>Gd}=7ltO|u(KiV%Z7MB6Sry* zKa|r4Z|MnZ=?&tuqooO6NBo)3g!{QW`k{|;t&{ZbX)5xs$M4?QuiqU1^XS=${g0zJ zN9&!Ya;5QT1{p?lwUf_Dh=$WJc?N-H!D}2tJy~AK@Hh8O6XPbT*QWY z&kj$Hem;KlD}vrpN)aw#CzdF5`Is@%`{B%q(9NtLYVctdTZ3r*zihYT;7IS6rQhezpYGFrx=;7%KHaDL dbf50ieY#Kg=|0`3``r8I{{c%gq(cB00RSnbp%VZA literal 8813 zcmbW3gH|OB0|2wN**0#rH(T4b*|u$Mw(DkhbKT9hHrv)_>;InNKW83c&Y2;LK|sK3 zuP}i6ax*ruGj}y~vUW1Jvvx2yG&Qz%0yudC+>BiyE`>i_*LfO0Rs&(Bv*HhkqT<&L zuJQ~Pa`ciPT{3vG{5GbP_IA}UsqYk%7)38C0zS950T9U$2#Kl8{L1pv%?+bNCO0h| zEBee!pV@E$vUzb}q;7%9I}G4e&s+Qla>DBsZ}~Rxv(js1J0BSEwaIIk6lu8GmqJsC zht-%NJs$U_2n2#3zvT7@6Z*F@-?vYGW3U=XpYy%e?aR<*lpEBGJSivm;05r_P<-MH z;l+@*3!H(h&ZfE+&f|97XOIWzAJlu(>hZ2^SdO&z^vpD(!*l5kfCqJ(W=fX)I)1gj zuGy0&fq5>`J}Re3e@h!O)c3*AwUHi@pMIO}nu|vt zetq0-+CD9wX9Na59)RA!mUa0Qb|)bE0KMrY08iSq`f~n$#&69F*lX@MP9(>zI>Wu0@UE{ogX8_^E5mrcr~$S1`N-1fiyWY}WKav;YcC5zJy8RTatDaRWW!Bvu%~U>d?CIjad9Qd}&UF`6nWju70RjT%Ga9#9W~9e?;mV#O@PNlhA^gWVtzU4j+gp(9gT&R?K6oa; z=jC;q41&o-fG8PV*i7D9VTC-qt%*-QS%CcFfl!FoMK@)YU*Efy3{aG(?Ro~3(av31z7|qE#VnIEZ`21tNRTi08}!G}4IQpMo%F2k&R(zG-R25em#xFV@e4NCMMofCOQ7X4D(mkC#LKz%a(a@K?U$*3VHpwHiA8R&A!6R zO|BGVJnn95*#GI}%+%Q#hFZQePSuwtGl#%y!t3m&E8JTwLzv(&$a?2av^7gVu1?SQ z-2#|tq&&Cy8U{^6$3KQJ6~L9AIn=vS8AA-#5AH+?1Plxa>41bbMzb+$cu&2t18nU~ zEc^tSwsBeUoSP{NvV-Kmzb^_zYlPhSfAZ+U-&bI>x>2M@F%f2@bpAHUsLK_H1Xbgh zLh0@mK4#4{jML4-R6>juPl_ZXzR)05q6|bao=iS{nsPi@GV~g5r~Vx=yANIXC;ml4 zmid|3Np^pT2;o@JzE44LU#gbAl`g?x03=CBpl+0;qstk*=E(V8K4nE2$Ydzc>jHOG zSRx@hr#(!_NBuF2WOD{>*dO?gA!&=TlRrVP1Bcy?#Kn>+tM*%wHZFPWVBuxUdnH2< zIselNTe_Hu6-{QNj2((#l3MLcF|u^`@Sy$c?1ky+^o}>->vbm)($s~P>3s{*|#W=uD6V&u(s$qH9;)o<|l!0h(#OH-RvkA;X*!wLyktw2HU z(%Boi4#0Rwj%-EXU?(wa z<<=++ay0SZXoKE-E>dDawm8GT3WO-P2{VB&H;3#L=%ecyDu(f6;~fRmVhB)}j=QYl z@Og|=QxtXdVrG!4F#EDy{8v0souN)VVy|51SUc|ZtOd^!u{9dXCh)X3&mC9Ym$sC_ z%lnZ9WL^weeS1W>i;~E_=o%27dep2kbH5cC#7l<_pWXhA?l-A=$8c~q?`Bp|u_q!c zACQf}F{|n{Cpy0JRR!;4VWkD4NwF%)cUR76=e0+@YS0G5A(D^ z;kUUoUY#Wns{ng+83vc&9*Bh5*4*BG%FhPYG7!%RC5uncH4?M7X1Ef&dw2UQcuLYk zmIrM4PUl(mtzp9&#D*KvtOfr;!M$Kk6veYRmq2Q@8k&*i zI?^!L)LtSzRy+e6?VQ*=1CzSf#0Y_YIga?SAr+NNfJr`SkIyH}!3*a`UZL0=zWSCS zeckRhq$Uxfu2&*vvJWH7W?+g*&Vc4mJP(&{yyqS@JYA@-HoKWoH|etbz8;%Ja>#y! zbmRUX@lMhc>U#M^T70Z@$CnmL+P-iffs|aQeb>pZR>V4=&#ZC)NXGK~gzoPf>{lMl zG+jbVa5}MCkgBIBE6)Ah^bFdS69Gk4xq&XE$i=%%kFeSwE^~pCVPf5UE=Mtq*>#4Q zqRGp)b5~^NntWS+|Bm3s#8tL;AMr!n)ZML&bLcA0ksFkb6b@%sg+wt>Mb}p|0W4R^M==7sC$`4dH6hm9lK=2g*)pcLmIJF4wb?@ zIN15_5i*)X=UXbj`HVOC=N**O!{SRV*)(;aOTsx2jkH7aPI%DL5V(>N-2Bt^0GUj` z7YjDDDSjzb{^4qdT1nnd5$UkT;_D+)baa%4I;0$u^;Ui7zXw^tX#>!|2gqyHu8Y#v z57KmNg=nqHQGP5y)Wk$Q-Uq&dBBA|Iuxx!Y1 z1JZ*zRCQdL)|i_XY0>@#9f?c^BgCnm@>^-J1CE2k6!zD@fA97iFq6VLU(@jT3bP%X z&5qzu$8TrCXTt!*Fh+;F;=HNHl&E>3`#b%NWq#BAyL_j$rS9p;Mm7Jed#tiinHtG* zAjPOm8*{mI;?y9v#OKdU6RTW9_-l;3YBx8pRpe=m z6Wq|132>LEdC4lA9r2x-JGDN7_Lsr}Ac*}Y>yT(baP1Ykx{LPZYn(yfd-VX5%lj+# zUq5`Qgn9PMbzp*`@pp`}URTlICo3?QE7vsFb#gnS2fa6DTW#o zID(1=y^r{!tJmSKZib7eMz@0{-}^Z>IJ?;<}^*47bx9?+`O0fnyrJ369nVE*wQXrqf!B_K&oI_@W z8bdRf$JtAqVj>Vj!mK87;0uJnkfpg_j*})yhh~`OaT6?)6TKVc5dF*Ox~pi?nYK4k zG9|CRx%gQzT{^0e3nN!BZ2hVeHrW|82W+K>xDi(4twcObCP@|gY_^&48NAv4@SESBl+G`A0Yt(r)Bo@ujdY-?R+PHF5^9pK*prW&zG zN;RtiX^BbGO$~XUPy8kMAi|W>Isk*zjqkAcUu^Xp1Vp4Sltsn5XQx4NG#V zXU{GIO)&*gB$=H8I@;3Kb*XS z=H)IfRX@9xceig>gyGbL;!PckP1bKJx*{%nUobDk$zDk(2ycynI8%2O!Uy(6B7cVc zD>BhZ-YV7zbO!~w_+35etg2VgQ|g|0(ykVeea^v2(H-dDnsnarWvYQSkoeXP!XAbJ zgYxdSn*u&M`ZXA`t4S zy(}Rx0-k=y@pV#384T!_mt%p34Chwix6#I-KsVCQ>48AY$ix?iBB!(_666$8%~~OL zzAbl0d}!xCTsjSxUeUhkV__=x$Ci&D);Axh0UkgR--o@!o_t}Djwj#S$Td`a%Jdr! zKDXPBF;%6ykcmoRtcNfh2(=z)4~Y~EURHncYH`xCv{9S zD(8fM_q$R(<4o_Z?mko>VN9dtx8y^~$8q3)FL-m6ORTrB&tJZ%& zRKyGuFRB0xWN)cp=i9%C;EgT;dJNd*EW8C8q3 zVGBUd$oV#1{?^_UP(x596k0006bg`ws0m3%^oo$@jcBB79wC}mlL;9Z2s77Ub==26vd*U zP#ToJCq>;NZ6>sqcO;#qbKx23S^wDK0@9|22(xj~Xjb+wk^=cd(snzO`%tXK-dKxH zpx%(Rf}=GoEV;eaw8bp|jNV(%cFgK1dSSC^K$e}{5L^N}1{8N4eaZ!CVb|y})~cqW z15CX`Qxr(y(MCoKqkA;?x)J#7ah_+0PB^m0B69iUCHy-m%@|yrt6yK>UdmHY8|GAP z6I%luKJqp@W4owgEYkI#fV2lw0f`S(-3!5Y8Hp%vu@}VofnSAto)T?Cl{rR!A@u2A zT)jvr{#yQX|1X5Uw_c-G5ds^X|2e>-f!PTN-_X?TDlq_M|p3t#@LJH!5Jf7ZM6> zpbN6x)NsDVTWcmLKcs%5XN8`mfZu zuGJY3vuX2cL0_$CGlVq3g(=2ANN@eDwSDy6$ZOHBmzf9D_bZ8C_6e{|0Pn%J+r1cX zHylFS8GmveM+B_k2F}4_k#_S*?j}{epG_@e#;w_Nnc;~LX7kgVAz_{{saIVA?{&a7 z@3ahFR3>_bUtT{&s51LxvZW?(IcI*Jo2eSKJ9UywPttAJu|x*0PW8aum&(>KXbA+d zh*=HroFa1X^xNBAgtzAP(rW3$k9u6T#$`<469J){t(sA6?CRANPcCr`rlfXQ(jxkQ z)UBFQNpP_*=^02Gm%fCb0GcOl?%lRNr=IL-m+jOoG-2DaC z?n_5i>z-$adtVqT=8-PhOZG?e>z9Dmi1j?Cw$7b;riH+J#n^B9^KrV-QPMc$sEqg5 z{mvM+uz(0h#X}^R`jg`xD*ZIMWw60c*>5BB`Szcr@nQnYwO&z{9 zYT?O(rRr;O&7S1;jAD33XY-ypDz`wlec1E&;xOC{+I(ilP5vCIk<(=6cDSo*p-(cY z^uv7G+Q4;av$$LFe+8a16U#Qwj^Tz{F%4jwHNvt#ml%c1YZd>ibT(qvxjod#jKEU4G}9-*c+1J;L0%+@n4nvk;{G)HOo+j{Feas&R#ITaF`9iT+EtiVB`L4o0d^AJQF`hj#o?}jEFji?~6iLAuQ zfC?4RHcBgZ8f~d=v@7i`hazTzIP${RWEW!APEm2&c);#;iRkzEi|XNgC~HD*Y%CET zqy9}918+u>O7m1~e{u4WueX-$welPdlaP7v2${1NpP36`myL5WtdC@q@)T7!A)op% zZjcK#GsT4w9TW*SJa>S9Pw;g?1{RVnNKd;L!wD5;SE-&98ec(ug+*Rf;Q(&0#|e9W zD+&b5eEVmK#YL%QfTUxEu1OqU8iqL(w#3S-$^<_ZtR8}}L@>N$190%W`b|0* zZ`n2UgQM%IImE^zo2-GX)^YGxP%DhTi~s(;%?Fre1JybRvj)hQn|08x+amZl&- zZuX1$65SmwhqF4Gj`wobyN-P-trmoilj;Nkv<7cGu8+FpVvqNQY^E420Ewnkuqx-UBALXVV%r z%jqgNJJ2>>M&jFXOcKPq8a@$mU>7zaK>vzRZ0b#0Lq~ zj^OwhFN8()H4_lU%`jNGcNf8vuv(82Cy;j=gKc#o<>gx_k_s42WBferhP&K z3!&Q4=BQBdAIhn9ncoxMy9ZJZ`#qO-8xcsyz@Gw(C*Ywo_@0;ZJ~DZmAAb_~v9Z0r zt`$H2B#8pg$Tr=7EPV6bO-4&hu)CMVu}T?8z(Lu`AN-0_hAT&pJn)6yScsW{N4ZJ# zBgSO?mG4{s?cEf|C8fWWYRs5dGib1*M0LBs&@B^Aqk{D>T=l1ZIw_+md#dIIBOcGt zfHM*F5}i~iLw0(|o=~{GwE51B{lI9()>(jd$*^wTTh&_v#%-j`@j}km^-0(lXWS8y zm;390ie5hwsWcF;6Pa8k@EJqoR;%}Je>DtNK>y|L-f!;hF2DO<9t0r%Fl?8XSD&`R zm`&2RO7@tF|v_1w*`>eLO84g z<8Lr~I;vZT%AW6%dX7m2j=$IiVUEc>I2XUg69^l1p6r@PdB$N~axi&jr9H_cbL3aa zBNlAiy!n^BAR$EocuJnZy+46_@;d*N6XM|6-x%~kg%gbnEGl-8GFSQ!ikMZ2Dth~- zlJTNpYn2L`+In=%(IrBfV|X<1Lh2+?zlrIp&lvctNZpx*DY$Ja|2e)84j46?7AoN; zY`C1UX~q0p*##(By0bs+Lw+UU|G+Rl6RTe0yqBgOwgSu7==yKgG;N&HmX3+tH_YV? z3Ax2UCVCaA^-X^i>Wl#red~fZ^yBwt05T2OKQ(>;tZU)V?MXeT9^73HR>smns^9Ihs}xMH5?1MczcU-^&BXx#NEgddZJYkGjM_dR)!ok&l@H1H$Yr|e0$h;kqA9=!mQQ7IIejH=> zu{tlIPuC&{Mi89cuc&pU6t@Zorp|&G-PGZhch z8I}|Tu>6r7`9Ws#BXWh%hFVL_cghsi;T}{3w3msUL#3!%eR<9eH%EK>*QPgWHF@;c zGf}VS&hOJeI3IIa!l1^3ol`itGj1LRBo+F3zuY|xs@Bs`zo~Mb({-Ax?0M%dz8N+@ zYt9+^lys#Jxz$|PM;}%BAt~R_aVsuJCE8$(gau`OKt#GHzSR`~vPUG|SVs-k- zbwRSNbjKt$+pCt$&G6N{#N_3B;*YwH1x(sEb>^Pz8u1 z3yOBa&(y&F+VrwNwQ~aB=wyYZ2+jDm&n(6I{oowZEv=|>=Jb2fE=0Rj2mx^;fWKYf zwpNAU4Jb*ocyx0U$jML=vENZ%V3(#@1WJL+@sYUad<}(c^V4MFl(^XspHD97l9WNZ zia9}PS94s9b=_UTO1SM$*O|!kT!0hll{Q4)mwFPio9`=QLt7EhT{5J|0uHz^qkj+f zJI6~ne2*azQZ0GIyX;mFd~7WZj}P9$80@Jt4E*a^5(ibHky*Qxkh1g&0VlQObm4;21j`qdVbp)4PEX4gL?K$=f zi@F6;GBFOeU^b6m*rjeP?HLbXK%Jsgxa>xE0VQqq(8J*ltDRYVH9*C_+r*PTT|gi2_aKmPou56m@pzH&Vu^tQWwtxY#DTkEnIbB|hKXD_1K7>|ykTwkf7t6lyi- zVuD<)-E6Tnk<=u>IDU(?EbV7&Gn1261f_)JKR-t6XX}(<%Om6RQlCRm+dAkY^AKa> zB3UqnAWjPHNzQtaVI;5%w3dc0`6$-pvNeQU9RN8d)aTR{*Cb0?*+BzFsj1`XjkRuZ zf4DCAS`v}}9bIm8z&l;-4NlYX8JcafZ(3wj&o26S;Z<9FGS_9RYuyCfWK0%IO(nbW zrTS_$ZMB1 z^^@g2ot*9Ke0$aE?Wa1mwukYonoF&n2h|mNDH-dyyz(6^Nf$Jjg;n)y#TX8F{=Fe* zl1_QpEk19fp^SU8wX<+X^r3BF=E_hX0QR^GV*W{no+h8Jk_I!e@{Wlwjz-cF;54Gj zwwSAxu8;?=Fc_PPbt@b}VMRLT4Ck@ic2UO+-Z}-FGJiE~0v~sbjyBG4VnuYVt)V`$+XLewhbrMD+yu zn{mfcuuYkYHYD!*kig0WcMhr&=%`|?G(`y<`{|H23R<$IdfK8b{Oh5xtlFp{E{Nks z#%Ohj|LB3kC|B;Gr{+%70-BL+Hu%*t)Umh-s`am_t=ja-@KJ*2&ra1VCh?2|Utyw)adqc#eJ#}1Hk|;0WpPYw{l6E=gYW}l$uTT3R)|lL;b?ro>Qs}O0 WFZcaFu{=Khf=ov19)}Qvg!mul7+O65 diff --git a/ml/automl/tables/kfp_e2e/tables_pipeline_kf.py b/ml/automl/tables/kfp_e2e/tables_pipeline_kf.py index 640a15b..07b703f 100644 --- a/ml/automl/tables/kfp_e2e/tables_pipeline_kf.py +++ b/ml/automl/tables/kfp_e2e/tables_pipeline_kf.py @@ -21,10 +21,6 @@ DEFAULT_SCHEMA = json.dumps({"end_station_id": ["CATEGORY", True], "start_station_id": ["CATEGORY", True], "loc_cross": ["CATEGORY", True], "bike_id": ["CATEGORY", True]}) -# DEFAULT_SCHEMA = json.dumps({"accepted_answer_id": ["CATEGORY", True], "id": ["CATEGORY", True], - # "last_editor_display_name": ["CATEGORY", True], "last_editor_user_id": ["CATEGORY", True], - # "owner_display_name": ["CATEGORY", True], "owner_user_id": ["CATEGORY", True], - # "parent_id": ["CATEGORY", True], "post_type_id": ["CATEGORY", True], "tags": ["CATEGORY", True]}) create_dataset_op = comp.load_component_from_file( @@ -65,11 +61,9 @@ def automl_tables( #pylint: disable=unused-argument model_prefix: str = 'bwmodel', # one of strings: [MAXIMIZE_AU_ROC, MAXIMIZE_AU_PRC, MINIMIZE_LOG_LOSS, MAXIMIZE_RECALL_AT_PRECISION, MAXIMIZE_PRECISION_AT_RECALL, MINIMIZE_RMSE, MINIMIZE_MAE, MINIMIZE_RMSLE] optimization_objective: str = '', # if not set, will use default - # ["title", "body", "answer_count", "comment_count", "creation_date", "favorite_count", "owner_user_id", "score", "view_count"] include_column_spec_names: str = '', exclude_column_spec_names: str = '', bucket_name: str = 'YOUR_BUCKET_NAME', - # thresholds: str = '{"au_prc": 0.9}', thresholds: str = '{"mean_absolute_error": 480}', ): diff --git a/ml/automl/tables/kfp_e2e/tables_pipeline_kf.py.tar.gz b/ml/automl/tables/kfp_e2e/tables_pipeline_kf.py.tar.gz index d3635b3c90ec819ba1011a8761bbf7f78ff48f5e..adbaec6eda203fa04a13739229ab23f1598e481a 100644 GIT binary patch literal 9199 zcmVPEfeCT^P3G|o8AIn!}GJd_B@tSM6E!;b5#|9*D?kOV;RA=yft zRGLXFk&DG^vAft^EEe8^`d&b-8)xo+wVBU8e4adh%>Keh{rhPD`$vzSWY75XgM%lJ zzyFHtfAtYQX`DC_A@|ZrxpSY%$60v)rcvyL!H_u7EL=q4AC?#HT^%_7;=(!Dx%2{e zNd6N>ms3By-kH?GiIV{8ct~y+ zg3w~?G96RIWrfjf?{d1ZWB1a&${Mk6ePnF->gOix<9gW3#vYHR>vwnRNju)uw^F z2)!U#3+4FF;ohF}M>=rn75tlrvE_$>3j{RwE@^CEQzyBgk)`}9w_^d%*Q(!5BSx=f zl)!;zFldv|Uw6PVjPji0vbYbNx9>YJ?sm-AGLYwcr0w^eXNRXpFOE-se-F43-+7`4^HaUEKmlbV{B0ZaGDE5o=k*-$`11U*2f!8R)ceq~3nt7mN<=aD{^FAp zj*;DXSGBZ}4M62GEs%S`#7|vDa_Kx^+?0`?HZIcqw_yVXe)^#j|J_*HPw$@nTFKGI z>EsgTdTmAqt7lG~z;?zlv^$|TjiNAmpI7kspZiN?)m}t2z6gCcZdcv8RMsGwI?L`7 z9{A3f`YVg!=dI%)}DLy=1=FKI}{}GQG}md;wHHWgKzWSgS7ZM$-n+z-o|MZ z&Iu4G5GL{Fi!e$E>W1!Vy71{v4hE}3(1$G=nl%XM!{+qS%b`KfROWd|Ko#q2ulYol zUYX{p1SC%bdr3y*br?_?hBqp2XRZYk*PA2&f4eN8cgf&Kso%ZsF&}oA;Im#=o;r8} z)$R64uPe=%^t6~L@$h_ixFa=7403!E8y==(7_2Rcbv2ay8j zNQb~B@Q(*tujBjpbryy*pTbaqehFj}2Dm(vFdmH#tS8ofR~{_3d40L|2{f-`||j;{p^>cXTNqqFF)u4Zw0A7 zxg4DWFWkMqZVe`Wn7RW_lcUkT^@F)bf2JNy_&VKq=gJex+gg6^hqD>1IOzpjha8+S znMoK?YvOyr#ceK8;;is9_Vk!PtU$p=Co3Qp?$yB<*1!RAV(~<7uxOD5O42L%N~7Mc z1!Vk+UQxf7y?pul=i^;Mz9y%~&yR;>?p#vhBLs?tsW$^{d8~9dgS@5yk^0agm`Cck zeAo!^4#L?0>C&_dJ2jW2F~uKZX03vVEg<6INS%qETyP}+*fn5*(xQi4=yOy(!81oH z|LluV8Wns~r&+yqBlV};1kftza05+J769lTY8_vbXN+GGCu8yC+Dk5A)+m{QMjH^p z@tHQF6wTmAqM?>5f_f&A(I*+hA5{o~U9EXQ_5`;c#2JIoW$ohixLE^~0@<3tBE8OY z(L^q-ZkHVkXnXGDZL{#j?rsYVC@m|>ThKi(>2>xxyJv%g^H%8OK{X6Hz!Gq~_1FR@ z2%B*+C=JsBSXt5oy&@~2y)V=cSS||jM4WoMdms%D7y&5-Bbua9Koo*w3XG{~py?D9 zDZJ&wgf5bjAmfjb5m_(TtHU=hpMg9-!i^OQB#j|u?_AlxkAa5KKQFv4H6&-VHw6u9H=BK>al3rjv6S?1SDR=tUpYW&FkoW_8<)6X zr8fJCs&e+qI*exN960J5_6Aw~Y}7quLLgfl8S_P1&DSr`k7R8fb-TNI6_(?=wj*%& zI{MzQ!w@8+k_nZK)ChHbA-1W@R4asMyYo@r^0)8ayg4~~`?g)(nH^QCH-EH=I!*i1 zy!Hx%>n7SE?`ZS-HLI+l-k%*mzFWn@rI3w7#_97mR7irXIk=E5KXlyqUR$N^)bqyJ z{HdKiEk-0j>&EC?9W%p<3g8v|>SeXDKkQ9UTB{lIsfSL7k%GZmgo|FO`m=LMlM*lE zTsf=RfB)kjGI9xt;Ay~%oCG*ah7)#0whaoum(aOwcXtUaaf$C0U}d%^HR~+`Ubl|3e20$I?Ch92@rFFNY&EhAFE!+XQ z9?(FDR-)trw5bS}b5Fvh@oga?kKZy`QO3G(l-)n!T5`Ccfs7QU?iwZxgv+#yVrpGS zp42007S3P{&hwdNzw-9Mwzse&-ExKT#qseA*#92BdGqS!v%^!i|2;W+e)RhE<>9Nh zeIlgk9;iJakoJ(*!WS3T8`K!@!DcRE3^;ZyT+>S(|Fc)e@1DPX{;PfX?(`Q4wf*x; zsP-84+$S%eAFW-FH+N=Gj2p#<-&)^`=T~TtroW-07)yPg#E~x;}oyU0XwLwdHllE_J4_HrVR3$Aj3t9H4j^lFR8LYi}#BJ9^K< z0JrqH&s+9dH+ZF)U&Q40FnOEnANCTj4&0t>}}%iq>Sl_bv|fjWTGz42!Sy97alpH z@jJ5%iFre}4y1Yq(m)IocN$lelKskBow9#;ko$)#@R*&+8b;6p}nw@SO>cRNTcyk`C$E-Z^^^lw%GnlT3hfusQ zW`e6iTN#cpB)%6Xh7y$l29+ghDRoFZW(UVelD6K#@~BDPz=BLm51Sl>CSRb@dH8kV zUKytz{O^!45Cm6#xmQFCY>ZZIc;kEH$ca4o4Ld_9?1n)%A=eJ~jkuwL91Y>_4Rpbq z5aEVn(@Cx`Xu#ga30;sV4Pxt~xGlc~N9EQv+2EQiwF_0Et;2Hbu-ufxQrO50GEk|x z{2oQ$?b5zo=GJ^!I&xb+`fQj>2mk9aTmDXkcc^=c0t%;}p&_?R%d#}ol|H+O#;{3@ zX~|W}%Jmw0(${QWHG*j6PeXm|1lgnIEe~EA@y`<(s~4q^m$s`9?-i zJ-eNJcEbhbvBioTFq=Rtf33`K6RxtGw~Vmtimt#4910ym5lTLCdf~-VJ#*&15m#j%=WGKFvP zh05!oB~TmCcHSxeq}QS|S*u)qbfCO47=^~hD}wiy7LkzYZvaq0*ZWtw+m>s7`6Ttl z66Y>|ZHUCRCn$*hpi9W-#D_eSm^}kR65K?WsE~u}QDBP&sN^R&&f^qb;ko6c)bz zh66aPh0J2Qb@dxv{Ux*E*3rLp^lu&gTStGrqhETZ_>c6L6;2@?tlq_A3N0`VMjJoj zpsf4kq|fN%=O~<`r@ZX1uW%kMR}6M&(IR8GKtuJb)m9Pop!l@b!`#q*t=-#Thffvk zpdKr};{koS(f4|ES#9g;=ari@fP*{48*DxMThD$iimXg=>g%3D z0ZL^ptx#4m?DgYqu)sE0;G>;x=89MDb!TcGlOAQRJTmz3b;lY5vA8svvu#j<>fLC> zN!$93y?>s+e~YiJ=SOc|9shpxylg1q>>nc)I8N$zOE78_!fzt@fH*#A748km3$_eJ zFwKNs3GaX$>(Cy%DcC`X@3luBms)%iOfI4@2w}Xg9WSZtK!=Y~HZ+#e@t*mpOYN_p z6k$G;PR_m#lb^#Bn8?cKWs90%?`=;JiJ_{atg5?I0I2iM?9>w%FQmA>`J>j(ZUMft zb!Fd^E1S=q*gO&u&bxJEZ{Cah`MR*zbhJHsP>wd!zVFL*vTYq{TSwZL>PTDN6!wvi z(;7^V8}crhYT_D`5xZo8+xmS>a(i=LAH^7?oAwPlS;e8#yEU87{{u15Q(#}1H6-#T zO~*zVsAN&*1ha>a*=b;5Z5|AYG_6&K7;^y4K0fzz+D^SZnTdWPF$c0!FF${r*Dm1@ z{i?E;7kC7qz|nYel2gk4+R)jK3WYW*L%1+ec-w%|=9>|wVf9eJ+MK79wVSeP(WDFH z=wMq!KX`Fshh^(+z8%V;jV66Y2Vx*)TYgR-tSC{8^~Ra3g%cH&VetH1MpgMq$=i;T z0t1_LFlp}(OXb950ZurmAr&!{d;DZyEyrw>um~sA&QTzSed{}H1!{8>A2cOz>TwhF zj^tN>&jWP{tdgVj^T%2IAy*G+Axi<>0Z+rQ3oJ5Nn+=$tFro=o1yBV_1N&0Z833wUmsO~>Cwttg1q&MCZ5y!Wnj53TwjMQ@nRX+W11hMI zSzp2yj{AKSqO{Pw5chWL4eOWebkUVO|A?<-g7&855-X{md0=g(87w_64{Lds4YlQi zc6m;IBv)k^y(hOzfV*A(j}Z3gD_}OOyk7sFAmb)BbKQE&LgNRpK{?yd+ zQoEn7-tNX$w`_9`##KTEPUC4pDYyrQrsQ8$xZ3`?C0H5Kl?-OcZQ6YmhL<>AxC&I9 zGCQkD_R0z}Q&qw9hUY_(Dd92*ul1w{7ZkL}C{AFilkyUwRjp?ZIr%}EDhe*aRX)ZL z(0eVbVg^QUaqxP4psB2a55Q~iAuIC3IBtzGD^Rwkq2-W`e+w*<-kK@V?B)s+Yk;}% zETf|P}D?#eX!ESOAn4Gd+wI%6~VMn;zIP#zu%@*-vAsaYUWV zo#v-1$I0qaRGNI*?|wo71FNKOVK-aKvDo2Q?)H!TH5VOdwV&!7%OW21Fl z?P#1%aBnqDegB5xp{~9W*Hza1jn+K_i^$nMEHS<#5gUW4g)t00RiWphA&fl%JR6=; zfI+TJn*s6wpbIDIvgA4>@@5w(Rc1YG6_-5XoIBB_8(s&-+Lz}` z(`X12fpIXz>n{2L%r&zxU{-TwmxFXZrV$HY0m!DEdj%3c5Kz8za`>N)xqGFD%Z3${ z#w;`}qX=g3k*J7Se%y4vh-l9E4X4ql&!|6Aw2gp1eB|(SHBiJTlwB+ItGDW!{)fU0Xe(Hji;AQobmA z;|&FsDi_;|WG?cfjRcey47{j6uinJnvxZDzvwy!yWXy3NMR0F5`^?(Z7&>vUamb|* z6H`-V_3V>0bIpk1g9jBctb~zal$HC(3Sj&?F%H?}g(A|3xbywu|P%z4R(Wom|qSt_BW~ zmu20G2g14_=~40CLrH2jd0C~UmfWE4tGzv*Pq3tE1zRxc~xrq}x z=V=;PJW*B>BaB#LbUuqL)gKw(5Y%Z|W*WSn1ztO>llrCT%qOlSe8?gM*u5->i>JAf zsc zux8tN<{|OG5=n#0cWRv(aKInLc`2 zK{fTw&o~aZOEDLqEy0-FF0D?jgSO@T?3NND6*Hu(E37+Qd1WxY-P(*jKjEN3v%Og0 z^lym6f)YL8Sj0)>96l9JW>zI{KKoiU&g@pwfOKlklSSDsEE3O;dz=9J1}&%Aj(eY( zsH$@i4O^r|`X*-1KNbgPb;viQIsH=Rf@B&H(}Oe`ywBcYgjWK($`vI=rY)9Jh+3OX z$W_LZnLMc5Y0RNzDqW;q>S%qO;dRBJ7N!v>Wq9LKxF9niNXHjU7TvOHaCoLfmRuD~ z$>iyBErHW6kbnT?uS7!Kx``$t<^uuIZNbM^-Y4YxvieluMmkeao}0U1#n8B31aln4 z)UdsTsRicO%i(v)Q*yAck2rgsgf{FCuVGNs5{>hvl|IdPDX<$?av zUW9oakrT{l5BbQA9*P2eviP}y9{h>Dzv!rUu-_+73~*y7y67>)ee$7C9Di})z((-9 zGUQDH+m1Mcy(t64&J~6Fdfj}$G?-j{fj(5~-$m)ZG8cVW#V&JUQ@|UyF<^aCxR;`7 zDJU)y*k+U_9-lVMeBqAn7DZ~@(U~*4`xGI6e0o70_s6Gu{LeUaZ{X3vk1x1{6+Dc* z%y}0;9{l(|cJUKWNUW--WT$+VvG9b{sHM8UqR~OBQ~fY!qkCQtt0eC#>Q~AQZQS6-E6Sz@l8VGo)}|gICDC*^9hi6qEW%l z{GnROZKU?SMrv;pAb)lVkfm^xymCo%-PmaJfEH=2<4s27f3h^sZBHP4s(|;;AwK`h z34H&85@m10?YH6f+ho{P$*|=R&X1KSdqXkvD;_p52Y$B4nrrArtZF3ZHsAJL5?8BQ zUPFU2?d%gJ^DgZ!Dx>eElM7iLGG6KA!n*PLc~Kc}PM8lQ5jYpcO!JgjS3}&ruvwZy zzVVe5W4}gpz7+Z`Md`PU&KLIytMU2u5&p)j!8x`yLfm(xu(^h8;)Z%Z_5GY2itjpr z8iXF&Q!G?jyBQJesHPfYi1B(6$fn`B_d7jJmDGmz-o!2AeYY9ATVCS%QM&VhF7a#^ z^QrmrnI%%XM?1HnSEgyHTSm>LU=IV_%{Coi=yW=-Fm?%MIg2l1En-ocn#&*d@wh)c zF9qae2r&GH(GkbN9Xd2?w2n@@jj%i@JI#F`83!cqu=raS>A@NwPLs5$Rx*W+p9=w^0(K z6{^=A|Jl{Jb(w2QDqcDMdIgR-_GW*kG`g{wPHd-S*1h#b#-py3B^L9X!~=o&s^#cQ}hkKy0V7j3ezj?JH#M?Q@AllQ0A+osKT z>Af_c#Z|mG#OO7|Q#nk5xuiG3d`|c*Gl>qY85U0D+;E^XT<;)aQ{P9SK_lp$eTZA%;U~<%BY6_*yQAN9bZ?$SbAFo7%fFZ|Ts$c(nZ;T=7}xYL z%AT%Oiz07;wz_R#GY&G+%;yy!ShzK=zqd4DDdldLEJF<%*x)O=Te50uvzt_QS?W=a zX}8IeZJ4wBG-y{fcGQ02@?MQ&|C&9o4&JaPdlf%pIMN*+4Nn$m0Aq6 z7`Ao`V~s^oNY{LL*?8;n%2ZT{#MKKbVzOLZUk={1s+q&GCsm_Y91f+hDgDnK2)2e~ zE847~(0$A~-C*I&-QPeu`e2zRrsYr-ML*&|ncCZOn4oCOLE1XZzZ{48!*Z949SGZO zh_jyjhEi#3qPkMwF;Yds<=13gUw9LqAJ(NYdJF_ka_#VRTbXxJn?0%KVdYi85>VAO zz}1=2hl!OvCRXs7JW^&^N?MA^DiNt|OzSWhC8e^ZtSen3E)6?Q zEs}$KkGEdq4S0=_urmVZi)?$UxY?6y^x$sZbE_E_i>t5HRlK3vCG%unvm_lhu`TW9^&S^qb5*2{jg()D1PPpa@h|&92hCpo&&|8P`6IdO9mXvyN%dD>$#`$2yS0d>^t8NE%WyH5roya*h?o@+ z8UJeNs>2v$=U`PpLn(}*ysuC~7$lr$RTKLlLKmb{z4yc6>hczBnEUag?2Cv_y$_|A zINx*cFP`r{93y(~iq?cX-4|Oe-O593~`LrAVen$3x|$uTC_Tgp;cPZDKK&CT!+FvO264@hwK| zx_Pk6+w?ib=i9ov#J zzH-W)DZ6MFYGty+sw!d$rN+u@w3@LuOibw}&+6GRf|{qOg2z21mbG$TD7i{yo+?n} zvvtS&JdLea@{!jKg?j(!XDDB5lHV9fwE@1CUdu|>R>(dCAd;Yi!XdQ@tdyIFJ69N6{BLMiOtj8k!H*JIQ|>UC1bYjix+0S5;L z;x@g-8%CkBj!Eqhnplktlfef3+MMN5&Mq^`FZAkk*HahzCgpIeL)@1R@qiX?6%jv_ z6rd|w!dhB``08k>g4Y3m<}2ZT?!ABLG_FmOK0QrM{{8sfiT&o}_+LlQPVHZgPL7s4 zP3=nK!2~Ld9RENAuGI)wP0tTc58oc0+OH3P(`zzQPFI6cb++-J!@WJ{j}*xG3jWQ* z*z#emEw;r(!8Pogl6s}SM7>xh` literal 8992 zcmV+*Bj4N~iwFoKx~N_P|8!wuY-Mv_aA|O5Y-w&~Uu$MAaCt6tVR8WNUH^02HnQ)} z{wpxbbR^wKblmoO@ARVH_!765yR^ zCr-(kHWs;9EWYh7upa;`e?uf(}rPbemG zqIhwby>^ooon~zDx$VqoGL8LJ$|dT&pi5wv#$HN^7n0+38vXo=oOzRgCgfTO(ixDB zAFkFZB_~T(rN1YGyD>< zjJ*K9rtzA>oAhSIXy2x>A1*p~1ETO>C&MWXLB8Nf=N!`o7GfK&OzIcMXUA_(&fH&) ze^#W%3QuV2&irH*csDM4zfCSOQ|JdS4QH#!57VufO@1969(w;>4`=iW{#!7gF(H3Dw;;YZmK8sm-K$zZa;LOe%*u* z5a@0nu&;jTJU>1=`RVk{zdrz1`#;ZS?pn~wP?%#|2a2l*b$CZ za6@3{W7g!Gw3bzAZ^Fc?pEqG5souIVD=F`amdh1vWj8daYoV=CO`^)V;clZ@sUlHA z<12qkkEhdU9j2-V&gKHccpZjp>~>PR1nrbku4qB_X%wc=%fJ&DWun4BPV!$vS>oXk z^yN_K;9;(V6`I>yK3{MOrS7Pj`|i%pThYah`0^Asotk-RPhr?){Lc$>`^Tb4Rm@Zd`oG3ZWL`G`%qc+7f2E|TY)VR$OWLfw#nI8Jl za_Qkj*2G|SlU_vOx5gr_jc=#L+x+D`j+O*m6L3k0zg$IeO0d7r&ep4d9u($<&c!fs zMd31$=TYqPk<-hmb+=+fU`jmDKJz|S`ef=LB_g?%*+()azeFLGX}D0ioY~xeL}vap z1^U~&g1S$JKS<^7bx(Q2bO+>puPb)|cmvt>2c*}PT3LEqG^KcX-X9%EVu?ylZW6=G z^#m$yN)!28@zMd6(|Zpo9r}?@*Q|g8(j6`#r!MFv{KpT$v<3nGxrm}gK%uI@(}Io^ zg{WlGC>f8BobR2dUAeOG=?;fsG`#lWaLCs*hG2u4CF-8*v^2aL-*#WWeC@t?`S$p) zuTEV2`=2Lo-o8Bj#eM#_ljr~Fj!5^Y3;r~?57W!>8QA6i?us>>2GM#plV&GmKO<)Wh5ib#MWPyc54tPbc;7=O&`VMsCSM-Voz3k)5Uw%C86Y_+doxV67 zk)?M@Nq`gtEZ*E-fDbxRiknehQ=nJ}AQ7}9^=7=<2Q7k&8YFbI z2a%Kk0A7}w#}o3L=}Y2eDxO^X=>@bJB@6IzLn1Ujk=zNzB!GNeP)9XEJ(I~8kWAr^ zEry`4i3i=D;2ei|6EMq6E}qXU6F^CjjR{QB>%0&ma%**a*Kwe4FTGqg2S4=t4LG2+ zw4`^Scz)XJ9CrHehezj)*vXY@WGq)&BaXQt?Nm{JNDX<}TB1p|^rHt}Hq#?$^9|eu zh07=!@#O8xX2x};NhqkeSz^psas(Vf*RSRldSO)0i&1HuR-nq#9{3em3hjI0e!y^1 zI#0}dPxlb4;SuXVO2U|?>o_Dzhhq+mNi^_uii{N9@^MO6X~~fB*Qkh`pV+74*Ds%g zJwL(O85$%_A!YT2_8R`}QMHgTg$FAkxu?R={Wr;{IoxG+VJ7E$f&UwIu_W-H%jS&C zXJk$v{HtaS79fmZ7$(f+2&c42#r?TaN%L}X&Tdwem8aMJCsLCRG%vY;!r{sv|0UTa zVC8hP)>nk<^J>Rb(#yWq(4xI^f>LF`j1%TXNuf$z_7h#@?2~gW77bssFR1DlimlrP$9K7OJ9NKvnGx7*jVaJ<>f%@!nk9esw|VGPo7IiQ7Q9;2@>%r>1d*9z&` zitIR-{O!Bfuiu=!ecLSWJhoNjo4?veo~9Xcp1UO=whuYv8FZe%7CSTK`=b{m_sdu; zP-pd!2VOH7ES4Y>2M4kfMBXgfsZ{F3KNrU3Z{6%|Q6nKP+l;@}IWw;40A9hbUX~mC zhkfZuV>M$w_b~)KRya7TXw@ra|NdO^q{PQ0cg`y2-+%an%v@q3bQw?4Yr*!GM-9CXKF6q63tgQAVvEFBYpO>UA)NjlvRTheF9&G(B|9D9em-<-TS`Q_~8@vFBvp=o87kV9~ez-%3&MutCTFe*W= z{ z;4oEb?u~Z2RU_3_<6FV`?=TupSwN~Bx~}pT8CR}u)C!hYRqHB1ntC*?Uvry@$Y810 zi4YLaCwE(*6h%~sNaZM^I;lVRrfL1is^LGBN+@+_lg?-qXxL7OqV6`3!5q7|uLx3{ zeG?(n7>&@#bpVJ;b;`nlsHJ5+!{7X3D4?3WjRk)0u|O7&A1Zc@`{t2yIFXYr`weN5d~AZz@Kd> zwcKvilzeDpG+lvuQNnZP9&Tf61+|Pw;3ugeg%!bKO;I@jM#N_U5<~$t4I7thO$rki zUApwLE^=%N5*wq+Pl~wPWbMQMJu-m~LD*!5hs#98CK&F3FM&Uaz1WA}{u~p}qOhBi zYmW!bW|6`iBP6pMD1tvFB0kHSlU!fWkbO;3x+3v9Oq>UaXMYJ1>^5dy6SFSK#a!k# zlHEqK_Y}z%i>C$~sN`J!j;8PS&b__M!#uKexD3U#m&2 ze#F@NWuUL#S{PWr$PDbQ+sRuuT2T(=R^r2113LLRnS1P0M*my?p-O->bng0d^ z1$A}coyYUJ=a)CqKn!t?1<}U1K(~mjv{BAu_=XWojEUO_`U6DJd0V#e^EQ6oJY&8r zWh{PqaPL=$e2=1>eg1MU8Jic+auKZC9rq%P@C=6Bun*C&53{|BT%M|ZlKX@+2$!Qm zvZ{TS`;@G5MX~^a)abB0#=01c;T+u; zlBRyIa0^Y@3Ywyxf^qMYVRm-D#;y+3xaSEPh_yM6WA8@gw&kfA8n-GNPK zGRWL3>ibc1%KLzW_^fDO&>)@ruv{4%z+&nG?%NBr5k$xokH1gmTh)SI`c16j_D#fm zk00Y`*^#|8qs$w$T?=g5JFuy|0@E0Fv}teo%+EjA7PW(CysVSYAA>Wat01}&Q|NL) zYwGEQCLDDuz`oKZbZ5NK)J@i|I4R;%+G{mtCj?xZe zSlqAOA}+B_&E+#n@9S=|l!)nkUk+^7)44+m5#e!-ZDvsq^PMW#e)|PB^LM zy!r2+s_EErGU$*gb#oTT;i>aSlmZ>^X#h^GKlgdG;6Mri!Sk6$Z#FqY`f;2Zt&%2b z6oXG&BHBwin}>CjeL5Av9`Rx2PcMVL?UFT)4p$trULeeS#YgMDtgbY?kb$yLs4-Xf zN}@^u?-4hpA!sY7by)@Xm^#5MH04M6N^Qapgjcnf&>8S0PyGJq+>tWD_iGLkQ!S9G zWQ!=31Y|tr!pV8)b73Ll`)(XXcwQ4IoaEOv==&MhH{EmGk|soOEJPNT2KA+~GZ5IM z%QDpD$zC^-U`1oR7YN2&y)Zg$<6483X;yPNp~4!K_2qzE@erNTDIHu1N_ze37|12t z&tKN$zKx%UvsVLz#0j?ldZNnobi= z!44Ffaz3`A)t;U^LX}~^lF1BbPj_EM(IwUwjslgZtj;P9<7Nrjs$!w@M&~1OV98|| zUF&C$Tu|^L<0OTqPRm1t+N5W(Q~5=?R}@-;yL?O`AkaW|#SDtx;pEl)z*E_TACOn@ zLsb++Nz#~OmY}@wiHp3f{u^+S{MO8gW_Ku=R0Fhy?-(83AI3%pcvL;gx-p%i|Ar;@ z9sZ*x#R{nWg!y4CQU1Fo-TcT+5XK^sWj~FnCNcFc59;5l8Yjz3acS~l-~WUL21ZHI zyS^l|P=+FuO7pB?X`X%B(lj&}MG!=tK7%_DPmF;OwV=s*igT;^ItXqUA3D<~;=0S4 zf6sDn6KvKHh9ov^DtE>t%HE(MDWFUKYjV8M# zE>G@ZmDrk^SZzc1(oi#9u^QSXBP~`fR+Npk+09m2M!05!3wGUsg)myfJO=5WkT<2S zJ^B!S2>fe|2~A}@>saykVl?NFTR{b*P$`@EDWR7hFBsnR)+Z8#mJC3XNK5j zIa`Yyu9p)UGo%qv*3H~4On5~=`p%o@X6xlD z=@mEq^f%+V_6c9MmQjNC(90_VtMkBHCN`(U2RIbG%Hpf9EZ+6!jy46?@qOtBXf-MuGY^Gzxg;l|pYk|u+WM6`w@Nj`C z#tYk$G+s~nBmf8c0AYN5{prHHqH@e{M(0pas|D6S-e(J;w76} zPR8D&7w+=pOj%N(gNupmhTOztlfy@OE>Zo?hHOkV z+1l_>+YmO~CNPKEckv3UcBep-odRq{n(#A^(4|!c>UO*T!po*upU!S`U~U&bHfFvK zr|6grBn||e7Iq~TMZL%lR>;Um-eF1M?coYuN`rvkXmyjMbji6< z+3~1kazk)t#EzoEySCu7#|p_OS;%1FU(t}Q7^^3|WH=Lc$rkeB=UPcwI#77^Iu&eS z*AM;Fb$ba7=3H`b&g(_ z>CbMz;eHb}t>lWT9PGz@H9F@@X}exGS0TGO+UxgrP25B53XX6J zeTNalfPp{1DV6k#+Hg~TFN%HeyE%}pL7t+?b>us!!N{ArlOUS(BsC8@k43E6x}Gdm*==v*G8i?&Df@g3;JO*v` zL1Pgwl@IN*KowY)TzvLtA)GllsS)YDIX@lFoxvdSgQUkhz`)?;)JFq$dGwt9(6Ld2 zbI`s{SO}S+DV+}an$(}SmIbnz2gLj!tq<<9bchp7!B@GWq&QlS9U?}Z{hq*A=98^1 z(pjhW&^%M>;+(aPRz|2If^OkD2B!?K&x=-M0Ugo{gp$SJzPiOAb0SLtFXm+OGv*z^ z(=C{Q0p*{>DfCqhO$2fX2{HJ^>sOv9eyR`AAtrm#FW7r{wDxLyox94FK;y@aL( z<=4yU`{WrpdaBnr`Leayc1o4@M)p^!$U`?tEK}a3 zFzrY(+M6;#;$2b5uh-2hOry!&7bt_(|1MGwl(rbiE_PW9n_!cCjREtMK!b{>rLee2 zVVY5Dczj#a@&%}&TO_FfHDr*6?lYwP;n@ZCWQemPd$ zA{iYXLhVVFt}wOoQQI&Fy&}DE`xW@7l&#*B6{zPb#B&+#oMVHp>161wRt9LW42E7H z!0*70f|f)+sD>Bs8d|(ZSn>WK#oNM(+jH96bJ}+&lG2~bP6@EzX!?Di#R_tZE+(Y- zx8PQ6_DdShPW#6Fy4{p6S5Rf^)Gaplfky9KcT5&4w@=*TV?vHTgxcm@)mdC649HS! zzU70G6CS7m0c4JyRgIJ!ly}4yk&p_vRFAiOjYu4A_i){cN49e6uuzMZ*x?~fXL&jE z;~xK!7+ess)Pbv7Cm4-PjG)L8B(gehR$1=kCZ$PF)Ui^Xc|UaX29R{3(ZSB*p<2oo z@4r*Le|zHjqdW0jg4N`iOP=fAYMaAuq`FS_xv%4so$KCoAIqoOHt`sCd3-tBC%&MQ z@!L%s?WT=()5haI{a&*3?J@cLIVN9qrE0{;U7-D+ykBzHt@}ml_R#ELUr9bI+jf)4 zE>G(JbUQ_MvU{rr(7jp4dBbdwu>p?#2sdch_5_%a_()r|8RIc+^uRVtU_Q9bRu7T( zG2O&$68ZDLUztP=&a@;5$oAaRl}%06--G%F}kMxD4kB{6++RV zl{2^)lZb(CH3Y{xV34)*622;9fM_b#k2nt=pQD-3Iy>z)*>a^E)R%p1tdP9G;@>LZ z4ii3_D}^@!pd`t#T$%i%(%C{U1U?H(t|rRCXHF%$^LZgj8RR8#bd1;Z+IeXhG@Vm5 z60M4{6_F-qQF*s;1#b+&qpNRcjMfb)%4dD@J`2Nq1y)5)WQ=)m>Ib;0qYaS%UIXNk z)zc+lE_0{db?lF|xHC9k zmDTlx{6qzy7KV`z-WmCcbyMgNXd~ZpBoKM`=4GPczq2f{_wZWas%X&ZKTPpUNpX#( z5NllgX}v$E^P0O=g?H{a4mrc~<)$7CLsSbMxxPSnj#4c34s@;pV1!01T=zyLz;DiT ztQo&R70I*Q=u&i*%bTojvxSI}k(2RULY32LE_uly=K~H9iB~BIK_Gf%g~y{=Gp9OzSv# z?xwNPUa0;V28}HXJ#FN;jU2a;<2G` z0pl$r#&YCx)1;^yK-O6k+DwCtwQz{#fq`2Q%6&^ycH-mh9Xmh@7fj(#44-A$R31KL zEwWUioKwFpGGS=5T?Qy@!j79aTwbbiMT_MPl>MPagtQRQ#-WVtZtKojjvEl&-5kp3 z_br1%R9mxwvBjV$tZQDqtiE-7WlAbc;^qkzGg(fq4+np46LVCKjBEWBt3yd_I-Tnt z9k!y&T4dH(=$L`L-(cblzERV4^vW_dOv9=wl0INXndEI)O^~!Mjl_OLG-BAgcf@Yqr1_+;CDD zD0uc6bYKRYAd1KPgm_HRu_+yt)?;SnkeL;HC)bqOmXd~M(q;VOU9Xspbe2}%9{_7+2#)$m>Bk|huXK}xwI*Rv{yRY8Y(Sk2p@I?#0cy!(l{qlDFZ-Un-ds20GLEb7CORD}*!ZbN}N#GM3pi%^FSY(*KQL1QboMzWl5Ew7!{PQ*pQlFA-#v z_-+WLaxbEFoZLgIO1`o_nNi#mOv9;9<)J%lXs0((O#1O#o2n&Cdb?68yCH&p&TdgQ z)g&{_j_rION4ObI&e^FygXm5@v)!%`o1bR1mBQ8Tm*28qiSUWZu2G` z$B^g?F=5*tY*#OzG6F38^Kr0~yPa!_T0V4dn@ToaZT-7-A~zbZwc2}=6Vf&tx`vdk z3);G%yKq5EhdOMQp>l-XmLsZaqPFpFp2h>KGI=|?+R?R((N$hL-zHzvexEug*SHkF zm8Waj##ARjX?cIk?2UVpHq|tBx?-a>*~TiynzVJkADi>t`oN8R;Nb>yR#RE=eoh}( zkcLO~qs&NreL+K=uwz=!5b{*?BAIN+FYCO!>);*4;_~3m!$pD{pSb>cgp3Ufn-Yu7z(LCAdM{Vrl^`!h8L`C22g!)P2=mD??T*wQ}6J2UQc z6^2eDTfVQ_r+wO|ecGpe+NXWmr+wO|ecGpe+NXWmr+wO|ecGpe+NXVX^ZY-H@Di*5 GKmh=I0-{0y From 86bfed6ff5351fc4800a8ffe840b7879e374d5ef Mon Sep 17 00:00:00 2001 From: Amy Unruh Date: Tue, 28 Jul 2020 10:53:16 -0700 Subject: [PATCH 10/10] checkpointing, tf-serving stuff --- .../bikesw_training/bw_hptune_standalone.py | 4 +- .../containers/tf-serving/Dockerfile | 49 ++++++++ .../containers/tf-serving/build.sh | 31 +++++ .../tf-serving/deploy-tfserve.py | 107 ++++++++++++++++++ .../tf-serving/tf-serve-template.yaml | 64 +++++++++++ .../components/serve_component_a.yaml | 37 ++++++ .../example_pipelines/bw_ktune.py | 31 ++--- 7 files changed, 306 insertions(+), 17 deletions(-) create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/Dockerfile create mode 100755 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/build.sh create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/deploy-tfserve.py create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/tf-serve-template.yaml create mode 100644 ml/kubeflow-pipelines/bikes_weather/components/serve_component_a.yaml diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py index d185aa5..6f30520 100644 --- a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/bikesw_training/bw_hptune_standalone.py @@ -105,8 +105,8 @@ def create_model(hp): sparse = { 'day_of_week': tf.feature_column.categorical_column_with_vocabulary_list('day_of_week', vocabulary_list='1,2,3,4,5,6,7'.split(',')), - # 'end_station_id' : tf.feature_column.categorical_column_with_hash_bucket('end_station_id', hash_bucket_size=800), - # 'start_station_id' : tf.feature_column.categorical_column_with_hash_bucket('start_station_id', hash_bucket_size=800), + 'end_station_id' : tf.feature_column.categorical_column_with_hash_bucket('end_station_id', hash_bucket_size=800), + 'start_station_id' : tf.feature_column.categorical_column_with_hash_bucket('start_station_id', hash_bucket_size=800), 'loc_cross' : tf.feature_column.categorical_column_with_hash_bucket('loc_cross', hash_bucket_size=21000), # 'bike_id' : tf.feature_column.categorical_column_with_hash_bucket('bike_id', hash_bucket_size=14000) } diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/Dockerfile b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/Dockerfile new file mode 100644 index 0000000..fb33367 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/Dockerfile @@ -0,0 +1,49 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM tensorflow/tensorflow:2.1.0-gpu-py3 + +RUN apt-get update -y + +RUN apt-get install --no-install-recommends -y -q ca-certificates python-dev python-setuptools wget unzip + +RUN easy_install pip + +RUN pip install pyyaml==3.12 six requests==2.18.4 + +RUN wget -nv https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip && \ + unzip -qq google-cloud-sdk.zip -d tools && \ + rm google-cloud-sdk.zip && \ + tools/google-cloud-sdk/install.sh --usage-reporting=false \ + --path-update=false --bash-completion=false \ + --disable-installation-options && \ + tools/google-cloud-sdk/bin/gcloud -q components update \ + gcloud core gsutil && \ + tools/google-cloud-sdk/bin/gcloud -q components install kubectl && \ + tools/google-cloud-sdk/bin/gcloud config set component_manager/disable_update_check true && \ + touch /tools/google-cloud-sdk/lib/third_party/google.py + +# RUN wget -nv https://github.com/ksonnet/ksonnet/releases/download/v0.11.0/ks_0.11.0_linux_amd64.tar.gz && \ +# tar -xvzf ks_0.11.0_linux_amd64.tar.gz && \ +# mkdir -p /tools/ks/bin && \ +# cp ./ks_0.11.0_linux_amd64/ks /tools/ks/bin && \ +# rm ks_0.11.0_linux_amd64.tar.gz && \ +# rm -r ks_0.11.0_linux_amd64 + +ENV PATH $PATH:/tools/google-cloud-sdk/bin:/tools/ks/bin + +ADD build /ml + +ENTRYPOINT ["python", "/ml/deploy-tfserve.py"] + diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/build.sh b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/build.sh new file mode 100755 index 0000000..3b6f848 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/containers/tf-serving/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +if [ -z "$1" ] + then + PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)") +else + PROJECT_ID=$1 +fi + +mkdir -p ./build +rsync -arvp "../../tf-serving"/ ./build/ + +docker build -t ml-pipeline-tfserve . +rm -rf ./build + +docker tag ml-pipeline-tfserve gcr.io/${PROJECT_ID}/ml-pipeline-tfserve +docker push gcr.io/${PROJECT_ID}/ml-pipeline-tfserve diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/deploy-tfserve.py b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/deploy-tfserve.py new file mode 100644 index 0000000..e8014a7 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/deploy-tfserve.py @@ -0,0 +1,107 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import os +import time +import logging +import subprocess +import requests + +from tensorflow.python.lib.io import file_io #pylint: disable=no-name-in-module + + +def main(): + parser = argparse.ArgumentParser(description='ML Trainer') + parser.add_argument( + '--model_name', + required=True) + + parser.add_argument( + '--model_path', + required=True) + parser.add_argument( + '--namespace', + default='kubeflow') + + parser.add_argument('--cluster', type=str, + help='GKE cluster set up for kubeflow. If set, zone must be provided. ' + + 'If not set, assuming this runs in a GKE container and current ' + + 'cluster is used.') + parser.add_argument('--zone', type=str, help='zone of the kubeflow cluster.') + args = parser.parse_args() + + # KUBEFLOW_NAMESPACE = 'kubeflow' + ts = str(int(time.time())) + + # Make sure model dir exists before proceeding + retries = 0 + sleeptime = 5 + while retries < 20: + try: + model_dir = os.path.join(args.model_path, file_io.list_directory(args.model_path)[-1]) + print("model subdir: %s" % model_dir) + break + except Exception as e: #pylint: disable=broad-except + print(e) + print("Sleeping %s seconds to sync with GCS..." % sleeptime) + time.sleep(sleeptime) + retries += 1 + sleeptime *= 2 + if retries >= 20: + print("could not get model subdir from %s, exiting" % args.model_path) + exit(1) + + logging.getLogger().setLevel(logging.INFO) + args_dict = vars(args) + if args.cluster and args.zone: + cluster = args_dict.pop('cluster') #pylint: disable=unused-variable + zone = args_dict.pop('zone') #pylint: disable=unused-variable + else: + # Get cluster name and zone from metadata + metadata_server = "http://metadata/computeMetadata/v1/instance/" + metadata_flavor = {'Metadata-Flavor' : 'Google'} + cluster = requests.get(metadata_server + "attributes/cluster-name", + headers=metadata_flavor).text + zone = requests.get(metadata_server + "zone", + headers=metadata_flavor).text.split('/')[-1] + + # logging.info('Getting credentials for GKE cluster %s.' % cluster) + # subprocess.call(['gcloud', 'container', 'clusters', 'get-credentials', cluster, + # '--zone', zone]) + + logging.info('Generating training template.') + + template_file = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'tf-serve-template.yaml') + target_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tf-serve.yaml') + + with open(template_file, 'r') as f: + with open(target_file, "w") as target: + data = f.read() + changed = data.replace('MODEL_NAME', args.model_name) + changed1 = changed.replace('KUBEFLOW_NAMESPACE', args.namespace) + changed2 = changed1.replace('MODEL_PATH', args.model_path) + changed3 = changed2.replace('SERVICE_NAME', args.model_name + ts) + target.write(changed3) + logging.info("template: %s", changed3) + + + logging.info('deploying model serving.') + subprocess.call(['kubectl', 'create', '-f', '/ml/tf-serve.yaml']) + + +if __name__ == "__main__": + main() diff --git a/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/tf-serve-template.yaml b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/tf-serve-template.yaml new file mode 100644 index 0000000..de7bd37 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/kubeflow-resources/tf-serving/tf-serve-template.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: SERVICE_NAME + name: SERVICE_NAME + namespace: KUBEFLOW_NAMESPACE +spec: + ports: + - name: grpc-tf-serving + port: 9000 + targetPort: 9000 + - name: tf-serving-builtin-http + port: 8500 + targetPort: 8500 + selector: + app: SERVICE_NAME + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: SERVICE_NAME + name: SERVICE_NAME + namespace: KUBEFLOW_NAMESPACE +spec: + replicas: 1 + selector: + matchLabels: + app: SERVICE_NAME + template: + metadata: + labels: + app: SERVICE_NAME + version: v1 + spec: + containers: + - args: + - --port=9000 + - --rest_api_port=8500 + - --model_name=MODEL_NAME + - --model_base_path=MODEL_PATH + command: + - /usr/bin/tensorflow_model_server + image: tensorflow/serving + imagePullPolicy: Always + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 30 + tcpSocket: + port: 9000 + name: MODEL_NAME + ports: + - containerPort: 9000 + - containerPort: 8500 + resources: + limits: + cpu: "4" + memory: 4Gi + requests: + cpu: "1" + memory: 1Gi diff --git a/ml/kubeflow-pipelines/bikes_weather/components/serve_component_a.yaml b/ml/kubeflow-pipelines/bikes_weather/components/serve_component_a.yaml new file mode 100644 index 0000000..cb096b2 --- /dev/null +++ b/ml/kubeflow-pipelines/bikes_weather/components/serve_component_a.yaml @@ -0,0 +1,37 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Serve TF model +description: | + A Kubeflow Pipeline component to deploy a tf-serving service +metadata: + labels: + add-pod-env: 'true' +inputs: + - name: model_name + type: String + - name: model_path + type: GCSPath + - name: namespace + type: String +implementation: + container: + image: gcr.io/aju-vtests2/ml-pipeline-tfserve:v1 + args: [ + --model_name, {inputValue: model_name}, + --model_path, {inputValue: model_path}, + --namespace, {inputValue: namespace} + ] + env: + KFP_POD_NAME: "{{pod.name}}" diff --git a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py index 1437bc6..1e0a7c6 100644 --- a/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py +++ b/ml/kubeflow-pipelines/bikes_weather/example_pipelines/bw_ktune.py @@ -22,8 +22,8 @@ # train_op = comp.load_component_from_url( # 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/train_component.yaml' # pylint: disable=line-too-long # ) -serve_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/amygdala/code-snippets/master/ml/kubeflow-pipelines/sbtb/components/serve_component.yaml' # pylint: disable=line-too-long +serve_op = comp.load_component_from_file( + '/Users/amyu/devrel/code-snippets/ml/kubeflow-pipelines/bikes_weather/components/serve_component_a.yaml' # pylint: disable=line-too-long ) @@ -39,7 +39,7 @@ def bikes_weather_hptune( #pylint: disable=unused-argument tuner_dir_prefix: str = 'hptest', tuner_proj: str = 'p1', max_trials: int = 32, - working_dir: str = 'gs://YOUR_GCS_DIR_HERE', + working_dir: str = 'gs://aju-pipelines/ktune5', data_dir: str = 'gs://aju-dev-demos-codelabs/bikes_weather/', steps_per_epoch: int = -1 , # if -1, don't override normal calcs based on dataset size # load_checkpoint: str = '' @@ -47,37 +47,30 @@ def bikes_weather_hptune( #pylint: disable=unused-argument hptune = dsl.ContainerOp( name='ktune', - image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v5', + image='gcr.io/aju-vtests2/ml-pipeline-bikes-dep:v6', arguments=['--epochs', tune_epochs, '--num-tuners', num_tuners, '--tuner-dir', '{}_{}'.format(tuner_dir_prefix, int(time.time())), '--tuner-proj', tuner_proj, '--bucket-name', bucket_name, '--max-trials', max_trials, + '--namespace', 'default', '--deploy' ], file_outputs={'hps': '/tmp/hps.json'}, ) train = dsl.ContainerOp( name='train', - image='gcr.io/aju-vtests2/ml-pl-bikes-train:v1', + image='gcr.io/aju-vtests2/ml-pl-bikes-train:v2', arguments=[ '--data-dir', data_dir, '--steps-per-epoch', steps_per_epoch, '--workdir', '%s/%s' % (working_dir, dsl.RUN_ID_PLACEHOLDER), - # '--load-checkpoint', load_checkpoint, '--epochs', train_epochs, '--hptune-results', hptune.outputs['hps'] ], file_outputs={'train_output_path': '/tmp/train_output_path.txt'}, ) - # train = train_op( - # data_dir=data_dir, - # workdir='%s/%s' % (working_dir, dsl.RUN_ID_PLACEHOLDER), - # epochs=epochs, steps_per_epoch=steps_per_epoch, - # load_checkpoint=load_checkpoint - # ).apply(gcp.use_gcp_secret('user-gcp-sa')) - - serve = serve_op( model_path=train.outputs['train_output_path'], - model_name='bikesw' + model_name='bikesw', + namespace='kubeflow' ) train.set_gpu_limit(2) @@ -85,3 +78,11 @@ def bikes_weather_hptune( #pylint: disable=unused-argument if __name__ == '__main__': import kfp.compiler as compiler compiler.Compiler().compile(bikes_weather_hptune, __file__ + '.tar.gz') + + + # train = train_op( + # data_dir=data_dir, + # workdir='%s/%s' % (working_dir, dsl.RUN_ID_PLACEHOLDER), + # epochs=epochs, steps_per_epoch=steps_per_epoch, + # load_checkpoint=load_checkpoint + # ).apply(gcp.use_gcp_secret('user-gcp-sa')) \ No newline at end of file