Create a GPU docker container for Datalab.

Based off of the nvidia Ubuntu 16.04 container. Also switching the non-GPU container to an Ubuntu 16.04 base image for consistency.
googledatalab · chmeyers · Jun 1, 2017 · May 15, 2017 · May 17, 2017 · May 22, 2017
commit 45c20de987de864771e7becac1dff38c2eae9585
diff --git a/containers/base/Dockerfile.gpu b/containers/base/Dockerfile.gpu
@@ -0,0 +1,22 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM datalab-core-gpu
+MAINTAINER Google Cloud DataLab
+
+# Download and Install GPU specific packages
+RUN pip install -U --upgrade-strategy only-if-needed --no-cache-dir tensorflow-gpu==1.0.1 && \
+    pip3 install -U --upgrade-strategy only-if-needed --no-cache-dir tensorflow-gpu==1.0.1
+
+
diff --git a/containers/base/Dockerfile → containers/base/Dockerfile.in b/containers/base/Dockerfile → containers/base/Dockerfile.in
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:jessie
+FROM _base_image_
 MAINTAINER Google Cloud DataLab
 
 # Container configuration
@@ -34,7 +34,7 @@ RUN echo "deb-src http://ftp.us.debian.org/debian testing main" >> /etc/apt/sour
 # Save GPL source packages
     mkdir -p /srcs && \
     cd /srcs && \
-    apt-get source -d wget git python-zmq ca-certificates pkg-config libpng-dev && \
+    apt-get source --allow-unauthenticated -d wget git python-zmq ca-certificates pkg-config libpng-dev && \
     wget --progress=dot:mega https://mirrors.kernel.org/gnu/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2 && \
     cd / && \
 
@@ -168,11 +168,9 @@ RUN echo "deb-src http://ftp.us.debian.org/debian testing main" >> /etc/apt/sour
     tools/google-cloud-sdk/bin/gcloud config set component_manager/disable_update_check true && \
     touch /tools/google-cloud-sdk/lib/third_party/google.py && \
 
-# locale
-    DEBIAN_FRONTEND=noninteractive apt-get install -y locales && \
-    sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
-    echo 'LANG="en_US.UTF-8"'>/etc/default/locale && \
-    dpkg-reconfigure --frontend=noninteractive locales && \
+# Set our locale to en_US.UTF-8.
+    apt-get install -y locales && \
+    locale-gen en_US.UTF-8 && \
     update-locale LANG=en_US.UTF-8 && \
 
 # Add some unchanging bits - specifically node modules (that need to be kept in sync
@@ -191,7 +189,7 @@ RUN echo "deb-src http://ftp.us.debian.org/debian testing main" >> /etc/apt/sour
     /tools/node/bin/npm install -g forever && \
 
 # Clean up
-    apt-get purge -y build-essential bzip2 cpp cpp-4.9 python-setuptools pkg-config libfreetype6-dev && \
+    apt-get purge -y build-essential bzip2 cpp pkg-config libfreetype6-dev && \
     apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/lib/dpkg/info/* && \

diff --git a/containers/base/build.gpu.sh b/containers/base/build.gpu.sh
@@ -0,0 +1,39 @@
+#!/bin/bash -e
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds the Google Cloud DataLab base docker image. Usage:
+#   build.sh [path_of_pydatalab_dir]
+# If [path_of_pydatalab_dir] is provided, it will copy the content of that dir into image.
+# Otherwise, it will get the pydatalab by "git clone" from pydatalab repo.
+
+# Build the docker image
+if [ -n "$1" ]; then
+  src_pydatalab=$(realpath "$1")
+
+  cd $(dirname $0)
+  rsync -avp "$src_pydatalab"/ pydatalab
+else
+  # Create empty dir to make docker build happy.
+  cd $(dirname $0)
+  mkdir -p pydatalab
+fi
+
+trap 'rm -rf pydatalab' exit
+
+BASE_IMAGE_SUBSTITUTION="s/_base_image_/nvidia\/cuda:8.0-cudnn5-devel-ubuntu16.04/"
+cat Dockerfile.in | sed $BASE_IMAGE_SUBSTITUTION > Dockerfile
+docker build ${DOCKER_BUILD_ARGS} -t datalab-core-gpu .
+rm Dockerfile
+docker build ${DOCKER_BUILD_ARGS} -f Dockerfile.gpu -t datalab-base-gpu .
diff --git a/containers/base/build.sh b/containers/base/build.sh
@@ -31,4 +31,8 @@ else
 fi
 
 trap 'rm -rf pydatalab' exit
+
+BASE_IMAGE_SUBSTITUTION="s/_base_image_/ubuntu:16.04/"
+cat Dockerfile.in | sed $BASE_IMAGE_SUBSTITUTION > Dockerfile
 docker build ${DOCKER_BUILD_ARGS} -t datalab-base .
+rm Dockerfile
diff --git a/containers/datalab/Dockerfile.in b/containers/datalab/Dockerfile.in
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM datalab-base
+FROM _base_image_
 MAINTAINER Google Cloud DataLab
 
 # Add build artifacts

diff --git a/containers/datalab/build.gpu.sh b/containers/datalab/build.gpu.sh
@@ -0,0 +1,61 @@
+#!/bin/bash -e
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds the Google Cloud DataLab docker image. Usage:
+#   build.sh [path_of_pydatalab_dir]
+# If [path_of_pydatalab_dir] is provided, it will copy the content of that dir into image.
+# Otherwise, it will get the pydatalab by "git clone" from pydatalab repo.
+
+# Create a versioned Dockerfile based on current date and git commit hash
+VERSION=`date +%Y%m%d`
+VERSION_SUBSTITUTION="s/_version_/0.5.$VERSION/"
+
+COMMIT=`git log --pretty=format:'%H' -n 1`
+COMMIT_SUBSTITUTION="s/_commit_/$COMMIT/"
+
+BASE_IMAGE_SUBSTITUTION="s/_base_image_/datalab-base-gpu/"
+
+if [ -z "$1" ]; then
+  pydatalabPath=''
+else
+  pydatalabPath=$(realpath "$1")
+fi
+
+cd $(dirname $0)
+
+cat Dockerfile.in | sed $VERSION_SUBSTITUTION | sed $COMMIT_SUBSTITUTION | sed $BASE_IMAGE_SUBSTITUTION > Dockerfile
+
+# Set up our required environment
+source ../../tools/initenv.sh
+
+# Build the datalab frontend
+../../sources/web/build.sh
+
+# Copy build outputs as a dependency of the Dockerfile
+rsync -avp ../../build/ build
+
+# Copy the license file into the container
+cp ../../third_party/license.txt content/license.txt
+
+# Build the base docker image
+../base/build.gpu.sh "$pydatalabPath"
+
+# Build the docker image
+docker build ${DOCKER_BUILD_ARGS} -t datalab-gpu .
+
+# Finally cleanup
+rm -rf build
+rm content/license.txt
+rm Dockerfile
diff --git a/containers/datalab/build.sh b/containers/datalab/build.sh
@@ -25,6 +25,8 @@ VERSION_SUBSTITUTION="s/_version_/0.5.$VERSION/"
 COMMIT=`git log --pretty=format:'%H' -n 1`
 COMMIT_SUBSTITUTION="s/_commit_/$COMMIT/"
 
+BASE_IMAGE_SUBSTITUTION="s/_base_image_/datalab-base/"
+
 if [ -z "$1" ]; then
   pydatalabPath=''
 else
@@ -33,7 +35,7 @@ fi
 
 cd $(dirname $0)
 
-cat Dockerfile.in | sed $VERSION_SUBSTITUTION | sed $COMMIT_SUBSTITUTION > Dockerfile
+cat Dockerfile.in | sed $VERSION_SUBSTITUTION | sed $COMMIT_SUBSTITUTION | sed $BASE_IMAGE_SUBSTITUTION > Dockerfile
 
 # Set up our required environment
 source ../../tools/initenv.sh