sql-server-samples/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda8.ipynb

In [2]:

%%configure -f
{
    "executorMemory": "4g",
    "driverMemory": "4g",
    "executorCores": 4,
    "driverCores": 2,
    "numExecutors": 1
}

In [3]:

# For informational purposes,
# print the hostname of the container
# where the Spark driver is running
import subprocess

stdout = subprocess.check_output(
    "hostname",
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)

In [4]:

# Install NVIDIA GPU libraries and TensorFlow for GPU
# in the container where the Spark driver is running
import subprocess

stdout = subprocess.check_output(
'''
echo $CUDA_VERSION
export CUDA_PKG_VERSION="8-0=$CUDA_VERSION-1"
echo $CUDA_PKG_VERSION

export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64

# nvidia-container-runtime
export NVIDIA_VISIBLE_DEVICES=all
export NVIDIA_DRIVER_CAPABILITIES="compute,utility"
export NVIDIA_REQUIRE_CUDA="cuda>=8.0"

apt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \\
    rm -rf /var/lib/apt/lists/* && \\
    NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \\
    NVIDIA_GPGKEY_FPR=ae09fe4bbd223a84b2ccfce3f60f4b3d7fa2af80 && \\
    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \\
    apt-key adv --export --no-emit-version -a $NVIDIA_GPGKEY_FPR | tail -n +5 > cudasign.pub && \\
    echo "$NVIDIA_GPGKEY_SUM  cudasign.pub" | sha256sum -c --strict - && rm cudasign.pub && \\
    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/cuda.list

apt-get update && apt-get install -y --no-install-recommends \\
        cuda-nvrtc-$CUDA_PKG_VERSION \\
        cuda-nvgraph-$CUDA_PKG_VERSION \\
        cuda-cusolver-$CUDA_PKG_VERSION \\
        cuda-cublas-8-0=8.0.61.2-1 \\
        cuda-cufft-$CUDA_PKG_VERSION \\
        cuda-curand-$CUDA_PKG_VERSION \\
        cuda-cusparse-$CUDA_PKG_VERSION \\
        cuda-npp-$CUDA_PKG_VERSION \\
        cuda-cudart-$CUDA_PKG_VERSION && \\
    ln -s cuda-8.0 /usr/local/cuda && \\
    rm -rf /var/lib/apt/lists/*

# Install tensorflow
pip3 install tensorflow-gpu==1.4.0

# add cudnn 6
echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list

export CUDNN_VERSION=6.0.21
#LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"

apt-get update && apt-get install -y --no-install-recommends \\
    libcudnn6=$CUDNN_VERSION-1+cuda8.0 && \\
    rm -rf /var/lib/apt/lists/*
''',
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)

In [5]:

# List CPU and GPU devices
from tensorflow.python.client import device_lib

device_lib.list_local_devices()

In [11]:

# Fit and evaluate TensorFlow model on MNIST data
import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),  # input_shape needed for older tensorflow
  tf.keras.layers.Dense(512, activation=tf.nn.relu),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=5)
print("\n")
metrics = model.evaluate(x_test, y_test)
print("\n")
print(metrics)

In [12]:

# Check available disk space
import subprocess

stdout = subprocess.check_output(
'''
df -h
''',
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)

In [13]:

# Download code for the CIFAR 10 benchmark
import subprocess
import os

if os.path.isdir("/tmp/models"):
    print("CIFAR 10 repo already cloned")
else:
    stdout = subprocess.check_output(
'''
apt-get update && apt-get install -y git
pip3 install --upgrade pip setuptools
pip3 install tensorflow-datasets
cd /tmp
git clone https://github.com/tensorflow/models.git
''',
        stderr=subprocess.STDOUT,
        shell=True).decode("utf-8")
    print(stdout)

In [14]:

# Run the CIFAR 10 benchmark
import subprocess

stdout = subprocess.check_output(
'''
python3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100 2>&1
''',
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)