sql-server-samples/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda9.ipynb

In [2]:

%%configure -f
{
    "executorMemory": "4g",
    "driverMemory": "4g",
    "executorCores": 4,
    "driverCores": 2,
    "numExecutors": 1
}

In [3]:

# For informational purposes,
# print the hostname of the container
# where the Spark driver is running
import subprocess

stdout = subprocess.check_output(
    "hostname",
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)

In [4]:

# Check that the CUDA_VERSION environment variable is set.
# Its precise value does not matter: one can install CUDA 9 even if the 
# CUDA_VERSION environment variable is set to 8.0.61.
import os
print(os.environ['CUDA_VERSION'])

In [5]:

# Install NVIDIA GPU libraries and TensorFlow for GPU
# in the container where the Spark driver is running
# per https://www.tensorflow.org/install/gpu#ubuntu_1604_cuda_90_for_tensorflow_1130
import subprocess

stdout = subprocess.check_output(
'''
# Add NVIDIA package repository
apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub

wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_9.1.85-1_amd64.deb

chown _apt cuda-repo-ubuntu1604_9.1.85-1_amd64.deb
chmod u+rwx cuda-repo-ubuntu1604_9.1.85-1_amd64.deb

apt install ./cuda-repo-ubuntu1604_9.1.85-1_amd64.deb

wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb

chown _apt nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
chmod u+rwx nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb

apt install ./nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb

apt update

# Install CUDA and tools. Include optional NCCL 2.x
apt install -y cuda9.0 cuda-cublas-9-0 cuda-cufft-9-0 cuda-curand-9-0 \\
    cuda-cusolver-9-0 cuda-cusparse-9-0 libcudnn7=7.2.1.38-1+cuda9.0 \\
    libnccl2=2.2.13-1+cuda9.0 cuda-command-line-tools-9-0

# Optional: Install the TensorRT runtime (must be after CUDA install)
apt update
apt install libnvinfer4=4.1.2-1+cuda9.0

pip3 install tensorflow-gpu==1.12.0
''',
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)

In [6]:

# List CPU and GPU devices
from tensorflow.python.client import device_lib

device_lib.list_local_devices()

In [7]:

# Fit and evaluate TensorFlow model on MNIST data
import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(),
  tf.keras.layers.Dense(512, activation=tf.nn.relu),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=5)
print("\n")
metrics = model.evaluate(x_test, y_test)
print("\n")
print(metrics)

In [8]:

# Check available disk space
import subprocess

stdout = subprocess.check_output(
'''
df -h
''',
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)

In [9]:

# Download code for the CIFAR 10 benchmark
import subprocess
import os

if os.path.isdir("/tmp/models"):
    print("CIFAR 10 repo already cloned")
else:
    stdout = subprocess.check_output(
'''
apt-get update && apt-get install -y git
pip3 install --upgrade pip setuptools
pip3 install tensorflow-datasets
cd /tmp
git clone https://github.com/tensorflow/models.git
''',
        stderr=subprocess.STDOUT,
        shell=True).decode("utf-8")
    print(stdout)

In [10]:

# Run the CIFAR 10 benchmark
import subprocess

stdout = subprocess.check_output(
'''
python3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100 2>&1
''',
    stderr=subprocess.STDOUT,
    shell=True).decode("utf-8")
print(stdout)