From 59e43277649ef6cdb2c935f34365bff92b844da8 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 18:14:06 -0700 Subject: [PATCH] CTP 3.2 updates --- .../oracle/customer-oracle.sql | 2 +- .../oracle/inventory-oracle.sql | 14 +- .../web-clickstreams-hdfs-parquet.sql | 4 +- .../sql-big-data-cluster/deployment/README.md | 8 +- .../deployment/kubeadm/README.md | 28 +- .../kubeadm/ubuntu-single-node-vm/README.md | 41 +++ .../ubuntu-single-node-vm/cleanup-bdc.sh | 75 ++++ .../ubuntu-single-node-vm/setup-bdc.sh | 337 ++++++++++++++++++ .../deployment/kubeadm/ubuntu/README.md | 30 ++ .../kubeadm/ubuntu/setup-k8s-prereqs.sh | 2 +- .../python/book-click-prediction-mml-py.sql | 4 +- 11 files changed, 504 insertions(+), 41 deletions(-) create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql index c2f5b3c9..6e3d7c86 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql @@ -49,7 +49,7 @@ GO SELECT pr.pr_item_sk, pc.pr_review_content, pr.pr_user_sk AS customerid FROM dbo.product_reviews as pr JOIN (SELECT TOP(100) * FROM dbo.product_reviews_hdfs_csv) AS pc ON pc.pr_review_sk = pr.pr_review_sk -JOIN dbo.customer_ora AS c ON c.c_customer_sk = pr.pr_user_sk +JOIN dbo.customer_ora AS c ON c.C_CUSTOMER_SK = pr.pr_user_sk JOIN dbo.item AS i ON i.i_item_sk = pr.pr_item_sk INNER JOIN ( SELECT diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql index 9b579521..4217484d 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql @@ -20,22 +20,22 @@ IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'OracleSalesS -- As a result, the names are case-sensitive so specify the name in the external table definition -- that matches the exact case of the table and column names in the Oracle metadata. CREATE EXTERNAL TABLE [inventory_ora] - ([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL, - [inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0)) + ([INV_DATE] DECIMAL(10,0) NOT NULL, [INV_ITEM] DECIMAL(10,0) NOT NULL, + [INV_WAREHOUSE] DECIMAL(10,0) NOT NULL, [INV_QUANTITY_ON_HAND] DECIMAL(10,0)) WITH (DATA_SOURCE=[OracleSalesSrvr], LOCATION='.SALES.INVENTORY'); GO -- Find quantity of certain items from inventory for a specific category -- -SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity +SELECT TOP(100) w.w_warehouse_name, i.INV_ITEM, SUM(i.INV_QUANTITY_ON_HAND) as total_quantity FROM [inventory_ora] as i JOIN item as it - ON it.i_item_sk = i.inv_item + ON it.i_item_sk = i.INV_ITEM JOIN warehouse as w - ON w.w_warehouse_sk = i.inv_warehouse - WHERE it.i_category = 'Movies & TV' and i.inv_item BETWEEN 17401 and 17402 --> get items within specific range - GROUP BY w.w_warehouse_name, i.inv_item; + ON w.w_warehouse_sk = i.INV_WAREHOUSE + WHERE it.i_category = 'Movies & TV' and i.INV_ITEM BETWEEN 17401 and 17402 --> get items within specific range + GROUP BY w.w_warehouse_name, i.INV_ITEM; GO -- Cleanup diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index 6b6b4753..9a1c1380 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -90,8 +90,8 @@ AS AND w.wcs_user_sk IS NOT NULL) GROUP BY w.wcs_user_sk ) AS q - INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk - INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk; + INNER JOIN customer_ora as c ON q.wcs_user_sk = c.C_CUSTOMER_SK + INNER JOIN customer_demographics as cd ON c.C_CURRENT_CDEMO_SK = cd.cd_demo_sk; GO diff --git a/samples/features/sql-big-data-cluster/deployment/README.md b/samples/features/sql-big-data-cluster/deployment/README.md index b53442f4..8caf7756 100644 --- a/samples/features/sql-big-data-cluster/deployment/README.md +++ b/samples/features/sql-big-data-cluster/deployment/README.md @@ -3,13 +3,13 @@ SQL Server 2019 big data cluster is deployed as docker containers on a Kubernetes cluster. These samples provide scripts that can be used to provision a Kubernetes clusters using different environments. -## Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS +## Deploy a Kubernetes cluster using kubeadm -Use the scripts in the **kubeadm** folder to deploy Kubernetes over multiple Linux machines (physical or virtualized) using `kubeadm` utility. +Use the scripts in the **kubeadm** folder to deploy a Kubernetes cluster over one or more Linux machines (physical or virtualized) using `kubeadm` utility. -## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS) +## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS) -Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it. +Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it. ## Push SQL Server big data cluster images to your own private Docker repository diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md index 2d24672b..020f5114 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md @@ -1,30 +1,10 @@ # Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS -In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions. +## __[ubuntu](ubuntu/)__ -## Pre-requisites +This folder contains scripts that provide a template for deploying a Kubernetes cluster using kubeadm on one or more Linux machines. -1. Multiple Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines -1. Designate one machine as the Kubernetes master -1. Rest of the machines will be used as the Kubernetes agents +## __[ubuntu-single-node-vm](ubuntu-single-node-vm/)__ -**NOTE: Ensure there is sufficient local storage on your agents. Each volume will use up to 10GB by default. The script creates 25 volumes. Not all of the volumes will be used since it depends on the number of pods being deployed on each agent node. It is recommended to have at least 200 GB of storage on the agent nodes** - -### Useful resources - -[Deploy SQL Server 2019 big data cluster on Kubernetes](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) - -[Creating a cluster using kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) - -[Troubleshooting kubeadm](https://kubernetes.io/docs/setup/independent/troubleshooting-kubeadm/) - -### Instructions - -1. Start a sudo shell context -1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine -1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master -1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine -1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage -1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. -1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) +This folder contains a sample script that can be used to deploy a single-node Kubernetes cluster on a Linux machine. diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md new file mode 100644 index 00000000..767e065a --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -0,0 +1,41 @@ + +# Deploy a SQL Server big data cluster on single node Kubernetes cluster (kubeadm) + +Using this sample bash script, you will deploy a single node Kubernetes cluster using kubeadm and a SQL Server big data cluster on top of it. The script must be run from the VM you are planning to use for your kubeadm deployment. + +## Pre-requisites + +1. A vanilla Ubuntu 16.04 or 18.04 VM. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported. +1. VM should have at least 8CPUs, 64GB RAM and 100GB disk space.After installing the images you will be left with 50GB for data/logs across all components. + +## Instructions + +1. Download the script on the VM you are planning to use for the deployment + +``` + +curl --output kickstarter-azdata.sh http://rima-5.guest.corp.microsoft.com/kickstarter-azdata.sh +``` + +1. Make the script executable + +``` + +chmod +x kickstarter-azdata.sh +``` + +1. Run the script (make sure you are running with sudo) + +``` + +sudo ./kickstarter-azdata.sh +``` + +1. Refresh alias setup for azdata + +``` + +source ~/.bashrc +``` + +When prompted, provide your input for the password that will be used for all external endpoints: controller, SQL Server master and gateway. The password should be sufficiently complex based on existing rules for SQL Server password. The controller username is defaulted to *admin*. diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh new file mode 100644 index 00000000..0c4baba5 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi +DIR_PREFIX=$1 + +kubeadm reset --force + +systemctl stop kubelet +rm -rf /var/lib/cni/ +rm -rf /var/lib/etcd/ +rm -rf /run/flannel/ +rm -rf /var/lib/kubelet/* +rm -rf /etc/cni/ +rm -rf /etc/kubernetes/* +ip link set cni0 down +#brctl delbr cni0 +ip link set flannel.1 down +#brctl delbr flannel.1 +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X + +rm -rf .azdata/ + +SERVICE_STOP_FAILED=0 + +systemctl | grep "/var/lib/kubelet/pods" | while read -r line; do + + # Retrieve the mount path + # + MOUNT_PATH=`echo "$line" | grep -v echo | egrep -oh -m 1 "(/var/lib/kubelet/pods).+"` + + if [ -z "$MOUNT_PATH" ]; then + continue + fi + + if [[ ! -d "$MOUNT_PATH" ]] && [[ ! -f "$MOUNT_PATH" ]]; then + + SERVICE=$(echo $line | cut -f1 -d' ') + + echo "Mount "$MOUNT_PATH" no longer exists." + echo "Stopping orphaned mount service: '$SERVICE'" + + systemctl stop $SERVICE + + if [ $? -ne 0 ]; then + SERVICE_STOP_FAILED=1 + fi + + echo "" + fi +done + +if [ $SERVICE_STOP_FAILED -ne 0 ]; then + echo "Not all services were stopped successfully. Please check the above output for more inforamtion." +else + echo "All orphaned services successfully stopped." +fi + +for i in $(seq 1 30); do + + vol="vol$i" + + sudo umount /mnt/local-storage/$vol + + sudo rm -rf /mnt/local-storage/$vol + +done + + +kubeadm reset -y +sudo apt-get -y purge kubeadm kubectl kubelet kubernetes-cni kube* +sudo apt-get autoremove +sudo rm -rf ~/.kube diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh new file mode 100644 index 00000000..71720013 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh @@ -0,0 +1,337 @@ +#!/bin/bash +set -Eeuo pipefail + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi + +# This is a script to create single-node Kubernetes cluster & deploy BDC on it. +# +export BDCDEPLOY_DIR=bdcdeploy + +# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and knox. +# +while true; do + read -s -p "Create Password for Big Data Cluster: " password + echo + read -s -p "Confirm your Password: " password2 + echo + [ "$password" = "$password2" ] && break + echo "Password mismatch. Please try again." +done + + +# Name of virtualenv variable used. +# +export VIRTUALENV_NAME="bdcvenv" +export LOG_FILE="bdcdeploy.log" +export DEBIAN_FRONTEND=noninteractive + +# Requirements file. +# +export REQUIREMENTS_LINK="https://aka.ms/azdata" + +# Kube version. +# +KUBE_DPKG_VERSION=1.15.0-00 +KUBE_VERSION=1.15.0 + +# Wait for 5 minutes for the cluster to be ready. +# +TIMEOUT=600 +RETRY_INTERVAL=5 + +# Variables for pulling dockers. +# +export DOCKER_REGISTRY="mcr.microsoft.com" +export DOCKER_REPOSITORY="mssql/bdc" +export DOCKER_TAG="2019-CTP3.2-ubuntu" + +# Variables used for azdata cluster creation. +# +export CONTROLLER_USERNAME=admin +export CONTROLLER_PASSWORD=$password +export MSSQL_SA_PASSWORD=$password +export KNOX_PASSWORD=$password +export ACCEPT_EULA=yes +export CLUSTER_NAME=mssql-cluster +export STORAGE_CLASS=local-storage +export PV_COUNT="30" + +IMAGES=( + mssql-app-service-proxy + mssql-appdeploy-init + mssql-controller + mssql-hadoop + mssql-mleap-serving-runtime + mssql-mlserver-py-runtime + mssql-mlserver-r-runtime + mssql-monitor-collectd + mssql-monitor-elasticsearch + mssql-monitor-fluentbit + mssql-monitor-grafana + mssql-monitor-influxdb + mssql-monitor-kibana + mssql-monitor-telegraf + mssql-security-knox + mssql-security-support + mssql-server-controller + mssql-server-data + mssql-service-proxy + mssql-ssis-app-runtime +) + + +# Make a directory for installing the scripts and logs. +# +mkdir -p $BDCDEPLOY_DIR +cd $BDCDEPLOY_DIR/ +touch $LOG_FILE + +{ +# Install all necessary packages: kuberenetes, docker, python3, python3-pip, request, azdata. +# +echo "" +echo "######################################################################################" +echo "Starting installing packages..." + +# Install docker. +# +apt-get update -q + +apt --yes install \ + software-properties-common \ + apt-transport-https \ + ca-certificates \ + curl + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + +apt update -q +apt-get install -q --yes docker-ce=18.06.2~ce~3-0~ubuntu --allow-downgrades +apt-mark hold docker-ce + +usermod --append --groups docker $USER + +# Install python3, python3-pip, requests. +# +apt-get install -q -y python3 +apt-get install -q -y python3-pip + +pip3 install requests --upgrade + +# Install and create virtualenv. +# +pip3 install --upgrade virtualenv +virtualenv -p python3 $VIRTUALENV_NAME +source $VIRTUALENV_NAME/bin/activate + +# Install azdata cli. +# +pip3 install -r $REQUIREMENTS_LINK +echo "Packages installed." + +# Load all prereqs for kubernetes. +# +echo "###########################################################################" +echo "Starting to setup pre-requisites for kubernetes..." + +# Setup the kubernetes preprequisites. +# +echo $(hostname -i) $(hostname) >> /etc/hosts + +swapoff -a +sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab + +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + +cat </etc/apt/sources.list.d/kubernetes.list + +deb http://apt.kubernetes.io/ kubernetes-xenial main + +EOF + +# Install docker and packages to allow apt to use a repository over HTTPS. +# +apt-get update -q + +apt-get install -q -y ebtables ethtool + +#apt-get install -y docker.ce + +apt-get install -q -y apt-transport-https + +# Setup daemon. +# +cat > /etc/docker/daemon.json < /etc/sysctl.conf +echo net.ipv6.conf.default.disable_ipv6=1 > /etc/sysctl.conf +echo net.ipv6.conf.lo.disable_ipv6=1 > /etc/sysctl.conf + + +sysctl net.bridge.bridge-nf-call-iptables=1 + +# Setting up the persistent volumes for the kubernetes. +# +for i in $(seq 1 $PV_COUNT); do + + vol="vol$i" + + mkdir -p /mnt/local-storage/$vol + + mount --bind /mnt/local-storage/$vol /mnt/local-storage/$vol + +done +echo "Kubernetes pre-requisites have been completed." + +# Setup kubernetes cluster including remove taint on master. +# +echo "" +echo "#############################################################################" +echo "Starting to setup Kubernetes master..." + +# Initialize a kubernetes cluster on the current node. +# +sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION + +mkdir -p $HOME/.kube +mkdir -p /home/$SUDO_USER/.kube + +sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.kube/config + +# To enable a single node cluster remove the taint that limits the first node to master only service. +# +master_node=`kubectl get nodes --no-headers=true --output=custom-columns=NAME:.metadata.name` +kubectl taint nodes ${master_node} node-role.kubernetes.io/master:NoSchedule- + +# Local storage provisioning. +# +kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml + +# Install the software defined network. +# +kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml + +# helm init + +kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/rbac.yaml + +# Verify that the cluster is ready to be used. +# +echo "Verifying that the cluster is ready for use..." +while true ; do + + if [[ "$TIMEOUT" -le 0 ]]; then + echo "Cluster node failed to reach the 'Ready' state. Kubeadm setup failed." + exit 1 + fi + + status=`kubectl get nodes --no-headers=true | awk '{print $2}'` + + if [ "$status" == "Ready" ]; then + break + fi + + sleep "$RETRY_INTERVAL" + + TIMEOUT=$(($TIMEOUT-$RETRY_INTERVAL)) + + echo "Cluster not ready. Retrying..." +done + + +# Install the dashbaord for kubernetes. +# +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml + +kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard +echo "Kubernetes master setup done." + +# Pull docker images of azdata. +# +echo "" +echo "############################################################################" +echo "Starting to pull docker images..." +echo "Pulling images from repository: " $DOCKER_REGISTRY"/"$DOCKER_REPOSITORY + +for image in "${IMAGES[@]}"; +do + docker pull $DOCKER_REGISTRY/$DOCKER_REPOSITORY/$image:$DOCKER_TAG + echo "Docker image" $image " pulled." +done +echo "Docker images pulled." + +# Deploy azdata bdc create cluster. +# +echo "" +echo "############################################################################" +echo "Starting to deploy azdata cluster..." + +# Command to create cluster for single node cluster. +# +azdata bdc config init --source kubeadm-dev-test --target kubeadm-custom -f +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.repository=$DOCKER_REPOSITORY" +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.registry=$DOCKER_REGISTRY" +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.imageTag=$DOCKER_TAG" +azdata bdc config replace -c kubeadm-custom/cluster.json -j "$.spec.pools[?(@.spec.type == "Data")].spec.replicas=1" +azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.data.className=$STORAGE_CLASS" +azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.logs.className=$STORAGE_CLASS" +azdata bdc create -c kubeadm-custom --accept-eula $ACCEPT_EULA +echo "Azdata cluster created." + +# Setting context to cluster. +# +kubectl config set-context --current --namespace $CLUSTER_NAME + +# Login and get endpoint list for the cluster. +# +azdata login -n $CLUSTER_NAME +azdata bdc endpoint list --output table + +if [ -d "$HOME/.azdata/" ]; then + sudo chown -R $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.azdata/ +fi + +echo "alias azdata='$BDCDEPLOY_DIR/$VIRTUALENV_NAME/bin/azdata'" >> $HOME/.bashrc +source ~/.bashrc +}| tee $LOG_FILE diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md new file mode 100644 index 00000000..2d24672b --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md @@ -0,0 +1,30 @@ +# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS + + +In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions. + +## Pre-requisites + +1. Multiple Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines +1. Designate one machine as the Kubernetes master +1. Rest of the machines will be used as the Kubernetes agents + +**NOTE: Ensure there is sufficient local storage on your agents. Each volume will use up to 10GB by default. The script creates 25 volumes. Not all of the volumes will be used since it depends on the number of pods being deployed on each agent node. It is recommended to have at least 200 GB of storage on the agent nodes** + +### Useful resources + +[Deploy SQL Server 2019 big data cluster on Kubernetes](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) + +[Creating a cluster using kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) + +[Troubleshooting kubeadm](https://kubernetes.io/docs/setup/independent/troubleshooting-kubeadm/) + +### Instructions + +1. Start a sudo shell context +1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine +1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master +1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine +1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage +1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. +1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh index dfcccf06..eb7e6f31 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh @@ -19,7 +19,7 @@ apt-get install -y kubelet=$KUBE_DPKG_VERSION kubeadm=$KUBE_DPKG_VERSION kubectl curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash . /etc/os-release -if [ "$VERSION_CODENAME" == "bionic" ]; then +if [ "$UBUNTU_CODENAME" == "bionic" ]; then modprobe br_netfilter fi sysctl net.bridge.bridge-nf-call-iptables=1 diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql index 0852b41c..de601dee 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql +++ b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql @@ -25,7 +25,7 @@ SELECT TOP(80) PERCENT SIGN(q.clicks_in_category) AS book_category , q.clicks_in_7 , q.clicks_in_8 , q.clicks_in_9 - FROM web_clickstreams_book_clicks as q + FROM web_clickstreams_hdfs_book_clicks as q '; -- Training R script that uses rxLogit function from RevoScaleR package (Microsoft R Server) to generate model to predict book_category click(s). SET @train_script = N' @@ -91,7 +91,7 @@ SELECT TOP(@top_count_value) PERCENT SIGN(q.clicks_in_category) AS book_category , q.clicks_in_7 , q.clicks_in_8 , q.clicks_in_9 - FROM web_clickstreams_book_clicks as q + FROM web_clickstreams_hdfs_book_clicks as q '; -- Scoring script that uses sklearn logistic regression model to predict book_category click(s)