Setup script for BDC with AD integration

This commit is contained in:
Umachandar Jayachandran
2019-08-12 09:02:40 -07:00
parent c8bc392c60
commit b1f6ce27ab
5 changed files with 572 additions and 0 deletions
@@ -0,0 +1,57 @@
# Deploy a SQL Server big data cluster on single node Kubernetes cluster (kubeadm)
Using this sample bash script, you will deploy a single node Kubernetes cluster using kubeadm and a SQL Server big data cluster that is integrated with Active Directory domain. The script must be run from the VM you are planning to use for your kubeadm deployment.
## Pre-requisites
1. A vanilla Ubuntu 16.04 or 18.04 virtual or physical machine on your corporate network. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported.
1. Machine should have at least 8 CPUs, 64GB RAM and 100GB disk space. After installing the images you will be left with 50GB for data/logs across all components.
1. Update existing packages using commands below to ensure that the OS image is up to date
``` bash
sudo apt update&&apt upgrade -y
sudo systemctl reboot
```
## Recommended Virtual Machine settings
1. Use static memory configuration for the virtual machine. For example, in hyper-v installations do not use dynamic memory allocation but instead allocate the recommended 64 GB or higher.
1. Use checkpoint or snapshot capability in your hyper visor so that you can rollback the virtual machine to a clean state.
## Instructions to deploy SQL Server big data cluster
1. Download the script on the VM you are planning to use for the deployment
``` bash
curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc.sh
curl --output endpoint-patch.json https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json
curl --output security-patch.json https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json
```
1. Edit and modify the endpoint-patch.json & security-patch.json files to supply the values specific to your Active Directory environment
1. Make the script executable
``` bash
chmod +x setup-bdc-ad.sh
```
1. Run the script (make sure you are running with sudo)
``` bash
sudo ./setup-bdc-ad.sh
```
1. Refresh alias setup for azdata
``` bash
source ~/.bashrc
```
When prompted, provide your input for the password that will be used for all external endpoints: controller, SQL Server master and gateway. The password should be sufficiently complex based on existing rules for SQL Server password. The controller username is defaulted to *admin*.
## Cleanup
1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capabiility in your hyper-visor to rollback the virtual machine to a clean state.
@@ -0,0 +1,87 @@
#!/bin/bash
if [ "$EUID" -ne 0 ]
then echo "Please run as root"
exit
fi
DIR_PREFIX=$1
kubeadm reset --force
unalias azdata
systemctl stop kubelet
rm -rf /var/lib/cni/
rm -rf /var/lib/etcd/
rm -rf /run/flannel/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
rm -rf /etc/kubernetes/
ip link set cni0 down
#brctl delbr cni0
ip link set flannel.1 down
#brctl delbr flannel.1
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
rm -rf .azdata/
rm -rf bdcdeploy/
# Remove mounts.
#
SERVICE_STOP_FAILED=0
systemctl | grep "/var/lib/kubelet/pods" | while read -r line; do
# Retrieve the mount path
#
MOUNT_PATH=`echo "$line" | grep -v echo | egrep -oh -m 1 "(/var/lib/kubelet/pods).+"`
if [ -z "$MOUNT_PATH" ]; then
continue
fi
if [[ ! -d "$MOUNT_PATH" ]] && [[ ! -f "$MOUNT_PATH" ]]; then
SERVICE=$(echo $line | cut -f1 -d' ')
echo "Mount "$MOUNT_PATH" no longer exists."
echo "Stopping orphaned mount service: '$SERVICE'"
systemctl stop $SERVICE
if [ $? -ne 0 ]; then
SERVICE_STOP_FAILED=1
fi
echo ""
fi
done
if [ $SERVICE_STOP_FAILED -ne 0 ]; then
echo "Not all services were stopped successfully. Please check the above output for more inforamtion."
else
echo "All orphaned services successfully stopped."
fi
# Clean the mounted volumes.
#
for i in $(seq 1 30); do
vol="vol$i"
sudo umount /mnt/local-storage/$vol
sudo rm -rf /mnt/local-storage/$vol
done
# Reset kube
#
sudo apt-get purge -y kubeadm --allow-change-held-packages
sudo apt-get purge -y kubectl --allow-change-held-packages
sudo apt-get purge -y kubelet --allow-change-held-packages
sudo apt-get purge -y kubernetes-cni --allow-change-held-packages
sudo apt-get purge -y kube* --allow-change-held-packages
sudo apt -y autoremove
sudo rm -rf ~/.kube
@@ -0,0 +1,23 @@
{
"patch": [
{
"op": "replace",
"path": "spec.pools[?(@.spec.type=='Master')].spec",
"value": {
"type": "Master",
"dnsName": "mastersql.contoso.local",
"replicas": 1,
"endpoints": [
{
"name": "Master",
"serviceType": "NodePort",
"port": 31433
}
]
}
}
]
}
@@ -0,0 +1,31 @@
{
"patch": [
{
"op": "add",
"path": "security",
"value": {
"useInternalDomain": false,
"ouDistinguishedName":"OU=bdc,DC=contoso,DC=local",
"dnsIpAddresses": ["11.11.111.11"],
"domainControllerFullyQualifiedDns": ["VM.CONTOSO.LOCAL"],
"realm":"CONTOSO.LOCAL",
"domainDnsName":"contoso.local",
"bdcAdminPrincipals": [
"Domain Admins", "Enterprise Admins"
],
"bdcUserPrincipals": [
"Domain Users"
]
}
},
{
"op": "add",
"path": "spec.endpoints/0",
"value": {
"name": "Kerberos",
"serviceType": "NodePort",
"port": 30088
}
}
]
}
@@ -0,0 +1,374 @@
#!/bin/bash
set -Eeuo pipefail
if [ "$EUID" -ne 0 ]
then echo "Please run as root"
exit
fi
# This is a script to create single-node Kubernetes cluster and deploy BDC on it.
#
export BDCDEPLOY_DIR=bdcdeploy
# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox.
#
while true; do
read -s -p "Create Password for Big Data Cluster: " password
echo
read -s -p "Confirm Password for Big Data Cluster: " password2
echo
[ "$password" = "$password2" ] && break
echo "Password mismatch. Please try again."
done
echo ""
# Get docker credentials for private release.
#
read -p "Enter Docker username: " DOCKER_USERNAME
while true; do
read -s -p "Enter Docker Password: " docker_password
echo
read -s -p "Confirm Docker Password: " docker_password2
echo
[ "$docker_password" = "$docker_password2" ] && break
echo "Password mismatch. Please try again."
done
export DOCKER_PASSWORD=$docker_password
echo ""
# Get Domain Service Account Username and Password.
#
read -p "Enter the Domain Service Account Username: " ds_username
export DOMAIN_SERVICE_ACCOUNT_USERNAME=$ds_username
while true; do
read -s -p "Enter Password for Domain Service Account: " ds_password
echo
read -s -p "Confirm your Domain Service Account Password: " ds_password2
echo
[ "$ds_password" = "$ds_password2" ] && break
echo "Password mismatch. Please try again."
done
export DOMAIN_SERVICE_ACCOUNT_PASSWORD=$ds_password
echo ""
# Name of virtualenv variable used.
#
export VIRTUALENV_NAME="bdcvenv"
export LOG_FILE="bdcdeploy.log"
export DEBIAN_FRONTEND=noninteractive
# Requirements file.
#
export REQUIREMENTS_LINK="https://aka.ms/azdata"
# Kube version.
#
KUBE_DPKG_VERSION=1.15.0-00
KUBE_VERSION=1.15.0
# Wait for 5 minutes for the cluster to be ready.
#
TIMEOUT=600
RETRY_INTERVAL=5
# Variables for pulling dockers.
#
export DOCKER_REGISTRY="private-repo.microsoft.com"
export DOCKER_REPOSITORY="mssql-private-preview"
export DOCKER_TAG="ctp3.2.1"
# Variables used for azdata cluster creation.
#
export CONTROLLER_USERNAME=admin
export CONTROLLER_PASSWORD=$password
export MSSQL_SA_PASSWORD=$password
export KNOX_PASSWORD=$password
export ACCEPT_EULA=yes
export CLUSTER_NAME=mssql-cluster
export STORAGE_CLASS=local-storage
export PV_COUNT="30"
IMAGES=(
mssql-app-service-proxy
mssql-appdeploy-init
mssql-controller
mssql-hadoop
mssql-mleap-serving-runtime
mssql-mlserver-py-runtime
mssql-mlserver-r-runtime
mssql-monitor-collectd
mssql-monitor-elasticsearch
mssql-monitor-fluentbit
mssql-monitor-grafana
mssql-monitor-influxdb
mssql-monitor-kibana
mssql-monitor-telegraf
mssql-security-knox
mssql-security-support
mssql-server-controller
mssql-server-data
mssql-service-proxy
mssql-ssis-app-runtime
)
# Make a directory for installing the scripts and logs.
#
mkdir -p $BDCDEPLOY_DIR
cd $BDCDEPLOY_DIR/
touch $LOG_FILE
{
# Install all necessary packages: kuberenetes, docker, python3, python3-pip, request, azdata.
#
echo ""
echo "######################################################################################"
echo "Starting installing packages..."
# Install docker.
#
apt-get update -q
apt --yes install \
software-properties-common \
apt-transport-https \
ca-certificates \
curl
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
apt update -q
apt-get install -q --yes docker-ce=18.06.2~ce~3-0~ubuntu --allow-downgrades
apt-mark hold docker-ce
usermod --append --groups docker $USER
# Install python3, python3-pip, requests.
#
apt-get install -q -y python3
apt-get install -q -y python3-pip
pip3 install requests --upgrade
# Install and create virtualenv.
#
pip3 install --upgrade virtualenv
virtualenv -p python3 $VIRTUALENV_NAME
source $VIRTUALENV_NAME/bin/activate
# Install azdata cli.
#
pip3 install -r $REQUIREMENTS_LINK
echo "Packages installed."
# Load all pre-requisites for Kubernetes.
#
echo "###########################################################################"
echo "Starting to setup pre-requisites for kubernetes..."
# Setup the kubernetes preprequisites.
#
echo $(hostname -i) $(hostname) >> /etc/hosts
swapoff -a
sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
# Install docker and packages to allow apt to use a repository over HTTPS.
#
apt-get update -q
apt-get install -q -y ebtables ethtool
#apt-get install -y docker.ce
apt-get install -q -y apt-transport-https
# Setup daemon.
#
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
mkdir -p /etc/systemd/system/docker.service.d
# Restart docker.
#
systemctl daemon-reload
systemctl restart docker
apt-get install -q -y kubelet=$KUBE_DPKG_VERSION kubeadm=$KUBE_DPKG_VERSION kubectl=$KUBE_DPKG_VERSION
# Holding the version of kube packages.
#
apt-mark hold kubelet kubeadm kubectl
curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
. /etc/os-release
if [ "$UBUNTU_CODENAME" == "bionic" ]; then
modprobe br_netfilter
fi
# Disable Ipv6 for cluster endpoints.
#
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
echo net.ipv6.conf.all.disable_ipv6=1 > /etc/sysctl.conf
echo net.ipv6.conf.default.disable_ipv6=1 > /etc/sysctl.conf
echo net.ipv6.conf.lo.disable_ipv6=1 > /etc/sysctl.conf
sysctl net.bridge.bridge-nf-call-iptables=1
# Setting up the persistent volumes for the kubernetes.
#
for i in $(seq 1 $PV_COUNT); do
vol="vol$i"
mkdir -p /mnt/local-storage/$vol
mount --bind /mnt/local-storage/$vol /mnt/local-storage/$vol
done
echo "Kubernetes pre-requisites have been completed."
# Setup kubernetes cluster including remove taint on master.
#
echo ""
echo "#############################################################################"
echo "Starting to setup Kubernetes master..."
# Initialize a kubernetes cluster on the current node.
#
sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION
mkdir -p $HOME/.kube
mkdir -p /home/$SUDO_USER/.kube
sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.kube/config
# To enable a single node cluster remove the taint that limits the first node to master only service.
#
master_node=`kubectl get nodes --no-headers=true --output=custom-columns=NAME:.metadata.name`
kubectl taint nodes ${master_node} node-role.kubernetes.io/master:NoSchedule-
# Local storage provisioning.
#
kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml
# Install the software defined network.
#
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
# helm init
kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/rbac.yaml
# Verify that the cluster is ready to be used.
#
echo "Verifying that the cluster is ready for use..."
while true ; do
if [[ "$TIMEOUT" -le 0 ]]; then
echo "Cluster node failed to reach the 'Ready' state. Kubeadm setup failed."
exit 1
fi
status=`kubectl get nodes --no-headers=true | awk '{print $2}'`
if [ "$status" == "Ready" ]; then
break
fi
sleep "$RETRY_INTERVAL"
TIMEOUT=$(($TIMEOUT-$RETRY_INTERVAL))
echo "Cluster not ready. Retrying..."
done
# Install the dashboard for Kubernetes.
#
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard
echo "Kubernetes master setup done."
# Pull docker images of SQL Server big data cluster.
#
echo ""
echo "############################################################################"
echo "Starting to pull docker images..."
echo "Pulling images from repository: " $DOCKER_REGISTRY"/"$DOCKER_REPOSITORY
docker login $DOCKER_REGISTRY -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
for image in "${IMAGES[@]}";
do
docker pull $DOCKER_REGISTRY/$DOCKER_REPOSITORY/$image:$DOCKER_TAG
echo "Docker image" $image " pulled."
done
docker logout $DOCKER_REGISTRY
echo "Docker images pulled."
# Deploy azdata bdc create cluster.
#
echo ""
echo "############################################################################"
echo "Starting to deploy azdata cluster..."
# Command to create cluster for single node cluster.
#
azdata bdc config init --source kubeadm-dev-test --target kubeadm-custom -f
azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.repository=$DOCKER_REPOSITORY"
azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.registry=$DOCKER_REGISTRY"
azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.imageTag=$DOCKER_TAG"
azdata bdc config replace -c kubeadm-custom/cluster.json -j "$.spec.pools[?(@.spec.type == "Data")].spec.replicas=1"
azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.data.className=$STORAGE_CLASS"
azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.logs.className=$STORAGE_CLASS"
azdata bdc config patch -c kubeadm-custom/control.json -p $STARTUP_PATH/security-patch.json
azdata bdc config patch -c kubeadm-custom/cluster.json -p $STARTUP_PATH/endpoint-patch.json
azdata bdc create -c kubeadm-custom --accept-eula $ACCEPT_EULA
echo "Azdata cluster created."
# Setting context to cluster.
#
kubectl config set-context --current --namespace $CLUSTER_NAME
# Login and get endpoint list for the cluster.
#
azdata login -n $CLUSTER_NAME
azdata bdc endpoint list --output table
if [ -d "$HOME/.azdata/" ]; then
sudo chown -R $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.azdata/
fi
if [ -d "$HOME/bdcdeploy/" ]; then
sudo chown -R $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/bdcdeploy/
fi
echo "alias azdata='$BDCDEPLOY_DIR/$VIRTUALENV_NAME/bin/azdata'" >> $HOME/.bashrc
}| tee $LOG_FILE