Merge pull request #959 from cloudmelon/bdc-platform-ops

Refresh deployment scripts with the tested configurations on BDC release notes
2025-12-08 14:58:54 +00:00 · 2021-08-31 10:51:14 -07:00
parent a82e76f782 796e135039
commit d3a13a6fd9
9 changed files with 336 additions and 9 deletions
@@ -0,0 +1,91 @@
+# Deploy BDC on Azure Kubernetes Service cluster
+
+SQL Server Big Data Clusters allow you to deploy scalable clusters of SQL Server, Spark, and HDFS containers running on Kubernetes, it allows you to easily combine and analyze your high-value relational data with high-volume big data.
+
+This repository contains the scripts that you can use to deploy a BDC cluster on Azure Kubernetes Service (AKS)  cluster with basic networking ( Kubenet ) and advanced networking ( CNI ). 
+
+This repository contains 3 bash scripts : 
+- **deploy-cni-aks.sh** : You can use it to deploy AKS cluster using CNI networking, it fits the use case that you need to deploy BDC with AKS cluster with CNI networking plugin for integration with existing virtual networks in Azure, and this network model allows greater separation of resources and controls in an enterprise environment.
+
+- **deploy-kubenet-aks.sh** : You can use it to deploy AKS cluster using kubenet networking, it fits the use case that you need to deploy BDC with AKS cluster with kubenet networking. Kubenet is a basic network plugin, on Linux only. AKS cluster by default is on kubenet networking, after provisioning it, it also creates an Azure virtual network and a subnet, where your nodes get an IP address from the subnet and all pods receive an IP address from a logically different address space to the subnet of the nodes. 
+
+- **deploy-bdc.sh** : You can use it to deploy Big Data Clusters ( BDC ) AKS cluster. Please find the inline comments about the deployment steps and configurations which allows your customization. 
+
+## Above all
+
+SQL Server Big Data Clusters is a fully containerized solution orchestrated by Kubernetes. Starting with CU12, each release of SQL Server Big Data Clusters is tested against a fixed configuration of components. The configuration is evaluated with each release and adjustments are made to stay in-line with the ecosystem as Kubernetes continues to evolve. Further information see [Tested Configurations from SQL Server Big Data Clusters platform release notes](https://docs.microsoft.com/en-us/sql/big-data-cluster/release-notes-big-data-cluster?view=sql-server-ver15#tested-configurations). 
+
+Please note that a no later than 1.13 version for Kubernetes server to deploy your big data clusters. Therefore you need to use --kubernetes-version parameter to specify a version different than the default for AKS.
+
+## Prerequisites
+
+You can run those scripts on the following client environment with Linux OS or WSL/WSL2.
+
+The following link listed common big data cluster tools and how to install them:
+
+https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-big-data-tools?view=sql-server-ver15
+
+
+## Instructions
+
+### deploy-cni-aks.sh
+
+1. Download the script on the location that you are planning to use for the deployment
+
+``` bash
+curl --output deploy-cni-aks.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/platform-ops/scripts/deploy-cni-aks.sh
+```
+
+2. Make the script executable
+
+``` bash
+chmod +x deploy-cni-aks.sh
+```
+
+3. Run the script (make sure you are running with sudo)
+
+``` bash
+sudo ./deploy-cni-aks.sh
+```
+
+### deploy-kubenet-aks.sh
+
+1. Download the script on the location that you are planning to use for the deployment
+
+``` bash
+curl --output deploy-kubenet-aks.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/platform-ops/scripts/deploy-kubenet-aks.sh
+```
+
+2. Make the script executable
+
+``` bash
+chmod +x deploy-kubenet-aks.sh
+```
+
+3. Run the script (make sure you are running with sudo)
+
+``` bash
+sudo ./deploy-kubenet-aks.sh
+```
+
+### deploy-bdc-aks.sh
+
+
+1. Download the script on the location that you are planning to use for the deployment
+
+``` bash
+curl --output deploy-bdc-aks.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/platform-ops/scripts/deploy-bdc-aks.sh
+```
+
+2. Make the script executable
+
+``` bash
+chmod +x deploy-bdc-aks.sh
+```
+
+3. Run the script (make sure you are running with sudo)
+
+``` bash
+sudo ./deploy-bdc-aks.sh
+```
+
@@ -0,0 +1,63 @@
+#!/bin/bash
+#Define a set of environment variables to be used in resource creations.
+#
+
+#!/bin/bash
+#Get Subscription ID and resource groups. It is used as default for controller, SQL Server Master instance (sa account) and Knox.
+#
+
+read -p "Your Azure Subscription: " subscription
+echo
+read -p "Your Resource Group Name: " resourcegroup
+echo
+read -p "In which region you're deploying: " region
+echo
+
+
+#Define a set of environment variables to be used in resource creations.
+export SUBID=$subscription
+
+export REGION_NAME=$region
+export RESOURCE_GROUP=$resourcegroup
+export KUBERNETES_VERSION=$version
+export SUBNET_NAME=aks-subnet
+export VNET_NAME=bdc-vnet
+export AKS_NAME=bdcakscluster
+ 
+#Set Azure subscription current in use
+az account set --subscription $subscription
+
+#Create Azure Resource Group
+az group create -n $RESOURCE_GROUP -l $REGION_NAME
+ 
+#Create Azure Virtual Network to host your AKS clus
+az network vnet create \
+    --resource-group $RESOURCE_GROUP \
+    --location $REGION_NAME \
+    --name $VNET_NAME \
+    --address-prefixes 10.0.0.0/8 \
+    --subnet-name $SUBNET_NAME \
+    --subnet-prefix 10.1.0.0/16
+
+SUBNET_ID=$(az network vnet subnet show \
+    --resource-group $RESOURCE_GROUP \
+    --vnet-name $VNET_NAME \
+    --name $SUBNET_NAME \
+    --query id -o tsv)
+ 
+#Create AKS Cluster
+az aks create \
+    --resource-group $RESOURCE_GROUP \
+    --name $AKS_NAME \
+    --kubernetes-version $version \
+    --load-balancer-sku standard \
+    --network-plugin azure \
+    --vnet-subnet-id $SUBNET_ID \
+    --docker-bridge-address 172.17.0.1/16 \
+    --dns-service-ip 10.2.0.10 \
+    --service-cidr 10.2.0.0/24 \
+    --node-vm-size Standard_D13_v2 \
+    --node-count 2 \
+    --generate-ssh-keys
+
+az aks get-credentials -g $RESOURCE_GROUP -n $AKS_NAME
@@ -0,0 +1,63 @@
+#!/bin/bash
+#Define a set of environment variables to be used in resource creations.
+#
+
+#!/bin/bash
+#Get Subscription ID and resource groups. It is used as default for controller, SQL Server Master instance (sa account) and Knox.
+#
+
+read -p "Your Azure Subscription: " subscription
+echo
+read -p "Your Resource Group Name: " resourcegroup
+echo
+read -p "In which region you're deploying: " region
+echo
+
+
+#Define a set of environment variables to be used in resource creations.
+export SUBID=$subscription
+
+export REGION_NAME=$region
+export RESOURCE_GROUP=$resourcegroup
+export KUBERNETES_VERSION=$version
+export SUBNET_NAME=aks-subnet
+export VNET_NAME=bdc-vnet
+export AKS_NAME=bdcakscluster
+ 
+#Set Azure subscription current in use
+az account set --subscription $subscription
+
+#Create Azure Resource Group
+az group create -n $RESOURCE_GROUP -l $REGION_NAME
+ 
+#Create Azure Virtual Network to host your AKS clus
+az network vnet create \
+    --resource-group $RESOURCE_GROUP \
+    --location $REGION_NAME \
+    --name $VNET_NAME \
+    --address-prefixes 10.0.0.0/8 \
+    --subnet-name $SUBNET_NAME \
+    --subnet-prefix 10.1.0.0/16
+
+SUBNET_ID=$(az network vnet subnet show \
+    --resource-group $RESOURCE_GROUP \
+    --vnet-name $VNET_NAME \
+    --name $SUBNET_NAME \
+    --query id -o tsv)
+ 
+#Create AKS Cluster
+az aks create \
+    --resource-group $RESOURCE_GROUP \
+    --name $AKS_NAME \
+    --kubernetes-version $version \
+    --load-balancer-sku standard \
+    --network-plugin azure \
+    --vnet-subnet-id $SUBNET_ID \
+    --docker-bridge-address 172.17.0.1/16 \
+    --dns-service-ip 10.2.0.10 \
+    --service-cidr 10.2.0.0/24 \
+    --node-vm-size Standard_D13_v2 \
+    --node-count 2 \
+    --generate-ssh-keys
+
+az aks get-credentials -g $RESOURCE_GROUP -n $AKS_NAME
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox.
+#
+
+while true; do
+    read -p "Create Admin username for Big Data Cluster: " bdcadmin
+    echo
+    read -s -p "Create Password for Big Data Cluster: " password
+    echo
+    read -s -p "Confirm your Password: " password2
+    echo
+    [ "$password" = "$password2" ] && break
+    echo "Password mismatch. Please try again."
+done
+
+
+#Create BDC custom profile
+azdata bdc config init --source aks-dev-test --target bdc-aks --force
+
+#Configurations for BDC deployment
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"
+
+
+azdata bdc create --config-profile bdc-aks --accept-eula yes
+
+#Login and get endpoint list for the cluster.
+#
+azdata login -n mssql-cluster
+
+azdata bdc endpoint list --output table
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox.
+#
+
+while true; do
+    read -p "Create Admin username for Big Data Cluster: " bdcadmin
+    echo
+    read -s -p "Create Password for Big Data Cluster: " password
+    echo
+    read -s -p "Confirm your Password: " password2
+    echo
+    [ "$password" = "$password2" ] && break
+    echo "Password mismatch. Please try again."
+done
+
+
+#Create BDC custom profile
+azdata bdc config init --source aks-dev-test --target bdc-aks --force
+
+#Configurations for BDC deployment
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"
+
+
+azdata bdc create --config-profile bdc-aks --accept-eula yes
+
+#Login and get endpoint list for the cluster.
+#
+azdata login -n mssql-cluster
+
+azdata bdc endpoint list --output table
@@ -0,0 +1,40 @@
+#!/bin/bash
+#Define a set of environment variables to be used in resource creations.
+#
+
+#!/bin/bash
+#Get Subscription ID and resource groups. It is used as default for controller, SQL Server Master instance (sa account) and Knox.
+#
+
+read -p "Your Azure Subscription: " subscription
+echo
+read -p "Your Resource Group Name: " resourcegroup
+echo
+read -p "In which region you're deploying: " region
+echo
+
+
+#Define a set of environment variables to be used in resource creations.
+export SUBID=$subscription
+
+export REGION_NAME=$region
+export RESOURCE_GROUP=$resourcegroup
+export KUBERNETES_VERSION=$version
+export AKS_NAME=bdcakscluster
+ 
+#Set Azure subscription current in use
+az account set --subscription $subscription
+
+#Create Azure Resource Group
+az group create -n $RESOURCE_GROUP -l $REGION_NAME
+ 
+#Create AKS Cluster
+az aks create \
+    --resource-group $RESOURCE_GROUP \
+    --name $AKS_NAME \
+    --kubernetes-version $version \
+    --node-vm-size Standard_D13_v2 \
+    --node-count 2 \
+    --generate-ssh-keys
+
+az aks get-credentials -g $RESOURCE_GROUP -n $AKS_NAME
@@ -1,5 +1,7 @@
 # Deploy BDC in private AKS cluster with Advanced Networking (CNI)

+SQL Server Big Data Clusters allow you to deploy scalable clusters of SQL Server, Spark, and HDFS containers running on Kubernetes, it allows you to easily combine and analyze your high-value relational data with high-volume big data.
+
 This repository contains the scripts that you can use to deploy a BDC cluster in Azure Kubernetes Service (AKS) private cluster with advanced networking ( CNI ). 

 This repository contains 3 bash scripts : 
@@ -7,7 +9,7 @@ This repository contains 3 bash scripts :

 - **deploy-private-aks-udr.sh** : You can use it to deploy private AKS cluster with private endpoint, it fits the use case that you need to deploy BDC with AKS private cluster and limit egress traffic with UDR ( User-defined Routes ). 

- **deploy-bdc.sh** : You can use it to deploy Big Data Clusters ( BDC ) in private deployment mode on private AKS cluster with or without User-defined routes based on your project requirements.  **Note** : Please use this scripts in the Azure VM which manages your AKS private cluster. 
+- **deploy-bdc.sh** : You can use it to deploy Big Data Clusters ( BDC ) in private deployment mode on private AKS cluster with or without user-defined routes based on your project requirements.  **Note** : Please use this scripts in the Azure VM or Azure Bastion instance which manages your AKS private cluster. 


 ## Prerequisites
@@ -18,16 +18,16 @@ done
 azdata bdc config init --source aks-dev-test --target private-bdc-aks --force

 #Configurations for BDC deployment
-azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU6-ubuntu-16.04"
-azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
-azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"

-azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.endpoints[0].serviceType=NodePort"
-azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.endpoints[1].serviceType=NodePort"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.endpoints[0].serviceType=NodePort"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.endpoints[1].serviceType=NodePort"

-azdata bdc config replace -c private-bdc-aks/bdc.json -j "$.spec.resources.master.spec.endpoints[0].serviceType=NodePort"
-azdata bdc config replace -c private-bdc-aks/bdc.json -j "$.spec.resources.gateway.spec.endpoints[0].serviceType=NodePort"
-azdata bdc config replace -c private-bdc-aks/bdc.json -j "$.spec.resources.appproxy.spec.endpoints[0].serviceType=NodePort"
+azdata bdc config replace -p private-bdc-aks/bdc.json -j "$.spec.resources.master.spec.endpoints[0].serviceType=NodePort"
+azdata bdc config replace -p private-bdc-aks/bdc.json -j "$.spec.resources.gateway.spec.endpoints[0].serviceType=NodePort"
+azdata bdc config replace -p private-bdc-aks/bdc.json -j "$.spec.resources.appproxy.spec.endpoints[0].serviceType=NodePort"

 #In case you're deploying BDC in HA mode ( aks-dev-test-ha profile ) please also use the following command 
 #azdata bdc config replace -c private-bdc-aks /bdc.json -j "$.spec.resources.master.spec.endpoints[1].serviceType=NodePort"
@@ -19,6 +19,7 @@ export SUBID=$subscription

 export REGION_NAME=$region
 export RESOURCE_GROUP=$resourcegroup
+export KUBERNETES_VERSION=$version
 export SUBNET_NAME=aks-subnet
 export VNET_NAME=bdc-vnet
 export AKS_NAME=bdcaksprivatecluster
@@ -50,6 +51,7 @@ az aks create \
    --name $AKS_NAME \
    --load-balancer-sku standard \
    --enable-private-cluster \
+    --kubernetes-version $version \
    --network-plugin azure \
    --vnet-subnet-id $SUBNET_ID \
    --docker-bridge-address 172.17.0.1/16 \