From 86cbe89bc67112e5ab3118e45123acf20f33c293 Mon Sep 17 00:00:00 2001 From: cloudmelon Date: Tue, 18 Aug 2020 15:30:12 +0100 Subject: [PATCH] add details for scripts --- .../sql-big-data-cluster/deployment/README.md | 7 ++- .../deployment/private-bdc/README.md | 51 +++++++++++++++++++ .../scripts/deploy-private-aks-udr.sh | 34 ++++++++++--- .../private-bdc/scripts/deploy-private-aks.sh | 5 +- .../private-bdc/scripts/deploy-private-bdc.sh | 20 +++++++- .../private-bdc/README.md | 10 ---- 6 files changed, 106 insertions(+), 21 deletions(-) create mode 100644 samples/features/sql-big-data-cluster/deployment/private-bdc/README.md rename samples/features/sql-big-data-cluster/{ => deployment}/private-bdc/scripts/deploy-private-aks-udr.sh (82%) rename samples/features/sql-big-data-cluster/{ => deployment}/private-bdc/scripts/deploy-private-aks.sh (91%) rename samples/features/sql-big-data-cluster/{ => deployment}/private-bdc/scripts/deploy-private-bdc.sh (69%) delete mode 100644 samples/features/sql-big-data-cluster/private-bdc/README.md diff --git a/samples/features/sql-big-data-cluster/deployment/README.md b/samples/features/sql-big-data-cluster/deployment/README.md index fe5ec277..0e85dd42 100644 --- a/samples/features/sql-big-data-cluster/deployment/README.md +++ b/samples/features/sql-big-data-cluster/deployment/README.md @@ -13,4 +13,9 @@ Using the sample Python script in **aks** folder, you will deploy a Kubernetes c ## __[Push SQL Server big data cluster images to your own private Docker repository](offline/)__ -Using the sample Python script in **offline** folder, you will push the necessary images required for the deployment to your own repository. \ No newline at end of file +Using the sample Python script in **offline** folder, you will push the necessary images required for the deployment to your own repository. + +## __[Deploy SQL Server big data cluster in private mode with Azure Kubernetes service (AKS) private cluster](private-bdc/)__ + +Using the sample Python script in **private-bdc** folder, you will Deploy SQL Server big data cluster in private mode with Azure Kubernetes service (AKS) private cluster. + diff --git a/samples/features/sql-big-data-cluster/deployment/private-bdc/README.md b/samples/features/sql-big-data-cluster/deployment/private-bdc/README.md new file mode 100644 index 00000000..47e3aa3f --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/private-bdc/README.md @@ -0,0 +1,51 @@ +# Deploy BDC in private AKS cluster with User-defined Route (UDR) + +This repository contains the scripts that you can use to deploy a private BDC cluster in Azure Kubernetes Service (AKS) with advanced networking ( CNI ). + +This repository contains 3 bash scripts : +- deploy-private-aks.sh : You can use it to deploy private AKS cluster with private endpoint, it fits the use case that you need to deploy BDC with a private endpoint with AKS private cluster. +- deploy-private-aks-udr.sh : You can use it to deploy private AKS cluster with private endpoint, it fits the use case that you need to deploy BDC with a private endpoint with AKS private cluster and limit egress traffic with UDR ( User-defined Routes ). +- deploy-private-bdc.sh : You can use it to deploy Big Data Clusters ( BDC ) in private deployment mode on private AKS cluster with or without User-defined routes based on your project requirements. + + +## Prerequisite + +The following table lists common big data cluster tools and how to install them: + +| Tool | Required | Description | Installation | +|---|---|---|---| +| `python` | Yes | Python is an interpreted, object-oriented, high-level programming language with dynamic semantics. Many parts of big data clusters for SQL Server use python. | [Install python](#python)| +| `azdata` | Yes | Command-line tool for installing and managing a big data cluster. | [Install](deploy-install-azdata.md) | +| `kubectl`1 | Yes | Command-line tool for monitoring the underlying Kubernetes cluster ([More info](https://kubernetes.io/docs/tasks/tools/install-kubectl/)). | [Windows](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-with-powershell-from-psgallery) \| [Linux](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-using-native-package-management) | +| **Azure Data Studio** | Yes | Cross-platform graphical tool for querying SQL Server. | [Install](https://aka.ms/getazuredatastudio) | +| **Data Virtualization extension** | Yes | Extension for Azure Data Studio that provides a Data Virtualization wizard. | [Install](../azure-data-studio/data-virtualization-extension.md) | +| **Azure CLI**2 | For AKS | Modern command-line interface for managing Azure services. Used with AKS big data cluster deployments ([More info](https://docs.microsoft.com/cli/azure/?view=azure-cli-latest)). | [Install](https://docs.microsoft.com/cli/azure/install-azure-cli?view=azure-cli-latest) | +| **mssql-cli** | Optional | Modern command-line interface for querying SQL Server ([More info](../tools/mssql-cli.md)). | [Windows](https://github.com/dbcli/mssql-cli/blob/master/doc/installation/windows.md) \| [Linux](https://github.com/dbcli/mssql-cli/blob/master/doc/installation/linux.md) | +| **sqlcmd** | For some scripts | Legacy command-line tool for querying SQL Server ([More info](https://docs.microsoft.com/sql/tools/sqlcmd-utility?view=sql-server-ver15)). You might need to install the Microsoft ODBC Driver 11 for SQL Server before installing the SQLCMD package. | [Windows](https://www.microsoft.com/download/details.aspx?id=36433) \| [Linux](../linux/sql-server-linux-setup-tools.md) | +| `curl` 3 | For some scripts | Command-line tool for transferring data with URLs. | [Windows](https://curl.haxx.se/windows/) \| Linux: install curl package | +| `oc` | Required for Red Hat OpenShift and Azure Redhat OpenShift deployments. |`oc` is the Open Shift command line interface (CLI). | [Installing the CLI](https://docs.openshift.com/container-platform/4.4/cli_reference/openshift_cli/getting-started-cli.html#installing-the-cli) + + + +## Instructions + +1. Download the script on the VM you are planning to use for the deployment + +``` bash +curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/private-bdc/ubuntu-single-node-vm/deploy-private-aks.sh +``` + +2. Make the script executable + +``` bash +chmod +x deploy-private-aks.sh +``` + +3. Run the script (make sure you are running with sudo) + +``` bash +sudo ./deploy-private-aks.sh +``` + + + diff --git a/samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-aks-udr.sh b/samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-aks-udr.sh similarity index 82% rename from samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-aks-udr.sh rename to samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-aks-udr.sh index 5102fded..e33ee6b2 100644 --- a/samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-aks-udr.sh +++ b/samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-aks-udr.sh @@ -1,6 +1,19 @@ #!/bin/bash -export SUBID= +# Get Subscription ID and service principles as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox. +# +while true; do + read -s -p "Your Azure Subscription: " subscription + echo + read -s -p "Your service principles ID: " sp_id + echo + read -s -p "Your service principles Password: " sp_pwd + +done + +# Define a set of environment variables to be used in resource creations. + +export SUBID=$subscription export REGION_NAME=northeurope export RESOURCE_GROUP=private-bdc-rg @@ -16,9 +29,10 @@ export FWROUTE_NAME=bdcaksroute export FWROUTE_NAME_INTERNET=bdcaksrouteinet - +# Create Azure Resource Group az group create -n $RESOURCE_GROUP -l $REGION_NAME +# Create Azure Virtual Network to host your AKS cluster az network vnet create \ --resource-group $RESOURCE_GROUP \ --location $REGION_NAME \ @@ -35,29 +49,34 @@ SUBNET_ID=$(az network vnet subnet show \ --query id -o tsv) +# Add Azure firewall extension az extension add --name azure-firewall # Dedicated subnet for Azure Firewall (Firewall name cannot be changed) - az network vnet subnet create \ --resource-group $RESOURCE_GROUP \ --vnet-name $VNET_NAME \ --name AzureFirewallSubnet \ --address-prefix 10.2.0.0/24 +# Create Azure firewall az network firewall create -g $RESOURCE_GROUP -n $FWNAME -l $REGION_NAME --enable-dns-proxy true +# Create public IP for Azure Firewall az network public-ip create -g $RESOURCE_GROUP -n $FWPUBIP -l $REGION_NAME --sku "Standard" +# Create IP configurations for Azure Firewall az network firewall ip-config create -g $RESOURCE_GROUP -f $FWNAME -n $FWIPCONFIG_NAME --public-ip-address $FWPUBIP --vnet-name $VNET_NAME - +# Getting public and private IP addresses for Azure Firewall export FWPUBLIC_IP=$(az network public-ip show -g $RESOURCE_GROUP -n $FWPUBIP --query "ipAddress" -o tsv) export FWPRIVATE_IP=$(az network firewall show -g $RESOURCE_GROUP -n $FWNAME --query "ipConfigurations[0].privateIpAddress" -o tsv) +## Create an User defined route table az network route-table create -g $RESOURCE_GROUP --name $FWROUTE_TABLE_NAME +# Create User defined routes az network route-table route create -g $RESOURCE_GROUP --name $FWROUTE_NAME --route-table-name $FWROUTE_TABLE_NAME --address-prefix 0.0.0.0/0 --next-hop-type VirtualAppliance --next-hop-ip-address $FWPRIVATE_IP --subscription $SUBID az network route-table route create -g $RESOURCE_GROUP --name $FWROUTE_NAME_INTERNET --route-table-name $FWROUTE_TABLE_NAME --address-prefix $FWPUBLIC_IP/32 --next-hop-type Internet @@ -84,19 +103,20 @@ az network vnet subnet update -g $RESOURCE_GROUP --vnet-name $VNET_NAME --name $ az ad sp create-for-rbac -n "bdcaks-sp" --skip-assignment -export APPID= -export PASSWORD=< your service principle password > +export APPID=$sp_id +export PASSWORD=$sp_pwd export VNETID=$(az network vnet show -g $RESOURCE_GROUP --name $VNET_NAME --query id -o tsv) # Assign SP Permission to VNET az role assignment create --assignee $APPID --scope $VNETID --role "Network Contributor" - +# Assign SP Permission to route table export RTID=$(az network route-table show -g $RESOURCE_GROUP -n $FWROUTE_TABLE_NAME --query id -o tsv) az role assignment create --assignee $APPID --scope $RTID --role "Network Contributor" +# Create AKS Cluster az aks create \ --resource-group $RESOURCE_GROUP \ --location $REGION_NAME \ diff --git a/samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-aks.sh b/samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-aks.sh similarity index 91% rename from samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-aks.sh rename to samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-aks.sh index 6ce64405..2469ea40 100644 --- a/samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-aks.sh +++ b/samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-aks.sh @@ -8,8 +8,11 @@ export SUBNET_NAME=aks-subnet export VNET_NAME=bdc-vnet export AKS_NAME=bdcaksprivatecluster + +# Create Azure Resource Group az group create -n $RESOURCE_GROUP -l $REGION_NAME +# Create Azure Virtual Network to host your AKS clus az network vnet create \ --resource-group $RESOURCE_GROUP \ --location $REGION_NAME \ @@ -17,7 +20,6 @@ az network vnet create \ --address-prefixes 10.0.0.0/8 \ --subnet-name $SUBNET_NAME \ --subnet-prefix 10.1.0.0/16 - SUBNET_ID=$(az network vnet subnet show \ --resource-group $RESOURCE_GROUP \ @@ -25,6 +27,7 @@ SUBNET_ID=$(az network vnet subnet show \ --name $SUBNET_NAME \ --query id -o tsv) +# Create AKS Cluster az aks create \ --resource-group $RESOURCE_GROUP \ --name $AKS_NAME \ diff --git a/samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-bdc.sh b/samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-bdc.sh similarity index 69% rename from samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-bdc.sh rename to samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-bdc.sh index f674b856..394bf9f7 100644 --- a/samples/features/sql-big-data-cluster/private-bdc/scripts/deploy-private-bdc.sh +++ b/samples/features/sql-big-data-cluster/deployment/private-bdc/scripts/deploy-private-bdc.sh @@ -1,7 +1,23 @@ #!/bin/bash +# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox. +# +while true; do + read -s -p "Create Admin username for Big Data Cluster: " bdcadmin + echo + read -s -p "Create Password for Big Data Cluster: " password + echo + read -s -p "Confirm your Password: " password2 + echo + [ "$password" = "$password2" ] && break + echo "Password mismatch. Please try again." +done + + +# Create BDC custom profile azdata bdc config init --source aks-dev-test --target private-bdc-aks --force +# Configurations for private BDC deployment azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU6-ubuntu-16.04" azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.storage.data.className=default" azdata bdc config replace -c private-bdc-aks/control.json -j "$.spec.storage.logs.className=default" @@ -17,8 +33,8 @@ azdata bdc config replace -c private-bdc-aks /bdc.json -j "$.spec.resources.appp # azdata bdc config replace -c private-bdc-aks /bdc.json -j "$.spec.resources.master.spec.endpoints[1].serviceType= NodePort" -export AZDATA_USERNAME= -export AZDATA_PASSWORD=< your bdcadmin password> +export AZDATA_USERNAME=$bdcadmin +export AZDATA_PASSWORD=$password export ACCEPT_EULA=yes #accept agreement azdata bdc create --config-profile private-bdc-aks --accept-eula yes diff --git a/samples/features/sql-big-data-cluster/private-bdc/README.md b/samples/features/sql-big-data-cluster/private-bdc/README.md deleted file mode 100644 index 42799f58..00000000 --- a/samples/features/sql-big-data-cluster/private-bdc/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Deploy BDC in private AKS cluster with User-defined Route (UDR) - -This repository contains : - -Bash Scripts : -- deploy-private-aks.sh To deploy private AKS cluster ( private endpoint ) -- deploy-private-aks-udr.sh To deploy private AKS cluster with UDR. -- deploy-private-bdc.sh To deploy Big Data Clusters ( BDC ) in private deployment mode. - -