From 0cc704ebc9fb1568b8d658a6981d70abd71edd01 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Tue, 23 Jul 2019 15:56:43 -0700 Subject: [PATCH 01/29] Update push-bdc-images-to-custom-private-repo.py --- .../push-bdc-images-to-custom-private-repo.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/offline/push-bdc-images-to-custom-private-repo.py b/samples/features/sql-big-data-cluster/deployment/offline/push-bdc-images-to-custom-private-repo.py index 0697de10..93c6db96 100644 --- a/samples/features/sql-big-data-cluster/deployment/offline/push-bdc-images-to-custom-private-repo.py +++ b/samples/features/sql-big-data-cluster/deployment/offline/push-bdc-images-to-custom-private-repo.py @@ -13,10 +13,11 @@ def execute_cmd (cmd): if (stderr is not None): raise Exception(stderr) -SOURCE_DOCKER_REGISTRY = input("Provide Docker registry source - press ENTER for using `private-repo.microsoft.com`:") or "private-repo.microsoft.com" -SOURCE_DOCKER_REPOSITORY = input("Provide Docker repository source - press ENTER for using `mssql-private-preview`:") or "mssql-private-preview" -SOURCE_DOCKER_USERNAME = input("Provide Docker username for the source registry:") -SOURCE_DOCKER_PASSWORD=getpass.getpass("Provide Docker password for the source registry:") +SOURCE_DOCKER_REGISTRY = input("Provide Docker registry source - press ENTER for using `mcr.microsoft.com`:") or "mcr.microsoft.com" +SOURCE_DOCKER_REPOSITORY = input("Provide Docker repository source - press ENTER for using `mssql/bdc`:") or "mssql/bdc" +# Use this only if your source is a private Docker registry +# SOURCE_DOCKER_USERNAME = input("Provide Docker username for the source registry:") +# SOURCE_DOCKER_PASSWORD=getpass.getpass("Provide Docker password for the source registry:") SOURCE_DOCKER_TAG = input("Provide Docker tag for the images at the source: ") or "latest" TARGET_DOCKER_REGISTRY = input("Provide Docker registry target:") @@ -43,12 +44,14 @@ images = [ 'mssql-appdeploy-init', 'mssql-service-proxy', 'mssql-app-service-proxy', 'mssql-ssis-app-runtime', - 'mssql-monitor-telegraf'] + 'mssql-monitor-telegraf', + 'mssql-security-support'] -print("Execute docker login to source registry: " + SOURCE_DOCKER_REGISTRY) -cmd = "docker login " + SOURCE_DOCKER_REGISTRY + " -u " + SOURCE_DOCKER_USERNAME + " -p " + SOURCE_DOCKER_PASSWORD -execute_cmd(cmd) -print("") +# Use this only if your source is a private Docker registry +# print("Execute docker login to source registry: " + SOURCE_DOCKER_REGISTRY) +# cmd = "docker login " + SOURCE_DOCKER_REGISTRY + " -u " + SOURCE_DOCKER_USERNAME + " -p " + SOURCE_DOCKER_PASSWORD +# execute_cmd(cmd) +# print("") print("Pulling images from source repository: " + SOURCE_DOCKER_REGISTRY + "/" + SOURCE_DOCKER_REPOSITORY) From 789332b81b234494574999f58f25bc5e33e68d4f Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Tue, 23 Jul 2019 15:57:03 -0700 Subject: [PATCH 02/29] Updates for CTP3.2 --- .../features/sql-big-data-cluster/deployment/offline/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/deployment/offline/Readme.md b/samples/features/sql-big-data-cluster/deployment/offline/Readme.md index c8e88233..b8f99f30 100644 --- a/samples/features/sql-big-data-cluster/deployment/offline/Readme.md +++ b/samples/features/sql-big-data-cluster/deployment/offline/Readme.md @@ -24,7 +24,7 @@ python push-bdc-images-to-custom-private-repo.py >``` When prompted, provide your input for: -- Docker registry, repository and credentials to access Microsoft private registry where the images will be pulled from (source) +- Docker registry, repository to access Microsoft registry where the images will be pulled from (source). Press enter to use default mcr registry. - Docker registry, repository and credentials to access your private registry where the images will be pushed to (target) ## Deploy with from your private repository From 394045b75d038de40a2f354b9b7dc35287783338 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Tue, 23 Jul 2019 15:57:11 -0700 Subject: [PATCH 03/29] Updates for CTP3.2 --- .../features/sql-big-data-cluster/deployment/aks/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/aks/README.md b/samples/features/sql-big-data-cluster/deployment/aks/README.md index 0c49e06b..01a95c55 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/README.md +++ b/samples/features/sql-big-data-cluster/deployment/aks/README.md @@ -16,9 +16,9 @@ Using this sample Python script, you will deploy a Kubernetes cluster in Azure u python -m pip install requests python -m pip install requests --upgrade ``` - - Install mssqlctl CLI latest version using . Run the command below using elevated priviledges (sudo or admin cmd window): + - Install latest version of the cluster management tool **azdata** (previously named mssqlctl) using below command. Run the command below using elevated priviledges (sudo or admin cmd window): ``` - pip3 install -r https://private-repo.microsoft.com/python/ctp3.1/mssqlctl/requirements.txt + pip3 install -r https://aka.ms/azdata ``` 1. Login into your Azure account. Run this command: ``` From c8b7594ec7c715d685e2c4fdadedd8ec0c423d82 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Tue, 23 Jul 2019 15:57:18 -0700 Subject: [PATCH 04/29] Updates for CTP3.2 --- .../deployment/aks/deploy-sql-big-data-aks.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py index edc2b433..e333794d 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py +++ b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py @@ -1,7 +1,7 @@ # # Prerequisites: # -# Azure CLI (https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), python3 (https://www.python.org/downloads), mssqlctl CLI (pip3 install -r https://private-repo.microsoft.com/python/ctp3.1/mssqlctl/requirements.txt) +# Azure CLI (https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), python3 (https://www.python.org/downloads), azdata CLI (pip3 install -r https://aka.ms/azdata) # # Run `az login` at least once BEFORE running this script # @@ -38,21 +38,24 @@ AKS_NODE_COUNT=input("Provide number of worker nodes for AKS cluster - Press ENT CLUSTER_NAME=input("Provide name of AKS cluster and SQL big data cluster - Press ENTER for using `sqlbigdata`:").strip() or "sqlbigdata" #This password will be use for Controller user, Knox user and SQL Server Master SA accounts +# CONTROLLER_USERNAME=input("Provide username to be used for Controller user - Press ENTER for using `admin`:").strip() or "admin" PASSWORD = getpass.getpass("Provide password to be used for Controller user, Knox user and SQL Server Master SA accounts - Press ENTER for using `MySQLBigData2019`").strip() or "MySQLBigData2019" -#docker registry details -DOCKER_REGISTRY="private-repo.microsoft.com" -DOCKER_REPOSITORY="mssql-private-preview" -DOCKER_IMAGE_TAG="ctp3.1" +# Docker registry details +# Use this only if you are using a private registry different than mcr. If so, make sure you are also setting the environment variables for DOCKER_USERNAME and DOCKER_PASSWORD +# DOCKER_REGISTRY="" +# DOCKER_REPOSITORY="" +# DOCKER_IMAGE_TAG="" print ('Setting environment variables') os.environ['MSSQL_SA_PASSWORD'] = PASSWORD os.environ['CONTROLLER_USERNAME'] = CONTROLLER_USERNAME os.environ['CONTROLLER_PASSWORD'] = PASSWORD os.environ['KNOX_PASSWORD'] = PASSWORD -os.environ['DOCKER_USERNAME']=DOCKER_USERNAME -os.environ['DOCKER_PASSWORD']=DOCKER_PASSWORD +# Use this only if you are using a private registry different than mcr. If so, you must set the environment variables for DOCKER_USERNAME and DOCKER_PASSWORD +# os.environ['DOCKER_USERNAME']=DOCKER_USERNAME +# os.environ['DOCKER_PASSWORD']=DOCKER_PASSWORD os.environ['ACCEPT_EULA']="Yes" print ("Set azure context to subcription: "+SUBSCRIPTION_ID) @@ -71,29 +74,30 @@ command = "az aks get-credentials --overwrite-existing --name "+CLUSTER_NAME+" - executeCmd (command) print("Creating SQL Big Data cluster:" +CLUSTER_NAME) -command="mssqlctl bdc config init --source aks-dev-test --target custom --force" +command="azdata bdc config init --source aks-dev-test --target custom --force" executeCmd (command) -command="mssqlctl bdc config section set -c custom -j ""metadata.name=" + CLUSTER_NAME + "" +command="azdata bdc config section set -c custom -j ""metadata.name=" + CLUSTER_NAME + "" executeCmd (command) -command="mssqlctl bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" +# Use this only if you are using a private registry different than default Micrososft registry (mcr). +# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" +# executeCmd (command) + +# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" +# executeCmd (command) + +# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" +# executeCmd (command) + +command="azdata bdc create -c custom --accept-eula yes" executeCmd (command) -command="mssqlctl bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" -executeCmd (command) - -command="mssqlctl bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" -executeCmd (command) - -command="mssqlctl bdc create -c custom --accept-eula yes" -executeCmd (command) - -command="mssqlctl login --cluster-name " + CLUSTER_NAME +command="azdata login --cluster-name " + CLUSTER_NAME executeCmd (command) print("") print("SQL Server big data cluster endpoints: ") -command="mssqlctl bdc endpoint list -o table" +command="azdata bdc endpoint list -o table" executeCmd(command) From 59e43277649ef6cdb2c935f34365bff92b844da8 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 18:14:06 -0700 Subject: [PATCH 05/29] CTP 3.2 updates --- .../oracle/customer-oracle.sql | 2 +- .../oracle/inventory-oracle.sql | 14 +- .../web-clickstreams-hdfs-parquet.sql | 4 +- .../sql-big-data-cluster/deployment/README.md | 8 +- .../deployment/kubeadm/README.md | 28 +- .../kubeadm/ubuntu-single-node-vm/README.md | 41 +++ .../ubuntu-single-node-vm/cleanup-bdc.sh | 75 ++++ .../ubuntu-single-node-vm/setup-bdc.sh | 337 ++++++++++++++++++ .../deployment/kubeadm/ubuntu/README.md | 30 ++ .../kubeadm/ubuntu/setup-k8s-prereqs.sh | 2 +- .../python/book-click-prediction-mml-py.sql | 4 +- 11 files changed, 504 insertions(+), 41 deletions(-) create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql index c2f5b3c9..6e3d7c86 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql @@ -49,7 +49,7 @@ GO SELECT pr.pr_item_sk, pc.pr_review_content, pr.pr_user_sk AS customerid FROM dbo.product_reviews as pr JOIN (SELECT TOP(100) * FROM dbo.product_reviews_hdfs_csv) AS pc ON pc.pr_review_sk = pr.pr_review_sk -JOIN dbo.customer_ora AS c ON c.c_customer_sk = pr.pr_user_sk +JOIN dbo.customer_ora AS c ON c.C_CUSTOMER_SK = pr.pr_user_sk JOIN dbo.item AS i ON i.i_item_sk = pr.pr_item_sk INNER JOIN ( SELECT diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql index 9b579521..4217484d 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql @@ -20,22 +20,22 @@ IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'OracleSalesS -- As a result, the names are case-sensitive so specify the name in the external table definition -- that matches the exact case of the table and column names in the Oracle metadata. CREATE EXTERNAL TABLE [inventory_ora] - ([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL, - [inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0)) + ([INV_DATE] DECIMAL(10,0) NOT NULL, [INV_ITEM] DECIMAL(10,0) NOT NULL, + [INV_WAREHOUSE] DECIMAL(10,0) NOT NULL, [INV_QUANTITY_ON_HAND] DECIMAL(10,0)) WITH (DATA_SOURCE=[OracleSalesSrvr], LOCATION='.SALES.INVENTORY'); GO -- Find quantity of certain items from inventory for a specific category -- -SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity +SELECT TOP(100) w.w_warehouse_name, i.INV_ITEM, SUM(i.INV_QUANTITY_ON_HAND) as total_quantity FROM [inventory_ora] as i JOIN item as it - ON it.i_item_sk = i.inv_item + ON it.i_item_sk = i.INV_ITEM JOIN warehouse as w - ON w.w_warehouse_sk = i.inv_warehouse - WHERE it.i_category = 'Movies & TV' and i.inv_item BETWEEN 17401 and 17402 --> get items within specific range - GROUP BY w.w_warehouse_name, i.inv_item; + ON w.w_warehouse_sk = i.INV_WAREHOUSE + WHERE it.i_category = 'Movies & TV' and i.INV_ITEM BETWEEN 17401 and 17402 --> get items within specific range + GROUP BY w.w_warehouse_name, i.INV_ITEM; GO -- Cleanup diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index 6b6b4753..9a1c1380 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -90,8 +90,8 @@ AS AND w.wcs_user_sk IS NOT NULL) GROUP BY w.wcs_user_sk ) AS q - INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk - INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk; + INNER JOIN customer_ora as c ON q.wcs_user_sk = c.C_CUSTOMER_SK + INNER JOIN customer_demographics as cd ON c.C_CURRENT_CDEMO_SK = cd.cd_demo_sk; GO diff --git a/samples/features/sql-big-data-cluster/deployment/README.md b/samples/features/sql-big-data-cluster/deployment/README.md index b53442f4..8caf7756 100644 --- a/samples/features/sql-big-data-cluster/deployment/README.md +++ b/samples/features/sql-big-data-cluster/deployment/README.md @@ -3,13 +3,13 @@ SQL Server 2019 big data cluster is deployed as docker containers on a Kubernetes cluster. These samples provide scripts that can be used to provision a Kubernetes clusters using different environments. -## Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS +## Deploy a Kubernetes cluster using kubeadm -Use the scripts in the **kubeadm** folder to deploy Kubernetes over multiple Linux machines (physical or virtualized) using `kubeadm` utility. +Use the scripts in the **kubeadm** folder to deploy a Kubernetes cluster over one or more Linux machines (physical or virtualized) using `kubeadm` utility. -## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS) +## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS) -Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it. +Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it. ## Push SQL Server big data cluster images to your own private Docker repository diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md index 2d24672b..020f5114 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md @@ -1,30 +1,10 @@ # Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS -In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions. +## __[ubuntu](ubuntu/)__ -## Pre-requisites +This folder contains scripts that provide a template for deploying a Kubernetes cluster using kubeadm on one or more Linux machines. -1. Multiple Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines -1. Designate one machine as the Kubernetes master -1. Rest of the machines will be used as the Kubernetes agents +## __[ubuntu-single-node-vm](ubuntu-single-node-vm/)__ -**NOTE: Ensure there is sufficient local storage on your agents. Each volume will use up to 10GB by default. The script creates 25 volumes. Not all of the volumes will be used since it depends on the number of pods being deployed on each agent node. It is recommended to have at least 200 GB of storage on the agent nodes** - -### Useful resources - -[Deploy SQL Server 2019 big data cluster on Kubernetes](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) - -[Creating a cluster using kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) - -[Troubleshooting kubeadm](https://kubernetes.io/docs/setup/independent/troubleshooting-kubeadm/) - -### Instructions - -1. Start a sudo shell context -1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine -1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master -1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine -1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage -1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. -1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) +This folder contains a sample script that can be used to deploy a single-node Kubernetes cluster on a Linux machine. diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md new file mode 100644 index 00000000..767e065a --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -0,0 +1,41 @@ + +# Deploy a SQL Server big data cluster on single node Kubernetes cluster (kubeadm) + +Using this sample bash script, you will deploy a single node Kubernetes cluster using kubeadm and a SQL Server big data cluster on top of it. The script must be run from the VM you are planning to use for your kubeadm deployment. + +## Pre-requisites + +1. A vanilla Ubuntu 16.04 or 18.04 VM. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported. +1. VM should have at least 8CPUs, 64GB RAM and 100GB disk space.After installing the images you will be left with 50GB for data/logs across all components. + +## Instructions + +1. Download the script on the VM you are planning to use for the deployment + +``` + +curl --output kickstarter-azdata.sh http://rima-5.guest.corp.microsoft.com/kickstarter-azdata.sh +``` + +1. Make the script executable + +``` + +chmod +x kickstarter-azdata.sh +``` + +1. Run the script (make sure you are running with sudo) + +``` + +sudo ./kickstarter-azdata.sh +``` + +1. Refresh alias setup for azdata + +``` + +source ~/.bashrc +``` + +When prompted, provide your input for the password that will be used for all external endpoints: controller, SQL Server master and gateway. The password should be sufficiently complex based on existing rules for SQL Server password. The controller username is defaulted to *admin*. diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh new file mode 100644 index 00000000..0c4baba5 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi +DIR_PREFIX=$1 + +kubeadm reset --force + +systemctl stop kubelet +rm -rf /var/lib/cni/ +rm -rf /var/lib/etcd/ +rm -rf /run/flannel/ +rm -rf /var/lib/kubelet/* +rm -rf /etc/cni/ +rm -rf /etc/kubernetes/* +ip link set cni0 down +#brctl delbr cni0 +ip link set flannel.1 down +#brctl delbr flannel.1 +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X + +rm -rf .azdata/ + +SERVICE_STOP_FAILED=0 + +systemctl | grep "/var/lib/kubelet/pods" | while read -r line; do + + # Retrieve the mount path + # + MOUNT_PATH=`echo "$line" | grep -v echo | egrep -oh -m 1 "(/var/lib/kubelet/pods).+"` + + if [ -z "$MOUNT_PATH" ]; then + continue + fi + + if [[ ! -d "$MOUNT_PATH" ]] && [[ ! -f "$MOUNT_PATH" ]]; then + + SERVICE=$(echo $line | cut -f1 -d' ') + + echo "Mount "$MOUNT_PATH" no longer exists." + echo "Stopping orphaned mount service: '$SERVICE'" + + systemctl stop $SERVICE + + if [ $? -ne 0 ]; then + SERVICE_STOP_FAILED=1 + fi + + echo "" + fi +done + +if [ $SERVICE_STOP_FAILED -ne 0 ]; then + echo "Not all services were stopped successfully. Please check the above output for more inforamtion." +else + echo "All orphaned services successfully stopped." +fi + +for i in $(seq 1 30); do + + vol="vol$i" + + sudo umount /mnt/local-storage/$vol + + sudo rm -rf /mnt/local-storage/$vol + +done + + +kubeadm reset -y +sudo apt-get -y purge kubeadm kubectl kubelet kubernetes-cni kube* +sudo apt-get autoremove +sudo rm -rf ~/.kube diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh new file mode 100644 index 00000000..71720013 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh @@ -0,0 +1,337 @@ +#!/bin/bash +set -Eeuo pipefail + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi + +# This is a script to create single-node Kubernetes cluster & deploy BDC on it. +# +export BDCDEPLOY_DIR=bdcdeploy + +# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and knox. +# +while true; do + read -s -p "Create Password for Big Data Cluster: " password + echo + read -s -p "Confirm your Password: " password2 + echo + [ "$password" = "$password2" ] && break + echo "Password mismatch. Please try again." +done + + +# Name of virtualenv variable used. +# +export VIRTUALENV_NAME="bdcvenv" +export LOG_FILE="bdcdeploy.log" +export DEBIAN_FRONTEND=noninteractive + +# Requirements file. +# +export REQUIREMENTS_LINK="https://aka.ms/azdata" + +# Kube version. +# +KUBE_DPKG_VERSION=1.15.0-00 +KUBE_VERSION=1.15.0 + +# Wait for 5 minutes for the cluster to be ready. +# +TIMEOUT=600 +RETRY_INTERVAL=5 + +# Variables for pulling dockers. +# +export DOCKER_REGISTRY="mcr.microsoft.com" +export DOCKER_REPOSITORY="mssql/bdc" +export DOCKER_TAG="2019-CTP3.2-ubuntu" + +# Variables used for azdata cluster creation. +# +export CONTROLLER_USERNAME=admin +export CONTROLLER_PASSWORD=$password +export MSSQL_SA_PASSWORD=$password +export KNOX_PASSWORD=$password +export ACCEPT_EULA=yes +export CLUSTER_NAME=mssql-cluster +export STORAGE_CLASS=local-storage +export PV_COUNT="30" + +IMAGES=( + mssql-app-service-proxy + mssql-appdeploy-init + mssql-controller + mssql-hadoop + mssql-mleap-serving-runtime + mssql-mlserver-py-runtime + mssql-mlserver-r-runtime + mssql-monitor-collectd + mssql-monitor-elasticsearch + mssql-monitor-fluentbit + mssql-monitor-grafana + mssql-monitor-influxdb + mssql-monitor-kibana + mssql-monitor-telegraf + mssql-security-knox + mssql-security-support + mssql-server-controller + mssql-server-data + mssql-service-proxy + mssql-ssis-app-runtime +) + + +# Make a directory for installing the scripts and logs. +# +mkdir -p $BDCDEPLOY_DIR +cd $BDCDEPLOY_DIR/ +touch $LOG_FILE + +{ +# Install all necessary packages: kuberenetes, docker, python3, python3-pip, request, azdata. +# +echo "" +echo "######################################################################################" +echo "Starting installing packages..." + +# Install docker. +# +apt-get update -q + +apt --yes install \ + software-properties-common \ + apt-transport-https \ + ca-certificates \ + curl + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + +apt update -q +apt-get install -q --yes docker-ce=18.06.2~ce~3-0~ubuntu --allow-downgrades +apt-mark hold docker-ce + +usermod --append --groups docker $USER + +# Install python3, python3-pip, requests. +# +apt-get install -q -y python3 +apt-get install -q -y python3-pip + +pip3 install requests --upgrade + +# Install and create virtualenv. +# +pip3 install --upgrade virtualenv +virtualenv -p python3 $VIRTUALENV_NAME +source $VIRTUALENV_NAME/bin/activate + +# Install azdata cli. +# +pip3 install -r $REQUIREMENTS_LINK +echo "Packages installed." + +# Load all prereqs for kubernetes. +# +echo "###########################################################################" +echo "Starting to setup pre-requisites for kubernetes..." + +# Setup the kubernetes preprequisites. +# +echo $(hostname -i) $(hostname) >> /etc/hosts + +swapoff -a +sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab + +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + +cat </etc/apt/sources.list.d/kubernetes.list + +deb http://apt.kubernetes.io/ kubernetes-xenial main + +EOF + +# Install docker and packages to allow apt to use a repository over HTTPS. +# +apt-get update -q + +apt-get install -q -y ebtables ethtool + +#apt-get install -y docker.ce + +apt-get install -q -y apt-transport-https + +# Setup daemon. +# +cat > /etc/docker/daemon.json < /etc/sysctl.conf +echo net.ipv6.conf.default.disable_ipv6=1 > /etc/sysctl.conf +echo net.ipv6.conf.lo.disable_ipv6=1 > /etc/sysctl.conf + + +sysctl net.bridge.bridge-nf-call-iptables=1 + +# Setting up the persistent volumes for the kubernetes. +# +for i in $(seq 1 $PV_COUNT); do + + vol="vol$i" + + mkdir -p /mnt/local-storage/$vol + + mount --bind /mnt/local-storage/$vol /mnt/local-storage/$vol + +done +echo "Kubernetes pre-requisites have been completed." + +# Setup kubernetes cluster including remove taint on master. +# +echo "" +echo "#############################################################################" +echo "Starting to setup Kubernetes master..." + +# Initialize a kubernetes cluster on the current node. +# +sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION + +mkdir -p $HOME/.kube +mkdir -p /home/$SUDO_USER/.kube + +sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.kube/config + +# To enable a single node cluster remove the taint that limits the first node to master only service. +# +master_node=`kubectl get nodes --no-headers=true --output=custom-columns=NAME:.metadata.name` +kubectl taint nodes ${master_node} node-role.kubernetes.io/master:NoSchedule- + +# Local storage provisioning. +# +kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml + +# Install the software defined network. +# +kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml + +# helm init + +kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/rbac.yaml + +# Verify that the cluster is ready to be used. +# +echo "Verifying that the cluster is ready for use..." +while true ; do + + if [[ "$TIMEOUT" -le 0 ]]; then + echo "Cluster node failed to reach the 'Ready' state. Kubeadm setup failed." + exit 1 + fi + + status=`kubectl get nodes --no-headers=true | awk '{print $2}'` + + if [ "$status" == "Ready" ]; then + break + fi + + sleep "$RETRY_INTERVAL" + + TIMEOUT=$(($TIMEOUT-$RETRY_INTERVAL)) + + echo "Cluster not ready. Retrying..." +done + + +# Install the dashbaord for kubernetes. +# +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml + +kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard +echo "Kubernetes master setup done." + +# Pull docker images of azdata. +# +echo "" +echo "############################################################################" +echo "Starting to pull docker images..." +echo "Pulling images from repository: " $DOCKER_REGISTRY"/"$DOCKER_REPOSITORY + +for image in "${IMAGES[@]}"; +do + docker pull $DOCKER_REGISTRY/$DOCKER_REPOSITORY/$image:$DOCKER_TAG + echo "Docker image" $image " pulled." +done +echo "Docker images pulled." + +# Deploy azdata bdc create cluster. +# +echo "" +echo "############################################################################" +echo "Starting to deploy azdata cluster..." + +# Command to create cluster for single node cluster. +# +azdata bdc config init --source kubeadm-dev-test --target kubeadm-custom -f +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.repository=$DOCKER_REPOSITORY" +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.registry=$DOCKER_REGISTRY" +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.imageTag=$DOCKER_TAG" +azdata bdc config replace -c kubeadm-custom/cluster.json -j "$.spec.pools[?(@.spec.type == "Data")].spec.replicas=1" +azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.data.className=$STORAGE_CLASS" +azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.logs.className=$STORAGE_CLASS" +azdata bdc create -c kubeadm-custom --accept-eula $ACCEPT_EULA +echo "Azdata cluster created." + +# Setting context to cluster. +# +kubectl config set-context --current --namespace $CLUSTER_NAME + +# Login and get endpoint list for the cluster. +# +azdata login -n $CLUSTER_NAME +azdata bdc endpoint list --output table + +if [ -d "$HOME/.azdata/" ]; then + sudo chown -R $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.azdata/ +fi + +echo "alias azdata='$BDCDEPLOY_DIR/$VIRTUALENV_NAME/bin/azdata'" >> $HOME/.bashrc +source ~/.bashrc +}| tee $LOG_FILE diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md new file mode 100644 index 00000000..2d24672b --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md @@ -0,0 +1,30 @@ +# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS + + +In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions. + +## Pre-requisites + +1. Multiple Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines +1. Designate one machine as the Kubernetes master +1. Rest of the machines will be used as the Kubernetes agents + +**NOTE: Ensure there is sufficient local storage on your agents. Each volume will use up to 10GB by default. The script creates 25 volumes. Not all of the volumes will be used since it depends on the number of pods being deployed on each agent node. It is recommended to have at least 200 GB of storage on the agent nodes** + +### Useful resources + +[Deploy SQL Server 2019 big data cluster on Kubernetes](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) + +[Creating a cluster using kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) + +[Troubleshooting kubeadm](https://kubernetes.io/docs/setup/independent/troubleshooting-kubeadm/) + +### Instructions + +1. Start a sudo shell context +1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine +1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master +1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine +1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage +1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. +1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh index dfcccf06..eb7e6f31 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh @@ -19,7 +19,7 @@ apt-get install -y kubelet=$KUBE_DPKG_VERSION kubeadm=$KUBE_DPKG_VERSION kubectl curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash . /etc/os-release -if [ "$VERSION_CODENAME" == "bionic" ]; then +if [ "$UBUNTU_CODENAME" == "bionic" ]; then modprobe br_netfilter fi sysctl net.bridge.bridge-nf-call-iptables=1 diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql index 0852b41c..de601dee 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql +++ b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql @@ -25,7 +25,7 @@ SELECT TOP(80) PERCENT SIGN(q.clicks_in_category) AS book_category , q.clicks_in_7 , q.clicks_in_8 , q.clicks_in_9 - FROM web_clickstreams_book_clicks as q + FROM web_clickstreams_hdfs_book_clicks as q '; -- Training R script that uses rxLogit function from RevoScaleR package (Microsoft R Server) to generate model to predict book_category click(s). SET @train_script = N' @@ -91,7 +91,7 @@ SELECT TOP(@top_count_value) PERCENT SIGN(q.clicks_in_category) AS book_category , q.clicks_in_7 , q.clicks_in_8 , q.clicks_in_9 - FROM web_clickstreams_book_clicks as q + FROM web_clickstreams_hdfs_book_clicks as q '; -- Scoring script that uses sklearn logistic regression model to predict book_category click(s) From a624a39ca7cf28237715d9e72afd33ad7d099ffb Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 18:20:15 -0700 Subject: [PATCH 06/29] CTP 3.2 updates --- .../features/sql-big-data-cluster/deployment/README.md | 6 +++--- .../deployment/kubeadm/ubuntu-single-node-vm/README.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/README.md b/samples/features/sql-big-data-cluster/deployment/README.md index 8caf7756..fe5ec277 100644 --- a/samples/features/sql-big-data-cluster/deployment/README.md +++ b/samples/features/sql-big-data-cluster/deployment/README.md @@ -3,14 +3,14 @@ SQL Server 2019 big data cluster is deployed as docker containers on a Kubernetes cluster. These samples provide scripts that can be used to provision a Kubernetes clusters using different environments. -## Deploy a Kubernetes cluster using kubeadm +## __[Deploy a Kubernetes cluster using kubeadm](kubeadm/)__ Use the scripts in the **kubeadm** folder to deploy a Kubernetes cluster over one or more Linux machines (physical or virtualized) using `kubeadm` utility. -## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS) +## __[Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS)](aks/)__ Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it. -## Push SQL Server big data cluster images to your own private Docker repository +## __[Push SQL Server big data cluster images to your own private Docker repository](offline/)__ Using the sample Python script in **offline** folder, you will push the necessary images required for the deployment to your own repository. \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md index 767e065a..7a07a541 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -6,7 +6,7 @@ Using this sample bash script, you will deploy a single node Kubernetes cluster ## Pre-requisites 1. A vanilla Ubuntu 16.04 or 18.04 VM. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported. -1. VM should have at least 8CPUs, 64GB RAM and 100GB disk space.After installing the images you will be left with 50GB for data/logs across all components. +1. VM should have at least 8 CPUs, 64GB RAM and 100GB disk space. After installing the images you will be left with 50GB for data/logs across all components. ## Instructions @@ -14,21 +14,21 @@ Using this sample bash script, you will deploy a single node Kubernetes cluster ``` -curl --output kickstarter-azdata.sh http://rima-5.guest.corp.microsoft.com/kickstarter-azdata.sh +curl --output setup-bdc.sh https://github.com/microsoft/sql-server-samples/blob/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh ``` 1. Make the script executable ``` -chmod +x kickstarter-azdata.sh +chmod +x setup-azdata.sh ``` 1. Run the script (make sure you are running with sudo) ``` -sudo ./kickstarter-azdata.sh +sudo ./setup-azdata.sh ``` 1. Refresh alias setup for azdata From 662cb4b266a7ec30431c7a75d3a599eace1d68c2 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Tue, 23 Jul 2019 19:11:12 -0700 Subject: [PATCH 07/29] Fix file name --- .../deployment/kubeadm/ubuntu-single-node-vm/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md index 7a07a541..3957fa84 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -21,14 +21,14 @@ curl --output setup-bdc.sh https://github.com/microsoft/sql-server-samples/blob/ ``` -chmod +x setup-azdata.sh +chmod +x setup-bdc.sh ``` 1. Run the script (make sure you are running with sudo) ``` -sudo ./setup-azdata.sh +sudo ./setup-bdc.sh ``` 1. Refresh alias setup for azdata From 639c30a4d7ec83163730d441548d602d2a1856d2 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 21:18:19 -0700 Subject: [PATCH 08/29] Fixed comments --- .../deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh index 71720013..0eafd86b 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh @@ -6,7 +6,7 @@ if [ "$EUID" -ne 0 ] exit fi -# This is a script to create single-node Kubernetes cluster & deploy BDC on it. +# This is a script to create single-node Kubernetes cluster and deploy BDC on it. # export BDCDEPLOY_DIR=bdcdeploy @@ -135,7 +135,7 @@ source $VIRTUALENV_NAME/bin/activate pip3 install -r $REQUIREMENTS_LINK echo "Packages installed." -# Load all prereqs for kubernetes. +# Load all pre-requisites for Kubernetes. # echo "###########################################################################" echo "Starting to setup pre-requisites for kubernetes..." @@ -280,14 +280,14 @@ while true ; do done -# Install the dashbaord for kubernetes. +# Install the dashboard for Kubernetes. # kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard echo "Kubernetes master setup done." -# Pull docker images of azdata. +# Pull docker images of SQL Server big data cluster. # echo "" echo "############################################################################" From 3ae560bc8265af2274971d2c6c467a9e9077ad0b Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 21:40:23 -0700 Subject: [PATCH 09/29] Updated url to raw github link --- .../deployment/kubeadm/ubuntu-single-node-vm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md index 3957fa84..62634630 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -14,7 +14,7 @@ Using this sample bash script, you will deploy a single node Kubernetes cluster ``` -curl --output setup-bdc.sh https://github.com/microsoft/sql-server-samples/blob/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh +curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh ``` 1. Make the script executable From 50b2aaadcd1d0b9ccaa76d17e39a730fd3657ff2 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 22:13:29 -0700 Subject: [PATCH 10/29] Fixed comments/readme --- .../ubuntu-single-node-vm/cleanup-bdc.sh | 22 ++++++++++++++----- .../ubuntu-single-node-vm/setup-bdc.sh | 3 +-- .../deployment/kubeadm/ubuntu/README.md | 11 +++++----- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh index 0c4baba5..9a53ae8c 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh @@ -7,6 +7,7 @@ fi DIR_PREFIX=$1 kubeadm reset --force +unalias azdata systemctl stop kubelet rm -rf /var/lib/cni/ @@ -14,7 +15,8 @@ rm -rf /var/lib/etcd/ rm -rf /run/flannel/ rm -rf /var/lib/kubelet/* rm -rf /etc/cni/ -rm -rf /etc/kubernetes/* +rm -rf /etc/kubernetes/ + ip link set cni0 down #brctl delbr cni0 ip link set flannel.1 down @@ -22,7 +24,10 @@ ip link set flannel.1 down iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X rm -rf .azdata/ +rm -rf bdcdeploy/ +# Remove mounts. +# SERVICE_STOP_FAILED=0 systemctl | grep "/var/lib/kubelet/pods" | while read -r line; do @@ -58,6 +63,9 @@ else echo "All orphaned services successfully stopped." fi +# Clean the mounted volumes. +# + for i in $(seq 1 30); do vol="vol$i" @@ -68,8 +76,12 @@ for i in $(seq 1 30); do done - -kubeadm reset -y -sudo apt-get -y purge kubeadm kubectl kubelet kubernetes-cni kube* -sudo apt-get autoremove +# Reset kube +# +sudo apt-get purge -y kubeadm --allow-change-held-packages +sudo apt-get purge -y kubectl --allow-change-held-packages +sudo apt-get purge -y kubelet --allow-change-held-packages +sudo apt-get purge -y kubernetes-cni --allow-change-held-packages +sudo apt-get purge -y kube* --allow-change-held-packages +sudo apt -y autoremove sudo rm -rf ~/.kube diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh index 0eafd86b..07112b25 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh @@ -10,7 +10,7 @@ fi # export BDCDEPLOY_DIR=bdcdeploy -# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and knox. +# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox. # while true; do read -s -p "Create Password for Big Data Cluster: " password @@ -333,5 +333,4 @@ if [ -d "$HOME/.azdata/" ]; then fi echo "alias azdata='$BDCDEPLOY_DIR/$VIRTUALENV_NAME/bin/azdata'" >> $HOME/.bashrc -source ~/.bashrc }| tee $LOG_FILE diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md index 2d24672b..9c9cfd66 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md @@ -1,11 +1,10 @@ # Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS - In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions. ## Pre-requisites -1. Multiple Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines +1. Multiple Ubuntu Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines 1. Designate one machine as the Kubernetes master 1. Rest of the machines will be used as the Kubernetes agents @@ -22,9 +21,9 @@ In this example, we will deploy Kubernetes over multiple Linux machines (physica ### Instructions 1. Start a sudo shell context -1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine -1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master +1. Execute [setup-k8s-prereqs.sh](setup-k8s-prereqs.sh/) script on each machine +1. Execute [setup-k8s-master.sh](setup-k8s-master.sh/) script on the machine designated as Kubernetes master 1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine -1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage -1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. +1. Execute [setup-volumes-agent.sh](setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage +1. Execute ***kubectl apply -f local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner. 1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions) From 99b3ca1919c77c3187df2cb23d934fd041e1af2a Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 23 Jul 2019 22:27:25 -0700 Subject: [PATCH 11/29] Update README.md --- .../deployment/kubeadm/ubuntu-single-node-vm/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md index 62634630..969fa87d 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -17,21 +17,21 @@ Using this sample bash script, you will deploy a single node Kubernetes cluster curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh ``` -1. Make the script executable +2. Make the script executable ``` chmod +x setup-bdc.sh ``` -1. Run the script (make sure you are running with sudo) +3. Run the script (make sure you are running with sudo) ``` sudo ./setup-bdc.sh ``` -1. Refresh alias setup for azdata +4. Refresh alias setup for azdata ``` From 8806368648d84b0e9c211db7969f27774544be57 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 24 Jul 2019 09:41:40 -0700 Subject: [PATCH 12/29] Added note about VM setting --- .../kubeadm/ubuntu-single-node-vm/README.md | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md index 62634630..e60bcd88 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -5,37 +5,49 @@ Using this sample bash script, you will deploy a single node Kubernetes cluster ## Pre-requisites -1. A vanilla Ubuntu 16.04 or 18.04 VM. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported. -1. VM should have at least 8 CPUs, 64GB RAM and 100GB disk space. After installing the images you will be left with 50GB for data/logs across all components. +1. A vanilla Ubuntu 16.04 or 18.04 virtual or physical machine. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported. +1. Machine should have at least 8 CPUs, 64GB RAM and 100GB disk space. After installing the images you will be left with 50GB for data/logs across all components. +1. Update existing packages using commands below to ensure that the OS image is up to date -## Instructions +``` bash +sudo apt update&&apt upgrade -y +sudo systemctl reboot +``` + +## Recommended Virtual Machine settings + +1. Use static memory configuration for the virtual machine. For example, in hyper-v installations do not use dynamic memory allocation but instead allocate the recommended 64 GB or higher. + +1. Use checkpoint or snapshot capability in your hyper visor so that you can rollback the virtual machine to a clean state. + +## Instructions to deploy SQL Server big data cluster 1. Download the script on the VM you are planning to use for the deployment -``` - +``` bash curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh ``` 1. Make the script executable -``` - +``` bash chmod +x setup-bdc.sh ``` 1. Run the script (make sure you are running with sudo) -``` - +``` bash sudo ./setup-bdc.sh ``` 1. Refresh alias setup for azdata -``` - +``` bash source ~/.bashrc ``` When prompted, provide your input for the password that will be used for all external endpoints: controller, SQL Server master and gateway. The password should be sufficiently complex based on existing rules for SQL Server password. The controller username is defaulted to *admin*. + +## Cleanup + +1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capabiility in your hyper-visor to rollback the virtual machine to a clean state. From 05508da6688283135eeb2b4c0bf1b4c13160d3d7 Mon Sep 17 00:00:00 2001 From: Mark Ghanayem <22989000+markingmyname@users.noreply.github.com> Date: Thu, 25 Jul 2019 07:14:58 -0500 Subject: [PATCH 13/29] added JSON files to the sql-assessment-api folder --- .../DisablingBuiltInChecks_sample.json | 12 + .../MakingCustomChecks_sample.json | 60 + samples/manage/sql-assessment-api/README.md | 72 + samples/manage/sql-assessment-api/config.json | 1534 +++++++++++++++++ 4 files changed, 1678 insertions(+) create mode 100644 samples/manage/sql-assessment-api/DisablingBuiltInChecks_sample.json create mode 100644 samples/manage/sql-assessment-api/MakingCustomChecks_sample.json create mode 100644 samples/manage/sql-assessment-api/config.json diff --git a/samples/manage/sql-assessment-api/DisablingBuiltInChecks_sample.json b/samples/manage/sql-assessment-api/DisablingBuiltInChecks_sample.json new file mode 100644 index 00000000..24982831 --- /dev/null +++ b/samples/manage/sql-assessment-api/DisablingBuiltInChecks_sample.json @@ -0,0 +1,12 @@ +{ + "checks":[ + { + "id": "SqlServer.Database.AutoCreateStats", + "enabled": false + }, + { + "select": ["TraceFlag"], + "enabled": false + } + ] +} diff --git a/samples/manage/sql-assessment-api/MakingCustomChecks_sample.json b/samples/manage/sql-assessment-api/MakingCustomChecks_sample.json new file mode 100644 index 00000000..732753f5 --- /dev/null +++ b/samples/manage/sql-assessment-api/MakingCustomChecks_sample.json @@ -0,0 +1,60 @@ +{ + "checks":[ + { + "target": { + "type": "Database", + "platform": "Windows" + }, + "id": "Custom_SqlServer.Database.AutoUpdateStats", + "tags": [ "InternalBestPracticeSet", "Performance" ], + "displayName": "Auto-Update Statistics should be on", + "description": "The query optimizer needs up-to-date and accurate statistics in order to generate good plans. In most cases, it's best to let SQL Server maintain the statistics. If you turn 'Auto Create Stats' and 'Auto Update Stats' off, then it is up to you to keep the statistics up-to-date somehow. Failure to do so will lead to poor query performance. Most applications should have these options ON.\n \n When the Auto Update Statistics setting is ON, the query optimizer updates statistics when they are used by a query and when they might be out-of-date. Statistics become out-of-date after insert, update, delete, or merge operations change the data distribution in the table or indexed view. The query optimizer determines when statistics might be out-of-date by counting the number of data modifications since the last statistics update and comparing the number of modifications to a threshold. The threshold is based on the number of rows in the table or indexed view. The query optimizer checks for out-of-date statistics before compiling a query and before executing a cached query plan. Before compiling a query, the query optimizer uses the columns, tables, and indexed views in the query predicate to determine which statistics might be out-of-date. Before executing a cached query plan, the Database Engine verifies that the query plan references up-to-date statistics. The AUTO_UPDATE_STATISTICS option applies to statistics created for indexes, single-columns in query predicates, and statistics that are created by using the CREATE STATISTICS statement. This option also applies to filtered statistics.", + "message": "Turn Auto-Update Statistics option on to improve query performance.", + "helpLink": "https://blogs.msdn.microsoft.com/buckwoody/2009/08/18/sql-server-best-practices-auto-create-and-auto-update-statistics-should-be-on-most-of-the-time/", + "probes": [ "DatabaseConfiguration" ], + "condition": "@is_auto_update_stats_on" + }, + { + "target": { + "type": "Database", + "version": "[12.0,)", + "platform": "Windows", + "name": { "not": "/^(master|msdb)$/" } + }, + "id": "Custom_SqlServer.Database.QueryStoreOn", + "tags": [ "InternalBestPracticeSet", "Performance" ], + "displayName": "Query Store should be on", + "description": "The SQL Server Query Store feature provides you with insight on query plan choice and performance. It simplifies performance troubleshooting by helping you quickly find performance differences caused by query plan changes. Query Store automatically captures a history of queries, plans, and runtime statistics, and retains these for your review. It separates data by time windows so you can see database usage patterns and understand when query plan changes happened on the server.", + "message": "Turn Query Store option on to improve query performance troubleshooting.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store", + "probes": [ "DatabaseConfiguration" ], + "condition": "@is_query_store_on" + } + ], + "probes":{ + "DatabaseConfiguration": [ + { + "type": "SQL", + "target": { + "type": "Database", + "version": "(,12.0)", + "platform": "Windows" + }, + "implementation": { + "query": "SELECT db.[is_auto_create_stats_on] AS is_auto_create_stats_on, db.[is_auto_update_stats_on] AS is_auto_update_stats_on, 0 AS is_query_store_on FROM sys.databases AS db WHERE db.[name]='@DatabaseName'" + } + }, + { + "type": "SQL", + "target": { + "type": "Database", + "version": "[12.0,)", + "platform": "Windows" + }, + "implementation": { + "query": "SELECT db.[is_auto_create_stats_on] AS is_auto_create_stats_on, db.[is_auto_update_stats_on] AS is_auto_update_stats_on, db.[is_query_store_on] AS is_query_store_on FROM sys.databases AS db WHERE db.[name]='@DatabaseName'" + } + } + ] + } +} diff --git a/samples/manage/sql-assessment-api/README.md b/samples/manage/sql-assessment-api/README.md index e69de29b..0d93c9dc 100644 --- a/samples/manage/sql-assessment-api/README.md +++ b/samples/manage/sql-assessment-api/README.md @@ -0,0 +1,72 @@ +# SQL Assessment API samples + +Contains samples for customizing SQL Assessment API. Learn more about the API on the [SQL Assessment API docs page](https://docs.microsoft.com/en-us/sql/sql-assessment-api/sql-assessment-api-overview). + +## config.json + +This is the default set of checks shipped with SQL Assessment API. Feel free to open issues to have us fix or add checks. Also, we're happy to see your pull requests to this file. + +## DisablingBuiltInChecks_sample.json + +Contains two parts. First shows how you can disable a specified check by its ID. The second disables all the checks with the "TraceFlag" tag. + +## MakingCustomChecks_sample.json + +Demonstrates how to make a custom rule set containing two checks. The sample contains two sections: `checks` and `probes`. `Checks` is for check (or rule) definitions. Usually, checks or rules are best practices or a company's internal policies that should be applied to SQL Server. Here's one of the checks from this sample with comments on each property: + +``` +{ + "target": { //Object to describe which SQL Server object this check is applied. + "type": "Database", //This check targets at Database object. + "version": "[12.0,)", //Applies to SQL Server 2014 and higher. + //Another example: "[12.0,13.0)" reads as "any SQL Server with version >= 12.0 and < 13.0. + "platform": "Windows", //Applies to SQL Server on Windows. + "name": { "not": "/^(master|msdb)$/" } //Applies to any database but master and msdb. + }, + "id": "CustomCheck1", //Check ID. + "tags": [ "InternalBestPracticeSet", "Performance" ], //Tags combine checks in different subsets. + "displayName": "Query Store should be on", //Short name for check. + "description": "The SQL Server Query Store feature provides you with insight on query plan choice and performance. It simplifies performance troubleshooting by helping you quickly find performance differences caused by query plan changes. /n Query Store automatically captures a history of queries, plans, and runtime statistics, and retains these for your review. It separates data by time windows so you can see database usage patterns and understand when query plan changes happened on the server.", + //Some more detailed explanation of the best practice or policy. + "message": "Turn Query Store option on to improve query performance troubleshooting.", + //Usually, it's for recommendation what the user should do if the check fires up + "helpLink": "https://docs.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store", + //Reference material + "probes": [ "DatabaseConfiguration" ], //List of probes that are used to get the required data for this check. + //Probes will be explained below. + "condition": "@is_query_store_on" //Check will pass if condition is true. Otherwise, the check fires up. +} +``` + +`Probes` describe how and where get required data to perform a check. For this, you can use T-SQL queries as well as methods from assemblies. The probe below uses a T-SQL query. +``` +"probes":{ + "DatabaseConfiguration": [ //Probe name that is used to reference the probe from a check. + //Probe can have a few implementations that will be used for different targets. + //This probe has two implementations for different version of SQL Server. + { + "type": "SQL", //Probe uses a T-SQL query to get the required data + "target": { + "type": "Database", //Targets at database + "version": "(,12.0)", //This implementation is for SQL Server before 2014 + "platform": "Windows" //Targets at SQL on Windows + }, + "implementation": { //Implementation object with a T-SQL query. + //sys.databases of SQL Server before 2014 doesn't have the field is_query_store_on so we replace it with 0. + "query": "SELECT db.[is_auto_create_stats_on] AS is_auto_create_stats_on, db.[is_auto_update_stats_on] AS is_auto_update_stats_on, 0 AS is_query_store_on FROM sys.databases AS db WHERE db.[name]='@DatabaseName'" + } + }, + { //Second implementation + "type": "SQL", + "target": { + "type": "Database", + "version": "[12.0,)", //This implementation is for SQL Server 2014 and up. + "platform": "Windows" + }, + "implementation": { //Query of the second implementation. + "query": "SELECT db.[is_auto_create_stats_on] AS is_auto_create_stats_on, db.[is_auto_update_stats_on] AS is_auto_update_stats_on, db.[is_query_store_on] AS is_query_store_on FROM sys.databases AS db WHERE db.[name]='@DatabaseName'" + } + } + ] +} +``` diff --git a/samples/manage/sql-assessment-api/config.json b/samples/manage/sql-assessment-api/config.json new file mode 100644 index 00000000..cc14c393 --- /dev/null +++ b/samples/manage/sql-assessment-api/config.json @@ -0,0 +1,1534 @@ +{ + "version": "0.3", + "checks": [ + { + "target": { + "type": "Database", + "platform": "Windows" + }, + "id": "SqlServer.Database.AutoCreateStats", + "tags": [ "Performance" ], + "displayName": "Auto-Create Statistics should be on", + "description": "The Query Optimizer determines whether an index is useful for a specific query by evaluating the stored statistics. If the statistics become out of date and significant changes have occurred against the underlying data, this can result in less than optimal query performance. In most cases, it's best to let SQL Server maintain the statistics. If you turn 'Auto Create Stats' and 'Auto Update Stats' off, then it is up to you to keep the statistics up-to-date somehow. Failure to do so will lead to poor query performance. Most applications should have these options ON.\n \n When the Auto Create statistics setting is ON, the Query Optimizer creates statistics on one or more columns of a table or an indexed view, as necessary, to improve query plans and query performance.", + "message": "Turn Auto-Create Statistics option on to improve query performance.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/statistics/statistics#CreateStatistics", + "probes": [ "DatabaseConfiguration" ], + "condition": "@is_auto_create_stats_on" + }, + { + "target": { + "type": "Database", + "platform": "Windows" + }, + "id": "SqlServer.Database.AutoUpdateStats", + "tags": [ "Performance" ], + "displayName": "Auto-Update Statistics should be on", + "description": "The Query Optimizer determines whether an index is useful for a specific query by evaluating the stored statistics. If the statistics become out of date and significant changes have occurred against the underlying data, this can result in less than optimal query performance. In most cases, it's best to let SQL Server maintain the statistics. If you turn 'Auto Create Stats' and 'Auto Update Stats' off, then it is up to you to keep the statistics up-to-date somehow. Failure to do so will lead to poor query performance. Most applications should have these options ON.\n \n When the Auto Update Statistics setting is ON, the Query Optimizer updates statistics when they are used by a query and when they might be out-of-date. Statistics become out-of-date after insert, update, delete, or merge operations change the data distribution in the table or indexed view. The Query Optimizer determines when statistics might be out-of-date by counting the number of data modifications since the last statistics update and comparing the number of modifications to a threshold. The threshold is based on the number of rows in the table or indexed view. The Query Optimizer checks for out-of-date statistics before compiling a query and before executing a cached query plan. Before compiling a query, the Query Optimizer uses the columns, tables, and indexed views in the query predicate to determine which statistics might be out-of-date. Before executing a cached query plan, the Database Engine verifies that the query plan references up-to-date statistics. The AUTO_UPDATE_STATISTICS option applies to statistics created for indexes, single-columns in query predicates, and statistics that are created by using the CREATE STATISTICS statement. This option also applies to filtered statistics.", + "message": "Turn Auto-Update Statistics option on to improve query performance.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/statistics/statistics#UpdateStatistics", + "probes": [ "DatabaseConfiguration" ], + "condition": "@is_auto_update_stats_on" + }, + { + "target": { + "type": "Database", + "version": "[12.0,)", + "platform": "Windows", + "name": { "not": "/^(master|tempdb)$/" } + }, + "id": "SqlServer.Database.QueryStoreOn", + "tags": [ "Performance" ], + "displayName": "Query Store should be on", + "description": "The Query Store feature provides you with insight on query plan choice and performance. It simplifies performance troubleshooting by helping you quickly find performance differences caused by query plan changes. Query Store automatically captures a history of queries, plans, and runtime statistics, and retains these for your review. It separates data by time windows so you can see database usage patterns and understand when query plan changes happened on the server.", + "message": "Enable the Query Store to improve performance troubleshooting.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store", + "probes": [ "DatabaseConfiguration" ], + "condition": "@is_query_store_on" + }, + { + "target": { + "type": "Server", + "version": [ + "[11.0.3368,12.0)", + "[12.0.2480,13.0)", + "[13.0,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.174", + "tags": [ "TraceFlag", "Compression", "TF174", "Memory" ], + "displayName": "TF 174 increases the plan cache bucket count", + "description": "Trace Flag 174 increases the SQL Server plan cache bucket count from 40,009 to 160,001 on 64-bit systems.\n When the SQL Server plan cache reaches its entry limit, plans that have low cost must be evicted in order to insert new plans. This can cause severe contention on the SOS_CACHESTORE spinlock and a high CPU usage occurs in SQL Server.\n On 64-bit systems, the number of buckets for the SQL Server plan cache is 40,009. Therefore, the maximum number of entries that can fit inside the SQL Server plan cache is 160,036. Enabling trace flag 174 on high performance systems increases the size of the cache and can avoid SOS_CACHESTORE spinlock contention.", + "message": "Consider enabling trace flag 174 to increase the plan cache bucket count.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 174, "@TraceFlag" ] + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.634", + "tags": [ "TraceFlag", "Compression", "TF634", "Memory" ], + "displayName": "TF 634 disables background columnstore compression", + "description": "Trace Flag 634 disables the background columnstore compression task.\n SQL Server periodically runs the Tuple Mover background task that compresses columnstore index rowgroups with uncompressed data, one such rowgroup at a time.\n Columnstore compression improves query performance but also consumes system resources.\n You can control the timing of columnstore compression manually, by disabling the background compression task with trace flag 634, and then explicitly invoking ALTER INDEX...REORGANIZE or ALTER INDEX...REBUILD at the time of your choice.", + "message": "Trace Flag 634 disables the background columnstore compression task. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 634, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.652", + "tags": [ "TraceFlag", "TF652", "Performance", "Memory" ], + "displayName": "TF 652 disables page pre-fetching scans", + "description": "Trace Flag 652 disables page pre-fetching for scans.\n The read-ahead process reads the full extent of a referenced database page into the buffer pool, before those contiguous database pages are consumed by scans.\n Enabling the trace flag can lead to performance issues on queries that leverage pre-fetching.", + "message": "Trace Flag 652 disables Page Pre-fetching Scans. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "http://support.microsoft.com/kb/920093", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 652, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.661", + "tags": [ "TraceFlag", "TF661", "Performance" ], + "displayName": "TF 661 disables Ghost Cleanup Task", + "description": "Trace Flag 661 disables the ghost record removal process. A ghost record is the result of a delete operation.\n When you delete a record, the deleted record is kept as a ghost record. Later, the deleted record is purged by the ghost record removal process. When you disable this process, the deleted record is not purged.\n Therefore, the space that the deleted record consumes is not freed. This behavior affects space consumption and the performance of scan operations.\n If you turn off this trace Flag, the ghost record removal process works correctly.", + "message": "Trace flag 661 disables the background ghost cleanup task. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/920093", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 661, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "version": "[11.0,)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.834", + "tags": [ "TraceFlag", "TF834", "Performance" ], + "displayName": "TF 834 enables large-page allocations", + "description": "Trace Flag 834 causes the server to use large-page memory (LPM) model for the buffer pool allocations. Consider enabling TF 834 if you have a server that has a lot of memory, particularly for an analytical or data warehousing workload.\n trace flag 834 is NOT recommended on SQL Server instances that use columnstore indexes.", + "message": "Enable trace flag 834 to use large-page allocations to improve analytical and data warehousing workloads.", + "helpLink": "https://support.microsoft.com/help/4465518", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 834, "@TraceFlag" ] + } + }, + { + "target": { + "type": "Server", + "version": "(,11.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.834.1", + "tags": [ "TraceFlag", "TF834", "Performance", "Memory" ], + "displayName": "TF 834 enables large-page allocations", + "description": "Trace Flag 834 causes the server to use large-page memory (LPM) model for the buffer pool allocations. Consider enabling TF 834 if you have a server that has a lot of memory, particularly for an analytical or data warehousing workload.", + "message": "Consider enabling trace flag 834 to use large-page allocations to improve analytical and data warehousing workloads.", + "helpLink": "https://support.microsoft.com/help/2964518", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 834, "@TraceFlag" ] + } + }, + { + "target": { + "type": "Server", + "version": "[11.0,)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.845", + "tags": [ "TraceFlag", "TF845", "Memory" ], + "displayName": "TF 845 is not needed in SQL Server 2012 and higher versions", + "description": "Trace Flag 845 supports locking pages in memory in SQL Server Standard edition, which is needed together with trace flag 834 to use the large-page memory (LPM) model for the buffer pool allocations.\n Consider enabling TF 834 if you have a server that has a lot of memory, particularly for an analytical or data warehousing workload.\n Starting with SQL Server 2012 this behavior is enabled by default for Standard edition, and trace flag 845 must not be used.", + "message": "Disable trace flag 845 in SQL Server 2012 and higher versions.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 845, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.902", + "tags": [ "TraceFlag", "TF902", "Update" ], + "displayName": "TF 902: Database Upgrade Bypass should be disabled", + "description": "Trace Flag 902 bypasses execution of database upgrade script when installing a Cumulative Update or Service Pack.\n This trace flag is not supported to run it continuously in a production environment.\n If you encounter an error during script upgrade mode, it is recommended to contact Microsoft SQL Customer Service and Support (CSS) for further guidance.", + "message": "Disable trace flag 902 which bypasses database upgrade.", + "helpLink": "https://support.microsoft.com/help/2163980", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 902, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[13.0,]", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1117", + "tags": [ "TraceFlag", "TF1117", "FileGroup" ], + "displayName": "TF 1117 has no effect in SQL Server 2016 and higher versions", + "description": "Trace Flag 1117 initiates the growth of every file in the filegroup, when a file in the filegroup meets the autogrow threshold, and together with trace flag 1118, can help reduce allocation contention in the SQL Server TempDB database.\n Starting with SQL Server 2016, this behavior is controlled by the AUTOGROW_SINGLE_FILE and AUTOGROW_ALL_FILES options of ALTER DATABASE syntax.", + "message": "Disable trace flag 1117 in SQL Server 2016 and higher versions.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/statements/alter-database-transact-sql-file-and-filegroup-options", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1117, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[,13.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1117.1", + "tags": [ "TraceFlag", "TF1117", "FileGroup" ], + "displayName": "TF 1117 Enables Filegroup-level Autogrow", + "description": "Trace Flag 1117 initiates the growth of every file in the filegroup, when a file in the filegroup meets the autogrow threshold, and together with trace flag 1118, can help reduce allocation contention in the SQL Server TempDB database.\n This trace flag affects all databases and is recommended only if every database is safe to be grow all files in a filegroup by the same amount.", + "message": "Consider enabling trace flag 1117 to enable filegroup autogrow.", + "helpLink": "https://support.microsoft.com/help/2154845", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 1117, "@TraceFlag" ] + } + }, + + { + "target": { + "type": "Server", + "version": "[13.0,]", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1118", + "tags": [ "TraceFlag", "TF1118", "Memory", "Performance" ], + "displayName": "TF 1118 has no effect in SQL Server 2016 and higher versions", + "description": "Trace Flag 1118 forces page allocations on uniform extents instead of mixed extents, and together with trace flag 1117, can help reduce allocation contention in the SQL Server TempDB database.\n When a new object is created, by default, the first eight pages are allocated from different extents (mixed extents).\n Afterwards, when more pages are needed, those are allocated from that same extent (uniform extent).\n The SGAM page is used to track these mixed extents, so can quickly become a bottleneck when numerous mixed page allocations are occurring.\n This trace flag allocates all eight pages from the same extent when creating new objects, minimizing the need to scan the SGAM page and forces uniform extent allocations instead of mixed page allocations.\n Starting with SQL Server 2016, this behavior is controlled by the SET MIXED_PAGE_ALLOCATION option of ALTER DATABASE syntax.", + "message": "Disable trace flag 1118 in SQL Server 2016 and higher versions.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/statements/alter-database-transact-sql-file-and-filegroup-options", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1118, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[,13.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1118.1", + "tags": [ "TraceFlag", "TF1118", "Memory", "Performance" ], + "displayName": "TF 1118 Disables Single Page Allocations", + "description": "Trace Flag 1118 forces page allocations on uniform extents instead of mixed extents, and together with trace flag 1117, can help reduce allocation contention in the SQL Server TempDB database.\n When a new object is created, by default, the first eight pages are allocated from different extents (mixed extents).\n Afterwards, when more pages are needed, those are allocated from that same extent (uniform extent).\n The SGAM page is used to track these mixed extents, so can quickly become a bottleneck when numerous mixed page allocations are occurring.\n This trace flag allocates all eight pages from the same extent when creating new objects, minimizing the need to scan the SGAM page and forces uniform extent allocations instead of mixed page allocations.", + "message": "Consider enabling trace flag 1118 to force page allocations on uniform extents.", + "helpLink": "https://support.microsoft.com/help/328551", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 1118, "@TraceFlag" ] + } + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1204", + "tags": [ "TraceFlag", "TF1204", "Lock" ], + "displayName": "TF 1204 returns deadlock information", + "description": "Trace Flag 1204 returns the resources and types of locks that are participating in a deadlock, including the affected command.\n The resulting information is recorded in the SQL Server Errorlog.", + "message": "Trace Flag 1204 returns deadlock information.", + "helpLink": "https://support.microsoft.com/help/832524", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1204, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1211", + "tags": [ "TraceFlag", "TF1211", "Lock" ], + "displayName": "TF 1211: Lock Escalation should be enabled", + "description": "Trace Flag 1211 disables lock escalation based on memory pressure, or based on number of locks. Because SQL Server will not escalate row or page locks to table locks, this can generate an excessive number of locks. If the lock memory grows large enough, attempts to allocate additional locks for any query may fail and cause performance issues.\n This trace flag should not be used in order to avoid inability to allocate lock resources because of insufficient memory. If lock escalation needs to be disabled, use trace flag 1224 instead.", + "message": "Disable trace flag 1211 to allow lock escalation.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1211, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1222", + "tags": [ "TraceFlag", "TF1222", "Lock" ], + "displayName": "TF 1222 returns deadlock information", + "description": "Trace Flag 1222 returns the resources and types of locks that are participating in a deadlock, including the affected command.\n The resulting information is recorded in the SQL Server Errorlog, in XML format that does not comply with any XSD schema.", + "message": "Trace Flag 1222 returns deadlock information.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1222, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1224", + "tags": [ "TraceFlag", "TF1224", "Lock" ], + "displayName": "TF 1224 should be disabled", + "description": "Trace Flag 1224 disables lock escalation based on the number of locks, unless SQL Server is under memory pressure. Because SQL Server will not escalate row or page locks to table locks, this can generate an excessive number of locks.", + "message": "Disable trace flag 1224 to allow lock escalation.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1224, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1229", + "tags": [ "TraceFlag", "TF1229", "Lock" ], + "displayName": "TF 1229: Lock Partitioning should be enabled", + "description": "Trace Flag 1229 disables lock partitioning regardless of the number of CPUs. By default, SQL Server enables lock partitioning when a server has 16 or more CPUs, to improve the scalability characteristics of larger systems.\n Disabling lock partitioning may cause can cause spinlock contention and poor performance, or unexpected behaviors when switching partitions.", + "message": "Disable trace flag 1229 to allow lock partitioning.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1229, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": [ "[9.0,11.0.6020)", "[12.0,12.0.4100)" ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1236", + "tags": [ "TraceFlag", "TF1236", "Lock" ], + "displayName": "TF 1236: Database Lock Partitioning should be enabled", + "description": "Trace Flag 1236 enables database-level lock partitioning.\n Lock Partitioning is utilized to improve the scalability characteristics on larger systems.", + "message": "Enable trace Flag 1236 to allow database lock partitioning.", + "helpLink": "https://support.microsoft.com/help/2926217", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 1236, "@TraceFlag" ] + } + }, + + { + "target": { + "type": "Server", + "version": [ "[11.0.6020,12.0)", "[12.0.4100,)" ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1236.1", + "tags": [ "TraceFlag", "TF1236", "Lock" ], + "displayName": "TF 1236 is not needed in SQL Server 2012 SP3, SQL Server 2014 SP1 and higher versions.", + "description": "Trace Flag 1236 enables database-level lock partitioning.\n Starting with SQL Server 2012 SP3 and SQL Server 2014 SP1, this behavior is controlled by the engine and trace flag 1236 has no effect.", + "message": "Disable trace flag 1236 in SQL Server 2012 SP3, SQL Server 2014 SP1 and higher versions.", + "helpLink": "https://support.microsoft.com/help/2926217", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1236, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.1462", + "tags": [ "TraceFlag", "TF1462", "Compression" ], + "displayName": "TF 1462 should be disabled to allow Log Stream Compression", + "description": "Trace Flag 1462 disables log stream compression for asynchronous availability groups.\n This feature is enabled by default on asynchronous replicas to optimize network bandwidth.", + "message": "Disable trace flag 1462 to allow log stream compression.", + "helpLink": "https://docs.microsoft.com/sql/database-engine/availability-groups/windows/tune-compression-for-availability-group", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 1462, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "version": "[12.0,13.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2312", + "tags": [ "TraceFlag", "TF2312", "Optimizer" ], + "displayName": "TF 2312 sets the default Cardinality Estimation model", + "description": "Trace Flag 2312 sets the Query Optimizer cardinality estimation model to the default version of the database compability level of SQL Server 2014 and higher versions.\n If the database compatibility level is lower than 120, enabling trace flag 2312 uses the cardinality estimation model of SQL Server 2014 (120).", + "message": "Trace Flag 2312 enables the default cardinality estimation model to SQL Server 2014 or above versions, dependent of the compatibility level of the database.", + "helpLink": "https://support.microsoft.com/help/2801413", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2312, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[13.0,]", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2312.1", + "tags": [ "TraceFlag", "TF2312", "Optimizer" ], + "displayName": "TF 2312 sets the default Cardinality Estimation model", + "description": "Trace Flag 2312 sets the Query Optimizer cardinality estimation model to the default version of the database compability level of SQL Server 2014 and higher versions.\n If the database compatibility level is lower than 120, enabling trace flag 2312 uses the cardinality estimation model of SQL Server 2014 (120).", + "message": "Trace Flag 2312 enables the default cardinality estimation model to SQL Server 2014 or above versions, dependent of the compatibility level of the database.", + "helpLink": "https://support.microsoft.com/help/2801413", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2312, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[,12.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2312.2", + "tags": [ "TraceFlag", "TF2312", "Optimizer" ], + "displayName": "TF 2312 sets the default Cardinality Estimation model", + "description": "Trace Flag 2312 sets the Query Optimizer cardinality estimation model to the default version of the database compability level of SQL Server 2014 and higher versions.\n If the database compatibility level is lower than 120, enabling trace flag 2312 uses the cardinality estimation model of SQL Server 2014 (120).", + "message": "Trace Flag 2312 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2801413", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2312, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[9.0,10.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2330", + "tags": [ "TraceFlag", "TF2330" ], + "displayName": "TF 2330 disables recording of index usage stats", + "description": "Trace Flag 2330 disables recording of index usage stats, which could lead to a non-yielding condition in SQL 2005.", + "message": "Disable trace flag 2330 in SQL Server 2008 and higher versions.", + "helpLink": "", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2330, "@TraceFlag" ] + } + } + }, + + { + "target": { + "type": "Server", + "version": "[10.0,]", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2330.1", + "tags": [ "TraceFlag", "TF2330" ], + "displayName": "TF 2330 disables recording of index usage stats", + "description": "Trace Flag 2330 disables recording of index usage stats, which could lead to a non-yielding condition in SQL 2005.\n This trace flag should be disabled in SQL Server 2008 and higher versions.", + "message": "Disable trace flag 2330 in SQL Server 2008 and higher versions.", + "helpLink": "", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 2330, "@TraceFlag" ] } + } + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2340", + "tags": [ "TraceFlag", "TF2340", "Optimizer" ], + "displayName": "TF 2340 disables Batch Sorts for optimized nested loops joins", + "description": "Trace Flag 2340 causes SQL Server not to use a sort operation (batch sort) for optimized nested loops joins when generating a plan.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 2340 disables Batch Sorts for optimized nested loops joins.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2340, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "version": "[10.5.2500,13.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2371", + "tags": [ "TraceFlag", "TF2371", "Performance" ], + "displayName": "TF 2371 enables a linear recompilation threshold for statistics", + "description": "Trace Flag 2371 causes SQL Server to change the fixed update statistics threshold to a linear update statistics threshold.\n This is especially useful to keep statistics updated on large tables.", + "message": "Enable trace Flag 2371 to allow a linear recompilation threshold for statistics.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "in": [ 2371, "@TraceFlag" ] + } + }, + { + "target": { + "type": "Server", + "version": "[13.0,]", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2371.1", + "tags": [ "TraceFlag", "TF2371", "Performance" ], + "displayName": "TF 2371 has no effect in SQL Server 2016 and higher versions", + "description": "Trace Flag 2371 causes SQL Server to change the fixed update statistics threshold to a linear update statistics threshold.\n This is especially useful to keep statistics updated on large tables. \n Starting with SQL Server 2016, for databases using compatibility level 130 and above, this behavior is controlled by the engine and trace flag 2371 has no effect.", + "message": "Disable trace flag 2371 in SQL Server 2016 and higher versions when all databases are at compatibility level 130 and above.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2371, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2389", + "tags": [ "TraceFlag", "TF2389" ], + "displayName": "TF 2389 enables automatic statistics for Ascending Keys", + "description": "Trace Flag 2389 enables automatically generated quick statistics for ascending keys (histogram amendment).\n This trace flag is recommended to be set in cases where queries access newly inserted ascending key values (such as an IDENTITY column), but the new values are not yet updated in the statistics histogram for the affected columns. In this case the histogram used to estimate cardinality will be adjusted at query compile time.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 2389 enables automatic statistics for ascending keys. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2801413", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2389, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2390", + "tags": [ "TraceFlag", "TF2390" ], + "displayName": "TF 2390 enables automatic statistics for Ascending or Unknown Keys", + "description": "Trace Flag 2390 enables automatically generated quick statistics for ascending or unknown keys (histogram amendment).\n This trace flag is recommended to be set in cases where queries access newly inserted ascending key values (such as an IDENTITY column), but the new values are not yet updated in the statistics histogram for the affected columns. In this case the histogram used to estimate cardinality will be adjusted at query compile time.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 2390 enables automatic statistics for ascending or unknown keys. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2801413", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { + "in": [ 2390, "@TraceFlag" ] + } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2528", + "tags": [ "TraceFlag", "TF2528" ], + "displayName": "TF 2528 disables parallel operations for integrity checking", + "description": "Trace Flag 2528 disables parallel checking of objects by DBCC CHECKDB, DBCC CHECKFILEGROUP, and DBCC CHECKTABLE.\n By default, the degree of parallelism is automatically determined by the query processor. The maximum degree of parallelism is configured just like that of parallel queries.\n When parallel checks are disabled, the DBCC commands will take longer to complete.\n Starting with SQL Server 2014 SP2, a MAXDOP option is available to override the max degree of parallelism configuration option of sp_configure for the DBCC statements.", + "message": "Trace Flag 2528 disables parallel operations for integrity checking.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Warning", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 2528, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2549", + "tags": [ "TraceFlag", "TF2549" ], + "displayName": "TF 2549 optimizes PHYSICAL_ONLY option of DBCC CHECKDB", + "description": "Trace Flag 2549 forces the DBCC CHECKDB command to assume each database file is on a unique disk drive but treating different physical files as one logical file.\n This trace flag can improve the performance when using the PHYSICAL_ONLY option, but it's not recommended unless it is known that each file is based on a unique physical disk.", + "message": "Trace Flag 2549 forces DBCC CHECKDB to treat different physical files as one logical file.", + "helpLink": "https://support.microsoft.com/help/2634571", + "level": "Warning", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 2549, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2562", + "tags": [ "TraceFlag", "TF2562" ], + "displayName": "TF 2562 forces the DBCC CHECKDB command to execute in a single batch", + "description": "Trace Flag 2562 forces the DBCC CHECKDB command to execute in a single batch regardless of the number of indexes in the database.\n This trace flag can improve the performance when using the PHYSICAL_ONLY option, but space requirements for TempDB may increase.", + "message": "Trace Flag 2562 forces the DBCC CHECKDB command to execute in a single batch.", + "helpLink": "https://support.microsoft.com/help/2634571", + "level": "Warning", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 2562, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.2566", + "tags": [ "TraceFlag", "TF2566" ], + "displayName": "TF 2566 disables default data purity check", + "description": "Trace Flag 2566 forces the DBCC CHECKDB command to execute without data purity check unless the DATA_PURITY option is specified.\n For databases upgraded from earlier versions of SQL Server, it is recommended to run the DBCC CHECKDB WITH DATA_PURITY command at least once, to enable column-value integrity checks to happen by default.", + "message": "Trace Flag 2566 disables default data purity check.", + "helpLink": "https://support.microsoft.com/help/945770", + "level": "Warning", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 2566, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.3023", + "tags": [ "TraceFlag", "TF3023", "Backup" ], + "displayName": "TF 3023 enables Backup Checksum option by default", + "description": "Trace Flag 3023 enables the CHECKSUM option of the BACKUP command by default.\n This trace flag forces the CHECKSUM option for BACKUP command even when not explicitely set.\n Starting with SQL Server 2014, this behavior is controlled by setting the Backup Checksum Default configuration option.", + "message": "Trace Flag 3023 enables CHECKSUM option as default for BACKUP command.", + "helpLink": "https://support.microsoft.com/help/2656988", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 3023, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.3042", + "tags": [ "TraceFlag", "TF3042", "Backup" ], + "displayName": "TF 3042 bypasses the default backup compression pre-allocation algorithm", + "description": "Trace Flag 3042 bypasses the default backup compression pre-allocation algorithm to allow the backup file to grow only as needed to reach its final size.\n This trace flag can achieve space savings by allocating only the actual size required for the compressed backup, but might increase the overall backup operation time.", + "message": "Trace Flag 3042 enables the bypass of the default backup compression.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/backup-restore/backup-compression-sql-server", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 3042, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.3226", + "tags": [ "TraceFlag", "TF3226", "Backup" ], + "displayName": "TF 3226 disables ErrorLog entries for every successful backup operation", + "description": "Trace Flag 3226 prevents SQL Server from recording an entry to the Errorlog on every successful backup operation.\n Backup operations are registered in the Errorlog by default. Frequent backup operations can contribute to massive Errorlog files that become unwieldy.", + "message": "Trace Flag 3226 disables ErrorLog entries for every successful backup operation.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/database-console-commands/dbcc-traceon-trace-flags-transact-sql", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 3226, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "[9.0.4294,10.0)", + "[10.0.2766,10.50)", + "[10.50.1720,11.0)", + "[11.0.2316,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4136", + "tags": [ "TraceFlag", "TF4136", "Optimizer" ], + "displayName": "TF 4136 disables Parameter Sniffing", + "description": "Trace Flag 4136 disables parameter sniffing unless OPTION(RECOMPILE), WITH RECOMPILE or OPTIMIZE FOR value is used.\n Starting with SQL Server 2016, to accomplish this at the database level use the PARAMETER_SNIFFING option in ALTER DATABASE SCOPED CONFIGURATION.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 4136 disables parameter sniffing. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/980653", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4136, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "(,9.0.4294)", + "[10.0,10.0.2766)", + "[10.50,10.50.1720)", + "[11.0,11.0.2316)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4136.1", + "tags": [ "TraceFlag", "TF4136", "Optimizer" ], + "displayName": "TF 4136 disables Parameter Sniffing", + "description": "Trace Flag 4136 disables parameter sniffing unless OPTION(RECOMPILE), WITH RECOMPILE or OPTIMIZE FOR value is used, but does not apply to this SQL Server version.\n Starting with SQL Server 2016, to accomplish this at the database level use the PARAMETER_SNIFFING option in ALTER DATABASE SCOPED CONFIGURATION.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 4136 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/980653", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4136, "@TraceFlag" ] } + } + }, + + { + "target": { + "type": "Server", + "version": [ + "[10.0.4326,10.0.4371]", + "[10.0.5794,10.50)", + "[10.50.2806,11.0)", + "[11.0.2316,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4137", + "tags": [ "TraceFlag", "TF4137", "Optimizer" ], + "displayName": "TF 4137 causes plans to use partial correlation for filters", + "description": "Trace Flag 4137 causes SQL Server to generate a plan using minimum selectivity when estimating AND predicates for filters to account for partial correlation instead of independance, under the query optimizer cardinality estimation model of SQL Server 2012 and earlier versions (CE 70).\n Does not apply to CE version 120 or above. Use trace flag 9471 instead.\n Starting with SQL Server 2016 SP1, to accomplish this at the query level use the USE HINT 'ASSUME_MIN_SELECTIVITY_FOR_FILTER_ESTIMATES' query hint.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 4137 causes SQL Server to generate a plan using partial correlation of filters. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2658214", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4137, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "(,10.0.4326)", + "(10.0.4371,10.0.5794)", + "[10.50,10.50.2806)", + "[11.0,11.0.2316)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4137.1", + "tags": [ "TraceFlag", "TF4137", "Optimizer" ], + "displayName": "TF 4137 causes plans to use partial correlation for filters", + "description": "Trace Flag 4137 causes SQL Server to generate a plan using minimum selectivity when estimating AND predicates for filters to account for partial correlation instead of independance, under the query optimizer cardinality estimation model of SQL Server 2012 and earlier versions (CE 70). However it does not apply to this SQL Server version.\n Does not apply to CE version 120 or above. Use trace flag 9471 instead.\n Starting with SQL Server 2016 SP1, to accomplish this at the query level use the USE HINT 'ASSUME_MIN_SELECTIVITY_FOR_FILTER_ESTIMATES' query hint.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace flag 4137 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2658214", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4137, "@TraceFlag" ] } + } + }, + + { + "target": { + "type": "Server", + "version": [ + "[10.50.2817,10.50.2881]", + "[10.50.4260,11.0)", + "[11.0.2325,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4138", + "tags": [ "TraceFlag", "TF4138", "Optimizer" ], + "displayName": "TF 4138 causes plans to not use Row Goal adjustments", + "description": "Trace Flag 4138 causes SQL Server to generate a plan that does not use row goal adjustments with queries that contain TOP, OPTION (FAST N), IN, or EXISTS keywords.\n Starting with SQL Server 2016 SP1, to accomplish this at the query level use the USE HINT 'DISABLE_OPTIMIZER_ROWGOAL' query hint.\n Verify need to set a Non-default trace flag with current system build and configuration.", + "message": "Trace Flag 4138 enables plan that without row goal adjustments. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2667211", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4138, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "(,10.50.2817)", + "(10.50.2881,10.50.4260)", + "[11.0,11.0.2325)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4138.1", + "tags": [ "TraceFlag", "TF4138", "Optimizer" ], + "displayName": "TF 4138 causes plans to not use Row Goal adjustments", + "description": "Trace Flag 4138 causes SQL Server to generate a plan that does not use row goal adjustments with queries that contain TOP, OPTION (FAST N), IN, or EXISTS keywords.\n Starting with SQL Server 2016 SP1, to accomplish this at the query level use the USE HINT 'DISABLE_OPTIMIZER_ROWGOAL' query hint. However it does not apply to this SQL Server version.\n Verify need to set a Non-default trace flag with current system build and configuration.", + "message": "Trace flag 4138 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2667211", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4138, "@TraceFlag" ] } + } + }, + + { + "target": { + "type": "Server", + "version": [ + "[11.0.3431,11.0.5058)", + "[11.0.5532,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4139", + "tags": [ "TraceFlag", "TF4139" ], + "displayName": "TF 4139 enables automatic statistics for any key ordering", + "description": "Trace Flag 4139 enables automatically generated quick statistics (histogram amendment) regardless of key column status.\n This trace flag is recommended to be set in cases where queries access newly inserted key values that may change plan shape, but the new values are not yet updated in the statistics histogram for the affected columns. In this case, regardless of the leading statistics column status (ascending, descending, or stationary), the histogram used to estimate cardinality will be adjusted at query compile time.\n This trace flag does not apply to the cardinality estimation model of SQL Server 2012 and earlier versions (CE 70).\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace Flag 4139 enables automatic statistics for any key ordering. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2952101", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4139, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "(,11.0.3431)", + "[11.0.5058,11.0.5532)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4139.1", + "tags": [ "TraceFlag", "TF4139" ], + "displayName": "TF 4139 enables automatic statistics for any key ordering", + "description": "Trace Flag 4139 enables automatically generated quick statistics (histogram amendment) regardless of key column status.\n However, this trace flag does not apply to this SQL Server version.\n Verify need to set a non-default trace flag with the current system build and configuration.", + "message": "Trace flag 4139 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/2952101", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4139, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "[9.0.4035,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.4199", + "tags": [ "TraceFlag", "TF4199", "Optimizer" ], + "displayName": "TF 4199 enables Query Optimizer fixes", + "description": "Trace Flag 4199 enables Query Optimizer fixes released in SQL Server Cumulative Updates and Service Packs.", + "message": "Trace Flag 4199 enables Query Optimizer fixes. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "http://support.microsoft.com/help/974006", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 4199, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "[12.0.2474,12.0.2480]", + "[12.0.4416,12.0.5000)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6498", + "tags": [ "TraceFlag", "TF6498", "Memory" ], + "displayName": "TF 6498 enables additional concurrent large queries", + "description": "Trace Flag 6498 enables more than one large query compilation to gain access to the big gateway when there is sufficient memory available.\n This trace flag can be used to keep memory usage for the compilation of incoming queries under control, avoiding compilation waits for concurrent large queries.\n Starting with SQL Server 2014 SP2 and SQL Server 2016, this behavior is controlled by the engine and trace flag 1236 has no effect.", + "message": "Trace Flag 6498 enables concurrent large queries.", + "helpLink": "https://support.microsoft.com/help/3024815", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 6498, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": "[12.0.5000,)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6498.1", + "tags": [ "TraceFlag", "TF6498", "Memory" ], + "displayName": "TF 6498 enables additional concurrent large queries", + "description": "Trace Flag 6498 enables more than one large query compilation to gain access to the big gateway when there is sufficient memory available.\n This trace flag can be used to keep memory usage for the compilation of incoming queries under control, avoiding compilation waits for concurrent large queries.\n Starting with SQL Server 2014 SP2 and SQL Server 2016, this behavior is controlled by the engine and trace flag 1236 has no effect.", + "message": "Disable trace flag 6498 in SQL Server 2014 SP2, SQL Server 2016 and higher versions.", + "helpLink": "https://support.microsoft.com/help/3024815", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 6498, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "(,12.0.2474)", + "(12.0.2480,12.0.4416)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6498.2", + "tags": [ "TraceFlag", "TF6498", "Memory" ], + "displayName": "TF 6498 enables additional concurrent large queries", + "description": "Trace Flag 6498 enables more than one large query compilation to gain access to the big gateway when there is sufficient memory available.\n However, this trace flag does not apply to this SQL Server version.\n This trace flag can be used to keep memory usage for the compilation of incoming queries under control, avoiding compilation waits for concurrent large queries.\n Starting with SQL Server 2014 SP2 and SQL Server 2016, this behavior is controlled by the engine and trace flag 1236 has no effect.", + "message": "Trace Flag 6498 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/3024815", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 6498, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": [ + "[11.0.6020,11.0.6518)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6532", + "tags": [ "TraceFlag", "TF6532", "Performance" ], + "displayName": "TF 6532 enables performance improvements for Spatial data", + "description": "Trace Flag 6532 enables performance improvements of query operations with spatial data types.\n The performance gain will vary, depending on the configuration, the types of queries, and the objects.", + "message": "Enable trace flag 6532 to enable performance improvements for Spatial data.", + "helpLink": "https://support.microsoft.com/help/3107399", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { "in": [ 6532, "@TraceFlag" ] } + }, + { + "target": { + "type": "Server", + "version": [ + "[11.0.6518,12.0)", + "[12.0.5000,13.0)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6532.6533", + "tags": [ "TraceFlag", "TF6532", "TF6533", "Performance" ], + "displayName": "TF 6532 and TF 6533 enable performance improvements for Spatial data", + "description": "Trace Flag 6532 and trace flag 6533 enable performance improvements of query operations with spatial data types.\n The performance gain will vary, depending on the configuration, the types of queries, and the objects.", + "message": "Enable trace flag 6532 and trace flag 6533 to enable performance improvements for Spatial data.", + "helpLink": "https://support.microsoft.com/help/3107399", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "and": [ + { "in": [ 6532, "@TraceFlag" ] }, + { "in": [ 6533, "@TraceFlag" ] } + ] + } + }, + { + "target": { + "type": "Server", + "version": "[13.0,)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6532.6533.1", + "tags": [ "TraceFlag", "TF6532", "TF6533", "Performance" ], + "displayName": "TF 6532 and TF 6533 enable performance improvements for Spatial data", + "description": "Trace flags 6532 and 6533 enable performance improvements of query operations with spatial data types.\n However, this trace flag does not apply to this SQL Server version.\n Starting with SQL Server 2016, this behavior is controlled by the engine and trace flags 6532 and 6533 have no effect.", + "message": "Disable trace flag 6532 in SQL Server 2016 and higher versions.", + "helpLink": "https://support.microsoft.com/help/3107399", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "and": [ + { "not": { "in": [ 6532, "@TraceFlag" ] } }, + { "not": { "in": [ 6533, "@TraceFlag" ] } } + ] + } + }, + { + "target": { + "type": "Server", + "version": [ + "(,11.0.6020)", + "[12.0,12.0.5000)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6532.6533.2", + "tags": [ "TraceFlag", "TF6532", "TF6533", "Performance" ], + "displayName": "TF 6532 and TF 6533 enable performance improvements for Spatial data", + "description": "Trace flags 6532 and 6533 enable performance improvements of query operations with spatial data types.\n However, this trace flag does not apply to this SQL Server version.\n Starting with SQL Server 2016, this behavior is controlled by the engine and trace flags 6532 and 6533 have no effect.", + "message": "Trace flags 6532 and 6533 do not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/3107399", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "and": [ + { "not": { "in": [ 6532, "@TraceFlag" ] } }, + { "not": { "in": [ 6533, "@TraceFlag" ] } } + ] + } + }, + + { + "target": { + "type": "Server", + "version": [ + "[11.0.6020,12.0)", + "[12.0.5000,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6534", + "tags": [ "TraceFlag", "TF6534", "Performance" ], + "displayName": "TF 6534 enables performance improvements for Spatial data", + "description": "Trace Flag 6534 enable performance improvements of query operations with spatial data types.\n The performance gain will vary, depending on the configuration, the types of queries, and the objects.", + "message": "Enable trace Flag 6534 to enable performance improvement of query operations with spatial data types.", + "helpLink": "https://support.microsoft.com/help/3107399", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { "in": [ 6534, "@TraceFlag" ] } + }, + { + "target": { + "type": "Server", + "version": [ + "(,11.0.6020)", + "[12.0,12.0.5000)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.6534.1", + "tags": [ "TraceFlag", "TF6534", "Performance" ], + "displayName": "TF 6534 enables performance improvements for Spatial data", + "description": "Trace Flag 6534 enables performance improvement of query operations with spatial data types.\n However, this trace flag does not apply to this SQL Server version.\n Verify need to set a Non-default trace flag with current system build and configuration.", + "message": "Trace flag 6534 does not apply to this SQL Server version. Verify need to set a non-default trace flag with the current system build and configuration.", + "helpLink": "https://support.microsoft.com/help/3107399", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { + "not": { "in": [ 6534, "@TraceFlag" ] } + } + }, + { + "target": { + "type": "Server", + "version": "[13.0.4001,15.0)", + "platform": "Windows" + }, + "id": "SqlServer.Server.TraceFlag.7412", + "tags": [ "TraceFlag", "TF7412", "Performance" ], + "displayName": "TF 7412 enables the lightweight profiling infrastructure for live query performance troubleshooting", + "description": "Trace Flag 7412 enables the lightweight profiling infrastructure for live query performance troubleshooting. Consider enabling TF 7412 to have the ability to access runtime information on query execution plans for any session.", + "message": "Enable trace flag 7412 to enables the lightweight profiling infrastructure for live query performance troubleshooting.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/performance/query-profiling-infrastructure", + "level": "Information", + "probes": [ "EnabledGlobalTraceFlags" ], + "condition": { "in": [ 7412, "@TraceFlag" ] + } + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.DeprecatedOrDiscontinuedFeatures", + "tags": [ "deprecated_feature", "discontinued_feature" ], + "displayName": "Deprecated or discontinued features should not be used", + "description": "This check detects deprecated or discontinued features used on target SQL Server instance. Deprecated features may be removed in a future release of SQL Server. Discontinued features have been removed from specific versions of SQL Server.", + "message": "Detected deprecated or discontinued feature uses: @{DeprecatedFeature}. We recommend to replace them with features actual for SQL Server version @{ProductVersion}.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/performance-monitor/sql-server-deprecated-features-object", + "probes": [ "Microsoft.DeprecatedFeaturesStats", "SQLServerVersion" ], + "condition": { + "equal": [ "@DeprecatedFeature", "" ] + } + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.SystemHealthSessionNotActive", + "tags": [ "XEvent", "SystemHealth" ], + "displayName": "system_health XEvent session is not active", + "description": "This session starts automatically when the SQL Server Database Engine starts, and runs without any noticeable performance effects. The session collects system data that you can use to help troubleshoot performance issues in the Database Engine.\n We recommend that you do not stop, alter, or delete the system health session.", + "message": "Start the session by using the ALTER EVENT SESSION statement or by using the Extended Events node in Object Explorer.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/extended-events/use-the-system-health-session", + "level": "Warning", + "probes": [ "Microsoft.XEvent.SystemHealth" ], + "condition": "@XEvent.SystemHealthCount" + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.SPServerDiagnosticsSessionNotActive", + "tags": [ "XEvent", "SPServerDiagnostics", "sp_server_diagnostics" ], + "displayName": "sp_server_diagnostics xEvent session is not active", + "description": "This session starts automatically when the SQL Server Database Engine starts, and runs without any noticeable performance effects. The session collects system data that you can use to help troubleshoot performance issues in the Database Engine.\n We recommend that you do not stop, alter, or delete the system health session.", + "message": "Start the session by using the ALTER EVENT SESSION statement or by using the Extended Events node in Object Explorer.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/system-stored-procedures/sp-server-diagnostics-transact-sql", + "level": "Warning", + "probes": [ "Microsoft.XEvent.SPServerDiagnostics" ], + "condition": "@XEvent.SPServerDiagnostics" + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.BlackboxTraceConfiguredRunning", + "tags": [ "DefaultCollections", "BlackBox", "Trace" ], + "displayName": "Blackbox trace is configured and running", + "description": "This trace is designed to behave similarly to an airplane black box, to help you diagnose intermittent server crashes. It consumes more resources than the default trace and should not be running for extended periods of time.", + "message": "Make sure you still need the blackbox trace and stop it otherwise.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/system-stored-procedures/sp-trace-create-transact-sql", + "level": "Warning", + "probes": [ "Microsoft.CountBlackBoxTraces" ], + "condition": { "not": "@BlackBoxTraceCount" } + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.NoDefaultTraceFoundIsNotActive", + "tags": [ "DefaultCollections", "DefaultTrace", "Trace" ], + "displayName": "No default trace was found or is not active", + "description": "Default trace provides troubleshooting assistance to database administrators by ensuring that they have the log data necessary to diagnose problems the first time they occur.", + "message": "Make sure that there is enough space for SQL Server to write the default trace file. Then have the default trace run by disabling and re-enabling it.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/policy-based-management/default-trace-log-files-disabled", + "level": "Warning", + "probes": [ "Microsoft.CountDefaultTraces" ], + "condition": "@DefaultTraceCount" + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.HintsStatistics", + "tags": [ "Hints", "Instance" ], + "displayName": "Hints are being used", + "description": "Hints are options or strategies specified for enforcement by the SQL Server query processor on SELECT, INSERT, UPDATE, or DELETE statements. The hints override any execution plan the Query Optimizer might select for a query. Because the SQL Server Query Optimizer typically selects the best execution plan for a query, we recommend that , , and be used only as a last resort by experienced developers and database administrators.", + "message": "We recommend that , , and be used only as a last resort by experienced developers and database administrators, because the SQL Server Query Optimizer typically selects the best execution plan for a query.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/queries/hints-transact-sql", + "level": "Information", + "probes": [ "Microsoft.Hints.InstanceStatistics" ], + "condition": "@HintUsageCount" + }, + + { + "target": { + "type": "Database", + "platform": "Windows" + }, + "id": "SqlServer.Database.HintsUsageInModules", + "tags": [ "Hints", "Database", "Module" ], + "displayName": "Hints usage in modules", + "enabled": false, + "description": "Hints are options or strategies specified for enforcement by the SQL Server query processor on SELECT, INSERT, UPDATE, or DELETE statements. The hints override any execution plan the Query Optimizer might select for a query. Because the SQL Server Query Optimizer typically selects the best execution plan for a query, we recommend that , , and be used only as a last resort by experienced developers and database administrators.", + "message": "@{Hint} was used in @{type_desc} @{Schema_Name}.@{Object_Name}.", + "helpLink": "https://docs.microsoft.com/sql/t-sql/queries/hints-transact-sql", + "level": "Information", + "probes": [ "Microsoft.Hints.ModuleUsage" ] + }, + + { + "target": { + "type": "Server", + "platform": "Windows" + }, + "id": "SqlServer.Server.PlansUseRatio", + "tags": [ "Plan", "Usage" ], + "displayName": "Amount of single use plans in cache is high", + "description": "Single use plans waste system resources.", + "message": "Amount of single use plans in cache is high (@{SingleUsePlansUseRatio:P0}). Consider enabling the Optimize for ad hoc workloads setting on heavy OLTP ad-hoc workloads to conserve resources.", + "helpLink": "https://docs.microsoft.com/sql/database-engine/configure-windows/optimize-for-ad-hoc-workloads-server-configuration-option", + "level": "Warning", + "probes": [ "Microsoft.CachedPlans.SingleUseRatio" ], + "condition": { "less": [ "@SingleUsePlansUseRatio", 0.5 ] } + }, + + { + "target": { + "type": "Database", + "platform": "Windows" + }, + "id": "SqlServer.Database.HypotheticalIndices", + "tags": [ "Hypothetical", "Index" ], + "displayName": "Hypothetical indexes found", + "description": "Database has indexes that are marked as hypothetical. Hypothetical indexes are created by the Database Tuning Assistant (DTA) during its tests. If a DTA session was interrupted, these indexes may not be deleted. It is recommended to drop these objects as soon as possible.", + "message": "Index @{IndexName} for @{Schema}.@{Object} is marked as hypothetical. It is recommended to drop hypothetical index @{IndexName} as soon as possible.", + "helpLink": "", + "level": "Warning", + "probes": [ "Microsoft.Hypothetical.Indices" ] + }, + + { + "target": { + "type": "Database", + "platform": "Windows" + }, + "id": "SqlServer.Database.HypotheticalStatistics", + "tags": [ "Hypothetical", "Statistics" ], + "displayName": "Hypothetical statistics found", + "description": "Database has statistics that are marked as hypothetical. Hypothetical statistics are created by the Database Tuning Assistant (DTA) during its tests. If a DTA session was interrupted, these statistics may not be deleted. It is recommended to drop these objects as soon as possible.", + "message": "Statistics @{StatName} for @{Schema}.@{Object} is hypothetical. It is recommended to drop hypothetical statistics @{StatName} as soon as possible.", + "helpLink": "", + "level": "Warning", + "probes": [ "Microsoft.Hypothetical.Statistics" ] + }, + + { + "target": { + "type": "Server", + "version": [ + "[11.0,11.0.7001.0)", + "[12.0,12.0.6259.0)", + "[13.0,13.0.5337.0)", + "[14.0,14.0.3162.1)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.LatestCUInstalled", + "tags": [ "Update" ], + "displayName": "SQL Server instance is not up to date", + "description": "On an installed instance of SQL Server, we recommend that you apply the latest security updates and critical updates including general distribution releases (GDRs), service packs (SPs), and cumulative updates (CUs).", + "message": "Product version @{ProductVersion} is not the latest available. We recommend keeping your SQL Server up to date and install Service Packs and Cumulative Updates as they are released.", + "helpLink": "https://support.microsoft.com/help/321185/how-to-determine-the-version-edition-and-update-level-of-sql-server-an https://docs.microsoft.com/sql/database-engine/install-windows/install-sql-server-servicing-updates", + "probes": [ "SQLServerVersion" ] + }, + { + "target": { + "type": "Server", + "version": [ + "[11.0.7001.0, 12.0)", + "[12.0.6259.0, 13.0)", + "[13.0.5337.0, 14.0)", + "[14.0.3162.1,)" + ], + "platform": "Windows" + }, + "id": "SqlServer.Server.LatestCUInstalled.1", + "tags": [ "Update" ], + "displayName": "SQL Server instance is not up to date", + "description": "On an installed instance of SQL Server, we recommend that you apply the latest security updates and critical updates including general distribution releases (GDRs), service packs (SPs), and cumulative updates (CUs).", + "message": "Product version @{ProductVersion} is not the latest available. We recommend keeping your SQL Server up to date and install Service Packs and Cumulative Updates as they are released.", + "helpLink": "https://support.microsoft.com/help/321185/how-to-determine-the-version-edition-and-update-level-of-sql-server-an https://docs.microsoft.com/sql/database-engine/install-windows/install-sql-server-servicing-updates", + "condition": false + }, + + { + "target": { + "type": "Database", + "name": { "not": "msdb" } + }, + "id": "SqlServer.Database.ForeignKeysNoIndices", + "tags": [ "Performance", "Index" ], + "displayName": "Foreign key constraints should have corresponding indexes", + "description": "Unlike primary key constraints, creating a foreign key constraint does not automatically create a corresponding index. However, manually creating an index on a foreign key is often useful.", + "message": "There is no index on the following foreign key(s): @{ConstraintName}. We recommend you have a corresponding index on every foreign key.", + "helpLink": "https://docs.microsoft.com/sql/relational-databases/tables/primary-and-foreign-key-constraints", + "level": "Warning", + "probes": [ "FKNoIndices" ] + } + ], + "probes": { + "EnabledGlobalTraceFlags": [ + { + "type": "CLR", + "target": { + "type": "Server" + }, + "implementation": { + "class": "Microsoft.SqlServer.Management.Assessment.Probes.TransposeColumnProbe", + "query": "DECLARE @tracestatus TABLE (TraceFlag NVARCHAR(40), [Status] tinyint, [Global] tinyint, [Session] tinyint); INSERT INTO @tracestatus EXEC ('DBCC TRACESTATUS WITH NO_INFOMSGS'); SELECT [TraceFlag], [Status] FROM @tracestatus WHERE Global=1;", + "columnName": "TraceFlag" + } + } + ], + "Microsoft.CountBlackBoxTraces": [ + { + "type": "SQL", + "target": { + "type": "Server" + }, + "implementation": { + "query": "SELECT COUNT(id) AS [BlackBoxTraceCount] FROM [sys].[traces] WHERE [path] LIKE '%blackbox%.trc' AND status = 1;" + } + } + ], + "Microsoft.CountDefaultTraces": [ + { + "type": "SQL", + "target": { + "type": "Server" + }, + "implementation": { + "query": "SELECT COUNT(id) AS [DefaultTraceCount] FROM [sys].[traces] WHERE is_default = 1 AND status = 1;" + } + } + ], + "Microsoft.XEvent.SystemHealth": [ + { + "type": "SQL", + "target": { + "type": "Server" + }, + "implementation": { + "query": "SELECT COUNT(name) AS [XEvent.SystemHealthCount] FROM [sys].[dm_xe_sessions] WHERE [name] = 'system_health';" + } + } + ], + "Microsoft.XEvent.SPServerDiagnostics": [ + { + "type": "SQL", + "target": { + "type": "Server" + }, + "implementation": { + "query": "SELECT COUNT(name) AS [XEvent.SPServerDiagnostics] FROM sys.dm_xe_sessions WHERE [name] = 'sp_server_diagnostics session';" + } + } + ], + "Microsoft.DeprecatedFeaturesStats": [ + { + "type": "CLR", + "target": { + "type": "Server" + }, + "implementation": { + "class": "Microsoft.SqlServer.Management.Assessment.Probes.TransposeColumnProbe", + "query": "SELECT [instance_name] AS [DeprecatedFeature] FROM [sys].[dm_os_performance_counters] WHERE ([object_name] LIKE '%Deprecated Features%') AND ([cntr_value] > 0);", + "columnName": "DeprecatedFeature", + "convertToString": true + } + } + ], + "DatabaseConfiguration": [ + { + "type": "SQL", + "target": { + "type": "Database", + "version": "(,12.0)", + "platform": "Windows" + }, + "implementation": { + "query": "SELECT db.[is_auto_create_stats_on] AS is_auto_create_stats_on, db.[is_auto_update_stats_on] AS is_auto_update_stats_on, 0 AS is_query_store_on FROM [sys].[databases] AS db WHERE db.[name]='@DatabaseName'" + } + }, + { + "type": "SQL", + "target": { + "type": "Database", + "version": "[12.0,)", + "platform": "Windows" + }, + "implementation": { + "query": "SELECT db.[is_auto_create_stats_on] AS is_auto_create_stats_on, db.[is_auto_update_stats_on] AS is_auto_update_stats_on, db.[is_query_store_on] AS is_query_store_on FROM [sys].[databases] AS db WHERE db.[name]='@DatabaseName'" + } + } + ], + "FKNoIndices": [ + { + "type": "CLR", + "target": { + "type": "Database" + }, + "implementation": { + "class": "Microsoft.SqlServer.Management.Assessment.Probes.TransposeColumnProbe", + "query": "USE [@DatabaseName];SELECT QUOTENAME(fk.parent_schema_name)+ '.' + QUOTENAME(fk.parent_table_name) + '.' + QUOTENAME(fk.constraint_name) AS ConstraintName FROM ( SELECT t.name AS [parent_schema_name], OBJECT_NAME(FKC.parent_object_id) [parent_table_name], OBJECT_NAME(constraint_object_id) AS [constraint_name], SUBSTRING((SELECT ',' + RTRIM(COL_NAME(k.parent_object_id,parent_column_id)) AS [data()] FROM sys.foreign_key_columns k (NOLOCK) INNER JOIN sys.foreign_keys (NOLOCK) ON k.constraint_object_id = [object_id] AND k.constraint_object_id = FKC.constraint_object_id ORDER BY constraint_column_id FOR XML PATH('')), 2, 8000) AS [parent_columns] FROM sys.foreign_key_columns FKC (NOLOCK) INNER JOIN sys.objects o (NOLOCK) ON FKC.parent_object_id = o.[object_id] INNER JOIN sys.tables mst (NOLOCK) ON mst.[object_id] = o.[object_id] INNER JOIN sys.schemas t (NOLOCK) ON t.[schema_id] = mst.[schema_id] INNER JOIN sys.objects so (NOLOCK) ON FKC.referenced_object_id = so.[object_id] WHERE o.type = 'U' AND so.type = 'U' GROUP BY o.[schema_id],so.[schema_id],FKC.parent_object_id,constraint_object_id,referenced_object_id,t.name ) fk WHERE NOT EXISTS (SELECT 1 FROM ( SELECT t.name AS schemaName, OBJECT_NAME(mst.[object_id]) AS objectName, SUBSTRING(( SELECT ',' + RTRIM(ac.name) FROM sys.tables AS st INNER JOIN sys.indexes AS mi ON st.[object_id] = mi.[object_id] INNER JOIN sys.index_columns AS ic ON mi.[object_id] = ic.[object_id] AND mi.[index_id] = ic.[index_id] INNER JOIN sys.all_columns AS ac ON st.[object_id] = ac.[object_id] AND ic.[column_id] = ac.[column_id] WHERE i.[object_id] = mi.[object_id] AND i.index_id = mi.index_id AND ic.is_included_column = 0 ORDER BY ac.column_id FOR XML PATH('')), 2, 8000) AS KeyCols FROM sys.indexes AS i INNER JOIN sys.tables AS mst ON mst.[object_id] = i.[object_id] INNER JOIN sys.schemas AS t ON t.[schema_id] = mst.[schema_id] WHERE i.[type] IN (1,2,5,6) AND i.is_unique_constraint = 0 AND mst.is_ms_shipped = 0 ) ict WHERE fk.parent_schema_name = ict.schemaName AND fk.parent_table_name = ict.objectName AND REPLACE(fk.parent_columns,' ,',',') = ict.KeyCols);", + "columnName": "ConstraintName", + "convertToString": true + } + } + ], + "Microsoft.Hints.InstanceStatistics": [ + { + "type": "SQL", + "target": { + "type": "Server", + "platform": "Windows" + }, + "implementation": { + "query": "SELECT COUNT([counter]) AS [HintUsageCount] FROM [sys].[dm_exec_query_optimizer_info] WHERE ([counter] = 'order hint' OR [counter] = 'join hint') AND occurrence > 1;" + } + } + ], + "Microsoft.Hints.ModuleUsage": [ + { + "type": "SQL", + "target": { + "type": "Database", + "platform": "Windows" + }, + "implementation": { + "query": "USE [@DatabaseName]; SELECT ss.name AS [Schema_Name], so.name AS [Object_Name], so.type_desc, CASE WHEN sm.[definition] LIKE '%FORCE ORDER%' THEN '[FORCE ORDER Hint]' WHEN sm.[definition] LIKE '%MERGE JOIN%' OR sm.[definition] LIKE '%LOOP JOIN%' OR sm.[definition] LIKE '%HASH JOIN%' THEN '[JOIN Hint]' END AS Hint FROM [sys].[sql_modules] AS sm INNER JOIN [sys].[objects] AS so ON sm.[object_id] = so.[object_id] INNER JOIN [sys].[schemas] AS ss ON so.[schema_id] = ss.[schema_id] WHERE (sm.[definition] LIKE '%FORCE ORDER%' OR sm.[definition] LIKE '%MERGE JOIN%' OR sm.[definition] LIKE '%LOOP JOIN%' OR sm.[definition] LIKE '%HASH JOIN%') AND OBJECTPROPERTY(sm.[object_id],'IsMSShipped') = 0;" + } + } + ], + "Microsoft.CachedPlans.SingleUseRatio": [ + { + "type": "SQL", + "target": { + "type": "Server", + "platform": "Windows" + }, + "implementation": { + "query": "DECLARE @Size1 AS REAL, @Size2 AS REAL; SET @Size1=COALESCE((SELECT SUM(CAST(size_in_bytes AS bigint)) AS Size1 FROM [sys].[dm_exec_cached_plans] (NOLOCK) WHERE cacheobjtype LIKE '%Plan%' AND usecounts = 1),0);SET @Size2=COALESCE((SELECT SUM(CAST(size_in_bytes AS bigint)) AS Size2 FROM [sys].[dm_exec_cached_plans] (NOLOCK) WHERE cacheobjtype LIKE '%Plan%' AND usecounts > 1),0);SELECT @Size1/(@Size1+@Size2) AS [SingleUsePlansUseRatio];" + } + } + ], + "Microsoft.Hypothetical.Indices": [ + { + "type": "SQL", + "target": { + "type": "Database", + "platform": "Windows" + }, + "implementation": { + "query": "USE [@DatabaseName]; SELECT QUOTENAME(t.name) AS [Schema], QUOTENAME(o.[name]) AS [Object], i.name AS [IndexName] FROM [sys].[indexes] i INNER JOIN [sys].[objects] AS o ON o.[object_id] = i.[object_id] INNER JOIN [sys].[tables] AS mst ON mst.[object_id] = i.[object_id] INNER JOIN [sys].[schemas] AS t ON t.[schema_id] = mst.[schema_id] WHERE i.is_hypothetical = 1;" + } + } + ], + "Microsoft.Hypothetical.Statistics": [ + { + "type": "SQL", + "target": { + "type": "Database", + "platform": "Windows" + }, + "implementation": { + "query": "USE [@DatabaseName]; SELECT QUOTENAME(t.name) AS [Schema], QUOTENAME(o.[name]) AS [Object], s.name AS [StatName] FROM [sys].[stats] s INNER JOIN [sys].[objects] AS o (NOLOCK) ON o.[object_id] = s.[object_id] INNER JOIN [sys].[tables] AS mst (NOLOCK) ON mst.[object_id] = s.[object_id] INNER JOIN [sys].[schemas] AS t (NOLOCK) ON t.[schema_id] = mst.[schema_id] WHERE (s.name LIKE 'hind_%' OR s.name LIKE '_dta_stat%') AND auto_created = 0 AND s.name NOT IN (SELECT name FROM sys.indexes);" + } + } + ], + "SQLServerVersion": [ + { + "type": "CLR", + "name": "SQLServerVersion", + "implementation": { + "class": "Microsoft.SqlServer.Management.Assessment.Probes.ServerVersionProbe" + } + } + ] + } +} From 0f6f0408fa86fd1c9276d36f3f6d1cae2965e4fa Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Thu, 25 Jul 2019 08:51:57 -0700 Subject: [PATCH 14/29] Update deploy-sql-big-data-aks.py --- .../deployment/aks/deploy-sql-big-data-aks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py index e333794d..114c6c62 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py +++ b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py @@ -77,17 +77,17 @@ print("Creating SQL Big Data cluster:" +CLUSTER_NAME) command="azdata bdc config init --source aks-dev-test --target custom --force" executeCmd (command) -command="azdata bdc config section set -c custom -j ""metadata.name=" + CLUSTER_NAME + "" +command="azdata bdc config replace -c custom -j ""metadata.name=" + CLUSTER_NAME + "" executeCmd (command) # Use this only if you are using a private registry different than default Micrososft registry (mcr). -# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" +# command="azdata bdc config replace -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" # executeCmd (command) -# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" +# command="azdata bdc config replace -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" # executeCmd (command) -# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" +# command="azdata bdc config replace -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" # executeCmd (command) command="azdata bdc create -c custom --accept-eula yes" From 155c53bb23fde8faecee3f3721948ed1956a3feb Mon Sep 17 00:00:00 2001 From: Brandon Koch Date: Thu, 25 Jul 2019 10:30:33 -0700 Subject: [PATCH 15/29] update references from mssqlctl to azdata --- .../sql-big-data-cluster/app-deploy/README.md | 4 ++-- .../app-deploy/RollDice/README.md | 12 +++++------ .../app-deploy/SSIS/README.md | 12 +++++------ .../app-deploy/addpy/README.md | 20 +++++++++---------- .../app-deploy/magic8ball/README.md | 12 +++++------ .../app-deploy/mleap/README.md | 12 +++++------ .../app-deploy/sentiment-analysis/README.md | 12 +++++------ .../app-deploy/sumofsq/README.md | 12 +++++------ 8 files changed, 48 insertions(+), 48 deletions(-) diff --git a/samples/features/sql-big-data-cluster/app-deploy/README.md b/samples/features/sql-big-data-cluster/app-deploy/README.md index 4e3e928b..4c32e423 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/README.md @@ -7,10 +7,10 @@ Application deployment allows you to deploy applications into SQL Server big dat ## Pre-requisites * SQL Server big data cluster CTP 2.3 or later -* `mssqlctl` CLI familiarity. If you are unfamiliar with `mssqlctl` please refer to - [App Deployment in SQL Server big data cluster](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) for more information. +* `azdata` CLI familiarity. If you are unfamiliar with `azdata` please refer to - [App Deployment in SQL Server big data cluster](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) for more information. * Tip -**mssqlctl app -h** will display the various commands to manage the app +**azdata app -h** will display the various commands to manage the app ## Templates Templates are used by our [App Deploy add-ins](https://docs.microsoft.com/en-us/sql/big-data-cluster/app-deployment-extension?view=sqlallproducts-allversions) and can be used to quickly deploy applications. diff --git a/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md b/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md index 1e8c8c81..240d4d85 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md @@ -34,7 +34,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server 2019 big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server 2019 big data cluster. @@ -44,20 +44,20 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `roll-dice.R` files are located: ```bash - mssqlctl app create --spec ./RollDice + azdata app create --spec ./RollDice ``` 4. Check the deployment by running the following command: ```bash - mssqlctl app list -n roll-dice -v [version] + azdata app list -n roll-dice -v [version] ``` Once the app is listed as `Ready` you can continue to the next step. 5. Test the app by running the following command: ```bash - mssqlctl app run -n roll-dice -v [version] --input x=3 + azdata app run -n roll-dice -v [version] --input x=3 ``` You should get output like the example for three dice below. The results of the dice rolled are in the `result` data frame: ```json @@ -88,7 +88,7 @@ To run this sample, you need the following prerequisites. 6. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name roll-dice --version [version] + azdata app delete --name roll-dice --version [version] ``` diff --git a/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md b/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md index 7b77325a..57037d8c 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md @@ -23,7 +23,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server big data cluster. 3. Optional: to see the SSIS package itself, install Visual Studio 2017 if you don't have it already. After that download and install [SSDT](https://docs.microsoft.com/en-us/sql/ssdt/download-sql-server-data-tools-ssdt?view=sql-server-2017#ssdt-for-vs-2017-standalone-installer). 4. Optional: install [SSMS](https://docs.microsoft.com/en-us/sql/ssms/download-sql-server-management-studio-ssms?view=sql-server-2017) if it is not already installed. @@ -35,16 +35,16 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. Replace `[SA_PASSWORD]` in the `spec.yaml` file with the password for SQL user `sa`. 4. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `back-up-db.dtsx` files are located: ```bash - mssqlctl app create --spec ./SSIS + azdata app create --spec ./SSIS ``` 5. Check the deployment by running the following command: ```bash - mssqlctl app list --name back-up-db --version [version] + azdata app list --name back-up-db --version [version] ``` Once the app is listed as `Ready` the job should run within a minute. You can check if the backup is created by running: @@ -56,7 +56,7 @@ To run this sample, you need the following prerequisites. 6. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name back-up-db --version [version] + azdata app delete --name back-up-db --version [version] # delete backup files kubectl -n [your namespace] exec -it mssql-master-pool-0 -c mssql-server -- /bin/bash -c "rm /var/opt/mssql/data/*.DWConfigbak" ``` @@ -73,7 +73,7 @@ Here is the spec file for this application. This sample uses the `SSIS` runtime |Setting|Description| |-|-| |options|Specifies any command line parameters passed to the execution of the SSIS package| -|schedule|Specifies when the job should run. This follows cron expressions. A value of '*/1 * * * *' means the job runs *every minute*. If omitted the package will not run automatically and you can run the package on demand using `mssqlctl run -n back-up-db -v [version]` or making a call to the API.| +|schedule|Specifies when the job should run. This follows cron expressions. A value of '*/1 * * * *' means the job runs *every minute*. If omitted the package will not run automatically and you can run the package on demand using `azdata run -n back-up-db -v [version]` or making a call to the API.| ```yaml name: back-up-db diff --git a/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md b/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md index 449efbbb..cbb7bd95 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md @@ -35,7 +35,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server 2019 big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server 2019 big data cluster. @@ -45,20 +45,20 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `add.py` files are located: ```bash - mssqlctl app create --spec ./addpy + azdata app create --spec ./addpy ``` 4. Check the deployment by running the following command: ```bash - mssqlctl app list -n addpy -v [version] + azdata app list -n addpy -v [version] ``` Once the app is listed as `Ready` you can continue to the next step. 5. Test the app by running the following command: ```bash - mssqlctl app run -n addpy -v [version] --input x=3,y=5 + azdata app run -n addpy -v [version] --input x=3,y=5 ``` You should get output like the example below. The result of adding 3+5 are returned as `result`. ```json @@ -75,7 +75,7 @@ To run this sample, you need the following prerequisites. ``` 6. Any app you create is also accessible using a RESTful web service that is [Swagger](swagger.io) compliant. You can get the endpoint for the web service by running: ```bash - mssqlctl app describe --name addpy --version [version] + azdata app describe --name addpy --version [version] ``` This will return an output much like the following: ```json @@ -106,7 +106,7 @@ To run this sample, you need the following prerequisites. } ``` Note the IP address and the port number in this output. Open the following URL in your browser: - `https://[IP]:[PORT]/api/docs/swagger.json`. You will have to log in with the same credentials you used for `mssqlctl login`. The contents of the `swagger.json` you can paste into [Swagger Editor](https://editor.swagger.io) to understand what methods are available: + `https://[IP]:[PORT]/api/docs/swagger.json`. You will have to log in with the same credentials you used for `azdata login`. The contents of the `swagger.json` you can paste into [Swagger Editor](https://editor.swagger.io) to understand what methods are available: ![API Swagger](api_swagger.png) Notice the `app` GET method as well as the `token` POST method. Since the authentication for apps uses JWT tokens you will need to get a token my using your favorite tool to make a POST call to the `token` method. Here is an example of how to do just that in [Postman](https://www.getpostman.com/): @@ -114,20 +114,20 @@ To run this sample, you need the following prerequisites. The result of this request will give you an `access_token`, which you will need to call the URL to run the app. - > *Optional*: If you want, you can open the URL for the `swagger` that was returned when you ran `mssqlctl app describe --name addpy --version [version]` in your browser. You will have to log in with the same credentials you used for `mssqlctl login`. The contents of the `swagger.json` you can paste into [Swagger Editor](https://editor.swagger.io). You will see that the web service exposes the `run` method. + > *Optional*: If you want, you can open the URL for the `swagger` that was returned when you ran `azdata app describe --name addpy --version [version]` in your browser. You will have to log in with the same credentials you used for `azdata login`. The contents of the `swagger.json` you can paste into [Swagger Editor](https://editor.swagger.io). You will see that the web service exposes the `run` method. You can use your favorite tool to call the `run` method (`https://[IP]:30778/api/app/addpy/[version]/run`), passing in the parameters in the body of your POST request as json. In this example we will use [Postman](https://www.getpostman.com/). Before making the call, you will need to set the `Authorization` to `Bearer Token` and paste in the token you retrieved earlier. This will set a header on your request. See the screenshot below. ![Postman Run Headers](postman_run_1.png) Next, in the requests body, pass in the parameters to the app you are calling and set the `content-type` to `application/json`: ![Postman Run Body](postman_run_2.png) - When you send the request, you will get the same output as you did when you ran the app through `mssqlctl app run`: + When you send the request, you will get the same output as you did when you ran the app through `azdata app run`: ![Postman Run Result](postman_result.png) You have now successfully called the app through the web service! 7. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name addpy --version [version] + azdata app delete --name addpy --version [version] ``` diff --git a/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md b/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md index 82a4eaa0..0d526304 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md @@ -34,7 +34,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server 2019 big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server 2019 big data cluster. @@ -44,20 +44,20 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `magic8ball.py` files are located: ```bash - mssqlctl app create --spec ./magic8ball + azdata app create --spec ./magic8ball ``` 4. Check the deployment by running the following command: ```bash - mssqlctl app list -n magic8ball -v [version] + azdata app list -n magic8ball -v [version] ``` Once the app is listed as `Ready` you can continue to the next step. 5. Test the app by running the following command: ```bash - mssqlctl app run -n magic8ball -v [version] --input txt="Will it rain tomorrow?" + azdata app run -n magic8ball -v [version] --input txt="Will it rain tomorrow?" ``` You should get output like the example below. The answer to your question are returned as `result`. ```json @@ -78,7 +78,7 @@ To run this sample, you need the following prerequisites. 6. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name magic8ball --version [version] + azdata app delete --name magic8ball --version [version] ``` diff --git a/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md b/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md index df135f69..b666d9ac 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md @@ -31,7 +31,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server 2019 big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server 2019 big data cluster. @@ -41,17 +41,17 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. This example uses a TensorFlow Machine Learning Model that uses public US Census data predict income. [More details and information on the example are here](https://docs.microsoft.com/en-us/sql/big-data-cluster/train-and-create-machinelearning-models-with-spark?view=sqlallproducts-allversions). The application you will be deploying as part of this sample is a Random Forest Model that was built in Spark and has been [serialized as an MLeap bundle](https://docs.microsoft.com/en-us/sql/big-data-cluster/export-model-with-spark-mleap?view=sqlallproducts-allversions). Deploy the app using the `create` command and pass the location of the spec file. In the example below, the spec file is expected to be in the `mleap` folder: ```bash - mssqlctl app create --spec ./mleap/ + azdata app create --spec ./mleap/ ``` 1. Check the deployment by running the following command: ```bash - mssqlctl app list -n mleap-census -v [version] + azdata app list -n mleap-census -v [version] ``` Once the app is listed as `Ready` you can continue to the next step. 2. Now that the app has been deployed you can test if the app works correctly by passing in a sample input that is available in the `mleap` folder. The deployed app is a RESTful webservice that is [Swagger](swagger.io) compliant. For this sample we will show you how you can test this using the CLI. @@ -59,7 +59,7 @@ To run this sample, you need the following prerequisites. To test the app, run the command below. The input parameter is a `MLeapFrame`, a `json` file that describes the parameters and the values provided to the model for predicting income. Note that the input parameter has a special character '@' to indicate that a `json` file is being passed. This command needs to be run within the `mleap` folder. ```bash - mssqlctl app run --name mleap-census --version [version] --input schema=@census_frame.json + azdata app run --name mleap-census --version [version] --input schema=@census_frame.json ``` The result will be a json output that includes the predicted income along with additional data. @@ -69,7 +69,7 @@ To run this sample, you need the following prerequisites. 6. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name mleap-census --version [version] + azdata app delete --name mleap-census --version [version] ``` diff --git a/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md b/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md index 69decc53..df6ceb16 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md @@ -35,7 +35,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server 2019 big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server 2019 big data cluster. @@ -45,20 +45,20 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml`, `sentiment.rds` and `sentiment.R` files are located: ```bash - mssqlctl app create --spec ./sentiment-analysis + azdata app create --spec ./sentiment-analysis ``` 4. Check the deployment by running the following command: ```bash - mssqlctl app list -n sentiment-r -v [version] + azdata app list -n sentiment-r -v [version] ``` Once the app is listed as `Ready` you can continue to the next step. 5. Test the app by running the following command: ```bash - mssqlctl app run -n sentiment-r -v [version] --input reviewText="Absolutely the best movie experience I have ever had!" + azdata app run -n sentiment-r -v [version] --input reviewText="Absolutely the best movie experience I have ever had!" ``` You should get output like the example below. The result of the sentiment analysis scoring is returned as a data frame in `out`. A `PredictedLabel` equal to `1` indicates the sentiment is deemed positive, whereas a `PredictedLabel` of `0` indicates a negative sentiment. The `Probability.1` indicates the level of certainty for the `PredictedLabel` to be the true sentiment. ```json @@ -89,7 +89,7 @@ To run this sample, you need the following prerequisites. 6. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name sentiment-r --version [version] + azdata app delete --name sentiment-r --version [version] ``` diff --git a/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md b/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md index 130134c9..fb74dabb 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md @@ -35,7 +35,7 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** 1. SQL Server big data cluster CTP 2.3 or later. -2. `mssqlctl`. Refer to [installing mssqlctl](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-mssqlctl?view=sqlallproducts-allversions) document on setting up the `mssqlctl` and connecting to a SQL Server 2019 big data cluster. +2. `azdata`. Refer to [installing azdata](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-install-azdata?view=sqlallproducts-allversions) document on setting up the `azdata` and connecting to a SQL Server 2019 big data cluster. @@ -45,20 +45,20 @@ To run this sample, you need the following prerequisites. 2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - mssqlctl login -e https://:30777 -u -p + azdata login -e https://:30777 -u -p ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `sum_of_squares.R` files are located: ```bash - mssqlctl app create --spec ./sumofsq + azdata app create --spec ./sumofsq ``` 4. Check the deployment by running the following command: ```bash - mssqlctl app list -n sum-of-sq -v [version] + azdata app list -n sum-of-sq -v [version] ``` Once the app is listed as `Ready` you can continue to the next step. 5. Test the app by running the following command: ```bash - mssqlctl app run -n sum-of-sq -v [version] --input a=42,b=3435 + azdata app run -n sum-of-sq -v [version] --input a=42,b=3435 ``` You should get output like the example below. The answer to your question are returned as `result`. ```json @@ -79,7 +79,7 @@ To run this sample, you need the following prerequisites. 6. You can clean up the sample by running the following commands: ```bash # delete app - mssqlctl app delete --name sum-of-sq --version [version] + azdata app delete --name sum-of-sq --version [version] ``` From b47672230dc542153edb609dc03b3821be376f7e Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Fri, 26 Jul 2019 08:38:58 -0700 Subject: [PATCH 16/29] Update deploy-sql-big-data-aks.py --- .../deployment/aks/deploy-sql-big-data-aks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py index 114c6c62..74ceb6e8 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py +++ b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py @@ -77,17 +77,17 @@ print("Creating SQL Big Data cluster:" +CLUSTER_NAME) command="azdata bdc config init --source aks-dev-test --target custom --force" executeCmd (command) -command="azdata bdc config replace -c custom -j ""metadata.name=" + CLUSTER_NAME + "" +command="azdata bdc config replace -c custom/cluster.json -j ""metadata.name=" + CLUSTER_NAME + "" executeCmd (command) # Use this only if you are using a private registry different than default Micrososft registry (mcr). -# command="azdata bdc config replace -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" +# command="azdata bdc config replace -c custom/control.json -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" # executeCmd (command) -# command="azdata bdc config replace -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" +# command="azdata bdc config replace -c custom/control.json -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" # executeCmd (command) -# command="azdata bdc config replace -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" +# command="azdata bdc config replace -c custom/control.json -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" # executeCmd (command) command="azdata bdc create -c custom --accept-eula yes" From ed815cf4435874a25b043ffbe8096f9d726df401 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Fri, 26 Jul 2019 08:43:32 -0700 Subject: [PATCH 17/29] Update deploy-sql-big-data-aks.py --- .../deployment/aks/deploy-sql-big-data-aks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py index 74ceb6e8..1b0524e4 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py +++ b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py @@ -24,8 +24,9 @@ def executeCmd (cmd): # SUBSCRIPTION_ID = input("Provide your Azure subscription ID:").strip() GROUP_NAME = input("Provide Azure resource group name to be created:").strip() -DOCKER_USERNAME = input("Provide your Docker username:").strip() -DOCKER_PASSWORD = getpass.getpass("Provide your Docker password:").strip() +# Use this only if you are using a private registry different than default Micrososft registry (mcr). +#DOCKER_USERNAME = input("Provide your Docker username:").strip() +#DOCKER_PASSWORD = getpass.getpass("Provide your Docker password:").strip() # # Optionally change these configuration settings From c6585d32afda3ba7fa2f3e4f80a84074b61baafe Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Mon, 5 Aug 2019 11:00:16 -0700 Subject: [PATCH 18/29] Update links --- samples/features/sql-big-data-cluster/spark/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/features/sql-big-data-cluster/spark/README.md b/samples/features/sql-big-data-cluster/spark/README.md index 6dc2829f..f1442943 100644 --- a/samples/features/sql-big-data-cluster/spark/README.md +++ b/samples/features/sql-big-data-cluster/spark/README.md @@ -4,13 +4,13 @@ SQL Server Big Data cluster bundles Spark and HDFS together with SQL server. Azu ## Contents -[PySpark Hello World](dataloading/hello_PySpark.ipynb) +[PySpark Hello World](data-loading/hello_PySpark.ipynb) -[Scala Hello World ](dataloading/hello_Scala.ipynb) +[Scala Hello World ](data-loading/hello_Scala.ipynb) -[SparkR Hello World ](dataloading/hello_sparkR.ipynb) +[SparkR Hello World ](data-loading/hello_sparkR.ipynb) -[DataLoading - Transforming CSV to Parquet](dataloading/transform-csv-files.ipynb/) +[DataLoading - Transforming CSV to Parquet](data-loading/transform-csv-files.ipynb/) [Data Transfer - Spark to SQL using Spark JDBC connector](data-virtualization/spark_to_sql_jdbc.ipynb/) @@ -20,7 +20,7 @@ SQL Server Big Data cluster bundles Spark and HDFS together with SQL server. Azu [Install - Install 3rd party packages](config-install/installpackage_Spark.ipynb/) -[Restful-Access - Access Spark in BDC via restful Livy APIs](restful-api-accessn/accessing_spark_via_livy.ipynb/) +[Restful-Access - Access Spark in BDC via restful Livy APIs](restful-api-access/accessing_spark_via_livy.ipynb/) ## Instructions on how to run in Azure Data Studio From 32fd143655432f9c9699731b61b6dcedf91d1e25 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Mon, 5 Aug 2019 11:20:57 -0700 Subject: [PATCH 19/29] Fix links --- samples/features/sql-big-data-cluster/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/README.md b/samples/features/sql-big-data-cluster/README.md index 6a0ac345..f141aad7 100644 --- a/samples/features/sql-big-data-cluster/README.md +++ b/samples/features/sql-big-data-cluster/README.md @@ -9,7 +9,7 @@ Installation instructions for SQL Server 2019 big data clusters can be found [he ## Executing the sample scripts The scripts should be executed in a specific order to test the various features. Execute the scripts from each folder in below order: -1. __[spark/dataloading/transform-csv-files.ipynb](spark/dataloading/transform-csv-files.ipynb)__ +1. __[spark/data-loading/transform-csv-files.ipynb](spark/data-loading/transform-csv-files.ipynb)__ 1. __[data-virtualization/generic-odbc](data-virtualization/generic-odbc)__ 1. __[data-virtualization/hadoop](data-virtualization/hadoop)__ 1. __[data-virtualization/storage-pool](data-virtualization/storage-pool)__ From d21697f39edbfac1580257b0f9937df6f4c3ad74 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Mon, 5 Aug 2019 11:21:19 -0700 Subject: [PATCH 20/29] Update README.md --- samples/features/sql-big-data-cluster/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/spark/README.md b/samples/features/sql-big-data-cluster/spark/README.md index f1442943..6dff33a3 100644 --- a/samples/features/sql-big-data-cluster/spark/README.md +++ b/samples/features/sql-big-data-cluster/spark/README.md @@ -10,7 +10,7 @@ SQL Server Big Data cluster bundles Spark and HDFS together with SQL server. Azu [SparkR Hello World ](data-loading/hello_sparkR.ipynb) -[DataLoading - Transforming CSV to Parquet](data-loading/transform-csv-files.ipynb/) +[Data Loading - Transforming CSV to Parquet](data-loading/transform-csv-files.ipynb/) [Data Transfer - Spark to SQL using Spark JDBC connector](data-virtualization/spark_to_sql_jdbc.ipynb/) From e1c6e26b567f5d268b57a6cb02be039c10f57264 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Mon, 5 Aug 2019 11:26:06 -0700 Subject: [PATCH 21/29] Update README.md --- .../sql-big-data-cluster/machine-learning/sql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/README.md b/samples/features/sql-big-data-cluster/machine-learning/sql/README.md index d36e1e27..f21e061a 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/sql/README.md +++ b/samples/features/sql-big-data-cluster/machine-learning/sql/README.md @@ -12,6 +12,6 @@ This folder contains the R samples. This folder contains the R samples. -[java](java) +[java](https://github.com/microsoft/sql-server-language-extensions/tree/master/language-extensions/java/samples/regex) This folder contains the Java samples. From 94d28eec9ddbd87fdef178654d1ac8e2376d4aa8 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Mon, 5 Aug 2019 11:27:32 -0700 Subject: [PATCH 22/29] Update README.md --- .../data-virtualization/storage-pool/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md index d35a9a58..8a6b77c8 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md @@ -12,7 +12,7 @@ In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node 1. Connect to SQL Server Master instance. -1. Run the [../../spark/dataloading/transform-csv-files.ipynb](../../spark/dataloading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s). +1. Run the [../../spark/data-loading/transform-csv-files.ipynb](../../spark/data-loading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s). 1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS. From 3779327a32ae9ed77f97e7b412db6e855fbc5c40 Mon Sep 17 00:00:00 2001 From: Mihaela Blendea Date: Mon, 5 Aug 2019 11:28:44 -0700 Subject: [PATCH 23/29] Update README.md --- .../sql-big-data-cluster/data-virtualization/hadoop/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md index 3a0734ea..ec5df8e5 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md @@ -12,10 +12,10 @@ In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node 1. Connect to SQL Server Master instance. -1. Run the [../../spark/dataloading/transform-csv-files.ipynb](../../spark/dataloading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s). +1. Run the [../../spark/data-loading/transform-csv-files.ipynb](../../spark/data-loading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s). 1. Execute the [web-clickstreams-hdfs-orc.sql](web-clickstreams-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS. 1. Execute the [product-reviews-hdfs-orc.sql](product-reviews-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS. -1. Execute the [inventory-hdfs-rcfile.sql](inventory-hdfs-rcfile.sql). This script demonstrates how to export data from SQL Server into HDFS using PolyBase v1 syntax. This script will export data from SQL Server into RCFILE format. \ No newline at end of file +1. Execute the [inventory-hdfs-rcfile.sql](inventory-hdfs-rcfile.sql). This script demonstrates how to export data from SQL Server into HDFS using PolyBase v1 syntax. This script will export data from SQL Server into RCFILE format. From c846036409a7b04fc54023ab54de58d0fa67b007 Mon Sep 17 00:00:00 2001 From: Bill Liang Date: Mon, 5 Aug 2019 12:42:26 -0700 Subject: [PATCH 24/29] updated login endpoint to controller-svc-external --- .../sql-big-data-cluster/app-deploy/RollDice/README.md | 4 ++-- .../features/sql-big-data-cluster/app-deploy/SSIS/README.md | 4 ++-- .../features/sql-big-data-cluster/app-deploy/addpy/README.md | 4 ++-- .../sql-big-data-cluster/app-deploy/magic8ball/README.md | 4 ++-- .../features/sql-big-data-cluster/app-deploy/mleap/README.md | 4 ++-- .../app-deploy/sentiment-analysis/README.md | 4 ++-- .../sql-big-data-cluster/app-deploy/sumofsq/README.md | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md b/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md index 240d4d85..6df943bd 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/RollDice/README.md @@ -41,10 +41,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u -p ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `roll-dice.R` files are located: ```bash diff --git a/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md b/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md index 57037d8c..0eaa2c74 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/SSIS/README.md @@ -32,10 +32,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u ``` 3. Replace `[SA_PASSWORD]` in the `spec.yaml` file with the password for SQL user `sa`. 4. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `back-up-db.dtsx` files are located: diff --git a/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md b/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md index cbb7bd95..b0f6fa8d 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/addpy/README.md @@ -42,10 +42,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `add.py` files are located: ```bash diff --git a/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md b/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md index 0d526304..3cff9216 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/magic8ball/README.md @@ -41,10 +41,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `magic8ball.py` files are located: ```bash diff --git a/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md b/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md index b666d9ac..4ab1640d 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/mleap/README.md @@ -38,10 +38,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u ``` 3. This example uses a TensorFlow Machine Learning Model that uses public US Census data predict income. [More details and information on the example are here](https://docs.microsoft.com/en-us/sql/big-data-cluster/train-and-create-machinelearning-models-with-spark?view=sqlallproducts-allversions). The application you will be deploying as part of this sample is a Random Forest Model that was built in Spark and has been [serialized as an MLeap bundle](https://docs.microsoft.com/en-us/sql/big-data-cluster/export-model-with-spark-mleap?view=sqlallproducts-allversions). diff --git a/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md b/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md index df6ceb16..b548030c 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/README.md @@ -42,10 +42,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml`, `sentiment.rds` and `sentiment.R` files are located: ```bash diff --git a/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md b/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md index fb74dabb..b641a7e0 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md +++ b/samples/features/sql-big-data-cluster/app-deploy/sumofsq/README.md @@ -42,10 +42,10 @@ To run this sample, you need the following prerequisites. ## Run this sample 1. Clone or download this sample on your computer. -2. Log in to the SQL Server big data cluster using the command below using the IP address of the `mgmtproxy-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. +2. Log in to the SQL Server big data cluster using the command below using the IP address of the `controller-svc-external` in your cluster. If you are not familiar with `mssqltctl` you can refer to the [documentation](https://docs.microsoft.com/en-us/sql/big-data-cluster/big-data-cluster-create-apps?view=sqlallproducts-allversions) and then return to this sample. ```bash - azdata login -e https://:30777 -u -p + azdata login -e https://:30080 -u ``` 3. Deploy the application by running the following command, specifying the folder where your `spec.yaml` and `sum_of_squares.R` files are located: ```bash From 4dca48cdea2f262e9f48af4a6d0f1d025dfe89b7 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Thu, 8 Aug 2019 16:21:16 -0700 Subject: [PATCH 25/29] Fixed typo. --- .../sql-big-data-cluster/machine-learning/sql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/README.md b/samples/features/sql-big-data-cluster/machine-learning/sql/README.md index d36e1e27..5e59b386 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/sql/README.md +++ b/samples/features/sql-big-data-cluster/machine-learning/sql/README.md @@ -10,7 +10,7 @@ This folder contains the R samples. [python](python) -This folder contains the R samples. +This folder contains the Python samples. [java](java) From c8bc392c60311bfb416c0164fe90d34851babd7e Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 12 Aug 2019 08:27:35 -0700 Subject: [PATCH 26/29] Fixed typos --- .../sql-big-data-cluster/data-virtualization/oracle/README.md | 2 +- .../data-virtualization/oracle/setup/bootstrap-oracle.cmd | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/README.md b/samples/features/sql-big-data-cluster/data-virtualization/oracle/README.md index 0a473bb7..bbd568cf 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/README.md @@ -8,7 +8,7 @@ SQL Server 2019 introduces new ODBC connectors to data sources like SQL Server, In this example, you are going to create an external table in a SQL Server instance over the inventory table that sits on an Oracle server. If you are using a SQL Server 2019 big data cluster then the scripts can be executed on the SQL Server Master instance. -**Before you begin**, you need to have an Oracle instance and credentials. Follow the instruction in the [setup\README.md](setup\README.md). +**Before you begin**, you need to have an Oracle instance and credentials. Follow the instruction in the [setup\README.md](setup/README.md). ### Instructions diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/setup/bootstrap-oracle.cmd b/samples/features/sql-big-data-cluster/data-virtualization/oracle/setup/bootstrap-oracle.cmd index f3231982..6c13bebd 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/setup/bootstrap-oracle.cmd +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/setup/bootstrap-oracle.cmd @@ -9,14 +9,14 @@ if NOT DEFINED ORACLE_SERVER goto :usage if NOT DEFINED ORACLE_USER goto :usage if NOT DEFINED ORACLE_PASSWORD goto :usage -for %F in (sqlplus.exe sqlldr.exe) do ( +for %%F in (sqlplus.exe sqlldr.exe) do ( echo Verifying %%F is in path & CALL WHERE /Q %%F || GOTO exit ) for %%F in (sales-user.sql inventory.sql customer.sql) do ( echo Executing [%%F]... - echo exit | sqlplus -S %ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% @sales-user.sql || GOTO exit + echo exit | sqlplus -S %ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% @%%F || GOTO exit ) for %%F in (inventory.ctl customer.ctl) do ( From b1f6ce27ab888618ee7178b89c3390e89224bd5c Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 12 Aug 2019 09:02:40 -0700 Subject: [PATCH 27/29] Setup script for BDC with AD integration --- .../ubuntu-single-node-vm-ad/README.md | 57 +++ .../ubuntu-single-node-vm-ad/cleanup-bdc.sh | 87 ++++ .../endpoint-patch.json | 23 ++ .../security-patch.json | 31 ++ .../ubuntu-single-node-vm-ad/setup-bdc-ad.sh | 374 ++++++++++++++++++ 5 files changed, 572 insertions(+) create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/cleanup-bdc.sh create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json create mode 100644 samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md new file mode 100644 index 00000000..44d4ecf2 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md @@ -0,0 +1,57 @@ + +# Deploy a SQL Server big data cluster on single node Kubernetes cluster (kubeadm) + +Using this sample bash script, you will deploy a single node Kubernetes cluster using kubeadm and a SQL Server big data cluster that is integrated with Active Directory domain. The script must be run from the VM you are planning to use for your kubeadm deployment. + +## Pre-requisites + +1. A vanilla Ubuntu 16.04 or 18.04 virtual or physical machine on your corporate network. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported. +1. Machine should have at least 8 CPUs, 64GB RAM and 100GB disk space. After installing the images you will be left with 50GB for data/logs across all components. +1. Update existing packages using commands below to ensure that the OS image is up to date + +``` bash +sudo apt update&&apt upgrade -y +sudo systemctl reboot +``` + +## Recommended Virtual Machine settings + +1. Use static memory configuration for the virtual machine. For example, in hyper-v installations do not use dynamic memory allocation but instead allocate the recommended 64 GB or higher. + +1. Use checkpoint or snapshot capability in your hyper visor so that you can rollback the virtual machine to a clean state. + +## Instructions to deploy SQL Server big data cluster + +1. Download the script on the VM you are planning to use for the deployment + +``` bash +curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc.sh +curl --output endpoint-patch.json https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json +curl --output security-patch.json https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json +``` + +1. Edit and modify the endpoint-patch.json & security-patch.json files to supply the values specific to your Active Directory environment + +1. Make the script executable + +``` bash +chmod +x setup-bdc-ad.sh +``` + +1. Run the script (make sure you are running with sudo) + +``` bash +sudo ./setup-bdc-ad.sh +``` + +1. Refresh alias setup for azdata + +``` bash +source ~/.bashrc +``` + +When prompted, provide your input for the password that will be used for all external endpoints: controller, SQL Server master and gateway. The password should be sufficiently complex based on existing rules for SQL Server password. The controller username is defaulted to *admin*. + +## Cleanup + +1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capabiility in your hyper-visor to rollback the virtual machine to a clean state. diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/cleanup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/cleanup-bdc.sh new file mode 100644 index 00000000..9a53ae8c --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/cleanup-bdc.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi +DIR_PREFIX=$1 + +kubeadm reset --force +unalias azdata + +systemctl stop kubelet +rm -rf /var/lib/cni/ +rm -rf /var/lib/etcd/ +rm -rf /run/flannel/ +rm -rf /var/lib/kubelet/* +rm -rf /etc/cni/ +rm -rf /etc/kubernetes/ + +ip link set cni0 down +#brctl delbr cni0 +ip link set flannel.1 down +#brctl delbr flannel.1 +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X + +rm -rf .azdata/ +rm -rf bdcdeploy/ + +# Remove mounts. +# +SERVICE_STOP_FAILED=0 + +systemctl | grep "/var/lib/kubelet/pods" | while read -r line; do + + # Retrieve the mount path + # + MOUNT_PATH=`echo "$line" | grep -v echo | egrep -oh -m 1 "(/var/lib/kubelet/pods).+"` + + if [ -z "$MOUNT_PATH" ]; then + continue + fi + + if [[ ! -d "$MOUNT_PATH" ]] && [[ ! -f "$MOUNT_PATH" ]]; then + + SERVICE=$(echo $line | cut -f1 -d' ') + + echo "Mount "$MOUNT_PATH" no longer exists." + echo "Stopping orphaned mount service: '$SERVICE'" + + systemctl stop $SERVICE + + if [ $? -ne 0 ]; then + SERVICE_STOP_FAILED=1 + fi + + echo "" + fi +done + +if [ $SERVICE_STOP_FAILED -ne 0 ]; then + echo "Not all services were stopped successfully. Please check the above output for more inforamtion." +else + echo "All orphaned services successfully stopped." +fi + +# Clean the mounted volumes. +# + +for i in $(seq 1 30); do + + vol="vol$i" + + sudo umount /mnt/local-storage/$vol + + sudo rm -rf /mnt/local-storage/$vol + +done + +# Reset kube +# +sudo apt-get purge -y kubeadm --allow-change-held-packages +sudo apt-get purge -y kubectl --allow-change-held-packages +sudo apt-get purge -y kubelet --allow-change-held-packages +sudo apt-get purge -y kubernetes-cni --allow-change-held-packages +sudo apt-get purge -y kube* --allow-change-held-packages +sudo apt -y autoremove +sudo rm -rf ~/.kube diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json new file mode 100644 index 00000000..a2b713fc --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json @@ -0,0 +1,23 @@ +{ + "patch": [ + { + "op": "replace", + "path": "spec.pools[?(@.spec.type=='Master')].spec", + "value": { + "type": "Master", + "dnsName": "mastersql.contoso.local", + "replicas": 1, + "endpoints": [ + { + "name": "Master", + "serviceType": "NodePort", + "port": 31433 + } + ] + } + } + ] +} + + + diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json new file mode 100644 index 00000000..73f95397 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json @@ -0,0 +1,31 @@ +{ + "patch": [ + { + "op": "add", + "path": "security", + "value": { + "useInternalDomain": false, + "ouDistinguishedName":"OU=bdc,DC=contoso,DC=local", + "dnsIpAddresses": ["11.11.111.11"], + "domainControllerFullyQualifiedDns": ["VM.CONTOSO.LOCAL"], + "realm":"CONTOSO.LOCAL", + "domainDnsName":"contoso.local", + "bdcAdminPrincipals": [ + "Domain Admins", "Enterprise Admins" + ], + "bdcUserPrincipals": [ + "Domain Users" + ] + } + }, + { + "op": "add", + "path": "spec.endpoints/0", + "value": { + "name": "Kerberos", + "serviceType": "NodePort", + "port": 30088 + } + } + ] +} diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh new file mode 100644 index 00000000..de6f0d13 --- /dev/null +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh @@ -0,0 +1,374 @@ +#!/bin/bash +set -Eeuo pipefail + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi + +# This is a script to create single-node Kubernetes cluster and deploy BDC on it. +# +export BDCDEPLOY_DIR=bdcdeploy + +# Get password as input. It is used as default for controller, SQL Server Master instance (sa account) and Knox. +# +while true; do + read -s -p "Create Password for Big Data Cluster: " password + echo + read -s -p "Confirm Password for Big Data Cluster: " password2 + echo + [ "$password" = "$password2" ] && break + echo "Password mismatch. Please try again." +done +echo "" +# Get docker credentials for private release. +# +read -p "Enter Docker username: " DOCKER_USERNAME +while true; do + read -s -p "Enter Docker Password: " docker_password + echo + read -s -p "Confirm Docker Password: " docker_password2 + echo + [ "$docker_password" = "$docker_password2" ] && break + echo "Password mismatch. Please try again." +done +export DOCKER_PASSWORD=$docker_password +echo "" + +# Get Domain Service Account Username and Password. +# +read -p "Enter the Domain Service Account Username: " ds_username +export DOMAIN_SERVICE_ACCOUNT_USERNAME=$ds_username + +while true; do + read -s -p "Enter Password for Domain Service Account: " ds_password + echo + read -s -p "Confirm your Domain Service Account Password: " ds_password2 + echo + [ "$ds_password" = "$ds_password2" ] && break + echo "Password mismatch. Please try again." +done +export DOMAIN_SERVICE_ACCOUNT_PASSWORD=$ds_password +echo "" + +# Name of virtualenv variable used. +# +export VIRTUALENV_NAME="bdcvenv" +export LOG_FILE="bdcdeploy.log" +export DEBIAN_FRONTEND=noninteractive + +# Requirements file. +# +export REQUIREMENTS_LINK="https://aka.ms/azdata" + +# Kube version. +# +KUBE_DPKG_VERSION=1.15.0-00 +KUBE_VERSION=1.15.0 + +# Wait for 5 minutes for the cluster to be ready. +# +TIMEOUT=600 +RETRY_INTERVAL=5 + +# Variables for pulling dockers. +# +export DOCKER_REGISTRY="private-repo.microsoft.com" +export DOCKER_REPOSITORY="mssql-private-preview" +export DOCKER_TAG="ctp3.2.1" + +# Variables used for azdata cluster creation. +# +export CONTROLLER_USERNAME=admin +export CONTROLLER_PASSWORD=$password +export MSSQL_SA_PASSWORD=$password +export KNOX_PASSWORD=$password +export ACCEPT_EULA=yes +export CLUSTER_NAME=mssql-cluster +export STORAGE_CLASS=local-storage +export PV_COUNT="30" + +IMAGES=( + mssql-app-service-proxy + mssql-appdeploy-init + mssql-controller + mssql-hadoop + mssql-mleap-serving-runtime + mssql-mlserver-py-runtime + mssql-mlserver-r-runtime + mssql-monitor-collectd + mssql-monitor-elasticsearch + mssql-monitor-fluentbit + mssql-monitor-grafana + mssql-monitor-influxdb + mssql-monitor-kibana + mssql-monitor-telegraf + mssql-security-knox + mssql-security-support + mssql-server-controller + mssql-server-data + mssql-service-proxy + mssql-ssis-app-runtime +) + + +# Make a directory for installing the scripts and logs. +# +mkdir -p $BDCDEPLOY_DIR +cd $BDCDEPLOY_DIR/ +touch $LOG_FILE + +{ +# Install all necessary packages: kuberenetes, docker, python3, python3-pip, request, azdata. +# +echo "" +echo "######################################################################################" +echo "Starting installing packages..." + +# Install docker. +# +apt-get update -q + +apt --yes install \ + software-properties-common \ + apt-transport-https \ + ca-certificates \ + curl + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + +apt update -q +apt-get install -q --yes docker-ce=18.06.2~ce~3-0~ubuntu --allow-downgrades +apt-mark hold docker-ce + +usermod --append --groups docker $USER + +# Install python3, python3-pip, requests. +# +apt-get install -q -y python3 +apt-get install -q -y python3-pip + +pip3 install requests --upgrade + +# Install and create virtualenv. +# +pip3 install --upgrade virtualenv +virtualenv -p python3 $VIRTUALENV_NAME +source $VIRTUALENV_NAME/bin/activate + +# Install azdata cli. +# +pip3 install -r $REQUIREMENTS_LINK +echo "Packages installed." + +# Load all pre-requisites for Kubernetes. +# +echo "###########################################################################" +echo "Starting to setup pre-requisites for kubernetes..." + +# Setup the kubernetes preprequisites. +# +echo $(hostname -i) $(hostname) >> /etc/hosts + +swapoff -a +sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab + +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + +cat </etc/apt/sources.list.d/kubernetes.list + +deb http://apt.kubernetes.io/ kubernetes-xenial main + +EOF + +# Install docker and packages to allow apt to use a repository over HTTPS. +# +apt-get update -q + +apt-get install -q -y ebtables ethtool + +#apt-get install -y docker.ce + +apt-get install -q -y apt-transport-https + +# Setup daemon. +# +cat > /etc/docker/daemon.json < /etc/sysctl.conf +echo net.ipv6.conf.default.disable_ipv6=1 > /etc/sysctl.conf +echo net.ipv6.conf.lo.disable_ipv6=1 > /etc/sysctl.conf + + +sysctl net.bridge.bridge-nf-call-iptables=1 + +# Setting up the persistent volumes for the kubernetes. +# +for i in $(seq 1 $PV_COUNT); do + + vol="vol$i" + + mkdir -p /mnt/local-storage/$vol + + mount --bind /mnt/local-storage/$vol /mnt/local-storage/$vol + +done +echo "Kubernetes pre-requisites have been completed." + +# Setup kubernetes cluster including remove taint on master. +# +echo "" +echo "#############################################################################" +echo "Starting to setup Kubernetes master..." + +# Initialize a kubernetes cluster on the current node. +# +sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION + +mkdir -p $HOME/.kube +mkdir -p /home/$SUDO_USER/.kube + +sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.kube/config + +# To enable a single node cluster remove the taint that limits the first node to master only service. +# +master_node=`kubectl get nodes --no-headers=true --output=custom-columns=NAME:.metadata.name` +kubectl taint nodes ${master_node} node-role.kubernetes.io/master:NoSchedule- + +# Local storage provisioning. +# +kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/local-storage-provisioner.yaml + +# Install the software defined network. +# +kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml + +# helm init + +kubectl apply -f https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/rbac.yaml + +# Verify that the cluster is ready to be used. +# +echo "Verifying that the cluster is ready for use..." +while true ; do + + if [[ "$TIMEOUT" -le 0 ]]; then + echo "Cluster node failed to reach the 'Ready' state. Kubeadm setup failed." + exit 1 + fi + + status=`kubectl get nodes --no-headers=true | awk '{print $2}'` + + if [ "$status" == "Ready" ]; then + break + fi + + sleep "$RETRY_INTERVAL" + + TIMEOUT=$(($TIMEOUT-$RETRY_INTERVAL)) + + echo "Cluster not ready. Retrying..." +done + + +# Install the dashboard for Kubernetes. +# +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml + +kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard +echo "Kubernetes master setup done." + +# Pull docker images of SQL Server big data cluster. +# +echo "" +echo "############################################################################" +echo "Starting to pull docker images..." +echo "Pulling images from repository: " $DOCKER_REGISTRY"/"$DOCKER_REPOSITORY + +docker login $DOCKER_REGISTRY -u $DOCKER_USERNAME -p $DOCKER_PASSWORD +for image in "${IMAGES[@]}"; +do + docker pull $DOCKER_REGISTRY/$DOCKER_REPOSITORY/$image:$DOCKER_TAG + echo "Docker image" $image " pulled." +done +docker logout $DOCKER_REGISTRY +echo "Docker images pulled." + +# Deploy azdata bdc create cluster. +# +echo "" +echo "############################################################################" +echo "Starting to deploy azdata cluster..." + +# Command to create cluster for single node cluster. +# +azdata bdc config init --source kubeadm-dev-test --target kubeadm-custom -f +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.repository=$DOCKER_REPOSITORY" +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.registry=$DOCKER_REGISTRY" +azdata bdc config replace -c kubeadm-custom/control.json -j ".spec.docker.imageTag=$DOCKER_TAG" +azdata bdc config replace -c kubeadm-custom/cluster.json -j "$.spec.pools[?(@.spec.type == "Data")].spec.replicas=1" +azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.data.className=$STORAGE_CLASS" +azdata bdc config replace -c kubeadm-custom/control.json -j "spec.storage.logs.className=$STORAGE_CLASS" +azdata bdc config patch -c kubeadm-custom/control.json -p $STARTUP_PATH/security-patch.json +azdata bdc config patch -c kubeadm-custom/cluster.json -p $STARTUP_PATH/endpoint-patch.json +azdata bdc create -c kubeadm-custom --accept-eula $ACCEPT_EULA +echo "Azdata cluster created." + +# Setting context to cluster. +# +kubectl config set-context --current --namespace $CLUSTER_NAME + +# Login and get endpoint list for the cluster. +# +azdata login -n $CLUSTER_NAME +azdata bdc endpoint list --output table + +if [ -d "$HOME/.azdata/" ]; then + sudo chown -R $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/.azdata/ +fi + +if [ -d "$HOME/bdcdeploy/" ]; then + sudo chown -R $(id -u $SUDO_USER):$(id -g $SUDO_USER) $HOME/bdcdeploy/ +fi + +echo "alias azdata='$BDCDEPLOY_DIR/$VIRTUALENV_NAME/bin/azdata'" >> $HOME/.bashrc +}| tee $LOG_FILE + From 3d3968a71fbbffd23d6b5188944a9c392d48557b Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 12 Aug 2019 09:05:21 -0700 Subject: [PATCH 28/29] Updated README --- .../deployment/kubeadm/ubuntu-single-node-vm-ad/README.md | 6 +++--- .../deployment/kubeadm/ubuntu-single-node-vm/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md index 44d4ecf2..453de70a 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/README.md @@ -25,12 +25,12 @@ sudo systemctl reboot 1. Download the script on the VM you are planning to use for the deployment ``` bash -curl --output setup-bdc.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc.sh +curl --output setup-bdc-ad.sh https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh curl --output endpoint-patch.json https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/endpoint-patch.json curl --output security-patch.json https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/security-patch.json ``` -1. Edit and modify the endpoint-patch.json & security-patch.json files to supply the values specific to your Active Directory environment +1. Modify the endpoint-patch.json & security-patch.json files to supply the values specific to your Active Directory environment 1. Make the script executable @@ -54,4 +54,4 @@ When prompted, provide your input for the password that will be used for all ext ## Cleanup -1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capabiility in your hyper-visor to rollback the virtual machine to a clean state. +1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capability in your hyper-visor to rollback the virtual machine to a clean state. diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md index a6090a02..adb926cd 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md @@ -50,4 +50,4 @@ When prompted, provide your input for the password that will be used for all ext ## Cleanup -1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capabiility in your hyper-visor to rollback the virtual machine to a clean state. +1. The [cleanup-bdc.sh](cleanup-bdc.sh/) script is provided as convenience to reset the environment in case of errors. However, we recommend that you use a virtual machine for testing purposes and use the snapshot capability in your hyper-visor to rollback the virtual machine to a clean state. From f1e503e2f5b869fc1dfc1c3238a745676cd320c1 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 12 Aug 2019 09:12:25 -0700 Subject: [PATCH 29/29] Added STARTUP_PATH --- .../deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh index de6f0d13..62176458 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh @@ -6,6 +6,8 @@ if [ "$EUID" -ne 0 ] exit fi +STARTUP_PATH=$(pwd) + # This is a script to create single-node Kubernetes cluster and deploy BDC on it. # export BDCDEPLOY_DIR=bdcdeploy