diff --git a/.gitignore b/.gitignore index 9d7f1407..6735850e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ build/ bld/ [Bb]in/ [Oo]bj/ +out/ # Visual Studio 2015 cache/options directory .vs/ @@ -435,3 +436,8 @@ samples/databases/wide-world-importers/workload-drivers/order-insert/Multithread /samples/features/epm-framework/5.0/2Reporting/PolicyReports/bin/Debug /samples/features/epm-framework/5.0/2Reporting/PolicyReports/PolicyDashboard - Backup.rdl /samples/features/epm-framework/5.0/2Reporting/PolicyReports/PolicyDashboardFiltered.rdl.data +samples/features/sql-management-objects/src/out/CodeCoverage/CodeCoverage.config + +# Certificates +*.pem +*.p12 diff --git a/media/demos/sql-sampledb/sampledb1/README.md b/media/demos/sql-sampledb/sampledb1/README.md new file mode 100644 index 00000000..0dc98dd3 --- /dev/null +++ b/media/demos/sql-sampledb/sampledb1/README.md @@ -0,0 +1,3 @@ +# Sample Database 1 (Employee DB Dummy Data) + +This is a fundamental database that houses data based on a conventinal Sales environment. Sourced from progress.com with slight modifications. diff --git a/media/demos/sql-sampledb/sampledb1/sampleDB1.sql b/media/demos/sql-sampledb/sampledb1/sampleDB1.sql new file mode 100644 index 00000000..adb245be --- /dev/null +++ b/media/demos/sql-sampledb/sampledb1/sampleDB1.sql @@ -0,0 +1,53 @@ +/* +This is sourced from progress.com, variation by lightofodin. +Basic database housing a sales scenario, that features employees and departments. +*/ +CREATE TABLE emp ( +empno INT PRIMARY KEY, +ename VARCHAR(10), +job VARCHAR(9), +mgr INT NULL, +hiredate DATETIME, +sal NUMERIC(7,2), +comm NUMERIC(7,2) NULL, +dept INT) +begin +insert into emp values + (1,'JOHNSON','ADMIN',6,'12-17-1990',18000,NULL,4) +insert into emp values + (2,'HARDING','MANAGER',9,'02-02-1998',52000,300,3) +insert into emp values + (3,'TAFT','SALES I',2,'01-02-1996',25000,500,3) +insert into emp values + (4,'HOOVER','SALES I',2,'04-02-1990',27000,NULL,3) +insert into emp values + (5,'LINCOLN','TECH',6,'06-23-1994',22500,1400,4) +insert into emp values + (6,'GARFIELD','MANAGER',9,'05-01-1993',54000,NULL,4) +insert into emp values + (7,'POLK','TECH',6,'09-22-1997',25000,NULL,4) +insert into emp values + (8,'GRANT','ENGINEER',10,'03-30-1997',32000,NULL,2) +insert into emp values + (9,'JACKSON','CEO',NULL,'01-01-1990',75000,NULL,4) +insert into emp values + (10,'FILLMORE','MANAGER',9,'08-09-1994',56000,NULL,2) +insert into emp values + (11,'ADAMS','ENGINEER',10,'03-15-1996',34000,NULL,2) +insert into emp values + (12,'WASHINGTON','ADMIN',6,'04-16-1998',18000,NULL,4) +insert into emp values + (13,'MONROE','ENGINEER',10,'12-03-2000',30000,NULL,2) +insert into emp values + (14,'ROOSEVELT','CPA',9,'10-12-1995',35000,NULL,1) +end +CREATE TABLE dept ( +deptno INT NOT NULL, +dname VARCHAR(14), +loc VARCHAR(13)) +begin +insert into dept values (1,'ACCOUNTING','ST LOUIS') +insert into dept values (2,'RESEARCH','NEW YORK') +insert into dept values (3,'SALES','ATLANTA') +insert into dept values (4, 'OPERATIONS','SEATTLE') +end \ No newline at end of file diff --git a/samples/features/Azure Data Studio/Notebooks/Install cluster dependencies.ipynb b/samples/features/Azure Data Studio/Notebooks/Install cluster dependencies.ipynb deleted file mode 100644 index 7a137897..00000000 --- a/samples/features/Azure Data Studio/Notebooks/Install cluster dependencies.ipynb +++ /dev/null @@ -1,316 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install Dependencies to deploy SQL Server 2019 Big Data Clusters\n", - "----------------------------------------------------------------\n", - "\n", - "The following tools are most important for managing, connecting to, and\n", - "querying the cluster:\n", - "\n", - "- **Azure CLI**\n", - "- **kubectl**\n", - "- **mssqlctl**\n", - "\n", - "**The cell below tries to install these dependencies for you. If you\n", - "still experience any issues in executing this cell then please install\n", - "from the following Installation links below.**\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
ToolRequiredDescriptionInstallation
mssqlctlYesCommand-line tool for installing and managing a big data cluster.Install
kubectlYesCommand-line tool for monitoring the underlying Kuberentes cluster (More info).Windows | Linux
Azure CLIFor AKSModern command-line interface for managing Azure services. Used with AKS big data cluster deployments (More info).Install
\n", - "\n", - "Steps\n", - "-----\n", - "\n", - "### Provide overrides for any default installation parameters\n", - "\n", - "You don’t need to provide any installation parameters, we’ll provide a\n", - "set of default which will provide a good experience.\n", - "\n", - "However, feel free to override any of the defaults provided below here:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "mssql_version=\"\"\n", - "docker_registry=\"\"\n", - "\n", - "windows_azcli_url=\"\"\n", - "windows_kubectl_url=\"\"\n", - "\n", - "azure_cli_use_force_install_option = None\n", - "azure_cli_use_user_install_option = None\n", - "\n", - "mssqlctl_url=\"\" # url to download mssqlctl from (without ending filename)\n", - "mssqlctl_use_force_install_option = None\n", - "mssqlctl_use_user_install_option = None\n", - "\n", - "skip_mssqlctl_uninstall = None\n", - "skip_mssqlctl_install = None" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Provide default installation parameters\n", - "\n", - "A default set of parameters to provide a good initial experience of SQL\n", - "Server 2019 big data clusters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if mssql_version==\"\":\n", - " mssql_version=\"ctp-2.5\"\n", - "\n", - "if windows_azcli_url==\"\":\n", - " windows_azcli_url=\"https://aka.ms/installazurecliwindows\"\n", - "\n", - "if windows_kubectl_url==\"\":\n", - " windows_kubectl_url=\"https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/windows/amd64/kubectl.exe\"\n", - "\n", - "if docker_repository==\"\":\n", - " docker_repository=\"aris-p-release-candidate-gb\"\n", - "\n", - "if mssqlctl_url==\"\":\n", - " mssqlctl_url=\"\"http://helsinki/browse/packages/python/{0}/mssqlctl/{1}\".format(docker_repository, 'requirements.txt')\n", - "\n", - "if azure_cli_use_force_install_option is None:\n", - " azure_cli_use_force_install_option = False\n", - "\n", - "if azure_cli_use_user_install_option is None:\n", - " azure_cli_use_user_install_option = True\n", - "\n", - "if mssqlctl_use_force_install_option is None:\n", - " mssqlctl_use_force_install_option = False\n", - "\n", - "if mssqlctl_use_user_install_option is None:\n", - " mssqlctl_use_user_install_option = True\n", - "\n", - "if skip_mssqlctl_uninstall is None:\n", - " skip_mssqlctl_uninstall = True\n", - "\n", - "if skip_mssqlctl_install is None:\n", - " skip_mssqlctl_install = False\n", - "\n", - "print('mssql_version = ' + mssql_version)\n", - "print('docker_registry = ' + docker_registry)\n", - "print('azure_cli_use_force_install_option = ' + str(azure_cli_use_force_install_option))\n", - "print('azure_cli_use_user_install_option = ' + str(azure_cli_use_user_install_option))\n", - "print('mssqlctl_url = ' + mssqlctl_url)\n", - "print('mssqlctl_use_force_install_option = ' + str(mssqlctl_use_force_install_option))\n", - "print('mssqlctl_use_user_install_option = ' + str(mssqlctl_use_user_install_option))\n", - "print('skip_mssqlctl_uninstall = ' + str(skip_mssqlctl_uninstall))\n", - "print('skip_mssqlctl_install = ' + str(skip_mssqlctl_install))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Install Azure CLI" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "force_install_option='--force-install' if azure_cli_use_force_install_option else ''\n", - "user_install_option='--user' if azure_cli_use_user_install_option else ''\n", - "\n", - "print(f'START: !{sys.executable} -m pip install azure-cli {force_install_option} {user_install_option}\\n')\n", - "\n", - "!{sys.executable} -m pip install azure-cli {force_install_option} {user_install_option}\n", - "if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t!{sys.executable} -m pip install azure-cli {force_install_option} {user_install_option}\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - "print(f'\\nSUCCESS: !{sys.executable} -m pip install azure-cli {force_install_option} {user_install_option}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Install Kubernetes CLI" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import platform\n", - "import requests\n", - "import sys\n", - "\n", - "if platform.system()==\"Darwin\":\n", - " print(f'START: !brew update\\n')\n", - "\n", - " !brew update\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !brew update\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !brew update')\n", - " print(f'START: !brew install kubernetes-cli\\n')\n", - "\n", - " !brew install kubernetes-cli\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !brew install kubernetes-cli\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !brew install kubernetes-cli')\n", - "elif platform.system()==\"Windows\":\n", - " print(f'START: !curl -LO {windows_kubectl_url}\\n')\n", - "\n", - " !curl -LO {windows_kubectl_url}\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !curl -LO {windows_kubectl_url}\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !curl -LO {windows_kubectl_url}')\n", - "elif platform.system()==\"Linux\":\n", - " print(f'START: !sudo apt-get update\\n')\n", - "\n", - " !sudo apt-get update\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !sudo apt-get update\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !sudo apt-get update')\n", - " print(f'START: !sudo apt-get install -y kubectl\\n')\n", - "\n", - " !sudo apt-get install -y kubectl\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !sudo apt-get install -y kubectl\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !sudo apt-get install -y kubectl')\n", - "else:\n", - " raise SystemExit(\"Platform '{0}' is not recognized, must be 'Darwin', 'Windows' or 'Linux'\".format(platform.system()))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Uninstall MSSQLCTL CLI" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "if not skip_mssqlctl_uninstall:\n", - " print(f'START: !{sys.executable} -m pip uninstall -r {mssqlctl_url} --yes --trusted-host helsinki\\n')\n", - "\n", - " !{sys.executable} -m pip uninstall -r {mssqlctl_url} --yes --trusted-host helsinki\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !{sys.executable} -m pip uninstall -r {mssqlctl_url} --yes --trusted-host helsinki\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !{sys.executable} -m pip uninstall -r {mssqlctl_url} --yes --trusted-host helsinki')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Install MSSQLCTL CLI" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "force_install_option='--force-install' if mssqlctl_use_force_install_option else ''\n", - "user_install_option='--user' if mssqlctl_use_user_install_option else ''\n", - "\n", - "if not skip_mssqlctl_install:\n", - " print(f'START: !{sys.executable} -m pip install -r {mssqlctl_url} {force_install_option} {user_install_option} --trusted-host helsinki\\n')\n", - "\n", - " !{sys.executable} -m pip install -r {mssqlctl_url} {force_install_option} {user_install_option} --trusted-host helsinki\n", - " if _exit_code != 0:\n", - " raise SystemExit('Shell command:\\n\\n\\t !{sys.executable} -m pip install -r {mssqlctl_url} {force_install_option} {user_install_option} --trusted-host helsinki\\n\\nreturned non-zero exit code: ' + str(_exit_code) + '.\\n')\n", - "\n", - " print(f'\\nSUCCESS: !{sys.executable} -m pip install -r {mssqlctl_url} {force_install_option} {user_install_option} --trusted-host helsinki')" - ] - } - ], - "nbformat": 4, - "nbformat_minor": 5, - "metadata": { - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "azdata": { - "test": { - "strategy": "Sequential", - "dri": false, - "ci": false, - "gci": false - }, - "publish": { - "access": { - "current": "Internal", - "goal": "Public" - }, - "state": "Draft" - } - } - } -} - diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql index 9fd76a49..842613ba 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql @@ -3,7 +3,7 @@ -- bigger - so you can see more impactful -- Intelligent QP demonstrations (aka.ms/iqp) -- --- Script last updated 10/02/2018 +-- Script last updated 05/03/2019 -- -- Database backup source: aka.ms/wwibak -- @@ -100,4 +100,4 @@ GO UPDATE Fact.OrderHistoryExtended SET [WWI Order ID] = [Order Key]; -GO +GO \ No newline at end of file diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql index 26e6012c..2bf41899 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql @@ -9,7 +9,17 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- -USE WideWorldImportersDW; + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; GO -- Compare execution time and distinct counts diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql new file mode 100644 index 00000000..b5a70078 --- /dev/null +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql @@ -0,0 +1,54 @@ +-- ******************************************************** -- +-- Batch mode Adaptive Join + +-- See https://aka.ms/IQP for more background + +-- Demo scripts: https://aka.ms/IQPDemos + +-- This demo is on SQL Server 2017 and Azure SQL DB + +-- Email IntelligentQP@microsoft.com for questions\feedback +-- ******************************************************** -- + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +-- Show with Live Query Stats +SELECT [fo].[Order Key], [si].[Lead Time Days], [fo].[Quantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [fo].[Quantity] = 360; +GO + +-- Inserting quantity row that doesn't exist in the table yet +DELETE [Fact].[Order] +WHERE Quantity = 361; + +INSERT [Fact].[Order] +([City Key], [Customer Key], [Stock Item Key], [Order Date Key], [Picked Date Key], [Salesperson Key], [Picker Key], [WWI Order ID], [WWI Backorder ID], Description, Package, Quantity, [Unit Price], [Tax Rate], [Total Excluding Tax], [Tax Amount], [Total Including Tax], [Lineage Key]) +SELECT TOP 5 [City Key], [Customer Key], [Stock Item Key], + [Order Date Key], [Picked Date Key], [Salesperson Key], + [Picker Key], [WWI Order ID], [WWI Backorder ID], + Description, Package, 361, [Unit Price], [Tax Rate], + [Total Excluding Tax], [Tax Amount], [Total Including Tax], + [Lineage Key] +FROM [Fact].[Order]; +GO + +-- Show with Live Query Stats +SELECT [fo].[Order Key], [si].[Lead Time Days], [fo].[Quantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [fo].[Quantity] = 361; +GO \ No newline at end of file diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql new file mode 100644 index 00000000..1cd6cfc6 --- /dev/null +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql @@ -0,0 +1,46 @@ +-- ******************************************************** -- +-- Batch mode Memory Grant Feedback + +-- See https://aka.ms/IQP for more background + +-- Demo scripts: https://aka.ms/IQPDemos + +-- This demo is on SQL Server 2017 and Azure SQL DB + +-- Email IntelligentQP@microsoft.com for questions\feedback +-- ******************************************************** -- + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +-- Intentionally forcing a row underestimate +CREATE OR ALTER PROCEDURE [FactOrderByLineageKey] + @LineageKey INT +AS +SELECT [fo].[Order Key], [fo].[Description] +FROM [Fact].[Order] AS [fo] +INNER HASH JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [fo].[Lineage Key] = @LineageKey + AND [si].[Lead Time Days] > 0 +ORDER BY [fo].[Stock Item Key], [fo].[Order Date Key] DESC +OPTION (MAXDOP 1); +GO + +-- Compiled and executed using a lineage key that doesn't have rows +EXEC [FactOrderByLineageKey] 8; +GO + +-- Execute this query a few times - each time looking at +-- the plan to see impact on spills, memory grant size, and run time +EXEC [FactOrderByLineageKey] 9; +GO \ No newline at end of file diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql index 39dea339..0ec77748 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql @@ -10,12 +10,18 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- +USE [master]; +GO + ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; GO ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO +USE [WideWorldImportersDW]; +GO + -- Row mode due to hint SELECT [Tax Rate], [Lineage Key], diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql new file mode 100644 index 00000000..eb2c1161 --- /dev/null +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql @@ -0,0 +1,173 @@ +-- ******************************************************** -- +-- Interleaved Execution + +-- See https://aka.ms/IQP for more background + +-- Demo scripts: https://aka.ms/IQPDemos + +-- This demo is on SQL Server 2017 and Azure SQL DB + +-- Email IntelligentQP@microsoft.com for questions\feedback +-- ******************************************************** -- + +/* + Create MSTVF +*/ + +USE [WideWorldImportersDW]; +GO + +CREATE FUNCTION [Fact].[WhatIfOutlierEventQuantity](@event VARCHAR(15), @beginOrderDateKey DATE, @endOrderDateKey DATE) +RETURNS @OutlierEventQuantity TABLE ( + [Order Key] [bigint], + [City Key] [int] NOT NULL, + [Customer Key] [int] NOT NULL, + [Stock Item Key] [int] NOT NULL, + [Order Date Key] [date] NOT NULL, + [Picked Date Key] [date] NULL, + [Salesperson Key] [int] NOT NULL, + [Picker Key] [int] NULL, + [OutlierEventQuantity] [int] NOT NULL) +AS +BEGIN + +-- Valid @event values + -- 'Mild Recession' + -- 'Hurricane - South Atlantic' + -- 'Hurricane - East South Central' + -- 'Hurricane - West South Central' + IF @event = 'Mild Recession' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [o].[Quantity] > 2 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + + IF @event = 'Hurricane - South Atlantic' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [o].[Quantity] > 10 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + WHERE [c].[State Province] IN + ('Florida', 'Georgia', 'Maryland', 'North Carolina', + 'South Carolina', 'Virginia', 'West Virginia', + 'Delaware') + AND [o].[Order Date Key] BETWEEN @beginOrderDateKey AND @endOrderDateKey + + IF @event = 'Hurricane - East South Central' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [o].[Quantity] > 50 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + INNER JOIN [Dimension].[Stock Item] AS [si] + ON [si].[Stock Item Key] = [o].[Stock Item Key] + WHERE [c].[State Province] IN + ('Alabama', 'Kentucky', 'Mississippi', 'Tennessee') + AND [si].[Buying Package] = 'Carton' + AND [o].[Order Date Key] BETWEEN @beginOrderDateKey AND @endOrderDateKey + + IF @event = 'Hurricane - West South Central' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [cu].[Customer] = 'Unknown' THEN 0 + WHEN [cu].[Customer] <> 'Unknown' AND + [o].[Quantity] > 10 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + INNER JOIN [Dimension].[Customer] AS [cu] + ON [cu].[Customer Key] = [o].[Customer Key] + WHERE [c].[State Province] IN + ('Arkansas', 'Louisiana', 'Oklahoma', 'Texas') + AND [o].[Order Date Key] BETWEEN @beginOrderDateKey AND @endOrderDateKey + + RETURN +END +GO + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 130; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +SELECT [fo].[Order Key], [fo].[Description], [fo].[Package], + [fo].[Quantity], [foo].[OutlierEventQuantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Fact].[WhatIfOutlierEventQuantity]('Mild Recession', + '1-01-2013', + '10-15-2014') AS [foo] ON [fo].[Order Key] = [foo].[Order Key] + AND [fo].[City Key] = [foo].[City Key] + AND [fo].[Customer Key] = [foo].[Customer Key] + AND [fo].[Stock Item Key] = [foo].[Stock Item Key] + AND [fo].[Order Date Key] = [foo].[Order Date Key] + AND [fo].[Picked Date Key] = [foo].[Picked Date Key] + AND [fo].[Salesperson Key] = [foo].[Salesperson Key] + AND [fo].[Picker Key] = [foo].[Picker Key] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [si].[Lead Time Days] > 0 + AND [fo].[Quantity] > 50; +GO + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +SELECT [fo].[Order Key], [fo].[Description], [fo].[Package], + [fo].[Quantity], [foo].[OutlierEventQuantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Fact].[WhatIfOutlierEventQuantity]('Mild Recession', + '1-01-2013', + '10-15-2014') AS [foo] ON [fo].[Order Key] = [foo].[Order Key] + AND [fo].[City Key] = [foo].[City Key] + AND [fo].[Customer Key] = [foo].[Customer Key] + AND [fo].[Stock Item Key] = [foo].[Stock Item Key] + AND [fo].[Order Date Key] = [foo].[Order Date Key] + AND [fo].[Picked Date Key] = [foo].[Picked Date Key] + AND [fo].[Salesperson Key] = [foo].[Salesperson Key] + AND [fo].[Picker Key] = [foo].[Picker Key] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [si].[Lead Time Days] > 0 + AND [fo].[Quantity] > 50; +GO diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql index 419b96c4..698b32cc 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql @@ -10,13 +10,16 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- -ALTER DATABASE WideWorldImportersDW SET COMPATIBILITY_LEVEL = 150; +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; GO ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO -USE WideWorldImportersDW; +USE [WideWorldImportersDW]; GO -- Simulate out-of-date stats @@ -30,10 +33,10 @@ GO SELECT fo.[Order Key], fo.Description, si.[Lead Time Days] -FROM Fact.OrderHistory AS fo +FROM Fact.OrderHistory AS fo INNER HASH JOIN Dimension.[Stock Item] AS si ON fo.[Stock Item Key] = si.[Stock Item Key] -WHERE fo.[Lineage Key] = 9 +WHERE fo.[Lineage Key] = 9 AND si.[Lead Time Days] > 19; -- Cleanup diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql index 67dda7eb..e5b2124a 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql @@ -9,19 +9,21 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- -USE WideWorldImportersDW; +USE [master]; GO -ALTER DATABASE WideWorldImportersDW -SET COMPATIBILITY_LEVEL = 150; +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; GO -ALTER DATABASE SCOPED CONFIGURATION -CLEAR PROCEDURE_CACHE; +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO + +USE [WideWorldImportersDW]; +GO + /* Adapted from SQL Server Books Online -https://docs.microsoft.com/en-us/sql/relational-databases/user-defined-functions/scalar-udf-inlining?view=sqlallproducts-allversions +https://docs.microsoft.com/sql/relational-databases/user-defined-functions/scalar-udf-inlining?view=sqlallproducts-allversions */ CREATE OR ALTER FUNCTION dbo.customer_category(@CustomerKey INT) diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql index 3b1dd936..6277b4a1 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql @@ -13,7 +13,10 @@ USE [master]; GO -ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO USE [WideWorldImportersDW]; diff --git a/samples/features/intelligent-query-processing/readme.md b/samples/features/intelligent-query-processing/readme.md index 38ad3a31..1f79e71a 100644 --- a/samples/features/intelligent-query-processing/readme.md +++ b/samples/features/intelligent-query-processing/readme.md @@ -8,6 +8,9 @@ Here are the instructions to prepare for demonstrating Intelligent QP's latest r Demos (with more on the way!): +- [Batch Mode Adaptive Join](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Batch%20Mode%20Adaptive%20Join.sql) +- [Interleaved Execution](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Interleaved%20Execution.sql) +- [Batch mode memory grant feedback](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Batch%20Mode%20MGF.sql) - [Row mode memory grant feedback](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Row%20Mode%20MGF.sql) - [Batch mode on rowstore](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Batch%20Mode%20on%20Rowstore.sql) - [APPROX_COUNT_DISTINCT](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20APPROX_COUNT_DISTINCT.sql) diff --git a/samples/features/query-store/Auto-Param Analysis.sql b/samples/features/query-store/Auto-Param Analysis.sql new file mode 100644 index 00000000..201876eb --- /dev/null +++ b/samples/features/query-store/Auto-Param Analysis.sql @@ -0,0 +1,61 @@ +USE [AdventureWorks2016_EXT] +GO + +/* (1) Do cardinality analysis when suspect on ad-hoc workloads*/ +SELECT COUNT(*) AS CountQueryTextRows FROM sys.query_store_query_text; +SELECT COUNT(*) AS CountQueryRows FROM sys.query_store_query; +SELECT COUNT(DISTINCT query_hash) AS CountDifferentQueryRows FROM sys.query_store_query; +SELECT COUNT(*) AS CountPlanRows FROM sys.query_store_plan; +SELECT COUNT(DISTINCT query_plan_hash) AS CountDifferentPlanRows FROM sys.query_store_plan; + +/* (2) Get Compile Vs Execution times: ad-hoc workloads tend to spend lot of time in compilation*/ +EXEC sp_GetCompilAndExecutionTotalTime + +/* (3) See query pattern*/ +SELECT TOP 10 * FROM sys.query_store_query_text + + +/* (4) I'm not getting new queries? +Look at Query Store parameters - is Query Store in READ_ONLY mode? +*/ +SELECT current_storage_size_mb, max_storage_size_mb, desired_state, desired_state_desc, actual_state, actual_state_desc, readonly_reason, flush_interval_seconds, +interval_length_minutes, stale_query_threshold_days, max_plans_per_query, query_capture_mode, query_capture_mode_desc, size_based_cleanup_mode, +size_based_cleanup_mode_desc, actual_state_additional_info +FROM sys.database_query_store_options + +ALTER DATABASE [AdventureWorks2016_EXT] SET QUERY_STORE CLEAR; +ALTER DATABASE [AdventureWorks2016_EXT] SET QUERY_STORE = ON (OPERATION_MODE = READ_WRITE); +GO + +/* (5) How do we fix the Auto-Param problem?*/ + +/* At the query level: apply the plan guide for selected query template */ +DECLARE @stmt nvarchar(max); +DECLARE @params nvarchar(max); +EXEC sp_get_query_template + N'select * from part p join partdetails pp on p.partid = pp.partid where p.partid = 46911', + @stmt OUTPUT, + @params OUTPUT; + +EXEC sp_create_plan_guide + N'TemplateGuide1', + @stmt, + N'TEMPLATE', + NULL, + @params, + N'OPTION(PARAMETERIZATION FORCED)'; + +/*(6) Alternative (at the database level): force parametrization for all queries*/ +ALTER DATABASE [AdventureWorks2016_EXT] SET PARAMETERIZATION FORCED; + +/* Run analysis query (1), (2) again to see results of parametrization */ + +/*(7) Reset the DB state*/ +ALTER DATABASE [AdventureWorks2016_EXT] SET PARAMETERIZATION SIMPLE; +GO +EXEC sp_control_plan_guide N'DROP', N'TemplateGuide1'; +GO +ALTER DATABASE [AdventureWorks2016_EXT] SET QUERY_STORE CLEAR; +ALTER DATABASE [AdventureWorks2016_EXT] SET QUERY_STORE = ON (OPERATION_MODE = READ_WRITE); +SELECT * FROM sys.database_query_store_options +GO \ No newline at end of file diff --git a/samples/features/query-store/NewDatabaseSettings.sql b/samples/features/query-store/NewDatabaseSettings.sql new file mode 100644 index 00000000..a0e418aa Binary files /dev/null and b/samples/features/query-store/NewDatabaseSettings.sql differ diff --git a/samples/features/query-store/QS_SSMS.png b/samples/features/query-store/QS_SSMS.png new file mode 100644 index 00000000..47069425 Binary files /dev/null and b/samples/features/query-store/QS_SSMS.png differ diff --git a/samples/features/query-store/QueryStoreSimpleDemo.exe b/samples/features/query-store/QueryStoreSimpleDemo.exe new file mode 100644 index 00000000..d2611cff Binary files /dev/null and b/samples/features/query-store/QueryStoreSimpleDemo.exe differ diff --git a/samples/features/query-store/README.md b/samples/features/query-store/README.md new file mode 100644 index 00000000..155f1d30 --- /dev/null +++ b/samples/features/query-store/README.md @@ -0,0 +1,46 @@ +## Query Store demo + +This demo shows capabilities of Query Store. Usually we demo 3-4 scenarios: +1. How to turn on and initially configure Query Store +2. How Query Store collects & exposed data +3. Detecting and fixing query with plan choice regression +4. Detecting and fixing workload that is candidate for auto-parametrization + +## Prerequisites + +Restore AdventureWorks2016_EXT database from the provided BAK at https://github.com/Microsoft/sql-server-samples/releases/tag/adventureworks. + +After restoring the database AdventureWorks2016_EXT, go to Properties / Query Store tab, turn ON Query Store, and configure it according to best practices in https://docs.microsoft.com/sql/relational-databases/performance/best-practice-with-the-query-store. + +![Query Store in SSMS](./QS_SSMS.png) + +Use docs content to walk through main config settings: https://docs.microsoft.com//sql/relational-databases/performance/best-practice-with-the-query-store#Configure + +### How Query Store Works +Open ShowBasics.sql script and execute queries individually: +- Run simple `SELECT * FROM` Part +- Show where query ends in sys.query_store_query_text, sys.query_store_query, sys.query_store_plan, sys.query_store_runtime_stats +- Use custom view `vw_QueryStoreCompileInfo` to get info more easily. The main point here is: people can write their own scripts combining Query Store views +- Execute the same user query from the proc, using sp_executesql, trigger and show that containing object defines query identity in QDS (each instance of the same query text becomes separate query that can be monitored and tuned independently) +- Show what happens with query that gets auto-parametrized. It cannot be searched using the original query text because QDS stores query as parametrized. Hopefully, sys.fn_stmt_sql_handle_from_sql_stmt can be used to track down query using original query text +- Run `vw_QueryStoreRuntimeInfo` (again custom view) to show main runtime stats combined with query/plan info + +### Query with plan regression +1. Run QueryStoreSimpleDemo.exe with option R or option S +2. Open SSMS, analyze and explain - two execution plans that SQL Server use alternately (switches between 2 plan almost randomly). This is known as Parameter Sniffing problem - plan gets generated based on parameter available at the compilation time. When compilation happens frequently and randomly and data is skewed (not all parameter values are uniformly distributed PSP is likely to occur and degradations are common) +3. Force better plan, explain what happens (SSMS) +4. Summarize benefits for DBA – fixing performance quickly without knowing details about the query. Fully transparent to running apps + +### Detect and fix ad hoc workload that is candidate for parametrization +1. Run QueryStoreSimpleDemo.exe with option P and let it work for some time (15-20 sec) +2. Open “Auto-Param Analysis.sql” and run queries from groups (1) and (2). What we see is: +a. Large number of queries / plan entries, small number of different query/plan hashes indicates queries that are not parametrized although they are good candidates +b. Relatively big compile time shows that system wastes resources on compilation instead of execution +3. Open SSMS Top Resource Consuming queries – if you increase number of presented queries to 50 you’ll see that majority of queries has similar /negligible consumption – there’s nothing user can optimize/tune. This is what we call “death by a thousand of cuts” +4. Run (3) query to see query text pattern – it becomes obvious that queries differ only by provided literal value +5. If you run (1) you’ll notice that numbers do not change although workload is running. (4) gives us the answer – Query Store went to READ_ONLY due to large number of queries / plans. This is another point you should make: ad-hoc queries are not bad for SQL Server & execution but also for Query Store as it goes to RO mode which means we do not operate with latest facts! +6. Run (5) to parametrize query and clear Query Store. Workload is still running! +7. Run (1) again to see numbers now: ration between count(queries) and count(distinct query_hash) is now near to 1. +8. Open Open SSMS Top Resource Consuming queries: you’ll see dozen of different queries to tune +9. (6) show alternative solution – applying forced parametrization for the entire DB. Just mention, as a possible solution +10. Run (7) to reset DB to initial state. diff --git a/samples/features/query-store/ShowBasics.sql b/samples/features/query-store/ShowBasics.sql new file mode 100644 index 00000000..79acfd0c --- /dev/null +++ b/samples/features/query-store/ShowBasics.sql @@ -0,0 +1,96 @@ +/*Clear Query Store and procedure cache*/ +ALTER DATABASE AdventureWorks2016_EXT SET QUERY_STORE CLEAR; +ALTER DATABASE AdventureWorks2016_EXT SET QUERY_STORE = ON (QUERY_CAPTURE_MODE = ALL); +DBCC FREEPROCCACHE +GO +USE AdventureWorks2016_EXT; +GO + +/*Run simple query - what data is collected and where does it go to?*/ +SELECT * FROM Part; + +SELECT * FROM sys.query_store_query_text; +SELECT * FROM sys.query_store_query; +SELECT * FROM sys.query_store_plan; +SELECT * FROM sys.query_store_runtime_stats; + +/* + Combine all info + vw_QueryStoreCompileInfo is custom view (created for presentation) + +*/ +SELECT * FROM vw_QueryStoreCompileInfo +WHERE query_sql_text = 'SELECT * FROM Part' + +/*The same query from the proc*/ +DROP PROCEDURE IF EXISTS sp_GetParts +GO + +CREATE PROCEDURE sp_GetParts +AS +SELECT * FROM Part; +GO + +EXEC sp_GetParts; + +/*Again the same query, from sp_executesql*/ +EXEC sp_executesql N'SELECT * FROM Part' + +SELECT * FROM vw_QueryStoreCompileInfo +WHERE query_sql_text = 'SELECT * FROM Part' + +/*Finally trigger*/ +DROP TRIGGER IF EXISTS dbo.OnPartInsert +GO + +CREATE TRIGGER dbo.OnPartInsert + ON dbo.Part + AFTER INSERT +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + + SELECT * FROM Part; + +END +GO + +INSERT INTO Part VALUES (3000020, 'Part_300020'); + +SELECT * FROM vw_QueryStoreCompileInfo +WHERE query_sql_text = 'SELECT * FROM Part' + +/*What happens with parametrized query?*/ +SELECT * FROM Part WHERE PartId = 5; + +SELECT * FROM vw_QueryStoreCompileInfo +WHERE query_sql_text = 'SELECT * FROM Part = 5' + +/* Check sys.query_store_query_text */ + +SELECT * FROM sys.query_store_query_text; + +/*Try sys.fn_stmt_sql_handle_from_sql_stmt this instead*/ +SELECT * FROM sys.fn_stmt_sql_handle_from_sql_stmt +('SELECT * FROM Part WHERE PartId = 5', NULL) + +/*Changed searched criteria*/ +SELECT V.* FROM vw_QueryStoreCompileInfo V +JOIN sys.fn_stmt_sql_handle_from_sql_stmt +('SELECT * FROM Part WHERE PartId = 5', NULL) F +ON V.statement_sql_handle = F.statement_sql_handle + +/*Get runtime info for the queries*/ +SELECT * FROM vw_QueryStoreRuntimeInfo +WHERE query_sql_text = 'SELECT * FROM Part' +ORDER BY start_time DESC + +SELECT * FROM vw_QueryStoreRuntimeInfo V +JOIN sys.fn_stmt_sql_handle_from_sql_stmt +('SELECT * FROM Part WHERE PartId = 5', NULL) F +ON V.statement_sql_handle = F.statement_sql_handle +ORDER BY start_time DESC + + diff --git a/samples/features/query-store/sp_GetCompilAndExecutionTotalTime.sql b/samples/features/query-store/sp_GetCompilAndExecutionTotalTime.sql new file mode 100644 index 00000000..df102bb3 --- /dev/null +++ b/samples/features/query-store/sp_GetCompilAndExecutionTotalTime.sql @@ -0,0 +1,25 @@ +USE [AdventureWorks2016_EXT] +GO + +DROP PROCEDURE IF EXISTS sp_GetCompilAndExecutionTotalTime +GO + +CREATE PROCEDURE sp_GetCompilAndExecutionTotalTime +AS + +DECLARE @totalCompiles int +DECLARE @totalExecutions int +DECLARE @totalCompileTime decimal(18,4) +DECLARE @totalExecutionTime decimal(18,4) + +SELECT @totalCompiles = SUM(count_compiles), + @totalCompileTime = SUM(count_compiles * avg_compile_duration / 1000.) +FROM sys.query_store_plan; + +SELECT @totalExecutions = SUM(count_executions), + @totalExecutionTime = SUM(count_executions * avg_duration / 1000.) +FROM sys.query_store_runtime_stats + +SELECT @totalCompiles AS TotalCompiles, @totalExecutions AS TotalExecutions, +@totalCompileTime AS TotalCompileTime, @totalExecutionTime AS TotalDurationTime +GO \ No newline at end of file diff --git a/samples/features/query-store/vw_QueryStoreCompileInfo.sql b/samples/features/query-store/vw_QueryStoreCompileInfo.sql new file mode 100644 index 00000000..c281640e --- /dev/null +++ b/samples/features/query-store/vw_QueryStoreCompileInfo.sql @@ -0,0 +1,19 @@ +USE [AdventureWorks2016_EXT] +GO + +DROP VIEW IF EXISTS [vw_QueryStoreCompileInfo]; +GO + +CREATE VIEW [dbo].[vw_QueryStoreCompileInfo] +AS +SELECT qt.query_text_id, q.query_id, p.plan_id, qt.query_sql_text, s.name AS ContainingSchema, o.name AS ContainingObject, q.query_hash, qt.statement_sql_handle, q.is_internal_query, + q.query_parameterization_type_desc, q.count_compiles AS query_count_compiles, p.query_plan_hash, p.count_compiles AS plan_count_compiles, p.last_compile_start_time, p.engine_version, + p.compatibility_level, p.query_plan, p.is_trivial_plan, p.is_parallel_plan, p.is_forced_plan +FROM sys.query_store_query_text AS qt INNER JOIN + sys.query_store_query AS q ON qt.query_text_id = q.query_text_id INNER JOIN + sys.query_store_plan AS p ON q.query_id = p.query_id LEFT OUTER JOIN + sys.objects AS o ON q.object_id = o.object_id LEFT OUTER JOIN + sys.schemas AS s ON s.schema_id = o.schema_id +GO + + diff --git a/samples/features/query-store/vw_QueryStoreRuntimeInfo.sql b/samples/features/query-store/vw_QueryStoreRuntimeInfo.sql new file mode 100644 index 00000000..f6e4a813 --- /dev/null +++ b/samples/features/query-store/vw_QueryStoreRuntimeInfo.sql @@ -0,0 +1,21 @@ +USE [AdventureWorks2016_EXT] +GO + +DROP VIEW IF EXISTS [dbo].[vw_QueryStoreRuntimeInfo] +GO + +CREATE VIEW [dbo].[vw_QueryStoreRuntimeInfo] +AS +SELECT qt.query_text_id, q.query_id, p.plan_id, qt.query_sql_text, s.name AS ContainingSchema, o.name AS ContainingObject, qt.statement_sql_handle, rsi.start_time, rsi.end_time, rs.execution_type_desc, + rs.count_executions, rs.avg_duration, rs.max_duration, rs.avg_cpu_time, rs.max_cpu_time, rs.avg_logical_io_reads, rs.max_logical_io_reads, rs.avg_physical_io_reads, rs.max_physical_io_reads, + rs.avg_logical_io_writes, rs.max_logical_io_writes, rs.avg_query_max_used_memory, rs.max_query_max_used_memory, rs.avg_rowcount, rs.max_rowcount, rs.avg_dop, rs.max_dop +FROM sys.query_store_query_text AS qt INNER JOIN + sys.query_store_query AS q ON qt.query_text_id = q.query_text_id INNER JOIN + sys.query_store_plan AS p ON q.query_id = p.query_id LEFT OUTER JOIN + sys.objects AS o ON q.object_id = o.object_id LEFT OUTER JOIN + sys.schemas AS s ON s.schema_id = o.schema_id INNER JOIN + sys.query_store_runtime_stats AS rs ON rs.plan_id = p.plan_id INNER JOIN + sys.query_store_runtime_stats_interval AS rsi ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id +GO + + diff --git a/samples/features/readme.md b/samples/features/readme.md index 70ceb668..079341dc 100644 --- a/samples/features/readme.md +++ b/samples/features/readme.md @@ -28,8 +28,15 @@ Built-in temporal functions enable you to easily track history of changes in a t Graph tables enable you to add a non-relational capability to your database. +[SQL Management Objects (SMO)](sql-management-objects) + +The SQL Server Management Objects (SMO) Framework is a set of objects designed for programmatic management of Microsoft SQL Server and Microsoft Azure SQL Database. These code snippets demonstrate features of SMO and illustrate how to use SMO properties and collections without sacrificing performance. + ## Samples for Business Intelligence features within SQL Server [Reporting Services (SSRS)](reporting-services) Reporting Services provides reporting capabilities for your organziation. Reporting Services can be integrated with SharePoint Server or used as a standalone service. + + + diff --git a/samples/features/sql-big-data-cluster/app-deploy/SSIS/spec.yaml b/samples/features/sql-big-data-cluster/app-deploy/SSIS/spec.yaml index 18925ef3..e72747bd 100644 --- a/samples/features/sql-big-data-cluster/app-deploy/SSIS/spec.yaml +++ b/samples/features/sql-big-data-cluster/app-deploy/SSIS/spec.yaml @@ -2,5 +2,5 @@ name: back-up-db version: v1 runtime: SSIS entrypoint: ./back-up-db.dtsx -options: /REP V /CONN "MasterSQL"\;"\"Data Source=service-master-pool;User ID=sa;Initial Catalog=master;Password=[SA_PASSWORD]\"" -schedule: "*/1 * * * *" \ No newline at end of file +options: /REP V /CONN "MasterSQL"\;"\"Data Source=master-0;User ID=sa;Initial Catalog=master;Password=[SA_PASSWORD]\"" +schedule: "*/1 * * * *" diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index 600c21d5..01cf8df2 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -46,7 +46,7 @@ if /i "%CTP_VERSION%" EQU "CTP2.4" (set MASTER_POD_NAME=mssql-master-pool-0) els REM Copy the backup file, restore the database, create necessary objects and data file echo Copying sales database backup file to SQL Master instance... -%DEBUG% kubectl cp tpcxbb_1gb.bak %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit +%DEBUG% kubectl cp tpcxbb_1gb.bak %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data -c mssql-server || goto exit REM Download and copy the sample backup files if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( @@ -57,7 +57,7 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/%%f" -o %%f ) echo Copying %%f database backup file to SQL Master instance... - %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit + %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data -c mssql-server || goto exit ) set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak @@ -67,7 +67,7 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/%%f" -o %%f ) echo Copying %%f database backup file to SQL Master instance... - %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit + %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data -c mssql-server || goto exit ) ) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 04679c05..6f5be13f 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -60,8 +60,8 @@ else MASTER_POD_NAME=master-0 fi -echo Copying database backup file... -$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) +echo Copying sales database backup file... +$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) # $DEBUG rm tpcxbb_1gb.bak if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ] @@ -74,7 +74,7 @@ then $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/$file" -o $file fi echo Copying $file database backup file to SQL Master instance... - $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) done @@ -86,18 +86,18 @@ then $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/$file" -o $file fi echo Copying $file database backup file to SQL Master instance... - $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) done fi echo Configuring sample database... # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE" export SA_PASSWORD=$KNOX_PASSWORD -$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b < "$STARTUP_PATH/bootstrap-sample-db.sql" > "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2) +$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2) # remove files copied into the pod: echo Removing database backup files... -kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" +$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" for table in web_clickstreams inventory customer do diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 982b4835..64368706 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -70,38 +70,34 @@ GO CREATE OR ALTER PROCEDURE #create_data_sources AS BEGIN - -- Create database master key (required for database scoped credentials used in the samples) - IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1) + -- Create database master key (required for database scoped credentials used in the samples) + IF NOT EXISTS(SELECT * FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##') CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!'; -- Create default data sources for SQL Big Data Cluster IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); END; GO @@ -118,11 +114,17 @@ BEGIN FETCH @sample_dbs INTO @file; IF @@FETCH_STATUS < 0 BREAK; + -- Restore the sample databases: EXECUTE #restore_database @file; + + -- Get database name used in restore: SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql'); EXECUTE @proc N'#create_data_sources'; + -- Set compatibility level to 150: + EXECUTE @proc N'ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 150'; + -- Rename TPCx-BB database: IF DB_ID('tpcxbb_1gb') IS NOT NULL ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales; diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql index 3073f88b..5d457be5 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql @@ -4,8 +4,12 @@ GO -- Create external data source for Data Pool inside a SQL big data cluster -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); -- Create external table in a data pool in SQL Server 2019 big data cluster. -- The SqlDataPool data source is a special data source that is available in diff --git a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql index ed9a3569..3b387ff3 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql @@ -4,16 +4,15 @@ -- the Query menu, and "Specify Values for Template Parameters" option. IF NOT EXISTS(SELECT * FROM sys.database_scoped_credentials WHERE name = 'MySQL80-user') CREATE DATABASE SCOPED CREDENTIAL [MySQL80-user] - WITH IDENTITY = 'mssql-user' - , SECRET = 'sql19tw0mysql'; + WITH IDENTITY = '' + , SECRET = ''; --- Create external data source that points to MySQL server --- The tokens '%u' and '%p' is used to reference the credential information. +-- Create external data source that points to MySQL server. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'MySQL80') CREATE EXTERNAL DATA SOURCE MySQL80 - WITH (LOCATION = 'odbc://uc-win19-vm.redmond.corp.microsoft.com' - , CONNECTION_OPTIONS = 'Driver={MySQL ODBC 8.0 Unicode Driver};User name=%u;Passwword=%p;IGNORE_SPACE=1' + WITH (LOCATION = 'odbc://' + , CONNECTION_OPTIONS = 'Driver={MySQL ODBC 8.0 Unicode Driver};IGNORE_SPACE=1' , CREDENTIAL = [MySQL80-user]); -- Create external table over inventory table on MySQL server diff --git a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql index 75e7827f..62c2a61d 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql @@ -7,13 +7,12 @@ IF NOT EXISTS(SELECT * FROM sys.database_scoped_credentials WHERE name = 'Postgr WITH IDENTITY = '' , SECRET = ''; --- Create external data source that points to PostgreSQL server --- The tokens '%u' and '%p' is used to reference the credential information. +-- Create external data source that points to PostgreSQL server. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'PostgreSQL11') CREATE EXTERNAL DATA SOURCE PostgreSQL11 WITH (LOCATION = 'odbc://' - , CONNECTION_OPTIONS = 'Driver={PostgreSQL ODBC Driver(UNICODE)};User name=%u;Passwword=%p' + , CONNECTION_OPTIONS = 'Driver={PostgreSQL ODBC Driver(UNICODE)}' , CREDENTIAL = [PostgreSQL11-user]); -- Create external table over inventory table on PostgreSQL server diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql index 93936138..41d5a2d3 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql @@ -17,20 +17,12 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); -- Create file format for RCFILE with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql index 40b7bb06..983d3eef 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql @@ -7,20 +7,12 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); -- Create file format for orc file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql index 18422fcf..bd9a10ce 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql @@ -7,20 +7,12 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); -- Create file format for orc file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql index c8797fed..5ae3193a 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for CSV separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql index 5c8545dc..a83299cd 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for parquet file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql index 6e0175d9..df333057 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for tab separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql index 6f683a84..b07ca68b 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for CSV file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index 1164a26e..ed61d1fb 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for parquet file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/deployment/aks/README.md b/samples/features/sql-big-data-cluster/deployment/aks/README.md index 816f6e45..5f24919a 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/README.md +++ b/samples/features/sql-big-data-cluster/deployment/aks/README.md @@ -18,7 +18,7 @@ Using this sample Python script, you will deploy a Kubernetes cluster in Azure u ``` - Install mssqlctl CLI latest version using . Run the command below using elevated priviledges (sudo or admin cmd window): ``` - pip3 install -r https://private-repo.microsoft.com/python/ctp-2.3/mssqlctl/requirements.txt + pip3 install -r https://private-repo.microsoft.com/python/ctp3.0/mssqlctl/requirements.txt ``` 1. Login into your Azure account. Run this command: ``` diff --git a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py index 2f9aba4f..bb1d56a4 100644 --- a/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py +++ b/samples/features/sql-big-data-cluster/deployment/aks/deploy-sql-big-data-aks.py @@ -1,7 +1,7 @@ # # Prerequisites: # -# Azure CLI (https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), python3 (https://www.python.org/downloads), mssqlctl CLI (pip3 install -r https://private-repo.microsoft.com/python/ctp-2.4/mssqlctl/requirements.txt ) +# Azure CLI (https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), python3 (https://www.python.org/downloads), mssqlctl CLI (pip3 install -r https://private-repo.microsoft.com/python/ctp3.0/mssqlctl/requirements.txt ) # # Run `az login` at least once BEFORE running this script # @@ -33,8 +33,10 @@ DOCKER_PASSWORD = getpass.getpass("Provide your Docker password:") AZURE_REGION=input("Provide Azure region - Press ENTER for using `westus`:") or "westus" VM_SIZE=input("Provide VM size for the AKS cluster - Press ENTER for using `Standard_L8s`:") or "Standard_L8s" AKS_NODE_COUNT=input("Provide number of worker nodes for AKS cluster - Press ENTER for using `1`:") or "1" + #This is both Kubernetes cluster name and SQL Big Data cluster name CLUSTER_NAME=input("Provide name of AKS cluster and SQL big data cluster - Press ENTER for using `sqlbigdata`:") or "sqlbigdata" + #This password will be use for Controller user, Knox user and SQL Server Master SA accounts PASSWORD=input("Provide password to be used for Controller user, Knox user and SQL Server Master SA accounts - Press ENTER for using `MySQLBigData2019`:") or "MySQLBigData2019" CONTROLLER_USERNAME=input("Provide username to be used for Controller user - Press ENTER for using `admin`:") or "admin" @@ -42,24 +44,17 @@ CONTROLLER_USERNAME=input("Provide username to be used for Controller user - Pre # DOCKER_REGISTRY="private-repo.microsoft.com" DOCKER_REPOSITORY="mssql-private-preview" -DOCKER_IMAGE_TAG="ctp2.4" +DOCKER_IMAGE_TAG="ctp3.0" print ('Setting environment variables') os.environ['MSSQL_SA_PASSWORD'] = PASSWORD os.environ['CONTROLLER_USERNAME'] = CONTROLLER_USERNAME os.environ['CONTROLLER_PASSWORD'] = PASSWORD os.environ['KNOX_PASSWORD'] = PASSWORD -os.environ['DOCKER_REGISTRY'] = DOCKER_REGISTRY -os.environ['DOCKER_REPOSITORY'] = DOCKER_REPOSITORY os.environ['DOCKER_USERNAME']=DOCKER_USERNAME os.environ['DOCKER_PASSWORD']=DOCKER_PASSWORD -os.environ['DOCKER_EMAIL']=DOCKER_USERNAME -os.environ['DOCKER_IMAGE_TAG']=DOCKER_IMAGE_TAG os.environ['DOCKER_IMAGE_POLICY']="IfNotPresent" -os.environ['DOCKER_PRIVATE_REGISTRY']="1" -os.environ['CLUSTER_PLATFORM']="aks" -os.environ['ACCEPT_EULA']="yes" -os.environ['STORAGE_SIZE']="10Gi" +os.environ['ACCEPT_EULA']="Yes" print ("Set azure context to subcription: "+SUBSCRIPTION_ID) command = "az account set -s "+ SUBSCRIPTION_ID @@ -70,27 +65,44 @@ command="az group create --name "+GROUP_NAME+" --location "+AZURE_REGION executeCmd (command) print("Creating AKS cluster: "+CLUSTER_NAME) -command = "az aks create --name "+CLUSTER_NAME+" --resource-group "+GROUP_NAME+" --generate-ssh-keys --node-vm-size "+VM_SIZE+" --node-count "+AKS_NODE_COUNT+" --kubernetes-version 1.12.6" +command = "az aks create --name "+CLUSTER_NAME+" --resource-group "+GROUP_NAME+" --generate-ssh-keys --node-vm-size "+VM_SIZE+" --node-count "+AKS_NODE_COUNT+" --kubernetes-version 1.12.8" executeCmd (command) command = "az aks get-credentials --overwrite-existing --name "+CLUSTER_NAME+" --resource-group "+GROUP_NAME+" --admin" executeCmd (command) print("Creating SQL Big Data cluster:" +CLUSTER_NAME) -command="mssqlctl cluster create --name "+CLUSTER_NAME +command="mssqlctl cluster config init --src aks-dev-test.json --target custom.json --force" +executeCmd (command) + +command="mssqlctl cluster config section set -c custom.json -j ""metadata.name=" + CLUSTER_NAME + "" +executeCmd (command) + +command="mssqlctl cluster config section set -c custom.json -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + "" +executeCmd (command) +command="mssqlctl cluster config section set -c custom.json -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + "" +executeCmd (command) +command="mssqlctl cluster config section set -c custom.json -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + "" +executeCmd (command) + +command="mssqlctl cluster create -c custom.json --accept-eula yes" executeCmd (command) print("") print("SQL Server big data cluster connection endpoints: ") + print("SQL Server master instance:") -command="kubectl get service endpoint-master-pool -o=custom-columns=""IP:.status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME +command="kubectl get service master-svc-external -o=custom-columns=""IP:.status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME executeCmd(command) + print("") print("HDFS/KNOX:") -command="kubectl get service endpoint-security -o=custom-columns=""IP:status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME +command="kubectl get service gateway-svc-external -o=custom-columns=""IP:status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME executeCmd(command) + print("") print("Cluster administration portal (https://:):") -command="kubectl get service endpoint-service-proxy -o=custom-columns=""IP:status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME +command="kubectl get service mgmtproxy-svc-external -o=custom-columns=""IP:status.loadBalancer.ingress[0].ip,PORT:.spec.ports[0].port"" -n "+CLUSTER_NAME executeCmd(command) + print("") diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh index 6ebceb86..a8025373 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh @@ -10,5 +10,5 @@ sudo chown $(id -u):$(id -g) $HOME/.kube/config kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml helm init kubectl apply -f rbac.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard diff --git a/samples/features/sql-big-data-cluster/deployment/offline/Readme.md b/samples/features/sql-big-data-cluster/deployment/offline/Readme.md index f3bb5ef4..c8e88233 100644 --- a/samples/features/sql-big-data-cluster/deployment/offline/Readme.md +++ b/samples/features/sql-big-data-cluster/deployment/offline/Readme.md @@ -31,9 +31,16 @@ When prompted, provide your input for: To deploy from your private repository, use the steps described in the [deployment guide](deployment-guidance.md), but customize the following environment variables to match your private Docker repository. -- **DOCKER_REGISTRY** -- **DOCKER_REPOSITORY** - **DOCKER_USERNAME** - **DOCKER_PASSWORD** -- **DOCKER_EMAIL** -- **DOCKER_IMAGE_TAG** \ No newline at end of file + +You must also customize the deployment configuration file to point to the correct docker repository: + +``` + "docker": { + "registry": "", + "repository": "", + "imageTag": "", + "imagePullPolicy": "Always" + } +``` diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb index 8f334205..48e45da1 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb @@ -203,7 +203,7 @@ }, { "cell_type": "markdown", - "source": "# Configuration settings for scaling to larger data\n\n## Number and size of nodes in our Kubernetes cluster\nWe can control the number and size of nodes in our Kubernetes cluster via the node-vm-size and node-count switches in our `aks create` command:\n\n`az aks create --name mycluster --resource-group myrg --generate-ssh-keys --node-vm-size Standard_DS14_v2 --node-count 3 --kubernetes-version 1.10.9`\n\nMore information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n\n## Number of Spark pods\nWe can control the number of Spark pods via the CLUSTER_STORAGE_POOL_REPLICAS environment variable used by `mssqlctl create cluster`:\n\nSET CLUSTER_STORAGE_POOL_REPLICAS=2\n\n## YARN scheduler memory and cores\nWe can control the YARN scheduler memory and cores via the following environment variable used by `mssqlctl create cluster`:\n\n- YARN_SCHEDULER_MAX_MEMORY\n- YARN_SCHEDULER_MAX_VCORES\n- YARN_NODEMANAGER_RESOURCE_MEMORY\n- YARN_NODEMANAGER_RESOURCE_VCORES\n\nFurther information regarding mssqlctl environtment variables is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#define-environment-variables).\n\n## Livy timeout\nThe Livy timeout sets a limit on the runtime of a cell in a PySpark3 Jupyter notebook. In SQL Server 2019 Big Data CTP 2.1, the Livy timeout defaults to 1 hour. In CTP 2.2, it defaults to 24 days. One can modify this as follows:\n\n- Log into the mssql-master-pool-0 pod using this command (requires permission to run kubectl):\n\n```\nkubectl exec -it mssql-master-pool-0 -n -- /bin/bash\n```\n- To set the Livy timeout to 24 days, run the following command or edit /livy/conf/livy.conf accordingly:\n\n```\necho 'livy.server.session.timeout = 24d' | cat >> /livy/conf/livy.conf \n```\n- Then restart the Livy server by running the following command:\n\n```\nsupervisorctl restart livy\n```", + "source": "# Configuration settings for scaling to larger data\n\n## Number and size of nodes in our Kubernetes cluster\nWe can control the number and size of nodes in our Kubernetes cluster via the node-vm-size and node-count switches in our `aks create` command:\n\n`az aks create --name mycluster --resource-group myrg --generate-ssh-keys --node-vm-size Standard_DS14_v2 --node-count 3 --kubernetes-version 1.10.9`\n\nMore information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n\n## Number of Spark pods\nWe can control the number of Spark pods via the CLUSTER_STORAGE_POOL_REPLICAS environment variable used by `mssqlctl create cluster`:\n\nSET CLUSTER_STORAGE_POOL_REPLICAS=2\n\n## YARN scheduler memory and cores\nWe can control the YARN scheduler memory and cores via the following environment variable used by `mssqlctl create cluster`:\n\n- YARN_SCHEDULER_MAX_MEMORY\n- YARN_SCHEDULER_MAX_VCORES\n- YARN_NODEMANAGER_RESOURCE_MEMORY\n- YARN_NODEMANAGER_RESOURCE_VCORES\n\nFurther information regarding mssqlctl environtment variables is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#define-environment-variables).\n\nIn CTP 2.5 and later, these environment variables are replaced by similarly named properties in a JSON file. See [Custom configurations](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#customconfig).\n\n## Livy timeout\nThe Livy timeout sets a limit on the runtime of a cell in a PySpark3 Jupyter notebook. In SQL Server 2019 Big Data CTP 2.1, the Livy timeout defaults to 1 hour. In CTP 2.2, it defaults to 24 days. One can modify this as follows:\n\n- Log into the mssql-master-pool-0 pod using this command (requires permission to run kubectl):\n\n```\nkubectl exec -it mssql-master-pool-0 -n -- /bin/bash\n```\n- To set the Livy timeout to 24 days, run the following command or edit /livy/conf/livy.conf accordingly:\n\n```\necho 'livy.server.session.timeout = 24d' | cat >> /livy/conf/livy.conf \n```\n- Then restart the Livy server by running the following command:\n\n```\nsupervisorctl restart livy\n```", "metadata": {} }, { diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda8.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda8.ipynb index 18f09431..83db034b 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda8.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda8.ipynb @@ -19,7 +19,7 @@ "cells": [ { "cell_type": "code", - "source": "%%configure -f\r\n{\r\n \"executorMemory\": \"4g\",\r\n \"driverMemory\": \"4g\",\r\n \"executorCores\": 4,\r\n \"driverCores\": 2,\r\n \"numExecutors\": 1\r\n}", + "source": "%%configure -f\r\n{\r\n \"executorMemory\": \"4g\",\r\n \"driverMemory\": \"8g\",\r\n \"executorCores\": 4,\r\n \"driverCores\": 2,\r\n \"numExecutors\": 1\r\n}", "metadata": { "language": "python" }, @@ -29,65 +29,30 @@ { "cell_type": "code", "source": "# For informational purposes,\r\n# print the hostname of the container\r\n# where the Spark driver is running\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n \"hostname\",\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 3 }, { "cell_type": "code", "source": "# Install NVIDIA GPU libraries and TensorFlow for GPU\r\n# in the container where the Spark driver is running\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\necho $CUDA_VERSION\r\nexport CUDA_PKG_VERSION=\"8-0=$CUDA_VERSION-1\"\r\necho $CUDA_PKG_VERSION\r\n\r\nexport PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}\r\nexport LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64\r\n\r\n# nvidia-container-runtime\r\nexport NVIDIA_VISIBLE_DEVICES=all\r\nexport NVIDIA_DRIVER_CAPABILITIES=\"compute,utility\"\r\nexport NVIDIA_REQUIRE_CUDA=\"cuda>=8.0\"\r\n\r\napt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \\\\\r\n rm -rf /var/lib/apt/lists/* && \\\\\r\n NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \\\\\r\n NVIDIA_GPGKEY_FPR=ae09fe4bbd223a84b2ccfce3f60f4b3d7fa2af80 && \\\\\r\n apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \\\\\r\n apt-key adv --export --no-emit-version -a $NVIDIA_GPGKEY_FPR | tail -n +5 > cudasign.pub && \\\\\r\n echo \"$NVIDIA_GPGKEY_SUM cudasign.pub\" | sha256sum -c --strict - && rm cudasign.pub && \\\\\r\n echo \"deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /\" > /etc/apt/sources.list.d/cuda.list\r\n\r\napt-get update && apt-get install -y --no-install-recommends \\\\\r\n cuda-nvrtc-$CUDA_PKG_VERSION \\\\\r\n cuda-nvgraph-$CUDA_PKG_VERSION \\\\\r\n cuda-cusolver-$CUDA_PKG_VERSION \\\\\r\n cuda-cublas-8-0=8.0.61.2-1 \\\\\r\n cuda-cufft-$CUDA_PKG_VERSION \\\\\r\n cuda-curand-$CUDA_PKG_VERSION \\\\\r\n cuda-cusparse-$CUDA_PKG_VERSION \\\\\r\n cuda-npp-$CUDA_PKG_VERSION \\\\\r\n cuda-cudart-$CUDA_PKG_VERSION && \\\\\r\n ln -s cuda-8.0 /usr/local/cuda && \\\\\r\n rm -rf /var/lib/apt/lists/*\r\n\r\n# Install tensorflow\r\npip3 install tensorflow-gpu==1.4.0\r\n\r\n# add cudnn 6\r\necho \"deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /\" > /etc/apt/sources.list.d/nvidia-ml.list\r\n\r\nexport CUDNN_VERSION=6.0.21\r\n#LABEL com.nvidia.cudnn.version=\"${CUDNN_VERSION}\"\r\n\r\napt-get update && apt-get install -y --no-install-recommends \\\\\r\n libcudnn6=$CUDNN_VERSION-1+cuda8.0 && \\\\\r\n rm -rf /var/lib/apt/lists/*\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 4 }, { "cell_type": "code", "source": "# List CPU and GPU devices\r\nfrom tensorflow.python.client import device_lib\r\n\r\ndevice_lib.list_local_devices()", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 5 }, { "cell_type": "code", "source": "# Fit and evaluate TensorFlow model on MNIST data\r\nimport tensorflow as tf\r\nmnist = tf.keras.datasets.mnist\r\n\r\n(x_train, y_train),(x_test, y_test) = mnist.load_data()\r\nx_train, x_test = x_train / 255.0, x_test / 255.0\r\n\r\nmodel = tf.keras.models.Sequential([\r\n tf.keras.layers.Flatten(input_shape=(28, 28)), # input_shape needed for older tensorflow\r\n tf.keras.layers.Dense(512, activation=tf.nn.relu),\r\n tf.keras.layers.Dropout(0.2),\r\n tf.keras.layers.Dense(10, activation=tf.nn.softmax)\r\n])\r\nmodel.compile(optimizer='adam',\r\n loss='sparse_categorical_crossentropy',\r\n metrics=['accuracy'])\r\n\r\nmodel.fit(x_train, y_train, epochs=5)\r\nprint(\"\\n\")\r\nmetrics = model.evaluate(x_test, y_test)\r\nprint(\"\\n\")\r\nprint(metrics)", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 11 - }, - { - "cell_type": "code", - "source": "# Check available disk space\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\ndf -h\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, - "outputs": [], - "execution_count": 12 - }, - { - "cell_type": "code", - "source": "# Download code for the CIFAR 10 benchmark\r\nimport subprocess\r\nimport os\r\n\r\nif os.path.isdir(\"/tmp/models\"):\r\n print(\"CIFAR 10 repo already cloned\")\r\nelse:\r\n stdout = subprocess.check_output(\r\n'''\r\napt-get update && apt-get install -y git\r\ncd /tmp\r\ngit clone https://github.com/tensorflow/models.git\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\n print(stdout)", - "metadata": { - "language": "python" - }, - "outputs": [], - "execution_count": 13 - }, - { - "cell_type": "code", - "source": "# Run the CIFAR 10 benchmark\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\npython3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, - "outputs": [], - "execution_count": 14 } ] } \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda9.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda9.ipynb index 27e3502e..fd35b220 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda9.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda9.ipynb @@ -19,7 +19,7 @@ "cells": [ { "cell_type": "code", - "source": "%%configure -f\r\n{\r\n \"executorMemory\": \"4g\",\r\n \"driverMemory\": \"4g\",\r\n \"executorCores\": 4,\r\n \"driverCores\": 2,\r\n \"numExecutors\": 1\r\n}", + "source": "%%configure -f\r\n{\r\n \"executorMemory\": \"4g\",\r\n \"driverMemory\": \"8g\",\r\n \"executorCores\": 4,\r\n \"driverCores\": 2,\r\n \"numExecutors\": 1\r\n}", "metadata": { "language": "python" }, @@ -29,74 +29,37 @@ { "cell_type": "code", "source": "# For informational purposes,\r\n# print the hostname of the container\r\n# where the Spark driver is running\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n \"hostname\",\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 3 }, { "cell_type": "code", "source": "# Check that the CUDA_VERSION environment variable is set.\r\n# Its precise value does not matter: one can install CUDA 9 even if the \r\n# CUDA_VERSION environment variable is set to 8.0.61.\r\nimport os\r\nprint(os.environ['CUDA_VERSION'])", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 4 }, { "cell_type": "code", "source": "# Install NVIDIA GPU libraries and TensorFlow for GPU\r\n# in the container where the Spark driver is running\r\n# per https://www.tensorflow.org/install/gpu#ubuntu_1604_cuda_90_for_tensorflow_1130\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\n# Add NVIDIA package repository\r\napt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub\r\n\r\nwget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\n\r\nchown _apt cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\nchmod u+rwx cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\n\r\napt install ./cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\n\r\nwget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\n\r\nchown _apt nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\nchmod u+rwx nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\n\r\napt install ./nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\n\r\napt update\r\n\r\n# Install CUDA and tools. Include optional NCCL 2.x\r\napt install -y cuda9.0 cuda-cublas-9-0 cuda-cufft-9-0 cuda-curand-9-0 \\\\\r\n cuda-cusolver-9-0 cuda-cusparse-9-0 libcudnn7=7.2.1.38-1+cuda9.0 \\\\\r\n libnccl2=2.2.13-1+cuda9.0 cuda-command-line-tools-9-0\r\n\r\n# Optional: Install the TensorRT runtime (must be after CUDA install)\r\napt update\r\napt install libnvinfer4=4.1.2-1+cuda9.0\r\n\r\npip3 install tensorflow-gpu==1.12.0\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 5 }, { "cell_type": "code", "source": "# List CPU and GPU devices\r\nfrom tensorflow.python.client import device_lib\r\n\r\ndevice_lib.list_local_devices()", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 6 }, { "cell_type": "code", "source": "# Fit and evaluate TensorFlow model on MNIST data\r\nimport tensorflow as tf\r\nmnist = tf.keras.datasets.mnist\r\n\r\n(x_train, y_train),(x_test, y_test) = mnist.load_data()\r\nx_train, x_test = x_train / 255.0, x_test / 255.0\r\n\r\nmodel = tf.keras.models.Sequential([\r\n tf.keras.layers.Flatten(),\r\n tf.keras.layers.Dense(512, activation=tf.nn.relu),\r\n tf.keras.layers.Dropout(0.2),\r\n tf.keras.layers.Dense(10, activation=tf.nn.softmax)\r\n])\r\nmodel.compile(optimizer='adam',\r\n loss='sparse_categorical_crossentropy',\r\n metrics=['accuracy'])\r\n\r\nmodel.fit(x_train, y_train, epochs=5)\r\nprint(\"\\n\")\r\nmetrics = model.evaluate(x_test, y_test)\r\nprint(\"\\n\")\r\nprint(metrics)", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [], "execution_count": 7 - }, - { - "cell_type": "code", - "source": "# Check available disk space\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\ndf -h\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, - "outputs": [], - "execution_count": 8 - }, - { - "cell_type": "code", - "source": "# Download code for the CIFAR 10 benchmark\r\nimport subprocess\r\nimport os\r\n\r\nif os.path.isdir(\"/tmp/models\"):\r\n print(\"CIFAR 10 repo already cloned\")\r\nelse:\r\n stdout = subprocess.check_output(\r\n'''\r\napt-get update && apt-get install -y git\r\ncd /tmp\r\ngit clone https://github.com/tensorflow/models.git\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\n print(stdout)", - "metadata": { - "language": "python" - }, - "outputs": [], - "execution_count": 9 - }, - { - "cell_type": "code", - "source": "# Run the CIFAR 10 benchmark\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\npython3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)", - "metadata": { - "language": "python" - }, - "outputs": [], - "execution_count": 10 } ] } \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/spark/README.md b/samples/features/sql-big-data-cluster/spark/README.md index 571bf29a..199ab595 100644 --- a/samples/features/sql-big-data-cluster/spark/README.md +++ b/samples/features/sql-big-data-cluster/spark/README.md @@ -1,29 +1,27 @@ # SQL Server big data clusters -The new built-in notebooks in Azure Data Studio enables data scientists and data engineers to run Python, R, Scala, or Spark SQL code against the cluster. +SQL Server Big Data cluster bundles Spark and HDFS together with SQL server. Azure Data Studio IDE provides built in notebooks that enables data scientists and data engineers to run Spark notebooks and job in Python, R, or Scala code against the Big Data Cluster. This folder contains spark sample notebook on using Spark in SQL server Big data cluster -## Instructions to open a notebook from Azure Data Studio and execute the commands +## Folder contents -1. Connect to the SQL Server Master instance in a big data cluster +[PySpark Hello World](dataloading/hello_PySpark.ipynb) -1. Right-click on the server name, select **Manage**, switch to **SQL Server Big Data Cluster** tab, and use open Notebook. +[Scala Hello World ](dataloading/hello_Scala.ipynb) -1. Open the notebook in Azure Data Studio, wait for the “Kernel” and the target context (“Attach to”) to be populated. +[SparkR Hello World ](dataloading/hello_sparkR.ipynb) -1. Run each cell in the Notebook sequentially. +[DataLoading - Transforming CSV to Parquet](dataloading/transform-csv-files.ipynb/) -## __[data-loading](data-loading/)__ +[Data Transfer - Spark to SQL using Spark JDBC connector](data-virtualization/spark_to_sql_jdbc.ipynb/) -This folder contains samples that show how to load data using Spark and query them using SQL statements. +[Data Transfer - Spark to SQL using MSSQL Spark connector](spark_to_sql/mssql_spark_connector.ipynb/) + +## Instructions on how to run in Azure Data Studio [data-loading/transform-csv-files.ipynb](dataloading/transform-csv-files.ipynb/) -This samnple notebook shows how to transform CSV files in HDFS to parquet files. +2. From Azure Data Studio Connect to the SQL Server Master instance in a big data cluster. -[dataloading/spark-sql.ipynb](dataloading/spark-sql.ipynb/) +3. Right-click on the server name, select **Manage**, switch to **SQL Server Big Data Cluster** tab, and open the notebook in Azure Data Studio. Wait for the “Kernel” and the target context (“Attach to”) to be populated. If required set the relevant “Kernel” ( e.g **PySpark3** ) and **Attach to** needs to be the IP address of your big data cluster endpoint. -This samnple notebook shows how to query hive tables created from Spark. - -## __[data-virtualization](data-virtualization/)__ - -This folder contains samples that show how to integrate Spark with other data sources. +4. Run each cell in the Notebook sequentially. diff --git a/samples/features/sql-big-data-cluster/spark/data-virtualization/spark_to_sql_jdbc.ipynb b/samples/features/sql-big-data-cluster/spark/data-virtualization/spark_to_sql_jdbc.ipynb index 0531f80d..d574d792 100644 --- a/samples/features/sql-big-data-cluster/spark/data-virtualization/spark_to_sql_jdbc.ipynb +++ b/samples/features/sql-big-data-cluster/spark/data-virtualization/spark_to_sql_jdbc.ipynb @@ -19,7 +19,7 @@ "cells": [ { "cell_type": "markdown", - "source": "# Read and write from Spark to SQL\r\nA typical big data scenario is large scale ETL in Spark and writing the processed data to SQLServer. The following samples shows \r\n- reading a HDFS file, \r\n- some basic processing on it and \r\n- then processed data to SQL Server table.\r\n\r\nNeed a database precreated in SQL for this sample. Here we are using database name \"MyTestDatabase\" that can be created using SQL statements below.\r\n\r\n``` sql\r\nCreate DATABASE MyTestDatabase\r\nGO \r\n``` \r\n ", + "source": "# Read and write from Spark to SQL\r\nA typical big data scenario is large scale ETL in Spark and post processing the data is written out to SQLServer for access to LOB applications. This sample shows how to write to SQLServer from Spark. The main steps in the sample are \r\n- Reading a HDFS file, \r\n- Basic processing on it and \r\n- Then writing processed data to SQL Server table using JDBC\r\n\r\nPreReq : \r\n- The sample uses a SQL database named \"MyTestDatabase\". Create this before you run this sample. The database can be created as follows\r\n ``` sql\r\n Create DATABASE MyTestDatabase\r\n GO \r\n ``` \r\n- Download [AdultCensusIncome.csv]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ) to your local machine. Create a hdfs folder named spark_data and upload the file there. \r\n\r\n \r\n ", "metadata": {} }, { @@ -28,12 +28,30 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education-num| marital-status| occupation| relationship| race| sex|capital-gain|capital-loss|hours-per-week|native-country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows" + "text": "Starting Spark application\n", + "output_type": "stream" + }, + { + "data": { + "text/plain": "", + "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
2application_1554755839506_0003pyspark3idleLinkLink✔
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "SparkSession available as 'spark'.\n", + "output_type": "stream" + }, + { + "name": "stdout", + "text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education-num| marital-status| occupation| relationship| race| sex|capital-gain|capital-loss|hours-per-week|native-country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows", + "output_type": "stream" } ], - "execution_count": 8 + "execution_count": 3 }, { "cell_type": "code", @@ -41,25 +59,25 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows" + "text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows", + "output_type": "stream" } ], - "execution_count": 9 + "execution_count": 4 }, { "cell_type": "code", - "source": "#Write from Spark to SQL table using JDBC\r\nprint(\"Use build in JDBC connector to write to SQLServer master instance in Big data \")\r\n\r\nservername = \"jdbc:sqlserver://mssql-master-pool-0.service-master-pool\"\r\ndbname = \"MyTestDatabase\"\r\nurl = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n\r\nc = \"dbo.AdultCensus\"\r\nuser = \"sa\"\r\npassword = \"****\"\r\n\r\nprint(\"url is \", url)\r\n\r\ntry:\r\n df.write \\\r\n .format(\"jdbc\") \\\r\n .mode(\"overwrite\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password)\\\r\n .save()\r\nexcept ValueError as error :\r\n print(\"JDBC Write failed\", error)\r\n\r\nprint(\"JDBC Write done \")\r\n\r\n\r\n", + "source": "#Write from Spark to SQL table using JDBC\r\nprint(\"Use build in JDBC connector to write to SQLServer master instance in Big data \")\r\n\r\nservername = \"jdbc:sqlserver://master-0.master-svc\"\r\ndbname = \"MyTestDatabase\"\r\nurl = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n\r\ndbtable = \"dbo.AdultCensus\"\r\nuser = \"sa\"\r\npassword = \"Yukon900\"\r\n\r\nprint(\"url is \", url)\r\n\r\ntry:\r\n df.write \\\r\n .format(\"jdbc\") \\\r\n .mode(\"overwrite\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password)\\\r\n .save()\r\nexcept ValueError as error :\r\n print(\"JDBC Write failed\", error)\r\n\r\nprint(\"JDBC Write done \")\r\n\r\n\r\n", "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Use build in JDBC connector to write to SQLServer master instance in Big data \nurl is jdbc:sqlserver://mssql-master-pool-0.service-master-pool;databaseName=MyTestDatabase;\nJDBC Write done" + "text": "Use build in JDBC connector to write to SQLServer master instance in Big data \nurl is jdbc:sqlserver://master-0.master-svc;databaseName=MyTestDatabase;\nJDBC Write done", + "output_type": "stream" } ], - "execution_count": 10 + "execution_count": 9 }, { "cell_type": "code", @@ -67,12 +85,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows" + "text": "read data from SQL server table \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows", + "output_type": "stream" } ], - "execution_count": 13 + "execution_count": 11 } ] } \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb b/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb new file mode 100644 index 00000000..def8a13a --- /dev/null +++ b/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb @@ -0,0 +1,81 @@ +{ + "metadata": { + "kernelspec": { + "name": "pyspark3kernel", + "display_name": "PySpark3" + }, + "language_info": { + "name": "pyspark3", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "python", + "version": 3 + }, + "pygments_lexer": "python3" + } + }, + "nbformat_minor": 2, + "nbformat": 4, + "cells": [ + { + "cell_type": "markdown", + "source": "# Read and write from Spark to SQL using the MSSQL jdbc Connector\r\nA typical big data scenario a key usage pattern is high volume, velocity and variety data processing in Spark followed with batch/streaming writes to SQL for access to LOB applications. These usage patterns greatly benefit from a connector that utilizes key SQL optimizations and provides an efficient and reliable write to SQLServer Big Data Cluster or SQL DB. \r\n\r\nMSSQL JDBC connector, referenced by the name com.microsoft.sqlserver.jdbc.spark, uses [SQL Server Bulk copy APIS](https://docs.microsoft.com/en-us/sql/connect/jdbc/using-bulk-copy-with-the-jdbc-driver?view=sql-server-2017#sqlserverbulkcopyoptions) to implement an efficient write to SQL Server. The connector is based on Spark Data source APIs and provides a familiar JDBC interface for access.\r\n\r\nThe following sample shows how to use the MSSQL JDBC Connector for writing and reading to/from a SQL Source. In this sample we' ll \r\n- Read a file from HDFS and do some basic processing \r\n- post that we'll write the dataframe to SQL server table using the MSSQL Connector. \r\n- Followed by the write we'll read using the MSSQLConnector.\r\n\r\nPreReq : \r\n- The sample uses a SQL database named \"MyTestDatabase\". Create this before you run this sample. The database can be created as follows\r\n ``` sql\r\n Create DATABASE MyTestDatabase\r\n GO \r\n ``` \r\n- Download [AdultCensusIncome.csv]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ) to your local machine. Create a hdfs folder named spark_data and upload the file there. \r\n- Configure the spark session to use the MSSQL Connector jar. The jar can be found at /jar/spark-mssql-connector-assembly-1.0.0.jar post deployment of Big Data Cluster.\r\n\r\n``` sh\r\n %%configure -f\r\n {\"conf\": {\"spark.jars\": \"/jar/spark-mssql-connector-assembly-1.0.0.jar\"}}\r\n```\r\n\r\n \r\n ", + "metadata": {} + }, + { + "cell_type": "markdown", + "source": "# Configure the notebook to use the MSSQL Spark connector\r\nThis step woould be removed in subsequent CTPs. As of CTP2.5 this step is required to point the spark session to the relevant jar.\r\n ", + "metadata": {} + }, + { + "cell_type": "code", + "source": "%%configure -f\r\n{\"conf\": {\"spark.jars\": \"/jar/spark-mssql-connector-assembly-1.0.0.jar\"}}\r\n\r\n\r\n\r\n", + "metadata": {}, + "outputs": [], + "execution_count": 4 + }, + { + "cell_type": "markdown", + "source": "# Read data into a data frame\r\nIn this step we read the data into a data frame and do some basic clearup steps. \r\n\r\n", + "metadata": {} + }, + { + "cell_type": "code", + "source": "#Read a file and then write it to the SQL table\r\ndatafile = \"/spark_data/AdultCensusIncome.csv\"\r\ndf = spark.read.format('csv').options(header='true', inferSchema='true', ignoreLeadingWhiteSpace='true', ignoreTrailingWhiteSpace='true').load(datafile)\r\ndf.show(5)\r\n", + "metadata": {}, + "outputs": [], + "execution_count": 6 + }, + { + "cell_type": "code", + "source": "\r\n#Process this data. Very simple data cleanup steps. Replacing \"-\" with \"_\" in column names\r\ncolumns_new = [col.replace(\"-\", \"_\") for col in df.columns]\r\ndf = df.toDF(*columns_new)\r\ndf.show(5)\r\n\r\n", + "metadata": {}, + "outputs": [], + "execution_count": 8 + }, + { + "cell_type": "markdown", + "source": "# Write dataframe to SQL using MSSQL Spark Connector", + "metadata": {} + }, + { + "cell_type": "code", + "source": "#Write from Spark to SQL table using MSSQL Spark Connector\r\nprint(\"Use MSSQL connector to write to master SQL instance \")\r\n\r\nservername = \"jdbc:sqlserver://master-0.master-svc\"\r\ndbname = \"MyTestDatabase\"\r\nurl = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n\r\ndbtable = \"dbo.AdultCensus\"\r\nuser = \"sa\"\r\npassword = \"****\" # Please specify password here\r\n\r\n\r\ntry:\r\n df.write \\\r\n .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n .mode(\"overwrite\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password)\\\r\n .save()\r\nexcept ValueError as error :\r\n print(\"MSSQL Connector write failed\", error)\r\n\r\nprint(\"MSSQL Connector write succeeded \")\r\n\r\n\r\n", + "metadata": {}, + "outputs": [], + "execution_count": 10 + }, + { + "cell_type": "markdown", + "source": "# Read SQL Table using MSSQL Spark connector.\r\nThe following code uses the connetor to read the tables. To confirm the write about check table directly using SQL", + "metadata": {} + }, + { + "cell_type": "code", + "source": "#Read from SQL table using MSSQ Connector\r\nprint(\"read data from SQL server table \")\r\njdbcDF = spark.read \\\r\n .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password) \\\r\n .load()\r\n\r\njdbcDF.show(5)", + "metadata": {}, + "outputs": [], + "execution_count": 11 + } + ] +} \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb b/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb index 26567e6d..df4bd268 100644 --- a/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb +++ b/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "markdown", - "source": "## Step 1 - Explore your data\r\n### Load the data\r\nFor this example we'll use **AdultCensusIncome** data from [here]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ). From your Azure Data Studio connect to the HDFS/Spark gateway and create a directory called spark_ml under HDFS. \r\nDownload [AdultCensusIncome.csv]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ) to your local machine and upload to HDFS.Upload AdultCensusIncome.csv to the folder we created.\r\n\r\n### Exploratory Analysis\r\n- Baisc exploration on the data\r\n- Labels & Features\r\n1. **Label** - This refers to predicted value. This is represented as a column in the data. Label is **income** \r\n2. **Features** - This refers to the characteristics that are used to predict. **age** and **hours_per_week**\r\n\r\nNote : In reality features are chosen by applying some correlations techniques to understand what best characterize the Label we are predicting.\r\n\r\n### The Model we will build\r\nIn AdultCensusIncome.csv contains several columsn like Income range, age, hours-per-week, education, occupation etc. We'll build a model that can predict income range would be >50K or <50K.\r\n", + "source": "## Step 1 - Explore your data\r\n### Load the data\r\nFor this example we'll use **AdultCensusIncome** data from [here]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ). From your Azure Data Studio connect to the HDFS/Spark gateway and create a directory called spark_data under HDFS. \r\nDownload [AdultCensusIncome.csv]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ) to your local machine and upload to HDFS.Upload AdultCensusIncome.csv to the folder we created.\r\n\r\n### Exploratory Analysis\r\n- Baisc exploration on the data\r\n- Labels & Features\r\n1. **Label** - This refers to predicted value. This is represented as a column in the data. Label is **income** \r\n2. **Features** - This refers to the characteristics that are used to predict. **age** and **hours_per_week**\r\n\r\nNote : In reality features are chosen by applying some correlations techniques to understand what best characterize the Label we are predicting.\r\n\r\n### The Model we will build\r\nIn AdultCensusIncome.csv contains several columsn like Income range, age, hours-per-week, education, occupation etc. We'll build a model that can predict income range would be >50K or <50K.\r\n", "metadata": {} }, { diff --git a/samples/features/sql-management-objects/README.md b/samples/features/sql-management-objects/README.md new file mode 100644 index 00000000..e057e5e7 --- /dev/null +++ b/samples/features/sql-management-objects/README.md @@ -0,0 +1,62 @@ +# SmoSamples + +This unit test project is meant to demonstrate features of the Sql Management Objects framework and to help developers optimize performance of their SMO-based applications. + + +### Contents + +[About this sample](#about-this-sample)
+[Before you begin](#before-you-begin)
+[Run this sample](#run-this-sample)
+[Sample details](#sample-details)
+[Disclaimers](#disclaimers)
+[Related links](#related-links)
+ + + + +## About this sample + + +- **Applies to:** SQL Server 2016 (or higher), Azure SQL Database, Azure SQL Data Warehouse +- **Key features:** +- Unit tests and a docker file that demonstrate proper use of SMO features against a working SQL Server instance. +- **Programming Language:** +- C# + + + +## Before you begin + +To run this sample, you need the following prerequisites. + +**Software prerequisites:** + +1. SQL Server 2016 (or higher) or an Azure SQL Database with the full WideWorldImporters sample database, or +2. Docker +3. At minimum the dotnet 2.2 SDK, or Visual Studio 2017 + + + +## Run this sample +If using Docker, use runtests.sh or runtests.cmd as appropriate. If using a separate instance of SQL Server or Azure SQL Database, create a .runsettings file and run the unit tests using Visual Studio or "dotnet vstest". + + + +## Sample details + +Each unit test demonstrates a specific aspect of SMO-based application development, either in isolation or in conjunction with other SMO components.
+Feature areas tested include: +1. Efficient use of collections +2. Sql query capture +3. Events +4. URNs +5. Script generation + + + + +## Related Links +The SMO NuGet package is at https://www.nuget.org/packages/Microsoft.SqlServer.SqlManagementObjects/
+Documentation for the APIs is at https://docs.microsoft.com/sql/relational-databases/server-management-objects-smo/overview-smo
+The WideWorldImporters sample database can be found at https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImporters-Full.bak
\ No newline at end of file diff --git a/samples/features/sql-management-objects/prep/dockerfile b/samples/features/sql-management-objects/prep/dockerfile new file mode 100644 index 00000000..70dcbbd5 --- /dev/null +++ b/samples/features/sql-management-objects/prep/dockerfile @@ -0,0 +1,8 @@ +FROM mcr.microsoft.com/mssql/server:2017-latest +WORKDIR /tmp/backup +RUN wget -q https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImporters-Full.bak +COPY restore.sql . +COPY restore.sh . +COPY entrypoint.sh . +CMD ["/bin/bash", "/tmp/backup/entrypoint.sh"] + diff --git a/samples/features/sql-management-objects/prep/entrypoint.sh b/samples/features/sql-management-objects/prep/entrypoint.sh new file mode 100644 index 00000000..c7f19da1 --- /dev/null +++ b/samples/features/sql-management-objects/prep/entrypoint.sh @@ -0,0 +1,2 @@ +/opt/mssql/bin/sqlservr & /tmp/backup/restore.sh +tail -f /dev/null diff --git a/samples/features/sql-management-objects/prep/restore.sh b/samples/features/sql-management-objects/prep/restore.sh new file mode 100644 index 00000000..d6d34702 --- /dev/null +++ b/samples/features/sql-management-objects/prep/restore.sh @@ -0,0 +1,5 @@ +# Wait for SQL Server to start and be ready to accept connections +sleep 35s +echo sa_password is $SA_PASSWORD +/opt/mssql-tools/bin/sqlcmd -S . -U sa -P $SA_PASSWORD -i /tmp/backup/restore.sql + \ No newline at end of file diff --git a/samples/features/sql-management-objects/prep/restore.sql b/samples/features/sql-management-objects/prep/restore.sql new file mode 100644 index 00000000..4e1fddd1 --- /dev/null +++ b/samples/features/sql-management-objects/prep/restore.sql @@ -0,0 +1,5 @@ +RESTORE DATABASE WideWorldImporters FROM DISK = "/tmp/backup/WideWorldImporters-Full.bak" +WITH MOVE "WWI_Primary" TO "/var/opt/mssql/data/WideWorldImporters.mdf", +MOVE "WWI_Userdata" TO "/var/opt/mssql/data/WideWorldImporters_UserData.ndf", +MOVE "WWI_Log" TO "/var/opt/mssql/data/WideWorldImporters.ldf", MOVE "WWI_InMemory_Data_1" +TO "/var/opt/mssql/data/WideWorldImporters_InMemory_Data_1" \ No newline at end of file diff --git a/samples/features/sql-management-objects/runtests.cmd b/samples/features/sql-management-objects/runtests.cmd new file mode 100644 index 00000000..7cd30c14 --- /dev/null +++ b/samples/features/sql-management-objects/runtests.cmd @@ -0,0 +1,17 @@ +@echo off + set pwd=Passwd__%random% +echo Building the SQL Linux Docker container +docker pull mcr.microsoft.com/mssql/server:2017-latest +docker build -t sqllinux prep +echo Running the SQL linux docker image +start cmd /k docker run -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=%pwd%" -e "MSSQL_SA_PASSWORD=%pwd%" -h sqlserver --name sqlserver -p:1433:1433 --rm sqllinux +echo Waiting 90 seconds for SQL server to restore WideWorldImporters +timeout /t 90 +setlocal +echo running tests against SQL 2017 database WideWorldImporters +set TEST_PASSWORD=%pwd% +dotnet publish src -o out +dotnet vstest src\out\SmoSamples.dll /logger:console /settings:src\localhost.runsettings +endlocal +echo Terminating docker container +docker kill sqlserver diff --git a/samples/features/sql-management-objects/runtests.sh b/samples/features/sql-management-objects/runtests.sh new file mode 100644 index 00000000..2712c7f0 --- /dev/null +++ b/samples/features/sql-management-objects/runtests.sh @@ -0,0 +1,14 @@ +pwd=Pwd$RANDOM +echo Building the SQL Linux Docker container +docker pull mcr.microsoft.com/mssql/server:2017-latest +docker build -t sqllinux prep +echo Running the SQL linux docker image +docker run -e ACCEPT_EULA=Y -e SA_PASSWORD=$pwd -e MSSQL_SA_PASSWORD=$pwd -h sqlserver --name sqlserver -p:1433:1433 -d --rm sqllinux +echo Waiting 2 minutes for SQL server to restore WideWorldImporters +sleep 120 +echo running tests against SQL 2017 database WideWorldImporters +export TEST_PASSWORD=$pwd +dotnet publish src +dotnet vstest src/bin/Debug/netcoreapp2.1/SmoSamples.dll --logger:console --Settings:src/localhost.runsettings +echo Terminating docker container +docker kill sqlserver diff --git a/samples/features/sql-management-objects/src/CollectionSamples.cs b/samples/features/sql-management-objects/src/CollectionSamples.cs new file mode 100644 index 00000000..e8cc35ce --- /dev/null +++ b/samples/features/sql-management-objects/src/CollectionSamples.cs @@ -0,0 +1,74 @@ +namespace Microsoft.SqlServer.SmoSamples +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Text; + using Microsoft.SqlServer.Management.Smo; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using NUnit.Framework; + using Assert = NUnit.Framework.Assert; + + [TestClass] + public class CollectionSamples + { + public VisualStudio.TestTools.UnitTesting.TestContext TestContext { get; set; } + + /// + /// SetDefaultInitFields tells the Server object which properties to include in the initial query + /// to populate of a given object type when initialized a collection of that type. + /// The test demonstrates the effect of using this call to enumerate Tables when accessing the FileGroup + /// property of each Table object + /// + [TestMethod] + public void Collection_iteration_is_faster_with_SetDefaultInitFields() + { + using (var connectionMetrics = ConnectionMetrics.SetupMeasuredConnection(TestContext, 50)) + { + var server = new Management.Smo.Server(connectionMetrics.ServerConnection); + var database = server.Databases[TestContext.GetTestDatabaseName()]; + connectionMetrics.Reset(); + foreach (Table table in database.Tables) + { + // Accessing FileGroup triggers a query to fetch it + Trace.TraceInformation( + $"Unoptimized table Name: {table.Name}\tSchema:{table.Schema}\tFileGroup:{table.FileGroup}"); + } + + var unoptimizedMetrics = (QueryCount: connectionMetrics.QueryCount, + BytesSent: connectionMetrics.BytesSent, BytesRead: connectionMetrics.BytesRead, + ConnectionCount: connectionMetrics.ConnectionCount); + Trace.TraceInformation(string.Join($"{Environment.NewLine}\t", new[] + { + "Unoptimized metrics:", + $"QueryCount:{unoptimizedMetrics.QueryCount}", $"ConnectionCount:{unoptimizedMetrics.ConnectionCount}", + $"BytesSent:{unoptimizedMetrics.BytesSent}", $"BytesRead:{unoptimizedMetrics.BytesRead}" + })); + + connectionMetrics.Reset(); + server.SetDefaultInitFields(typeof(Table), "Name", "Schema", "FileGroup"); + database.Tables.Refresh(); + foreach (Table table in database.Tables) + { + // The FileGroup property is already populated, so no extra query is needed + Trace.TraceInformation( + $"Optimized table Name: {table.Name}\tSchema:{table.Schema}\tFileGroup:{table.FileGroup}"); + } + + var optimizedMetrics = (QueryCount: connectionMetrics.QueryCount, + BytesSent: connectionMetrics.BytesSent, BytesRead: connectionMetrics.BytesRead, + ConnectionCount: connectionMetrics.ConnectionCount); + Trace.TraceInformation(string.Join($"{Environment.NewLine}\t", new[] + { + "Optimized Metrics:", + $"QueryCount:{optimizedMetrics.QueryCount}", $"ConnectionCount:{optimizedMetrics.ConnectionCount}", + $"BytesSent:{optimizedMetrics.BytesSent}", $"BytesRead:{optimizedMetrics.BytesRead}" + })); + Assert.That(optimizedMetrics.BytesRead, Is.LessThan(unoptimizedMetrics.BytesRead), "BytesRead"); + Assert.That(optimizedMetrics.BytesSent, Is.LessThan(unoptimizedMetrics.BytesSent), "BytesSent"); + Assert.That(optimizedMetrics.ConnectionCount, Is.AtMost(unoptimizedMetrics.ConnectionCount), "ConnectionCount"); + Assert.That(optimizedMetrics.QueryCount, Is.LessThan(unoptimizedMetrics.QueryCount), "QueryCount"); + } + } + } +} diff --git a/samples/features/sql-management-objects/src/ConnectionHelpers.cs b/samples/features/sql-management-objects/src/ConnectionHelpers.cs new file mode 100644 index 00000000..f1988740 --- /dev/null +++ b/samples/features/sql-management-objects/src/ConnectionHelpers.cs @@ -0,0 +1,150 @@ +using Microsoft.SqlServer.Management.Common; +using Microsoft.SqlServer.Management.Smo; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Diagnostics; +using System.Reflection; +using System.Text; +using Assert = NUnit.Framework.Assert; +namespace Microsoft.SqlServer.SmoSamples +{ + /// + /// Used by test classes to initialize and retrieve a ServerConnection for use in the tests themselves + /// + static class ConnectionHelpers + { + + /// + /// Returns a ServerConnection based on the connectionString parameter defined in the runsettings file + /// + public static ServerConnection GetTestConnection(this VisualStudio.TestTools.UnitTesting.TestContext context, ConnectionType connectionType = ConnectionType.Default) + { + var connectionString = context.GetConnectionString(); + var connectionStrBuilder = new SqlConnectionStringBuilder(connectionString); + var instanceName = connectionStrBuilder.DataSource; + var sqlServerLogin = connectionStrBuilder.UserID; + var password = connectionStrBuilder.Password; + if (connectionType == ConnectionType.SqlConnection) + { + return new ServerConnection(new SqlConnection(connectionString)); + } + if (connectionType == ConnectionType.Integrated) + { + return new ServerConnection(instanceName); + } + if (connectionType == ConnectionType.SqlAuth ) + { + if (string.IsNullOrWhiteSpace(sqlServerLogin) || string.IsNullOrWhiteSpace(password)) + { + throw new ArgumentException("username and password values are missing from test connection string"); + } + return new ServerConnection(instanceName, sqlServerLogin, password); + } + if (string.IsNullOrEmpty(sqlServerLogin)) + { + return new ServerConnection(instanceName); + } + return new ServerConnection(instanceName, sqlServerLogin, password); + } + + /// + /// Returns a connection string based on the connectionString parameter defined in the runsettings file + /// Placeholders may be included in the connection string if the caller has set corresponding environment variables. + /// [hostname] -> TEST_HOSTNAME environment variable + /// [username] -> TEST_USERNAME + /// [password] -> TEST_PASSWORD + /// [database] -> TEST_DATABASE + /// + public static string GetConnectionString(this VisualStudio.TestTools.UnitTesting.TestContext context) + { + var connectionString = context.Properties["connectionString"].ToString(); + Assert.That(connectionString, Is.Not.Empty, "connectionString must be set"); + connectionString = connectionString.Replace("[hostname]", Environment.GetEnvironmentVariable("TEST_HOSTNAME")). + Replace("[username]", Environment.GetEnvironmentVariable("TEST_USERNAME")). + Replace("[password]", Environment.GetEnvironmentVariable("TEST_PASSWORD")). + Replace("[database]", Environment.GetEnvironmentVariable("TEST_DATABASE")); + Console.WriteLine("Connection string: {0}", connectionString); + return connectionString; + } + + /// + /// Returns the name of the database to use for the tests + /// If TEST_DATABASE environment variable is set, that value is used, otherwise the + /// testDatabase parameter from runsettings is used. + /// + /// + public static string GetTestDatabaseName(this VisualStudio.TestTools.UnitTesting.TestContext context) + { + var databaseName = Environment.GetEnvironmentVariable("TEST_DATABASE"); + if (string.IsNullOrEmpty(databaseName)) + { + databaseName = context.Properties["testDatabase"].ToString(); + } + Assert.That(databaseName, Is.Not.Empty, "testDatabase must be set"); + Console.WriteLine("Test database: {0}", databaseName); + return databaseName; + } + + /// + /// Returns the folder where result files should be written + /// + /// + /// + public static string GetResultsFolder(this VisualStudio.TestTools.UnitTesting.TestContext context) + { + var path = Environment.GetEnvironmentVariable("RESULTS_FOLDER"); + if (string.IsNullOrEmpty(path)) + { + path = context.Properties.ContainsKey("resultsFolder") ? context.Properties["resultsFolder"].ToString() : null; + } + if (string.IsNullOrEmpty(path)) + { + path = PathWrapper.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + path = PathWrapper.Combine(path, "results"); + } + return path; + } + + /// + /// creates a new database with a random name, runs the action, and drops the database + /// + /// + /// + /// + public static void ExecuteWithDbDrop(this VisualStudio.TestTools.UnitTesting.TestContext context, Action action, Action preCreateAction = null) + { + var dbName = string.Format("{0}{1}", context.TestName, new Random().Next()); + var serverConnection = context.GetTestConnection(); + var server = new Management.Smo.Server(serverConnection); + var database = new Database(server, dbName); + preCreateAction?.Invoke(database); + database.Create(); + try + { + action(database); + } + finally + { + try + { + database.Drop(); + } + catch (Exception e) + { + Trace.TraceError("Unable to drop database {0}: {1}", dbName, e); + } + } + } + } + + enum ConnectionType + { + Default, // whatever is specified in the config + Integrated, // integrated auth + SqlAuth, // SQL auth + SqlConnection // Create a SqlConnection first from the connection string + } +} diff --git a/samples/features/sql-management-objects/src/ConnectionMetrics.cs b/samples/features/sql-management-objects/src/ConnectionMetrics.cs new file mode 100644 index 00000000..02164844 --- /dev/null +++ b/samples/features/sql-management-objects/src/ConnectionMetrics.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Diagnostics; +using System.Text; +using System.Threading; +using Microsoft.SqlServer.Management.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.SqlServer.SmoSamples +{ + class ConnectionMetrics : IDisposable + { + public int ConnectionCount; + public long BytesRead; + public long BytesSent; + public int QueryCount; + public readonly ServerConnection ServerConnection; + private readonly GenericSqlProxy proxy; + + public ConnectionMetrics(ServerConnection serverConnection, GenericSqlProxy proxy) + { + this.proxy = proxy; + ServerConnection = serverConnection; + proxy.OnConnect += Proxy_OnConnect; + proxy.OnWriteHost += Proxy_OnWriteHost; + proxy.OnWriteClient += Proxy_OnWriteClient; + serverConnection.StatementExecuted += ServerConnection_StatementExecuted; + } + + public void Reset() + { + ConnectionCount = 0; + BytesRead = BytesSent = 0; + QueryCount = 0; + } + + private void ServerConnection_StatementExecuted(object sender, StatementEventArgs e) + { + QueryCount++; + } + + private void Proxy_OnWriteClient(object sender, StreamWriteEventArgs e) + { + BytesRead += e.BytesWritten; + } + + private void Proxy_OnWriteHost(object sender, StreamWriteEventArgs e) + { + BytesSent += e.BytesWritten; + } + + private void Proxy_OnConnect(object sender, ProxyConnectionEventArgs e) + { + ConnectionCount++; + } + + public void Dispose() + { + proxy.OnConnect -= Proxy_OnConnect; + proxy.OnWriteHost -= Proxy_OnWriteHost; + proxy.OnWriteClient -= Proxy_OnWriteClient; + ServerConnection.StatementExecuted -= ServerConnection_StatementExecuted; + ServerConnection.SqlConnectionObject.Dispose(); + proxy.Dispose(); + } + + public static ConnectionMetrics SetupMeasuredConnection(TestContext testContext, int latencyPaddingMs = 0) + { + var connectionString = testContext.GetConnectionString(); + var proxy = new GenericSqlProxy(connectionString); + if (latencyPaddingMs > 0) + { + proxy.OnWriteClient += (o,e) => DelayWrite(latencyPaddingMs, e); + } + // If running these tests in a container you may need to set a specific port + // and expose that port in the dockerfile + var port = testContext.Properties.ContainsKey("proxyPort") + ? Convert.ToInt32(testContext.Properties["proxyPort"]) + : 0; + var sqlConnection = new SqlConnection(proxy.Initialize(port)); + var serverConnection = new ServerConnection(sqlConnection); + return new ConnectionMetrics(serverConnection, proxy); + } + + static void DelayWrite(long delay, StreamWriteEventArgs args) + { + Thread.Sleep(Convert.ToInt32(delay)); + } + } + + +} diff --git a/samples/features/sql-management-objects/src/GenericSqlProxy.cs b/samples/features/sql-management-objects/src/GenericSqlProxy.cs new file mode 100644 index 00000000..eac86266 --- /dev/null +++ b/samples/features/sql-management-objects/src/GenericSqlProxy.cs @@ -0,0 +1,220 @@ +using System; +using System.Data.SqlClient; +using System.Net.Sockets; +using System.Net; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SqlServer.SmoSamples +{ + /// + /// Provides an in-memory proxy with callbacks that allow tests to run code before transmission and after receipt of + /// data on the wire + /// + [DebuggerDisplay("{connectionString}:[{Port}]")] + class GenericSqlProxy : IDisposable + { + // We pick a buffer size that's large enough to hold most single replies so we don't over-inject latency + private const int BufferSizeBytes = 128 * 1024; + readonly string connectionString; + volatile bool disposed; + private TcpListener listener = null; + private readonly CancellationTokenSource tokenSource = new CancellationTokenSource(); + + /// + /// Constructs a GenericSqlProxy for the local default sql instance + /// + public GenericSqlProxy() : this(".") + { + + } + + /// + /// Construct a new GenericSqlProxy for the given connection string + /// + /// + public GenericSqlProxy(string connectionString) + { + this.connectionString = connectionString; + } + + public int Port { get; private set; } + + /// + /// Initializes the proxy by opening the TCP listener and copying data between client and server + /// + /// local port number to use. 0 will use a random port + /// The connection string to use for the SqlConnection + public string Initialize(int localPort = 0) + { + var builder = new SqlConnectionStringBuilder(connectionString); + GetTcpInfoFromDataSource(builder.DataSource, out string hostName, out int port); + listener = new TcpListener(IPAddress.Loopback, localPort); + listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontLinger, true); + listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + listener.Start(); + Port = ((IPEndPoint) listener.LocalEndpoint).Port; + Trace.TraceInformation($"Starting TcpListener on port {Port}"); + Task.Factory.StartNew(() => { AsyncInit(listener, hostName, port); }); + return new SqlConnectionStringBuilder(builder.ConnectionString) + { + DataSource = $"tcp:127.0.0.1,{Port}" + }.ConnectionString; + } + + private void AsyncInit(TcpListener tcpListener, string hostName, int port) + { + + while (!disposed) + { + var accept = tcpListener.AcceptTcpClientAsync(); + if (accept.Wait(1000, tokenSource.Token) && !tokenSource.IsCancellationRequested) + { + var localClient = accept.GetAwaiter().GetResult(); + OnConnect?.Invoke(this, new ProxyConnectionEventArgs(localClient)); + var remoteClient = new TcpClient() {NoDelay = true}; + tokenSource.Token.Register(() => + { + localClient.Dispose(); + remoteClient.Dispose(); + }); + remoteClient.ConnectAsync(hostName, port).Wait(tokenSource.Token); + if (!tokenSource.IsCancellationRequested) + { + + + Task.Factory.StartNew(() => { ForwardToSql(localClient, remoteClient); }); + Task.Factory.StartNew(() => { ForwardToClient(localClient, remoteClient); }); + } + else + { + Trace.TraceInformation("AsyncInit aborted due to cancellation token set"); + } + } + } + } + + /// + /// Fires before the proxy writes a buffer to the host + /// + public event EventHandler OnWriteHost; + + /// + /// Fires before the proxy writes a buffer to the client + /// + public event EventHandler OnWriteClient; + + /// + /// Fires when a new connection to the proxy's port is accepted + /// + public event EventHandler OnConnect; + + private void ForwardToSql(TcpClient ourClient, TcpClient sqlClient) + { + long index = 0; + try + { + while (!disposed) + { + byte[] buffer = new byte[BufferSizeBytes]; + int bytesRead = ourClient.GetStream().ReadAsync(buffer, 0, buffer.Length, tokenSource.Token).Result; + if (!tokenSource.Token.IsCancellationRequested) + { + OnWriteHost?.Invoke(this, new StreamWriteEventArgs(index++, buffer, bytesRead)); + sqlClient.GetStream().Write(buffer, 0, bytesRead); + } + } + } + catch (Exception) + { + if (!disposed) + { + throw; + } + } + finally + { + Trace.TraceInformation("ForwardToSql exiting"); + } + } + + private void ForwardToClient(TcpClient ourClient, TcpClient sqlClient) + { + long index = 0; + try + { + while (!disposed) + { + byte[] buffer = new byte[BufferSizeBytes]; + int bytesRead = sqlClient.GetStream().ReadAsync(buffer, 0, buffer.Length, tokenSource.Token).Result; + if (!tokenSource.Token.IsCancellationRequested) + { + OnWriteClient?.Invoke(this, new StreamWriteEventArgs(index++, buffer, bytesRead)); + ourClient.GetStream().Write(buffer, 0, bytesRead); + } + } + } + catch (Exception) + { + if (!disposed) + { + throw; + } + } + finally + { + Trace.TraceInformation("ForwardToClient exiting"); + } + } + + private static void GetTcpInfoFromDataSource(string dataSource, out string hostName, out int port) + { + string[] dataSourceParts = dataSource.Split(','); + if (dataSourceParts.Length == 1) + { + hostName = dataSourceParts[0].Replace("tcp:", ""); + port = 1433; + } + else if (dataSourceParts.Length == 2) + { + hostName = dataSourceParts[0].Replace("tcp:", ""); + port = int.Parse(dataSourceParts[1]); + } + else + { + throw new InvalidOperationException("TCP Connection String not in correct format!"); + } + } + + public void Dispose() + { + disposed = true; + tokenSource.Cancel(); + Trace.TraceInformation("Disposing TcpListener on port {0}", Port); + listener?.Stop(); + } + } + + public class StreamWriteEventArgs : EventArgs + { + public StreamWriteEventArgs(long index, byte[]buffer, int bytesWritten) + { + Index = index; + Buffer = buffer; + BytesWritten = bytesWritten; + } + public long Index; + public byte[] Buffer; + public int BytesWritten; + } + + public class ProxyConnectionEventArgs : EventArgs + { + public ProxyConnectionEventArgs(TcpClient client) + { + Client = client; + } + public TcpClient Client; + } +} diff --git a/samples/features/sql-management-objects/src/SmoSamples.csproj b/samples/features/sql-management-objects/src/SmoSamples.csproj new file mode 100644 index 00000000..73090a98 --- /dev/null +++ b/samples/features/sql-management-objects/src/SmoSamples.csproj @@ -0,0 +1,29 @@ + + + Library + netcoreapp2.1 + false + false + + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + Microsoft.SqlSerer.SmoSamples + + + Microsoft.SqlServer.SmoSamples + + + + + + + + + + + + + + + + + diff --git a/samples/features/sql-management-objects/src/SmoSamples.sln b/samples/features/sql-management-objects/src/SmoSamples.sln new file mode 100644 index 00000000..559bb64e --- /dev/null +++ b/samples/features/sql-management-objects/src/SmoSamples.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28307.572 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SmoSamples", "SmoSamples.csproj", "{7923416F-F384-458E-991C-65AD376F54D0}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7923416F-F384-458E-991C-65AD376F54D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7923416F-F384-458E-991C-65AD376F54D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7923416F-F384-458E-991C-65AD376F54D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7923416F-F384-458E-991C-65AD376F54D0}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {D78231A7-6CE4-407D-B13E-DC6A7F972E3C} + EndGlobalSection +EndGlobal diff --git a/samples/features/sql-management-objects/src/Urn.cs b/samples/features/sql-management-objects/src/Urn.cs new file mode 100644 index 00000000..118206db --- /dev/null +++ b/samples/features/sql-management-objects/src/Urn.cs @@ -0,0 +1,51 @@ +using Microsoft.SqlServer.Management.Smo; + +namespace Microsoft.SqlServer.SmoSamples +{ + +using VisualStudio.TestTools.UnitTesting; +using Management.Sdk.Sfc; +using NUnit.Framework; +using Assert=NUnit.Framework.Assert; + + [TestClass] + public class UrnSamples + { + public VisualStudio.TestTools.UnitTesting.TestContext TestContext {get;set;} + + [TestMethod] + public virtual void Urn_attribute_values_require_escaping() + { + var connection = TestContext.GetTestConnection(); + var server = new Management.Smo.Server(connection); + TestContext.ExecuteWithDbDrop((database) => + { + var table = new Table(database, "Name'With'Quotes"); + table.Columns.Add(new Column(table, "col1", DataType.Int)); + table.Create(); + Assert.That(table.Urn.GetNameForType(Table.UrnSuffix), Is.EqualTo("Name'With'Quotes"), "Urn Value"); + Assert.Throws(() => + table = (Table) server.GetSmoObject( + $"Server/Database[@Name='{database.Name}']/Table[@Name='Name'With'Quotes']")); + table = (Table)server.GetSmoObject( + $"Server/Database[@Name='{database.Name}']/Table[@Name='{Urn.EscapeString("Name'With'Quotes")}']"); + Assert.That(table.Name, Is.EqualTo("Name'With'Quotes"), "Table with escaped name"); + }); + } + + [TestMethod] + public virtual void Server_Urn_has_Name_matching_InstanceName() + { + var connection = TestContext.GetTestConnection(); + var server = new Management.Smo.Server(connection); + Assert.That(server.Urn.Value, Is.EqualTo($"Server[@Name='{Urn.EscapeString(connection.TrueName)}']"), "Server URN"); + } + + [TestMethod] + public virtual void Urn_Type_is_the_last_item() + { + var urn = new Urn("Server[@Name='server']/Database[@Name='database']/Table[@Name='table']"); + Assert.That(urn.Type, Is.EqualTo(Table.UrnSuffix), "Urn Type"); + } + } +} \ No newline at end of file diff --git a/samples/features/sql-management-objects/src/localhost.runsettings b/samples/features/sql-management-objects/src/localhost.runsettings new file mode 100644 index 00000000..58f228d5 --- /dev/null +++ b/samples/features/sql-management-objects/src/localhost.runsettings @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md index af34d600..57a876ea 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md +++ b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md @@ -29,8 +29,8 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** -1. PowerShell 5.1 -2. Azure PowerShell 5.4.2 or higher +1. PowerShell 5.1 or PowerShell Core 6.0 +2. Azure PowerShell Az module **Azure prerequisites:** @@ -40,7 +40,7 @@ To run this sample, you need the following prerequisites. ## Run this sample -Run the script below from Windows PowerShell or Azure Cloud Shell +Run the script below from PowerShell or Azure Cloud Shell ```powershell diff --git a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 index 306de1cb..252e9c9b 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 @@ -1,47 +1,69 @@ $parameters = $args[0] +$scriptUrlBase = $args[1] $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualMachineName = $parameters['virtualMachineName'] $virtualNetworkName = $parameters['virtualNetworkName'] $managementSubnetName = $parameters['subnetName'] -$administratorLogin = $parameters['administratorLogin'] -$administratorLoginPassword = $parameters['administratorLoginPassword'] +$administratorLogin = $parameters['administratorLogin'] +$administratorLoginPassword = $parameters['administratorLoginPassword'] -$scriptUrlBase = $args[1] - -if($virtualMachineName -eq '' -or $virtualMachineName -eq $null) { +if ($virtualMachineName -eq '' -or ($null -eq $virtualMachineName)) { $virtualMachineName = 'Jumpbox' Write-Host "VM Name: 'Jumpbox'." -ForegroundColor Green } -if($managementSubnetName -eq '' -or $managementSubnetName -eq $null) { +if ($managementSubnetName -eq '' -or ($null -eq $managementSubnetName)) { $managementSubnetName = 'Management' Write-Host "Using subnet 'Management' to deploy jumpbox VM." -ForegroundColor Green } -function VerifyPSVersion -{ - Write-Host "Verifying PowerShell version, must be 5.0 or higher." - if($PSVersionTable.PSVersion.Major -ge 5) - { - Write-Host "PowerShell version verified." -ForegroundColor Green +function VerifyPSVersion { + Write-Host "Verifying PowerShell version." + if ($PSVersionTable.PSEdition -eq "Desktop") { + if (($PSVersionTable.PSVersion.Major -ge 6) -or + (($PSVersionTable.PSVersion.Major -eq 5) -and ($PSVersionTable.PSVersion.Minor -ge 1))) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 5.1 or heigher." -ForegroundColor Red + Break; + } } - else - { - Write-Host "You need to install PowerShell version 5.0 or heigher." -ForegroundColor Red - Break; + else { + if ($PSVersionTable.PSVersion.Major -ge 6) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 6.0 or heigher." -ForegroundColor Red + Break; + } } } -function EnsureLogin () -{ - $context = Get-AzureRmContext - If($null -eq $context.Subscription) - { +function EnsureAzModule { + Write-Host "Checking if Az module is imported." + $module = Get-Module Az + If ($null -eq $module) { + try { + Import-Module Az -ErrorAction Stop + Write-Host "Module Az imported." -ForegroundColor Green + } + catch { + Install-Module Az -AllowClobber + Write-Host "Module Az installed." -ForegroundColor Green + } + } else { + Write-Host "Module Az imported." -ForegroundColor Green + } +} + +function EnsureLogin () { + $context = Get-AzContext + If ($null -eq $context.Subscription) { Write-Host "Sign-in..." - If($null -eq (Login-AzureRmAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) - { + If ($null -eq (Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) { Write-Host ("Sign-in failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red Break } @@ -54,16 +76,14 @@ function SelectSubscriptionId { $subscriptionId ) Write-Host "Selecting subscription '$subscriptionId'..." - $context = Get-AzureRmContext - If($context.Subscription.Id -ne $subscriptionId) - { - Try - { - Write-Host "Switching subscription $context.Subscription.Id to '$subscriptionId'." -ForegroundColor Green - Select-AzureRmSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null + $context = Get-AzContext + If ($context.Subscription.Id -ne $subscriptionId) { + Try { + $currentSubscriptionId = $context.Subscription.Id + Write-Host "Switching subscription $currentSubscriptionId to '$subscriptionId'." -ForegroundColor Green + Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null } - Catch - { + Catch { Write-Host "Subscription selection failed: $_" -ForegroundColor Red Break } @@ -76,59 +96,52 @@ function LoadVirtualNetwork { $resourceGroupName, $virtualNetworkName ) - Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) - $virtualNetwork = Get-AzureRmVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue - $id = $virtualNetwork.Id - If($null -ne $id) - { - Write-Host "Virtual network with id $id is loaded." -ForegroundColor Green - If($virtualNetwork.VirtualNetworkPeerings.Count -gt 0) { - Write-Host "Virtual network is loaded, but it should not have peerings." -ForegroundColor Red - } - return $virtualNetwork - } - else - { - Write-Host "Virtual network $virtualNetworkName cannot be found." -ForegroundColor Red - Break + Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) + $virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue + $id = $virtualNetwork.Id + If ($null -ne $id) { + Write-Host "Virtual network with id $id is loaded." -ForegroundColor Green + If ($virtualNetwork.VirtualNetworkPeerings.Count -gt 0) { + Write-Host "Virtual network is loaded, but it should not have peerings." -ForegroundColor Red } + return $virtualNetwork + } + else { + Write-Host "Virtual network $virtualNetworkName cannot be found." -ForegroundColor Red + Break + } } -function SetVirtualNetwork -{ +function SetVirtualNetwork { param($virtualNetwork) Write-Host "Applying changes to the virtual network." - Try - { - Set-AzureRmVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null + Try { + Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null } - Catch - { + Catch { Write-Host "Failed to configure Virtual Network: $_" -ForegroundColor Red } } -function ConvertCidrToUint32Array -{ +function ConvertCidrToUint32Array { param($cidrRange) $cidrRangeParts = $cidrRange.Split("/") $ipParts = $cidrRangeParts[0].Split(".") $ipnum = ([Convert]::ToUInt32($ipParts[0]) -shl 24) -bor ` - ([Convert]::ToUInt32($ipParts[1]) -shl 16) -bor ` - ([Convert]::ToUInt32($ipParts[2]) -shl 8) -bor ` - [Convert]::ToUInt32($ipParts[3]) + ([Convert]::ToUInt32($ipParts[1]) -shl 16) -bor ` + ([Convert]::ToUInt32($ipParts[2]) -shl 8) -bor ` + [Convert]::ToUInt32($ipParts[3]) $maskbits = [System.Convert]::ToInt32($cidrRangeParts[1]) $mask = 0xffffffff - $mask = $mask -shl (32 -$maskbits) + $mask = $mask -shl (32 - $maskbits) $ipstart = $ipnum -band $mask $ipend = $ipnum -bor ($mask -bxor 0xffffffff) return @($ipstart, $ipend) } -function ConvertUInt32ToIPAddress -{ +function ConvertUInt32ToIPAddress { param($uint32IP) $v1 = $uint32IP -band 0xff $v2 = ($uint32IP -shr 8) -band 0xff @@ -137,16 +150,13 @@ function ConvertUInt32ToIPAddress return "$v4.$v3.$v2.$v1" } -function CalculateNextAddressPrefix -{ +function CalculateNextAddressPrefix { param($virtualNetwork, $prefixLength) Write-Host "Calculating address prefix with length $prefixLength..." $startIPAddress = 0 - ForEach($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) - { + ForEach ($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) { $endIPAddress = (ConvertCidrToUint32Array $addressPrefix)[1] - If($endIPAddress -gt $startIPAddress) - { + If ($endIPAddress -gt $startIPAddress) { $startIPAddress = $endIPAddress } } @@ -156,22 +166,20 @@ function CalculateNextAddressPrefix return $addressPrefixResult } -function CalculateVpnClientAddressPoolPrefix -{ +function CalculateVpnClientAddressPoolPrefix { param($gatewaySubnetPrefix) Write-Host "Calculating VPN client address pool prefix." - If($gatewaySubnetPrefix.StartsWith("10.")) - { + If ($gatewaySubnetPrefix.StartsWith("10.")) { return "192.168.0.0/24" } - else - { + else { return "172.16.0.0/24" } } VerifyPSVersion +EnsureAzModule EnsureLogin SelectSubscriptionId -subscriptionId $subscriptionId @@ -179,34 +187,36 @@ $virtualNetwork = LoadVirtualNetwork -resourceGroupName $resourceGroupName -virt $subnets = $virtualNetwork.Subnets.Name -If($false -eq $subnets.Contains($managementSubnetName)) -{ +If ($false -eq $subnets.Contains($managementSubnetName)) { Write-Host "$managementSubnetName is not one of the subnets in $subnets" -ForegroundColor Yellow - Write-Host "Creating subnet $managementSubnetName ($managementSubnetPrefix) in the VNet..." -ForegroundColor Green $managementSubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 + Write-Host "Creating subnet $managementSubnetName ($managementSubnetPrefix) in the virtual network ..." -ForegroundColor Green $virtualNetwork.AddressSpace.AddressPrefixes.Add($managementSubnetPrefix) - Add-AzureRmVirtualNetworkSubnetConfig -Name $managementSubnetName -VirtualNetwork $virtualNetwork -AddressPrefix $managementSubnetPrefix | Out-Null + Add-AzVirtualNetworkSubnetConfig -Name $managementSubnetName -VirtualNetwork $virtualNetwork -AddressPrefix $managementSubnetPrefix | Out-Null SetVirtualNetwork $virtualNetwork - Write-Host "Added subnet $managementSubnetName into VNet." -ForegroundColor Green -} else { - Write-Host "The subnet $managementSubnetName exists in the VNet." -ForegroundColor Green + Write-Host "Added subnet $managementSubnetName into virtual network." -ForegroundColor Green +} +else { + Write-Host "The subnet $managementSubnetName exists in the virtual network." -ForegroundColor Green } Write-Host # Start the deployment Write-Host "Starting deployment..." +Write-Host "Deployment will take about 20m." -ForegroundColor Yellow $templateParameters = @{ - virtualNetworkName = $virtualNetworkName - managementSubnetName = $managementSubnetName - virtualMachineName = $virtualMachineName - administratorLogin = $administratorLogin - administratorLoginPassword = $administratorLoginPassword + location = $virtualNetwork.Location + virtualNetworkName = $virtualNetworkName + managementSubnetName = $managementSubnetName + virtualMachineName = $virtualMachineName + administratorLogin = $administratorLogin + administratorLoginPassword = $administratorLoginPassword } -New-AzureRmResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase+'/azuredeploy.json?t='+ [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters +New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase + '/azuredeploy.json?t=' + [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters Write-Host "Deployment completed." diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md index e38acde9..69c9a0bb 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md @@ -29,8 +29,11 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** -1. PowerShell 5.1 -2. Azure PowerShell 5.4.2 or higher +1. PowerShell 5.1 or PowerShell Core 6.0 +2. Azure PowerShell Az module + +**Linux prerequisites** +1. strongSwan **Azure prerequisites:** @@ -40,7 +43,7 @@ To run this sample, you need the following prerequisites. ## Run this sample -Run the script below from Windows PowerShell +Run the script below from PowerShell ```powershell diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 index 14364d4e..84ddb7d4 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 @@ -1,56 +1,82 @@ $parameters = $args[0] +$scriptUrlBase = $args[1] $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualNetworkName = $parameters['virtualNetworkName'] $certificateNamePrefix = $parameters['certificateNamePrefix'] -$force = $parameters['force'] +$clientCertificatePassword = $parameters['clientCertificatePassword'] #used only when certificates are created using openssl -$scriptUrlBase = $args[1] +if ($clientCertificatePassword -eq '' -or ($null -eq $clientCertificatePassword)) { + $clientCertificatePassword = 'S0m3Str0nGP@ssw0rd' +} -function VerifyPSVersion -{ - Write-Host "Verifying PowerShell version, must be 5.0 or higher." - if($PSVersionTable.PSVersion.Major -ge 5) - { - Write-Host "PowerShell version verified." -ForegroundColor Green +function VerifyPSVersion { + Write-Host "Verifying PowerShell version." + if ($PSVersionTable.PSEdition -eq "Desktop") { + if (($PSVersionTable.PSVersion.Major -ge 6) -or + (($PSVersionTable.PSVersion.Major -eq 5) -and ($PSVersionTable.PSVersion.Minor -ge 1))) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 5.1 or heigher." -ForegroundColor Red + Break; + } } - else - { - Write-Host "You need to install PowerShell version 5.0 or heigher." -ForegroundColor Red - Break; + else { + if ($PSVersionTable.PSVersion.Major -ge 6) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 6.0 or heigher." -ForegroundColor Red + Break; + } } } -function Ensure-Login () -{ - $context = Get-AzureRmContext - If($context.Subscription -eq $null) - { - Write-Host "Loging in ..." - If((Login-AzureRmAccount -ErrorAction SilentlyContinue -ErrorVariable Errors) -eq $null) - { - Write-Host ("Login failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red +function EnsureAzModule { + Write-Host "Checking if Az module is imported." + $module = Get-Module Az + If ($null -eq $module) { + try { + Import-Module Az -ErrorAction Stop + Write-Host "Module Az imported." -ForegroundColor Green + } + catch { + Install-Module Az -AllowClobber + Write-Host "Module Az installed." -ForegroundColor Green + } + } + else { + Write-Host "Module Az imported." -ForegroundColor Green + } +} + +function EnsureLogin () { + $context = Get-AzContext + If ($null -eq $context.Subscription) { + Write-Host "Sign-in..." + If ($null -eq (Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) { + Write-Host ("Sign-in failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red Break } } - Write-Host "User logedin." -ForegroundColor Green + Write-Host "Sign-in successful." -ForegroundColor Green } -function Select-SubscriptionId { +function SelectSubscriptionId { param ( $subscriptionId ) - Write-Host "Selecting subscription '$subscriptionId'." - $context = Get-AzureRmContext - If($context.Subscription.Id -ne $subscriptionId) - { - Try - { - Select-AzureRmSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null + Write-Host "Selecting subscription '$subscriptionId'..." + $context = Get-AzContext + If ($context.Subscription.Id -ne $subscriptionId) { + Try { + $currentSubscriptionId = $context.Subscription.Id + Write-Host "Switching subscription $currentSubscriptionId to '$subscriptionId'." -ForegroundColor Green + Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null } - Catch - { + Catch { Write-Host "Subscription selection failed: $_" -ForegroundColor Red Break } @@ -58,78 +84,57 @@ function Select-SubscriptionId { Write-Host "Subscription selected." -ForegroundColor Green } -function Load-VirtualNetwork { +function LoadVirtualNetwork { param ( $resourceGroupName, $virtualNetworkName ) - Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) - $virtualNetwork = Get-AzureRmVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue - If($virtualNetwork.Id -ne $null) - { - Write-Host "Virtual network loaded." -ForegroundColor Green - return $virtualNetwork + Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) + $virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue + $id = $virtualNetwork.Id + If ($null -ne $id) { + Write-Host "Virtual network with id $id is loaded." -ForegroundColor Green + If ($virtualNetwork.VirtualNetworkPeerings.Count -gt 0) { + Write-Host "Virtual network is loaded, but it should not have peerings." -ForegroundColor Red } - else - { - Write-Host "Virtual network not found." -ForegroundColor Red - Break - } -} - -function Load-ResourceGroup { - param ( - $resourceGroupName - ) - Write-Host("Loading resource group '{0}'." -f $resourceGroupName) - $resourceGroup = Get-AzureRmResourceGroup -Name $resourceGroupName - If($resourceGroup.ResourceId -ne $null) - { - Write-Host "Resource group loaded." -ForegroundColor Green - return $resourceGroup + return $virtualNetwork } - else - { - Write-Host "Resource group not found." -ForegroundColor Red + else { + Write-Host "Virtual network $virtualNetworkName cannot be found." -ForegroundColor Red Break } } -function Set-VirtualNetwork -{ +function SetVirtualNetwork { param($virtualNetwork) Write-Host "Applying changes to the virtual network." - Try - { - Set-AzureRmVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null + Try { + Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null } - Catch - { - Write-Host "Failed: $_" -ForegroundColor Red + Catch { + Write-Host "Failed to configure Virtual Network: $_" -ForegroundColor Red } - } -function ConvertCidrToUint32Array -{ +function ConvertCidrToUint32Array { param($cidrRange) - $cidrRangeParts = $cidrRange.Split(@(".","/")) - $ipnum = ([Convert]::ToUInt32($cidrRangeParts[0]) -shl 24) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[1]) -shl 16) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[2]) -shl 8) -bor ` - [Convert]::ToUInt32($cidrRangeParts[3]) + $cidrRangeParts = $cidrRange.Split("/") + $ipParts = $cidrRangeParts[0].Split(".") + $ipnum = ([Convert]::ToUInt32($ipParts[0]) -shl 24) -bor ` + ([Convert]::ToUInt32($ipParts[1]) -shl 16) -bor ` + ([Convert]::ToUInt32($ipParts[2]) -shl 8) -bor ` + [Convert]::ToUInt32($ipParts[3]) - $maskbits = [System.Convert]::ToInt32($cidrRangeParts[4]) + $maskbits = [System.Convert]::ToInt32($cidrRangeParts[1]) $mask = 0xffffffff - $mask = $mask -shl (32 -$maskbits) + $mask = $mask -shl (32 - $maskbits) $ipstart = $ipnum -band $mask $ipend = $ipnum -bor ($mask -bxor 0xffffffff) return @($ipstart, $ipend) } -function ConvertUInt32ToIPAddress -{ +function ConvertUInt32ToIPAddress { param($uint32IP) $v1 = $uint32IP -band 0xff $v2 = ($uint32IP -shr 8) -band 0xff @@ -138,80 +143,126 @@ function ConvertUInt32ToIPAddress return "$v4.$v3.$v2.$v1" } -function CalculateNextAddressPrefix -{ +function CalculateNextAddressPrefix { param($virtualNetwork, $prefixLength) - Write-Host "Calculating address prefix." + Write-Host "Calculating address prefix with length $prefixLength..." $startIPAddress = 0 - ForEach($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) - { + ForEach ($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) { $endIPAddress = (ConvertCidrToUint32Array $addressPrefix)[1] - If($endIPAddress -gt $startIPAddress) - { + If ($endIPAddress -gt $startIPAddress) { $startIPAddress = $endIPAddress } } $startIPAddress += 1 - return (ConvertUInt32ToIPAddress $startIPAddress) + "/" + $prefixLength + $addressPrefixResult = (ConvertUInt32ToIPAddress $startIPAddress) + "/" + $prefixLength + Write-Host "Using address prefix $addressPrefixResult." -ForegroundColor Green + return $addressPrefixResult } -function CalculateVpnClientAddressPoolPrefix -{ +function CalculateVpnClientAddressPoolPrefix { param($gatewaySubnetPrefix) Write-Host "Calculating VPN client address pool prefix." - If($gatewaySubnetPrefix.StartsWith("10.")) - { + If ($gatewaySubnetPrefix.StartsWith("10.")) { return "192.168.0.0/24" } - else - { + else { return "172.16.0.0/24" } } +function CreateCerificateWindows() { + $certificate = New-SelfSignedCertificate -Type Custom -KeySpec Signature ` + -Subject ("CN=$certificateNamePrefix" + "P2SRoot") -KeyExportPolicy Exportable ` + -HashAlgorithm sha256 -KeyLength 2048 ` + -CertStoreLocation "Cert:\CurrentUser\My" -KeyUsageProperty Sign -KeyUsage CertSign + + $certificateThumbprint = $certificate.Thumbprint + + New-SelfSignedCertificate -Type Custom -DnsName ($certificateNamePrefix + "P2SChild") -KeySpec Signature ` + -Subject ("CN=$certificateNamePrefix" + "P2SChild") -KeyExportPolicy Exportable ` + -HashAlgorithm sha256 -KeyLength 2048 ` + -CertStoreLocation "Cert:\CurrentUser\My" ` + -Signer $certificate -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") | Out-null + + [Convert]::ToBase64String((Get-Item cert:\currentuser\my\$certificateThumbprint).RawData) +} + +function CreateCerificateOpenSsl() { + $dn = "CN=$certificateNamePrefix" + "P2SRoot" + ipsec pki --gen --outform pem > caKey.pem + ipsec pki --self --in caKey.pem --dn $dn --ca --outform pem > caCert.pem + + $dn = $certificateNamePrefix + "P2SChild" + ipsec pki --gen --outform pem > "$($dn)Key.pem" + ipsec pki --pub --in "$($dn)Key.pem" --outform pem > "$($dn)PubKey.pem" + ipsec pki --issue --in "$($dn)PubKey.pem" --cacert caCert.pem --cakey caKey.pem --dn "CN=$($dn)" --san $dn --flag clientAuth --outform pem > "$($dn)Cert.pem" + + openssl pkcs12 -in "$($dn)Cert.pem" -inkey "$($dn)Key.pem" -certfile caCert.pem -export -out "$($dn).p12" -password "pass:$($clientCertificatePassword)" + #openssl pkcs12 -in "$($dn).p12" -password "pass:$($clientCertificatePassword)" -nocerts -out "$($dn)PrivateKey.pem" -nodes + #openssl pkcs12 -in "$($dn).p12" -password "pass:$($clientCertificatePassword)" -nokeys -out "$($dn)PublicCert.pem" -nodes + + $publicRootCertData = openssl x509 -in caCert.pem -outform pem + $publicRootCertData = $publicRootCertData -replace "-----BEGIN CERTIFICATE-----", "" + $publicRootCertData = $publicRootCertData -replace "-----END CERTIFICATE-----", "" + [string]::Join("", $publicRootCertData.Split()) +} + +function CreateCertificate() { + Write-Host "Creating certificate." + if ($PSVersionTable.PSEdition -eq "Desktop") { + return CreateCerificateWindows + } + else { + return CreateCerificateOpenSsl + } +} + VerifyPSVersion -Ensure-Login -Select-SubscriptionId -subscriptionId $subscriptionId +EnsureAzModule +EnsureLogin +SelectSubscriptionId -subscriptionId $subscriptionId -$virtualNetwork = Load-VirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName +$virtualNetwork = LoadVirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName -$resourceGroup = Get-AzureRmResourceGroup -Name $resourceGroupName +$subnets = $virtualNetwork.Subnets.Name -$certificate = New-SelfSignedCertificate -Type Custom -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SRoot") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" -KeyUsageProperty Sign -KeyUsage CertSign +$gatewaySubnetName = "GatewaySubnet" -$certificateThumbprint = $certificate.Thumbprint +If ($false -eq $subnets.Contains($gatewaySubnetName)) { + Write-Host "$gatewaySubnetName is not one of the subnets in $subnets" -ForegroundColor Yellow + $gatewaySubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 + Write-Host "Creating subnet $gatewaySubnetName ($gatewaySubnetPrefix) in the virtual network ..." -ForegroundColor Green -New-SelfSignedCertificate -Type Custom -DnsName ($certificateNamePrefix+"P2SChild") -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SChild") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" ` - -Signer $certificate -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") | Out-null + $virtualNetwork.AddressSpace.AddressPrefixes.Add($gatewaySubnetPrefix) + Add-AzVirtualNetworkSubnetConfig -Name $gatewaySubnetName -VirtualNetwork $virtualNetwork -AddressPrefix $gatewaySubnetPrefix | Out-Null -$publicRootCertData = [Convert]::ToBase64String((Get-Item cert:\currentuser\my\$certificateThumbprint).RawData) - -$gatewaySubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 + SetVirtualNetwork $virtualNetwork + Write-Host "Added subnet $gatewaySubnetName into virtual network." -ForegroundColor Green +} +else { + Write-Host "The subnet $gatewaySubnetName exists in the virtual network." -ForegroundColor Green + $gatewaySubnet = Get-AzVirtualNetworkSubnetConfig -Name $gatewaySubnetName -VirtualNetwork $virtualNetwork + $gatewaySubnetPrefix = $gatewaySubnet.AddressPrefix[0] +} $vpnClientAddressPoolPrefix = CalculateVpnClientAddressPoolPrefix $gatewaySubnetPrefix - -$virtualNetwork.AddressSpace.AddressPrefixes.Add($gatewaySubnetPrefix) -Add-AzureRmVirtualNetworkSubnetConfig -Name GatewaySubnet -VirtualNetwork $virtualNetwork -AddressPrefix $gatewaySubnetPrefix | Out-Null - -Set-VirtualNetwork $virtualNetwork +$publicRootCertData = CreateCertificate Write-Host # Start the deployment Write-Host "Starting deployment..." +Write-Host "Deployment will take about 1h." -ForegroundColor Yellow $templateParameters = @{ - virtualNetworkName = $virtualNetworkName - gatewaySubnetPrefix = $gatewaySubnetPrefix - vpnClientAddressPoolPrefix = $vpnClientAddressPoolPrefix - publicRootCertData = $publicRootCertData - } + location = $virtualNetwork.Location + virtualNetworkName = $virtualNetworkName + gatewaySubnetPrefix = $gatewaySubnetPrefix + vpnClientAddressPoolPrefix = $vpnClientAddressPoolPrefix + publicRootCertData = $publicRootCertData +} -New-AzureRmResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase+'/azuredeploy.json?t='+ [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters +New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase + '/azuredeploy.json?t=' + [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters + +Write-Host "Deployment completed." \ No newline at end of file diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 deleted file mode 100644 index 6cfd2ab4..00000000 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 +++ /dev/null @@ -1,217 +0,0 @@ -$parameters = $args[0] - -$subscriptionId = $parameters['subscriptionId'] -$resourceGroupName = $parameters['resourceGroupName'] -$virtualNetworkName = $parameters['virtualNetworkName'] -$certificateNamePrefix = $parameters['certificateNamePrefix'] -$force = $parameters['force'] - -$scriptUrlBase = $args[1] - -function VerifyPSVersion -{ - Write-Host "Verifying PowerShell version, must be 5.0 or higher." - if($PSVersionTable.PSVersion.Major -ge 5) - { - Write-Host "PowerShell version verified." -ForegroundColor Green - } - else - { - Write-Host "You need to install PowerShell version 5.0 or heigher." -ForegroundColor Red - Break; - } -} - -function Ensure-Login () -{ - $context = Get-AzContext - If($context.Subscription -eq $null) - { - Write-Host "Loging in ..." - If((Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors) -eq $null) - { - Write-Host ("Login failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red - Break - } - } - Write-Host "User logedin." -ForegroundColor Green -} - -function Select-SubscriptionId { - param ( - $subscriptionId - ) - Write-Host "Selecting subscription '$subscriptionId'." - $context = Get-AzContext - If($context.Subscription.Id -ne $subscriptionId) - { - Try - { - Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null - } - Catch - { - Write-Host "Subscription selection failed: $_" -ForegroundColor Red - Break - } - } - Write-Host "Subscription selected." -ForegroundColor Green -} - -function Load-VirtualNetwork { - param ( - $resourceGroupName, - $virtualNetworkName - ) - Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) - $virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue - If($virtualNetwork.Id -ne $null) - { - Write-Host "Virtual network loaded." -ForegroundColor Green - return $virtualNetwork - } - else - { - Write-Host "Virtual network not found." -ForegroundColor Red - Break - } -} - -function Load-ResourceGroup { - param ( - $resourceGroupName - ) - Write-Host("Loading resource group '{0}'." -f $resourceGroupName) - $resourceGroup = Get-AzResourceGroup -Name $resourceGroupName - If($resourceGroup.ResourceId -ne $null) - { - Write-Host "Resource group loaded." -ForegroundColor Green - return $resourceGroup - } - else - { - Write-Host "Resource group not found." -ForegroundColor Red - Break - } -} - -function Set-VirtualNetwork -{ - param($virtualNetwork) - - Write-Host "Applying changes to the virtual network." - Try - { - Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null - } - Catch - { - Write-Host "Failed: $_" -ForegroundColor Red - } - -} - -function ConvertCidrToUint32Array -{ - param($cidrRange) - $cidrRangeParts = $cidrRange.Split(@(".","/")) - $ipnum = ([Convert]::ToUInt32($cidrRangeParts[0]) -shl 24) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[1]) -shl 16) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[2]) -shl 8) -bor ` - [Convert]::ToUInt32($cidrRangeParts[3]) - - $maskbits = [System.Convert]::ToInt32($cidrRangeParts[4]) - $mask = 0xffffffff - $mask = $mask -shl (32 -$maskbits) - $ipstart = $ipnum -band $mask - $ipend = $ipnum -bor ($mask -bxor 0xffffffff) - return @($ipstart, $ipend) -} - -function ConvertUInt32ToIPAddress -{ - param($uint32IP) - $v1 = $uint32IP -band 0xff - $v2 = ($uint32IP -shr 8) -band 0xff - $v3 = ($uint32IP -shr 16) -band 0xff - $v4 = ($uint32IP -shr 24) - return "$v4.$v3.$v2.$v1" -} - -function CalculateNextAddressPrefix -{ - param($virtualNetwork, $prefixLength) - Write-Host "Calculating address prefix." - $startIPAddress = 0 - ForEach($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) - { - $endIPAddress = (ConvertCidrToUint32Array $addressPrefix)[1] - If($endIPAddress -gt $startIPAddress) - { - $startIPAddress = $endIPAddress - } - } - $startIPAddress += 1 - return (ConvertUInt32ToIPAddress $startIPAddress) + "/" + $prefixLength -} - -function CalculateVpnClientAddressPoolPrefix -{ - param($gatewaySubnetPrefix) - Write-Host "Calculating VPN client address pool prefix." - If($gatewaySubnetPrefix.StartsWith("10.")) - { - return "192.168.0.0/24" - } - else - { - return "172.16.0.0/24" - } - -} - -VerifyPSVersion -Ensure-Login -Select-SubscriptionId -subscriptionId $subscriptionId - -$virtualNetwork = Load-VirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName - -$resourceGroup = Get-AzResourceGroup -Name $resourceGroupName - -$certificate = New-SelfSignedCertificate -Type Custom -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SRoot") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" -KeyUsageProperty Sign -KeyUsage CertSign - -$certificateThumbprint = $certificate.Thumbprint - -New-SelfSignedCertificate -Type Custom -DnsName ($certificateNamePrefix+"P2SChild") -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SChild") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" ` - -Signer $certificate -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") | Out-null - -$publicRootCertData = [Convert]::ToBase64String((Get-Item cert:\currentuser\my\$certificateThumbprint).RawData) - -$gatewaySubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 - -$vpnClientAddressPoolPrefix = CalculateVpnClientAddressPoolPrefix $gatewaySubnetPrefix - -$virtualNetwork.AddressSpace.AddressPrefixes.Add($gatewaySubnetPrefix) -Add-AzVirtualNetworkSubnetConfig -Name GatewaySubnet -VirtualNetwork $virtualNetwork -AddressPrefix $gatewaySubnetPrefix | Out-Null - -Set-VirtualNetwork $virtualNetwork - -Write-Host - -# Start the deployment -Write-Host "Starting deployment..." - -$templateParameters = @{ - virtualNetworkName = $virtualNetworkName - gatewaySubnetPrefix = $gatewaySubnetPrefix - vpnClientAddressPoolPrefix = $vpnClientAddressPoolPrefix - publicRootCertData = $publicRootCertData - } - -New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase+'/azuredeploy.json?t='+ [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json index 822a302c..ca5795f0 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json @@ -37,7 +37,6 @@ "variables": { "gatewayPublicIpAddressName": "[concat('GatewayIP-', uniqueString(resourceGroup().id))]", "gatewayName": "[concat('Gateway-', uniqueString(resourceGroup().id))]", - "gatewaySku": "Basic", "gatewaySubnetName": "GatewaySubnet", "clientRootCertName": "RootCert" }, @@ -46,7 +45,7 @@ "apiVersion": "2017-10-01", "type": "Microsoft.Network/publicIPAddresses", "name": "[variables('gatewayPublicIpAddressName')]", - "location": "[resourceGroup().location]", + "location": "[parameters('location')]", "properties": { "publicIPAllocationMethod": "Dynamic" } @@ -55,7 +54,7 @@ "apiVersion": "2017-10-01", "type": "Microsoft.Network/virtualNetworkGateways", "name": "[variables('gatewayName')]", - "location": "[resourceGroup().location]", + "location": "[parameters('location')]", "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', variables('gatewayPublicIpAddressName'))]" ], @@ -75,8 +74,8 @@ } ], "sku": { - "name": "[variables('gatewaySku')]", - "tier": "[variables('gatewaySku')]" + "name": "Standard", + "tier": "Standard" }, "gatewayType": "Vpn", "vpnType": "RouteBased", @@ -87,6 +86,10 @@ "[parameters('vpnClientAddressPoolPrefix')]" ] }, + "vpnClientProtocols": [ + "IkeV2", + "SSTP" + ], "vpnClientRootCertificates": [ { "name": "[variables('clientRootCertName')]", @@ -99,4 +102,4 @@ } } ] -} \ No newline at end of file +} diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql new file mode 100644 index 00000000..7f71a60c --- /dev/null +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql @@ -0,0 +1,50 @@ +declare @verbose int = 0; -- change to 1 to get more verbose comparison; + +declare @source xml = ''; +declare @target xml = ''; + +with +src as( +select property = x.v.value('name[1]', 'nvarchar(300)'), + value = x.v.value('value[1]', 'nvarchar(300)') +from @source.nodes('//row') x(v) +UNION ALL +select property = 'DB-CONFIG:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @source.nodes('//db') x(v) +cross apply x.v.nodes('*') y(v) +UNION ALL +select property = 'TEMPDB:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @source.nodes('//tempdb') x(v) +cross apply x.v.nodes('*') y(v) +), +tgt as( +select property = x.v.value('name[1]', 'nvarchar(300)'), + value = x.v.value('value[1]', 'nvarchar(300)') +from @target.nodes('//row') x(v) +UNION ALL +select property = 'DB-CONFIG:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @target.nodes('//db') x(v) +cross apply x.v.nodes('*') y(v) +UNION ALL +select property = 'TEMPDB:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @target.nodes('//tempdb') x(v) +cross apply x.v.nodes('*') y(v) +), +diff as ( +select property = isnull(src.property, tgt.property), + source = src.value, target = tgt.value, + is_missing = (case when src.value is null or tgt.value is null then 1 else 0 end) +from src full outer join tgt on src.property = tgt.property +where (src.value <> tgt.value +or src.value is null and tgt.value is not null +or src.value is not null and tgt.value is null) +) +select * +from diff +where is_missing = 0 or @verbose = 1 -- in the earlier versions you had to comment out this line. Now just set the value of the flag up +order by property + diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql new file mode 100644 index 00000000..4e02b26d --- /dev/null +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql @@ -0,0 +1,58 @@ +declare @db_name sysname = 'master' + +begin +declare @result NVARCHAR(MAX); +set @result = (select compatibility_level, recovery_model_desc, snapshot_isolation_state_desc, is_read_committed_snapshot_on, + is_auto_update_stats_on, is_auto_update_stats_async_on, delayed_durability_desc, + is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced +from sys.databases +where name = @db_name +for xml raw('db'), elements); +set @result += (select compatibility_level, snapshot_isolation_state_desc, is_read_committed_snapshot_on, + is_auto_update_stats_on, is_auto_update_stats_async_on, delayed_durability_desc, + is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced, + number_of_files = (select count(*) from master.sys.master_files where database_id = db_id('tempdb')) +from sys.databases +where name = 'tempdb' +for xml raw('tempdb'), elements); +set @result += ISNULL(( +select name = CONCAT('DB-CONFIG:',name), value +from sys.database_scoped_configurations +for xml raw, elements ),''); +declare @tf table (TraceFlag smallint, status bit,global bit, session bit) +insert into @tf execute('DBCC TRACESTATUS(-1)'); +set @result += ISNULL(( +select name=CONCAT('TF:',TraceFlag), value=status from @tf +where global=1 and session=0 +and (TraceFlag in (8690 -- https://blogs.msdn.microsoft.com/psssql/2015/12/15/spool-operator-and-trace-flag-8690/ +, 8744, 9347, 9349, 9471, 9476, 9488 -- Plan affecting TFs include others such as +, 9453, 9495 -- Execution related TFs +, 4199, 9481 /*force legacy CE*/, 2312 /* force default CE */ +--https://kohera.be/blog/sql-server/trace-flags-sql-servers-transformer-like-tuning/ +, 1118, 2371, 610, 1117, 8048, 1236, 8015, 834, 1224, 2335, + -- Taking care of Query-Hint-Hell +4136, 8602, 8722, 8755, +-- random trace flags aka.ms/traceflags +634, 3459, 3468, 3505, 9495, +9347, 9349, 9389, 9398, 9453 -- batch mode related + +) +or TraceFlag between 4100 and 4120 +) +for xml raw, elements +),''); +set @result += ( +select name = CONCAT('CONFIG:',name), value from sys.configurations +where name in ('cost threshold for parallelism','cursor threshold','fill factor (%)' +,'index create memory (KB)','lightweight pooling' +,'locks','max degree of parallelism','max full-text crawl range','max text repl size (B)' +,'max worker threads','min memory per query (KB)','nested triggers' +,'network packet size (B)','optimize for ad hoc workloads' +,'priority boost','query governor cost limit','query wait (s)','recovery interval (min)' +,'set working set size','user connections') +for xml raw, elements +); +set @result += (select name = 'version', value = @@VERSION for xml raw, elements) +select cast(@result as xml); +end; + diff --git a/samples/manage/azure-sql-db-managed-instance/prepare-subnet/prepareSubnet.ps1 b/samples/manage/azure-sql-db-managed-instance/prepare-subnet/prepareSubnet.ps1 index 22e9960c..7a848922 100644 --- a/samples/manage/azure-sql-db-managed-instance/prepare-subnet/prepareSubnet.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/prepare-subnet/prepareSubnet.ps1 @@ -182,39 +182,11 @@ function DefineSecurityRules{ $securityRules = New-Object "$NScollections.List``1[$NSnetworkModels.PSSecurityRule]" #begin NSG inbound rules $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-inbound-9000 ` - -Description "Allow inbound TCP traffic on port 9000" ` + -Name prepare-allow-management-inbound ` + -Description "Allow inbound TCP traffic on ports 9000,9003,1438,1440,1452" ` -Direction Inbound -Priority 110 -Access Allow -Protocol Tcp ` -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 9000 - $securityRules.Add($rule) - $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-inbound-9003 ` - -Description "Allow inbound TCP traffic on port 9003" ` - -Direction Inbound -Priority 120 -Access Allow -Protocol Tcp ` - -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 9003 - $securityRules.Add($rule) - $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-inbound-1438 ` - -Description "Allow inbound TCP traffic on port 1438" ` - -Direction Inbound -Priority 130 -Access Allow -Protocol Tcp ` - -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 1438 - $securityRules.Add($rule) - $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-inbound-1440 ` - -Description "Allow inbound TCP traffic on port 1440" ` - -Direction Inbound -Priority 140 -Access Allow -Protocol Tcp ` - -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 1440 - $securityRules.Add($rule) - $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-inbound-1452 ` - -Description "Allow inbound TCP traffic on port 1452" ` - -Direction Inbound -Priority 150 -Access Allow -Protocol Tcp ` - -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 1452 + -SourcePortRange * -DestinationPortRange @(9000, 9003, 1438, 1440, 1452) $securityRules.Add($rule) $rule = New-AzureRmNetworkSecurityRuleConfig ` -Name prepare-allow-mi_subnet-inbound ` @@ -225,7 +197,7 @@ function DefineSecurityRules{ $securityRules.Add($rule) $rule = New-AzureRmNetworkSecurityRuleConfig ` -Name prepare-allow-health_probe-inbound ` - -Description "Allow healt probe inbound" ` + -Description "Allow health probe inbound" ` -Direction Inbound -Priority 170 -Access Allow -Protocol * ` -SourceAddressPrefix AzureLoadBalancer -DestinationAddressPrefix * ` -SourcePortRange * -DestinationPortRange * @@ -233,25 +205,11 @@ function DefineSecurityRules{ #end NSG inbound rules #begin NSG outbound rules $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-outbound-80 ` - -Description "Allow outbound TCP traffic on port 80" ` + -Name prepare-allow-management-outbound ` + -Description "Allow outbound TCP traffic on port 80,443,12000" ` -Direction Outbound -Priority 110 -Access Allow -Protocol Tcp ` -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 80 - $securityRules.Add($rule) - $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-outbound-443 ` - -Description "Allow outbound TCP traffic on port 443" ` - -Direction Outbound -Priority 120 -Access Allow -Protocol Tcp ` - -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 443 - $securityRules.Add($rule) - $rule = New-AzureRmNetworkSecurityRuleConfig ` - -Name prepare-allow-management-outbound-12000 ` - -Description "Allow outbound TCP traffic on port 12000" ` - -Direction Outbound -Priority 130 -Access Allow -Protocol Tcp ` - -SourceAddressPrefix * -DestinationAddressPrefix * ` - -SourcePortRange * -DestinationPortRange 12000 + -SourcePortRange * -DestinationPortRange @(80, 443, 12000) $securityRules.Add($rule) $rule = New-AzureRmNetworkSecurityRuleConfig ` -Name prepare-allow-mi_subnet-outbound ` @@ -502,6 +460,9 @@ function VerifyNSG { } $result['success'] = $result['failedSecurityRules'].Count -eq 0 } + Else { + $result['failedSecurityRules'] = DefineSecurityRules + } If($true -eq $result['success']) { Write-Host "Passed Validation - Network security group." -ForegroundColor Green