From 644a8cfe66fc725ae43c398ea528b9ae8abd07dc Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Thu, 2 May 2019 20:21:03 -0700 Subject: [PATCH 01/16] Fixed kubectl command syntax --- .../sql-big-data-cluster/bootstrap-sample-db.cmd | 6 +++--- .../sql-big-data-cluster/bootstrap-sample-db.sh | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index 600c21d5..01cf8df2 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -46,7 +46,7 @@ if /i "%CTP_VERSION%" EQU "CTP2.4" (set MASTER_POD_NAME=mssql-master-pool-0) els REM Copy the backup file, restore the database, create necessary objects and data file echo Copying sales database backup file to SQL Master instance... -%DEBUG% kubectl cp tpcxbb_1gb.bak %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit +%DEBUG% kubectl cp tpcxbb_1gb.bak %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data -c mssql-server || goto exit REM Download and copy the sample backup files if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( @@ -57,7 +57,7 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/%%f" -o %%f ) echo Copying %%f database backup file to SQL Master instance... - %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit + %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data -c mssql-server || goto exit ) set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak @@ -67,7 +67,7 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/%%f" -o %%f ) echo Copying %%f database backup file to SQL Master instance... - %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit + %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data -c mssql-server || goto exit ) ) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 04679c05..023f7963 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -60,8 +60,8 @@ else MASTER_POD_NAME=master-0 fi -echo Copying database backup file... -$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) +echo Copying sales database backup file... +$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) # $DEBUG rm tpcxbb_1gb.bak if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ] @@ -74,7 +74,7 @@ then $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/$file" -o $file fi echo Copying $file database backup file to SQL Master instance... - $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) done @@ -86,7 +86,7 @@ then $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/$file" -o $file fi echo Copying $file database backup file to SQL Master instance... - $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) done fi @@ -97,7 +97,7 @@ $DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b < "$S # remove files copied into the pod: echo Removing database backup files... -kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" +$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" for table in web_clickstreams inventory customer do From cb52314bc46818c0b0e7e6494a0f40ac8407bf5b Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Thu, 2 May 2019 20:22:05 -0700 Subject: [PATCH 02/16] Simplified CONNECTION_OPTIONS for data source DDL --- .../generic-odbc/mysql/mysql_version.sql | 11 +++++------ .../generic-odbc/postgresql/pg_tables.sql | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql index ed9a3569..3b387ff3 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/mysql/mysql_version.sql @@ -4,16 +4,15 @@ -- the Query menu, and "Specify Values for Template Parameters" option. IF NOT EXISTS(SELECT * FROM sys.database_scoped_credentials WHERE name = 'MySQL80-user') CREATE DATABASE SCOPED CREDENTIAL [MySQL80-user] - WITH IDENTITY = 'mssql-user' - , SECRET = 'sql19tw0mysql'; + WITH IDENTITY = '' + , SECRET = ''; --- Create external data source that points to MySQL server --- The tokens '%u' and '%p' is used to reference the credential information. +-- Create external data source that points to MySQL server. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'MySQL80') CREATE EXTERNAL DATA SOURCE MySQL80 - WITH (LOCATION = 'odbc://uc-win19-vm.redmond.corp.microsoft.com' - , CONNECTION_OPTIONS = 'Driver={MySQL ODBC 8.0 Unicode Driver};User name=%u;Passwword=%p;IGNORE_SPACE=1' + WITH (LOCATION = 'odbc://' + , CONNECTION_OPTIONS = 'Driver={MySQL ODBC 8.0 Unicode Driver};IGNORE_SPACE=1' , CREDENTIAL = [MySQL80-user]); -- Create external table over inventory table on MySQL server diff --git a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql index 75e7827f..62c2a61d 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/generic-odbc/postgresql/pg_tables.sql @@ -7,13 +7,12 @@ IF NOT EXISTS(SELECT * FROM sys.database_scoped_credentials WHERE name = 'Postgr WITH IDENTITY = '' , SECRET = ''; --- Create external data source that points to PostgreSQL server --- The tokens '%u' and '%p' is used to reference the credential information. +-- Create external data source that points to PostgreSQL server. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'PostgreSQL11') CREATE EXTERNAL DATA SOURCE PostgreSQL11 WITH (LOCATION = 'odbc://' - , CONNECTION_OPTIONS = 'Driver={PostgreSQL ODBC Driver(UNICODE)};User name=%u;Passwword=%p' + , CONNECTION_OPTIONS = 'Driver={PostgreSQL ODBC Driver(UNICODE)}' , CREDENTIAL = [PostgreSQL11-user]); -- Create external table over inventory table on PostgreSQL server From 02b01d6d3ccf27f742fab511f42f244326270720 Mon Sep 17 00:00:00 2001 From: pmasl Date: Fri, 3 May 2019 09:25:44 -0700 Subject: [PATCH 03/16] Added AQP/IQP demos --- ...P Demos Enlarging WideWorldImportersDW.sql | 4 +- ...Public Preview - APPROX_COUNT_DISTINCT.sql | 12 +- ...lic Preview - Batch Mode Adaptive Join.sql | 54 ++++++ ...tersDW Public Preview - Batch Mode MGF.sql | 46 +++++ ...ublic Preview - Batch Mode on Rowstore.sql | 6 + ...Public Preview - Interleaved Execution.sql | 173 ++++++++++++++++++ ...ortersDW Public Preview - Row Mode MGF.sql | 11 +- ...W Public Preview - Scalar UDF Inlining.sql | 14 +- ...WorldImportersDW Public Preview - TVDC.sql | 5 +- .../intelligent-query-processing/readme.md | 3 + samples/features/query-store/README.md | 17 +- 11 files changed, 322 insertions(+), 23 deletions(-) create mode 100644 samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql create mode 100644 samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql create mode 100644 samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql index 9fd76a49..842613ba 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos Enlarging WideWorldImportersDW.sql @@ -3,7 +3,7 @@ -- bigger - so you can see more impactful -- Intelligent QP demonstrations (aka.ms/iqp) -- --- Script last updated 10/02/2018 +-- Script last updated 05/03/2019 -- -- Database backup source: aka.ms/wwibak -- @@ -100,4 +100,4 @@ GO UPDATE Fact.OrderHistoryExtended SET [WWI Order ID] = [Order Key]; -GO +GO \ No newline at end of file diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql index 26e6012c..2bf41899 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - APPROX_COUNT_DISTINCT.sql @@ -9,7 +9,17 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- -USE WideWorldImportersDW; + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; GO -- Compare execution time and distinct counts diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql new file mode 100644 index 00000000..b5a70078 --- /dev/null +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode Adaptive Join.sql @@ -0,0 +1,54 @@ +-- ******************************************************** -- +-- Batch mode Adaptive Join + +-- See https://aka.ms/IQP for more background + +-- Demo scripts: https://aka.ms/IQPDemos + +-- This demo is on SQL Server 2017 and Azure SQL DB + +-- Email IntelligentQP@microsoft.com for questions\feedback +-- ******************************************************** -- + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +-- Show with Live Query Stats +SELECT [fo].[Order Key], [si].[Lead Time Days], [fo].[Quantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [fo].[Quantity] = 360; +GO + +-- Inserting quantity row that doesn't exist in the table yet +DELETE [Fact].[Order] +WHERE Quantity = 361; + +INSERT [Fact].[Order] +([City Key], [Customer Key], [Stock Item Key], [Order Date Key], [Picked Date Key], [Salesperson Key], [Picker Key], [WWI Order ID], [WWI Backorder ID], Description, Package, Quantity, [Unit Price], [Tax Rate], [Total Excluding Tax], [Tax Amount], [Total Including Tax], [Lineage Key]) +SELECT TOP 5 [City Key], [Customer Key], [Stock Item Key], + [Order Date Key], [Picked Date Key], [Salesperson Key], + [Picker Key], [WWI Order ID], [WWI Backorder ID], + Description, Package, 361, [Unit Price], [Tax Rate], + [Total Excluding Tax], [Tax Amount], [Total Including Tax], + [Lineage Key] +FROM [Fact].[Order]; +GO + +-- Show with Live Query Stats +SELECT [fo].[Order Key], [si].[Lead Time Days], [fo].[Quantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [fo].[Quantity] = 361; +GO \ No newline at end of file diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql new file mode 100644 index 00000000..1cd6cfc6 --- /dev/null +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode MGF.sql @@ -0,0 +1,46 @@ +-- ******************************************************** -- +-- Batch mode Memory Grant Feedback + +-- See https://aka.ms/IQP for more background + +-- Demo scripts: https://aka.ms/IQPDemos + +-- This demo is on SQL Server 2017 and Azure SQL DB + +-- Email IntelligentQP@microsoft.com for questions\feedback +-- ******************************************************** -- + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +-- Intentionally forcing a row underestimate +CREATE OR ALTER PROCEDURE [FactOrderByLineageKey] + @LineageKey INT +AS +SELECT [fo].[Order Key], [fo].[Description] +FROM [Fact].[Order] AS [fo] +INNER HASH JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [fo].[Lineage Key] = @LineageKey + AND [si].[Lead Time Days] > 0 +ORDER BY [fo].[Stock Item Key], [fo].[Order Date Key] DESC +OPTION (MAXDOP 1); +GO + +-- Compiled and executed using a lineage key that doesn't have rows +EXEC [FactOrderByLineageKey] 8; +GO + +-- Execute this query a few times - each time looking at +-- the plan to see impact on spills, memory grant size, and run time +EXEC [FactOrderByLineageKey] 9; +GO \ No newline at end of file diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql index 39dea339..0ec77748 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Batch Mode on Rowstore.sql @@ -10,12 +10,18 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- +USE [master]; +GO + ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; GO ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO +USE [WideWorldImportersDW]; +GO + -- Row mode due to hint SELECT [Tax Rate], [Lineage Key], diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql new file mode 100644 index 00000000..eb2c1161 --- /dev/null +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Interleaved Execution.sql @@ -0,0 +1,173 @@ +-- ******************************************************** -- +-- Interleaved Execution + +-- See https://aka.ms/IQP for more background + +-- Demo scripts: https://aka.ms/IQPDemos + +-- This demo is on SQL Server 2017 and Azure SQL DB + +-- Email IntelligentQP@microsoft.com for questions\feedback +-- ******************************************************** -- + +/* + Create MSTVF +*/ + +USE [WideWorldImportersDW]; +GO + +CREATE FUNCTION [Fact].[WhatIfOutlierEventQuantity](@event VARCHAR(15), @beginOrderDateKey DATE, @endOrderDateKey DATE) +RETURNS @OutlierEventQuantity TABLE ( + [Order Key] [bigint], + [City Key] [int] NOT NULL, + [Customer Key] [int] NOT NULL, + [Stock Item Key] [int] NOT NULL, + [Order Date Key] [date] NOT NULL, + [Picked Date Key] [date] NULL, + [Salesperson Key] [int] NOT NULL, + [Picker Key] [int] NULL, + [OutlierEventQuantity] [int] NOT NULL) +AS +BEGIN + +-- Valid @event values + -- 'Mild Recession' + -- 'Hurricane - South Atlantic' + -- 'Hurricane - East South Central' + -- 'Hurricane - West South Central' + IF @event = 'Mild Recession' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [o].[Quantity] > 2 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + + IF @event = 'Hurricane - South Atlantic' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [o].[Quantity] > 10 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + WHERE [c].[State Province] IN + ('Florida', 'Georgia', 'Maryland', 'North Carolina', + 'South Carolina', 'Virginia', 'West Virginia', + 'Delaware') + AND [o].[Order Date Key] BETWEEN @beginOrderDateKey AND @endOrderDateKey + + IF @event = 'Hurricane - East South Central' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [o].[Quantity] > 50 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + INNER JOIN [Dimension].[Stock Item] AS [si] + ON [si].[Stock Item Key] = [o].[Stock Item Key] + WHERE [c].[State Province] IN + ('Alabama', 'Kentucky', 'Mississippi', 'Tennessee') + AND [si].[Buying Package] = 'Carton' + AND [o].[Order Date Key] BETWEEN @beginOrderDateKey AND @endOrderDateKey + + IF @event = 'Hurricane - West South Central' + INSERT @OutlierEventQuantity + SELECT [o].[Order Key], [o].[City Key], [o].[Customer Key], + [o].[Stock Item Key], [o].[Order Date Key], [o].[Picked Date Key], + [o].[Salesperson Key], [o].[Picker Key], + CASE + WHEN [cu].[Customer] = 'Unknown' THEN 0 + WHEN [cu].[Customer] <> 'Unknown' AND + [o].[Quantity] > 10 THEN [o].[Quantity] * .5 + ELSE [o].[Quantity] + END + FROM [Fact].[Order] AS [o] + INNER JOIN [Dimension].[City] AS [c] + ON [c].[City Key] = [o].[City Key] + INNER JOIN [Dimension].[Customer] AS [cu] + ON [cu].[Customer Key] = [o].[Customer Key] + WHERE [c].[State Province] IN + ('Arkansas', 'Louisiana', 'Oklahoma', 'Texas') + AND [o].[Order Date Key] BETWEEN @beginOrderDateKey AND @endOrderDateKey + + RETURN +END +GO + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 130; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +SELECT [fo].[Order Key], [fo].[Description], [fo].[Package], + [fo].[Quantity], [foo].[OutlierEventQuantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Fact].[WhatIfOutlierEventQuantity]('Mild Recession', + '1-01-2013', + '10-15-2014') AS [foo] ON [fo].[Order Key] = [foo].[Order Key] + AND [fo].[City Key] = [foo].[City Key] + AND [fo].[Customer Key] = [foo].[Customer Key] + AND [fo].[Stock Item Key] = [foo].[Stock Item Key] + AND [fo].[Order Date Key] = [foo].[Order Date Key] + AND [fo].[Picked Date Key] = [foo].[Picked Date Key] + AND [fo].[Salesperson Key] = [foo].[Salesperson Key] + AND [fo].[Picker Key] = [foo].[Picker Key] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [si].[Lead Time Days] > 0 + AND [fo].[Quantity] > 50; +GO + +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; +GO + +USE [WideWorldImportersDW]; +GO + +SELECT [fo].[Order Key], [fo].[Description], [fo].[Package], + [fo].[Quantity], [foo].[OutlierEventQuantity] +FROM [Fact].[Order] AS [fo] +INNER JOIN [Fact].[WhatIfOutlierEventQuantity]('Mild Recession', + '1-01-2013', + '10-15-2014') AS [foo] ON [fo].[Order Key] = [foo].[Order Key] + AND [fo].[City Key] = [foo].[City Key] + AND [fo].[Customer Key] = [foo].[Customer Key] + AND [fo].[Stock Item Key] = [foo].[Stock Item Key] + AND [fo].[Order Date Key] = [foo].[Order Date Key] + AND [fo].[Picked Date Key] = [foo].[Picked Date Key] + AND [fo].[Salesperson Key] = [foo].[Salesperson Key] + AND [fo].[Picker Key] = [foo].[Picker Key] +INNER JOIN [Dimension].[Stock Item] AS [si] + ON [fo].[Stock Item Key] = [si].[Stock Item Key] +WHERE [si].[Lead Time Days] > 0 + AND [fo].[Quantity] > 50; +GO diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql index 419b96c4..698b32cc 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Row Mode MGF.sql @@ -10,13 +10,16 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- -ALTER DATABASE WideWorldImportersDW SET COMPATIBILITY_LEVEL = 150; +USE [master]; +GO + +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; GO ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO -USE WideWorldImportersDW; +USE [WideWorldImportersDW]; GO -- Simulate out-of-date stats @@ -30,10 +33,10 @@ GO SELECT fo.[Order Key], fo.Description, si.[Lead Time Days] -FROM Fact.OrderHistory AS fo +FROM Fact.OrderHistory AS fo INNER HASH JOIN Dimension.[Stock Item] AS si ON fo.[Stock Item Key] = si.[Stock Item Key] -WHERE fo.[Lineage Key] = 9 +WHERE fo.[Lineage Key] = 9 AND si.[Lead Time Days] > 19; -- Cleanup diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql index 67dda7eb..e5b2124a 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - Scalar UDF Inlining.sql @@ -9,19 +9,21 @@ -- Email IntelligentQP@microsoft.com for questions\feedback -- ******************************************************** -- -USE WideWorldImportersDW; +USE [master]; GO -ALTER DATABASE WideWorldImportersDW -SET COMPATIBILITY_LEVEL = 150; +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; GO -ALTER DATABASE SCOPED CONFIGURATION -CLEAR PROCEDURE_CACHE; +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO + +USE [WideWorldImportersDW]; +GO + /* Adapted from SQL Server Books Online -https://docs.microsoft.com/en-us/sql/relational-databases/user-defined-functions/scalar-udf-inlining?view=sqlallproducts-allversions +https://docs.microsoft.com/sql/relational-databases/user-defined-functions/scalar-udf-inlining?view=sqlallproducts-allversions */ CREATE OR ALTER FUNCTION dbo.customer_category(@CustomerKey INT) diff --git a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql index 3b1dd936..6277b4a1 100644 --- a/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql +++ b/samples/features/intelligent-query-processing/Intelligent QP Demos WideWorldImportersDW Public Preview - TVDC.sql @@ -13,7 +13,10 @@ USE [master]; GO -ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 140; +ALTER DATABASE [WideWorldImportersDW] SET COMPATIBILITY_LEVEL = 150; +GO + +ALTER DATABASE SCOPED CONFIGURATION CLEAR PROCEDURE_CACHE; GO USE [WideWorldImportersDW]; diff --git a/samples/features/intelligent-query-processing/readme.md b/samples/features/intelligent-query-processing/readme.md index 38ad3a31..1f79e71a 100644 --- a/samples/features/intelligent-query-processing/readme.md +++ b/samples/features/intelligent-query-processing/readme.md @@ -8,6 +8,9 @@ Here are the instructions to prepare for demonstrating Intelligent QP's latest r Demos (with more on the way!): +- [Batch Mode Adaptive Join](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Batch%20Mode%20Adaptive%20Join.sql) +- [Interleaved Execution](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Interleaved%20Execution.sql) +- [Batch mode memory grant feedback](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Batch%20Mode%20MGF.sql) - [Row mode memory grant feedback](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Row%20Mode%20MGF.sql) - [Batch mode on rowstore](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20Batch%20Mode%20on%20Rowstore.sql) - [APPROX_COUNT_DISTINCT](Intelligent%20QP%20Demos%20WideWorldImportersDW%20Public%20Preview%20-%20APPROX_COUNT_DISTINCT.sql) diff --git a/samples/features/query-store/README.md b/samples/features/query-store/README.md index e369029a..4cc13077 100644 --- a/samples/features/query-store/README.md +++ b/samples/features/query-store/README.md @@ -1,4 +1,4 @@ -**Query Store demo** +## Query Store demo This demo shows capabilities of Query Store. Usually we demo 3-4 scenarios: 1. How to turn on and initially configure Query Store @@ -6,19 +6,17 @@ This demo shows capabilities of Query Store. Usually we demo 3-4 scenarios: 3. Detecting and fixing query with plan choice regression 4. Detecting and fixing workload that is candidate for auto-parametrization -**Prerequisites** +## Prerequisites Restore AdventureWorks2016_EXT database from the provided BAK at https://github.com/Microsoft/sql-server-samples/releases/tag/adventureworks. -Turn ON and Configure Query Store +After restoring the database AdventureWorks2016_EXT, go to Properties / Query Store tab, turn ON Query Store, and configure it according to best practices in https://docs.microsoft.com/sql/relational-databases/performance/best-practice-with-the-query-store. -After restoring the database, go to Properties / Query Store tab. - -![Query Store in SSMS](../QS_SSMS.png) +![Query Store in SSMS](./QS_SSMS.png) Use docs content to walk through main config settings: https://docs.microsoft.com//sql/relational-databases/performance/best-practice-with-the-query-store#Configure -**How Query Store Works** +### How Query Store Works Open ShowBasics.sql script and execute queries individually: - Run simple `SELECT * FROM` Part - Show where query ends in sys.query_store_query_text, sys.query_store_query, sys.query_store_plan, sys.query_store_runtime_stats @@ -27,12 +25,13 @@ Open ShowBasics.sql script and execute queries individually: - Show what happens with query that gets auto-parametrized. It cannot be searched using the original query text because QDS stores query as parametrized. Hopefully, sys.fn_stmt_sql_handle_from_sql_stmt can be used to track down query using original query text - Run `vw_QueryStoreRuntimeInfo` (again custom view) to show main runtime stats combined with query/plan info -**Query with plan regression** +### Query with plan regression 1. Run QueryStoreSimpleDemo.exe with option R or option S 2. Open SSMS, analyze and explain - two execution plans that SQL Server use alternately (switches between 2 plan almost randomly). This is known as Parameter Sniffing problem - plan gets generated based on parameter available at the compilation time. When compilation happens frequently and randomly and data is skewed (not all parameter values are uniformly distributed PSP is likely to occur and degradations are common) 3. Force better plan, explain what happens (SSMS) 4. Summarize benefits for DBA – fixing performance quickly without knowing details about the query. Fully transparent to running apps -Detect and fix ad hoc workload that is candidate for parametrization + +### Detect and fix ad hoc workload that is candidate for parametrization 1. Run QueryStoreSimpleDemo.exe with option P and let it work for some time (15-20 sec) 2. Open “Auto-Param Analysis.sql” and run queries from groups (1) and (2). What we see is: a. Large number of queries / plan entries, small number of different query/plan hashes indicates queries that are not parametrized although they are good candidates From 671d31a423330c8cb978da2ff70bae6a083adc38 Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sat, 4 May 2019 13:15:54 +0200 Subject: [PATCH 04/16] mod: switch to Az module --- .../attach-jumpbox/attachJumpbox.ps1 | 184 +++++++++--------- 1 file changed, 97 insertions(+), 87 deletions(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 index 306de1cb..252e9c9b 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/attachJumpbox.ps1 @@ -1,47 +1,69 @@ $parameters = $args[0] +$scriptUrlBase = $args[1] $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualMachineName = $parameters['virtualMachineName'] $virtualNetworkName = $parameters['virtualNetworkName'] $managementSubnetName = $parameters['subnetName'] -$administratorLogin = $parameters['administratorLogin'] -$administratorLoginPassword = $parameters['administratorLoginPassword'] +$administratorLogin = $parameters['administratorLogin'] +$administratorLoginPassword = $parameters['administratorLoginPassword'] -$scriptUrlBase = $args[1] - -if($virtualMachineName -eq '' -or $virtualMachineName -eq $null) { +if ($virtualMachineName -eq '' -or ($null -eq $virtualMachineName)) { $virtualMachineName = 'Jumpbox' Write-Host "VM Name: 'Jumpbox'." -ForegroundColor Green } -if($managementSubnetName -eq '' -or $managementSubnetName -eq $null) { +if ($managementSubnetName -eq '' -or ($null -eq $managementSubnetName)) { $managementSubnetName = 'Management' Write-Host "Using subnet 'Management' to deploy jumpbox VM." -ForegroundColor Green } -function VerifyPSVersion -{ - Write-Host "Verifying PowerShell version, must be 5.0 or higher." - if($PSVersionTable.PSVersion.Major -ge 5) - { - Write-Host "PowerShell version verified." -ForegroundColor Green +function VerifyPSVersion { + Write-Host "Verifying PowerShell version." + if ($PSVersionTable.PSEdition -eq "Desktop") { + if (($PSVersionTable.PSVersion.Major -ge 6) -or + (($PSVersionTable.PSVersion.Major -eq 5) -and ($PSVersionTable.PSVersion.Minor -ge 1))) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 5.1 or heigher." -ForegroundColor Red + Break; + } } - else - { - Write-Host "You need to install PowerShell version 5.0 or heigher." -ForegroundColor Red - Break; + else { + if ($PSVersionTable.PSVersion.Major -ge 6) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 6.0 or heigher." -ForegroundColor Red + Break; + } } } -function EnsureLogin () -{ - $context = Get-AzureRmContext - If($null -eq $context.Subscription) - { +function EnsureAzModule { + Write-Host "Checking if Az module is imported." + $module = Get-Module Az + If ($null -eq $module) { + try { + Import-Module Az -ErrorAction Stop + Write-Host "Module Az imported." -ForegroundColor Green + } + catch { + Install-Module Az -AllowClobber + Write-Host "Module Az installed." -ForegroundColor Green + } + } else { + Write-Host "Module Az imported." -ForegroundColor Green + } +} + +function EnsureLogin () { + $context = Get-AzContext + If ($null -eq $context.Subscription) { Write-Host "Sign-in..." - If($null -eq (Login-AzureRmAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) - { + If ($null -eq (Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) { Write-Host ("Sign-in failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red Break } @@ -54,16 +76,14 @@ function SelectSubscriptionId { $subscriptionId ) Write-Host "Selecting subscription '$subscriptionId'..." - $context = Get-AzureRmContext - If($context.Subscription.Id -ne $subscriptionId) - { - Try - { - Write-Host "Switching subscription $context.Subscription.Id to '$subscriptionId'." -ForegroundColor Green - Select-AzureRmSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null + $context = Get-AzContext + If ($context.Subscription.Id -ne $subscriptionId) { + Try { + $currentSubscriptionId = $context.Subscription.Id + Write-Host "Switching subscription $currentSubscriptionId to '$subscriptionId'." -ForegroundColor Green + Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null } - Catch - { + Catch { Write-Host "Subscription selection failed: $_" -ForegroundColor Red Break } @@ -76,59 +96,52 @@ function LoadVirtualNetwork { $resourceGroupName, $virtualNetworkName ) - Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) - $virtualNetwork = Get-AzureRmVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue - $id = $virtualNetwork.Id - If($null -ne $id) - { - Write-Host "Virtual network with id $id is loaded." -ForegroundColor Green - If($virtualNetwork.VirtualNetworkPeerings.Count -gt 0) { - Write-Host "Virtual network is loaded, but it should not have peerings." -ForegroundColor Red - } - return $virtualNetwork - } - else - { - Write-Host "Virtual network $virtualNetworkName cannot be found." -ForegroundColor Red - Break + Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) + $virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue + $id = $virtualNetwork.Id + If ($null -ne $id) { + Write-Host "Virtual network with id $id is loaded." -ForegroundColor Green + If ($virtualNetwork.VirtualNetworkPeerings.Count -gt 0) { + Write-Host "Virtual network is loaded, but it should not have peerings." -ForegroundColor Red } + return $virtualNetwork + } + else { + Write-Host "Virtual network $virtualNetworkName cannot be found." -ForegroundColor Red + Break + } } -function SetVirtualNetwork -{ +function SetVirtualNetwork { param($virtualNetwork) Write-Host "Applying changes to the virtual network." - Try - { - Set-AzureRmVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null + Try { + Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null } - Catch - { + Catch { Write-Host "Failed to configure Virtual Network: $_" -ForegroundColor Red } } -function ConvertCidrToUint32Array -{ +function ConvertCidrToUint32Array { param($cidrRange) $cidrRangeParts = $cidrRange.Split("/") $ipParts = $cidrRangeParts[0].Split(".") $ipnum = ([Convert]::ToUInt32($ipParts[0]) -shl 24) -bor ` - ([Convert]::ToUInt32($ipParts[1]) -shl 16) -bor ` - ([Convert]::ToUInt32($ipParts[2]) -shl 8) -bor ` - [Convert]::ToUInt32($ipParts[3]) + ([Convert]::ToUInt32($ipParts[1]) -shl 16) -bor ` + ([Convert]::ToUInt32($ipParts[2]) -shl 8) -bor ` + [Convert]::ToUInt32($ipParts[3]) $maskbits = [System.Convert]::ToInt32($cidrRangeParts[1]) $mask = 0xffffffff - $mask = $mask -shl (32 -$maskbits) + $mask = $mask -shl (32 - $maskbits) $ipstart = $ipnum -band $mask $ipend = $ipnum -bor ($mask -bxor 0xffffffff) return @($ipstart, $ipend) } -function ConvertUInt32ToIPAddress -{ +function ConvertUInt32ToIPAddress { param($uint32IP) $v1 = $uint32IP -band 0xff $v2 = ($uint32IP -shr 8) -band 0xff @@ -137,16 +150,13 @@ function ConvertUInt32ToIPAddress return "$v4.$v3.$v2.$v1" } -function CalculateNextAddressPrefix -{ +function CalculateNextAddressPrefix { param($virtualNetwork, $prefixLength) Write-Host "Calculating address prefix with length $prefixLength..." $startIPAddress = 0 - ForEach($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) - { + ForEach ($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) { $endIPAddress = (ConvertCidrToUint32Array $addressPrefix)[1] - If($endIPAddress -gt $startIPAddress) - { + If ($endIPAddress -gt $startIPAddress) { $startIPAddress = $endIPAddress } } @@ -156,22 +166,20 @@ function CalculateNextAddressPrefix return $addressPrefixResult } -function CalculateVpnClientAddressPoolPrefix -{ +function CalculateVpnClientAddressPoolPrefix { param($gatewaySubnetPrefix) Write-Host "Calculating VPN client address pool prefix." - If($gatewaySubnetPrefix.StartsWith("10.")) - { + If ($gatewaySubnetPrefix.StartsWith("10.")) { return "192.168.0.0/24" } - else - { + else { return "172.16.0.0/24" } } VerifyPSVersion +EnsureAzModule EnsureLogin SelectSubscriptionId -subscriptionId $subscriptionId @@ -179,34 +187,36 @@ $virtualNetwork = LoadVirtualNetwork -resourceGroupName $resourceGroupName -virt $subnets = $virtualNetwork.Subnets.Name -If($false -eq $subnets.Contains($managementSubnetName)) -{ +If ($false -eq $subnets.Contains($managementSubnetName)) { Write-Host "$managementSubnetName is not one of the subnets in $subnets" -ForegroundColor Yellow - Write-Host "Creating subnet $managementSubnetName ($managementSubnetPrefix) in the VNet..." -ForegroundColor Green $managementSubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 + Write-Host "Creating subnet $managementSubnetName ($managementSubnetPrefix) in the virtual network ..." -ForegroundColor Green $virtualNetwork.AddressSpace.AddressPrefixes.Add($managementSubnetPrefix) - Add-AzureRmVirtualNetworkSubnetConfig -Name $managementSubnetName -VirtualNetwork $virtualNetwork -AddressPrefix $managementSubnetPrefix | Out-Null + Add-AzVirtualNetworkSubnetConfig -Name $managementSubnetName -VirtualNetwork $virtualNetwork -AddressPrefix $managementSubnetPrefix | Out-Null SetVirtualNetwork $virtualNetwork - Write-Host "Added subnet $managementSubnetName into VNet." -ForegroundColor Green -} else { - Write-Host "The subnet $managementSubnetName exists in the VNet." -ForegroundColor Green + Write-Host "Added subnet $managementSubnetName into virtual network." -ForegroundColor Green +} +else { + Write-Host "The subnet $managementSubnetName exists in the virtual network." -ForegroundColor Green } Write-Host # Start the deployment Write-Host "Starting deployment..." +Write-Host "Deployment will take about 20m." -ForegroundColor Yellow $templateParameters = @{ - virtualNetworkName = $virtualNetworkName - managementSubnetName = $managementSubnetName - virtualMachineName = $virtualMachineName - administratorLogin = $administratorLogin - administratorLoginPassword = $administratorLoginPassword + location = $virtualNetwork.Location + virtualNetworkName = $virtualNetworkName + managementSubnetName = $managementSubnetName + virtualMachineName = $virtualMachineName + administratorLogin = $administratorLogin + administratorLoginPassword = $administratorLoginPassword } -New-AzureRmResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase+'/azuredeploy.json?t='+ [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters +New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase + '/azuredeploy.json?t=' + [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters Write-Host "Deployment completed." From 3fb2c5b27e456eab1966e09a913b335ecdb9cab7 Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sat, 4 May 2019 23:22:40 +0200 Subject: [PATCH 05/16] mod: swich GW to Az --- .gitignore | 5 + .../attach-vpn-gateway/attachVPNGateway.ps1 | 288 ++++++++++-------- .../attach-vpn-gateway/attachVPNGatewayAz.ps1 | 217 ------------- .../attach-vpn-gateway/azuredeploy.json | 2 +- 4 files changed, 173 insertions(+), 339 deletions(-) delete mode 100644 samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 diff --git a/.gitignore b/.gitignore index 9d7f1407..1d12d793 100644 --- a/.gitignore +++ b/.gitignore @@ -435,3 +435,8 @@ samples/databases/wide-world-importers/workload-drivers/order-insert/Multithread /samples/features/epm-framework/5.0/2Reporting/PolicyReports/bin/Debug /samples/features/epm-framework/5.0/2Reporting/PolicyReports/PolicyDashboard - Backup.rdl /samples/features/epm-framework/5.0/2Reporting/PolicyReports/PolicyDashboardFiltered.rdl.data + +# Certificates +*.pem +*.p12 + diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 index 14364d4e..7ec2c1ac 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 @@ -1,56 +1,82 @@ $parameters = $args[0] +$scriptUrlBase = $args[1] $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualNetworkName = $parameters['virtualNetworkName'] $certificateNamePrefix = $parameters['certificateNamePrefix'] -$force = $parameters['force'] +$clientCertificatePassword = $parameters['clientCertificatePassword'] -$scriptUrlBase = $args[1] +if ($clientCertificatePassword -eq '' -or ($null -eq $clientCertificatePassword)) { + $clientCertificatePassword = 'S0m3Str0nGP@ssw0rd' +} -function VerifyPSVersion -{ - Write-Host "Verifying PowerShell version, must be 5.0 or higher." - if($PSVersionTable.PSVersion.Major -ge 5) - { - Write-Host "PowerShell version verified." -ForegroundColor Green +function VerifyPSVersion { + Write-Host "Verifying PowerShell version." + if ($PSVersionTable.PSEdition -eq "Desktop") { + if (($PSVersionTable.PSVersion.Major -ge 6) -or + (($PSVersionTable.PSVersion.Major -eq 5) -and ($PSVersionTable.PSVersion.Minor -ge 1))) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 5.1 or heigher." -ForegroundColor Red + Break; + } } - else - { - Write-Host "You need to install PowerShell version 5.0 or heigher." -ForegroundColor Red - Break; + else { + if ($PSVersionTable.PSVersion.Major -ge 6) { + Write-Host "PowerShell version verified." -ForegroundColor Green + } + else { + Write-Host "You need to install PowerShell version 6.0 or heigher." -ForegroundColor Red + Break; + } } } -function Ensure-Login () -{ - $context = Get-AzureRmContext - If($context.Subscription -eq $null) - { - Write-Host "Loging in ..." - If((Login-AzureRmAccount -ErrorAction SilentlyContinue -ErrorVariable Errors) -eq $null) - { - Write-Host ("Login failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red +function EnsureAzModule { + Write-Host "Checking if Az module is imported." + $module = Get-Module Az + If ($null -eq $module) { + try { + Import-Module Az -ErrorAction Stop + Write-Host "Module Az imported." -ForegroundColor Green + } + catch { + Install-Module Az -AllowClobber + Write-Host "Module Az installed." -ForegroundColor Green + } + } + else { + Write-Host "Module Az imported." -ForegroundColor Green + } +} + +function EnsureLogin () { + $context = Get-AzContext + If ($null -eq $context.Subscription) { + Write-Host "Sign-in..." + If ($null -eq (Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) { + Write-Host ("Sign-in failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red Break } } - Write-Host "User logedin." -ForegroundColor Green + Write-Host "Sign-in successful." -ForegroundColor Green } -function Select-SubscriptionId { +function SelectSubscriptionId { param ( $subscriptionId ) - Write-Host "Selecting subscription '$subscriptionId'." - $context = Get-AzureRmContext - If($context.Subscription.Id -ne $subscriptionId) - { - Try - { - Select-AzureRmSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null + Write-Host "Selecting subscription '$subscriptionId'..." + $context = Get-AzContext + If ($context.Subscription.Id -ne $subscriptionId) { + Try { + $currentSubscriptionId = $context.Subscription.Id + Write-Host "Switching subscription $currentSubscriptionId to '$subscriptionId'." -ForegroundColor Green + Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null } - Catch - { + Catch { Write-Host "Subscription selection failed: $_" -ForegroundColor Red Break } @@ -58,78 +84,57 @@ function Select-SubscriptionId { Write-Host "Subscription selected." -ForegroundColor Green } -function Load-VirtualNetwork { +function LoadVirtualNetwork { param ( $resourceGroupName, $virtualNetworkName ) - Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) - $virtualNetwork = Get-AzureRmVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue - If($virtualNetwork.Id -ne $null) - { - Write-Host "Virtual network loaded." -ForegroundColor Green - return $virtualNetwork + Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) + $virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue + $id = $virtualNetwork.Id + If ($null -ne $id) { + Write-Host "Virtual network with id $id is loaded." -ForegroundColor Green + If ($virtualNetwork.VirtualNetworkPeerings.Count -gt 0) { + Write-Host "Virtual network is loaded, but it should not have peerings." -ForegroundColor Red } - else - { - Write-Host "Virtual network not found." -ForegroundColor Red - Break - } -} - -function Load-ResourceGroup { - param ( - $resourceGroupName - ) - Write-Host("Loading resource group '{0}'." -f $resourceGroupName) - $resourceGroup = Get-AzureRmResourceGroup -Name $resourceGroupName - If($resourceGroup.ResourceId -ne $null) - { - Write-Host "Resource group loaded." -ForegroundColor Green - return $resourceGroup + return $virtualNetwork } - else - { - Write-Host "Resource group not found." -ForegroundColor Red + else { + Write-Host "Virtual network $virtualNetworkName cannot be found." -ForegroundColor Red Break } } -function Set-VirtualNetwork -{ +function SetVirtualNetwork { param($virtualNetwork) Write-Host "Applying changes to the virtual network." - Try - { - Set-AzureRmVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null + Try { + Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null } - Catch - { - Write-Host "Failed: $_" -ForegroundColor Red + Catch { + Write-Host "Failed to configure Virtual Network: $_" -ForegroundColor Red } - } -function ConvertCidrToUint32Array -{ +function ConvertCidrToUint32Array { param($cidrRange) - $cidrRangeParts = $cidrRange.Split(@(".","/")) - $ipnum = ([Convert]::ToUInt32($cidrRangeParts[0]) -shl 24) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[1]) -shl 16) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[2]) -shl 8) -bor ` - [Convert]::ToUInt32($cidrRangeParts[3]) + $cidrRangeParts = $cidrRange.Split("/") + $ipParts = $cidrRangeParts[0].Split(".") + $ipnum = ([Convert]::ToUInt32($ipParts[0]) -shl 24) -bor ` + ([Convert]::ToUInt32($ipParts[1]) -shl 16) -bor ` + ([Convert]::ToUInt32($ipParts[2]) -shl 8) -bor ` + [Convert]::ToUInt32($ipParts[3]) - $maskbits = [System.Convert]::ToInt32($cidrRangeParts[4]) + $maskbits = [System.Convert]::ToInt32($cidrRangeParts[1]) $mask = 0xffffffff - $mask = $mask -shl (32 -$maskbits) + $mask = $mask -shl (32 - $maskbits) $ipstart = $ipnum -band $mask $ipend = $ipnum -bor ($mask -bxor 0xffffffff) return @($ipstart, $ipend) } -function ConvertUInt32ToIPAddress -{ +function ConvertUInt32ToIPAddress { param($uint32IP) $v1 = $uint32IP -band 0xff $v2 = ($uint32IP -shr 8) -band 0xff @@ -138,80 +143,121 @@ function ConvertUInt32ToIPAddress return "$v4.$v3.$v2.$v1" } -function CalculateNextAddressPrefix -{ +function CalculateNextAddressPrefix { param($virtualNetwork, $prefixLength) - Write-Host "Calculating address prefix." + Write-Host "Calculating address prefix with length $prefixLength..." $startIPAddress = 0 - ForEach($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) - { + ForEach ($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) { $endIPAddress = (ConvertCidrToUint32Array $addressPrefix)[1] - If($endIPAddress -gt $startIPAddress) - { + If ($endIPAddress -gt $startIPAddress) { $startIPAddress = $endIPAddress } } $startIPAddress += 1 - return (ConvertUInt32ToIPAddress $startIPAddress) + "/" + $prefixLength + $addressPrefixResult = (ConvertUInt32ToIPAddress $startIPAddress) + "/" + $prefixLength + Write-Host "Using address prefix $addressPrefixResult." -ForegroundColor Green + return $addressPrefixResult } -function CalculateVpnClientAddressPoolPrefix -{ +function CalculateVpnClientAddressPoolPrefix { param($gatewaySubnetPrefix) Write-Host "Calculating VPN client address pool prefix." - If($gatewaySubnetPrefix.StartsWith("10.")) - { + If ($gatewaySubnetPrefix.StartsWith("10.")) { return "192.168.0.0/24" } - else - { + else { return "172.16.0.0/24" } } +function CreateCerificateWindows() { + $certificate = New-SelfSignedCertificate -Type Custom -KeySpec Signature ` + -Subject ("CN=$certificateNamePrefix" + "P2SRoot") -KeyExportPolicy Exportable ` + -HashAlgorithm sha256 -KeyLength 2048 ` + -CertStoreLocation "Cert:\CurrentUser\My" -KeyUsageProperty Sign -KeyUsage CertSign + + $certificateThumbprint = $certificate.Thumbprint + + New-SelfSignedCertificate -Type Custom -DnsName ($certificateNamePrefix + "P2SChild") -KeySpec Signature ` + -Subject ("CN=$certificateNamePrefix" + "P2SChild") -KeyExportPolicy Exportable ` + -HashAlgorithm sha256 -KeyLength 2048 ` + -CertStoreLocation "Cert:\CurrentUser\My" ` + -Signer $certificate -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") | Out-null + + [Convert]::ToBase64String((Get-Item cert:\currentuser\my\$certificateThumbprint).RawData) +} + +function CreateCerificateOpenSsl() { + $dn = "CN=$certificateNamePrefix" + "P2SRoot" + ipsec pki --gen --outform pem > caKey.pem + ipsec pki --self --in caKey.pem --dn $dn --ca --outform pem > caCert.pem + + $dn = $certificateNamePrefix + "P2SChild" + ipsec pki --gen --outform pem > "$($dn)Key.pem" + ipsec pki --pub --in "$($dn)Key.pem" --outform pem > "$($dn)PubKey.pem" + ipsec pki --issue --in "$($dn)PubKey.pem" --cacert caCert.pem --cakey caKey.pem --dn "CN=$($dn)" --san $dn --flag clientAuth --outform pem > "$($dn)Cert.pem" + openssl pkcs12 -in "$($dn)Cert.pem" -inkey "$($dn)Key.pem" -certfile caCert.pem -export -out "$($dn).p12" -password "pass:$($clientCertificatePassword)" + + $publicRootCertData = openssl x509 -in caCert.pem -outform pem + $publicRootCertData = $publicRootCertData -replace "-----BEGIN CERTIFICATE-----", "" + $publicRootCertData = $publicRootCertData -replace "-----END CERTIFICATE-----", "" + [string]::Join("", $publicRootCertData.Split()) +} + +function CreateCertificate() { + Write-Host "Creating certificate." + if ($PSVersionTable.PSEdition -eq "Desktop") { + return CreateCerificateWindows + } + else { + return CreateCerificateOpenSsl + } +} + VerifyPSVersion -Ensure-Login -Select-SubscriptionId -subscriptionId $subscriptionId +EnsureAzModule +EnsureLogin +SelectSubscriptionId -subscriptionId $subscriptionId -$virtualNetwork = Load-VirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName +$virtualNetwork = LoadVirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName -$resourceGroup = Get-AzureRmResourceGroup -Name $resourceGroupName +$subnets = $virtualNetwork.Subnets.Name -$certificate = New-SelfSignedCertificate -Type Custom -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SRoot") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" -KeyUsageProperty Sign -KeyUsage CertSign +$gatewaySubnetName = "GatewaySubnet" -$certificateThumbprint = $certificate.Thumbprint +If ($false -eq $subnets.Contains($gatewaySubnetName)) { + Write-Host "$gatewaySubnetName is not one of the subnets in $subnets" -ForegroundColor Yellow + $gatewaySubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 + Write-Host "Creating subnet $gatewaySubnetName ($gatewaySubnetPrefix) in the virtual network ..." -ForegroundColor Green -New-SelfSignedCertificate -Type Custom -DnsName ($certificateNamePrefix+"P2SChild") -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SChild") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" ` - -Signer $certificate -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") | Out-null + $virtualNetwork.AddressSpace.AddressPrefixes.Add($gatewaySubnetPrefix) + Add-AzVirtualNetworkSubnetConfig -Name $gatewaySubnetName -VirtualNetwork $virtualNetwork -AddressPrefix $gatewaySubnetPrefix | Out-Null -$publicRootCertData = [Convert]::ToBase64String((Get-Item cert:\currentuser\my\$certificateThumbprint).RawData) - -$gatewaySubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 + SetVirtualNetwork $virtualNetwork + Write-Host "Added subnet $gatewaySubnetName into virtual network." -ForegroundColor Green +} +else { + Write-Host "The subnet $gatewaySubnetName exists in the virtual network." -ForegroundColor Green +} $vpnClientAddressPoolPrefix = CalculateVpnClientAddressPoolPrefix $gatewaySubnetPrefix - -$virtualNetwork.AddressSpace.AddressPrefixes.Add($gatewaySubnetPrefix) -Add-AzureRmVirtualNetworkSubnetConfig -Name GatewaySubnet -VirtualNetwork $virtualNetwork -AddressPrefix $gatewaySubnetPrefix | Out-Null - -Set-VirtualNetwork $virtualNetwork +$publicRootCertData = CreateCertificate Write-Host # Start the deployment Write-Host "Starting deployment..." +Write-Host "Deployment will take about 1h." -ForegroundColor Yellow $templateParameters = @{ - virtualNetworkName = $virtualNetworkName - gatewaySubnetPrefix = $gatewaySubnetPrefix - vpnClientAddressPoolPrefix = $vpnClientAddressPoolPrefix - publicRootCertData = $publicRootCertData - } + location = $virtualNetwork.Location + virtualNetworkName = $virtualNetworkName + gatewaySubnetPrefix = $gatewaySubnetPrefix + vpnClientAddressPoolPrefix = $vpnClientAddressPoolPrefix + publicRootCertData = $publicRootCertData +} -New-AzureRmResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase+'/azuredeploy.json?t='+ [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters +New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase + '/azuredeploy.json?t=' + [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters + +Write-Host "Deployment completed." \ No newline at end of file diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 deleted file mode 100644 index 6cfd2ab4..00000000 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGatewayAz.ps1 +++ /dev/null @@ -1,217 +0,0 @@ -$parameters = $args[0] - -$subscriptionId = $parameters['subscriptionId'] -$resourceGroupName = $parameters['resourceGroupName'] -$virtualNetworkName = $parameters['virtualNetworkName'] -$certificateNamePrefix = $parameters['certificateNamePrefix'] -$force = $parameters['force'] - -$scriptUrlBase = $args[1] - -function VerifyPSVersion -{ - Write-Host "Verifying PowerShell version, must be 5.0 or higher." - if($PSVersionTable.PSVersion.Major -ge 5) - { - Write-Host "PowerShell version verified." -ForegroundColor Green - } - else - { - Write-Host "You need to install PowerShell version 5.0 or heigher." -ForegroundColor Red - Break; - } -} - -function Ensure-Login () -{ - $context = Get-AzContext - If($context.Subscription -eq $null) - { - Write-Host "Loging in ..." - If((Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors) -eq $null) - { - Write-Host ("Login failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red - Break - } - } - Write-Host "User logedin." -ForegroundColor Green -} - -function Select-SubscriptionId { - param ( - $subscriptionId - ) - Write-Host "Selecting subscription '$subscriptionId'." - $context = Get-AzContext - If($context.Subscription.Id -ne $subscriptionId) - { - Try - { - Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null - } - Catch - { - Write-Host "Subscription selection failed: $_" -ForegroundColor Red - Break - } - } - Write-Host "Subscription selected." -ForegroundColor Green -} - -function Load-VirtualNetwork { - param ( - $resourceGroupName, - $virtualNetworkName - ) - Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName) - $virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue - If($virtualNetwork.Id -ne $null) - { - Write-Host "Virtual network loaded." -ForegroundColor Green - return $virtualNetwork - } - else - { - Write-Host "Virtual network not found." -ForegroundColor Red - Break - } -} - -function Load-ResourceGroup { - param ( - $resourceGroupName - ) - Write-Host("Loading resource group '{0}'." -f $resourceGroupName) - $resourceGroup = Get-AzResourceGroup -Name $resourceGroupName - If($resourceGroup.ResourceId -ne $null) - { - Write-Host "Resource group loaded." -ForegroundColor Green - return $resourceGroup - } - else - { - Write-Host "Resource group not found." -ForegroundColor Red - Break - } -} - -function Set-VirtualNetwork -{ - param($virtualNetwork) - - Write-Host "Applying changes to the virtual network." - Try - { - Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop | Out-Null - } - Catch - { - Write-Host "Failed: $_" -ForegroundColor Red - } - -} - -function ConvertCidrToUint32Array -{ - param($cidrRange) - $cidrRangeParts = $cidrRange.Split(@(".","/")) - $ipnum = ([Convert]::ToUInt32($cidrRangeParts[0]) -shl 24) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[1]) -shl 16) -bor ` - ([Convert]::ToUInt32($cidrRangeParts[2]) -shl 8) -bor ` - [Convert]::ToUInt32($cidrRangeParts[3]) - - $maskbits = [System.Convert]::ToInt32($cidrRangeParts[4]) - $mask = 0xffffffff - $mask = $mask -shl (32 -$maskbits) - $ipstart = $ipnum -band $mask - $ipend = $ipnum -bor ($mask -bxor 0xffffffff) - return @($ipstart, $ipend) -} - -function ConvertUInt32ToIPAddress -{ - param($uint32IP) - $v1 = $uint32IP -band 0xff - $v2 = ($uint32IP -shr 8) -band 0xff - $v3 = ($uint32IP -shr 16) -band 0xff - $v4 = ($uint32IP -shr 24) - return "$v4.$v3.$v2.$v1" -} - -function CalculateNextAddressPrefix -{ - param($virtualNetwork, $prefixLength) - Write-Host "Calculating address prefix." - $startIPAddress = 0 - ForEach($addressPrefix in $virtualNetwork.AddressSpace.AddressPrefixes) - { - $endIPAddress = (ConvertCidrToUint32Array $addressPrefix)[1] - If($endIPAddress -gt $startIPAddress) - { - $startIPAddress = $endIPAddress - } - } - $startIPAddress += 1 - return (ConvertUInt32ToIPAddress $startIPAddress) + "/" + $prefixLength -} - -function CalculateVpnClientAddressPoolPrefix -{ - param($gatewaySubnetPrefix) - Write-Host "Calculating VPN client address pool prefix." - If($gatewaySubnetPrefix.StartsWith("10.")) - { - return "192.168.0.0/24" - } - else - { - return "172.16.0.0/24" - } - -} - -VerifyPSVersion -Ensure-Login -Select-SubscriptionId -subscriptionId $subscriptionId - -$virtualNetwork = Load-VirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName - -$resourceGroup = Get-AzResourceGroup -Name $resourceGroupName - -$certificate = New-SelfSignedCertificate -Type Custom -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SRoot") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" -KeyUsageProperty Sign -KeyUsage CertSign - -$certificateThumbprint = $certificate.Thumbprint - -New-SelfSignedCertificate -Type Custom -DnsName ($certificateNamePrefix+"P2SChild") -KeySpec Signature ` - -Subject ("CN=$certificateNamePrefix"+"P2SChild") -KeyExportPolicy Exportable ` - -HashAlgorithm sha256 -KeyLength 2048 ` - -CertStoreLocation "Cert:\CurrentUser\My" ` - -Signer $certificate -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") | Out-null - -$publicRootCertData = [Convert]::ToBase64String((Get-Item cert:\currentuser\my\$certificateThumbprint).RawData) - -$gatewaySubnetPrefix = CalculateNextAddressPrefix $virtualNetwork 28 - -$vpnClientAddressPoolPrefix = CalculateVpnClientAddressPoolPrefix $gatewaySubnetPrefix - -$virtualNetwork.AddressSpace.AddressPrefixes.Add($gatewaySubnetPrefix) -Add-AzVirtualNetworkSubnetConfig -Name GatewaySubnet -VirtualNetwork $virtualNetwork -AddressPrefix $gatewaySubnetPrefix | Out-Null - -Set-VirtualNetwork $virtualNetwork - -Write-Host - -# Start the deployment -Write-Host "Starting deployment..." - -$templateParameters = @{ - virtualNetworkName = $virtualNetworkName - gatewaySubnetPrefix = $gatewaySubnetPrefix - vpnClientAddressPoolPrefix = $vpnClientAddressPoolPrefix - publicRootCertData = $publicRootCertData - } - -New-AzResourceGroupDeployment -ResourceGroupName $resourceGroupName -TemplateUri ($scriptUrlBase+'/azuredeploy.json?t='+ [DateTime]::Now.Ticks) -TemplateParameterObject $templateParameters diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json index 822a302c..db683af4 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json @@ -55,7 +55,7 @@ "apiVersion": "2017-10-01", "type": "Microsoft.Network/virtualNetworkGateways", "name": "[variables('gatewayName')]", - "location": "[resourceGroup().location]", + "location": "[parameters('location')]", "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', variables('gatewayPublicIpAddressName'))]" ], From 2943ec1eccc8de609ff28db9a12fdce857efe770 Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sat, 4 May 2019 23:26:11 +0200 Subject: [PATCH 06/16] mod: publicip location --- .../attach-vpn-gateway/attachVPNGateway.ps1 | 9 +++++++++ .../attach-vpn-gateway/azuredeploy.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 index 7ec2c1ac..f97d02c3 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 @@ -1,6 +1,15 @@ $parameters = $args[0] $scriptUrlBase = $args[1] +$scriptUrlBase = 'https://raw.githubusercontent.com/srdan-bozovic-msft/sql-server-samples/master/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway' + +$parameters = @{ + subscriptionId = 'a8c9a924-06c0-4bde-9788-e7b1370969e1' + resourceGroupName = 'srki_test' + virtualNetworkName = 'vnet-testlinuxps' + certificateNamePrefix = 'cert4testlinuxps' +} + $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualNetworkName = $parameters['virtualNetworkName'] diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json index db683af4..9532f57e 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json @@ -46,7 +46,7 @@ "apiVersion": "2017-10-01", "type": "Microsoft.Network/publicIPAddresses", "name": "[variables('gatewayPublicIpAddressName')]", - "location": "[resourceGroup().location]", + "location": "[parameters('location')]", "properties": { "publicIPAllocationMethod": "Dynamic" } From 8519432dae210b0fbb44751e03673ff34875aa9d Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sat, 4 May 2019 23:35:26 +0200 Subject: [PATCH 07/16] rem: test script --- .../attach-vpn-gateway/attachVPNGateway.ps1 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 index f97d02c3..7ec2c1ac 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 @@ -1,15 +1,6 @@ $parameters = $args[0] $scriptUrlBase = $args[1] -$scriptUrlBase = 'https://raw.githubusercontent.com/srdan-bozovic-msft/sql-server-samples/master/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway' - -$parameters = @{ - subscriptionId = 'a8c9a924-06c0-4bde-9788-e7b1370969e1' - resourceGroupName = 'srki_test' - virtualNetworkName = 'vnet-testlinuxps' - certificateNamePrefix = 'cert4testlinuxps' -} - $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualNetworkName = $parameters['virtualNetworkName'] From 3d0df911fdadf1de7cb66304f5c14fed7e6836fe Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sat, 4 May 2019 23:44:44 +0200 Subject: [PATCH 08/16] mod: documentation --- .../attach-jumpbox/README.md | 6 +++--- .../attach-vpn-gateway/README.md | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md index af34d600..57a876ea 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md +++ b/samples/manage/azure-sql-db-managed-instance/attach-jumpbox/README.md @@ -29,8 +29,8 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** -1. PowerShell 5.1 -2. Azure PowerShell 5.4.2 or higher +1. PowerShell 5.1 or PowerShell Core 6.0 +2. Azure PowerShell Az module **Azure prerequisites:** @@ -40,7 +40,7 @@ To run this sample, you need the following prerequisites. ## Run this sample -Run the script below from Windows PowerShell or Azure Cloud Shell +Run the script below from PowerShell or Azure Cloud Shell ```powershell diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md index e38acde9..69c9a0bb 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/README.md @@ -29,8 +29,11 @@ To run this sample, you need the following prerequisites. **Software prerequisites:** -1. PowerShell 5.1 -2. Azure PowerShell 5.4.2 or higher +1. PowerShell 5.1 or PowerShell Core 6.0 +2. Azure PowerShell Az module + +**Linux prerequisites** +1. strongSwan **Azure prerequisites:** @@ -40,7 +43,7 @@ To run this sample, you need the following prerequisites. ## Run this sample -Run the script below from Windows PowerShell +Run the script below from PowerShell ```powershell From d1ca907b9bd100c366e22efae98851a464117408 Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sun, 5 May 2019 10:20:57 +0200 Subject: [PATCH 09/16] mod: deployment to support standard gw sku --- .../attach-vpn-gateway/azuredeploy.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json index 9532f57e..eb603ed7 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json @@ -37,7 +37,6 @@ "variables": { "gatewayPublicIpAddressName": "[concat('GatewayIP-', uniqueString(resourceGroup().id))]", "gatewayName": "[concat('Gateway-', uniqueString(resourceGroup().id))]", - "gatewaySku": "Basic", "gatewaySubnetName": "GatewaySubnet", "clientRootCertName": "RootCert" }, @@ -75,9 +74,13 @@ } ], "sku": { - "name": "[variables('gatewaySku')]", - "tier": "[variables('gatewaySku')]" + "name": "Standard", + "tier": "Standard" }, + "vpnClientProtocols": [ + "IkeV2", + "SSTP" + ], "gatewayType": "Vpn", "vpnType": "RouteBased", "enableBgp": "false", From 2adc780721627e4b25ce43ad9eb0bfd099f09d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sr=C4=91an=20Bo=C5=BEovi=C4=87?= Date: Sun, 5 May 2019 21:48:14 +0200 Subject: [PATCH 10/16] mod template for gw configuration --- .../attach-vpn-gateway/azuredeploy.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json index eb603ed7..ca5795f0 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/azuredeploy.json @@ -77,10 +77,6 @@ "name": "Standard", "tier": "Standard" }, - "vpnClientProtocols": [ - "IkeV2", - "SSTP" - ], "gatewayType": "Vpn", "vpnType": "RouteBased", "enableBgp": "false", @@ -90,6 +86,10 @@ "[parameters('vpnClientAddressPoolPrefix')]" ] }, + "vpnClientProtocols": [ + "IkeV2", + "SSTP" + ], "vpnClientRootCertificates": [ { "name": "[variables('clientRootCertName')]", @@ -102,4 +102,4 @@ } } ] -} \ No newline at end of file +} From 79a09aede499be6d136ef1f1a760693539da15e3 Mon Sep 17 00:00:00 2001 From: srdan-bozovic-msft Date: Sun, 5 May 2019 21:55:38 +0200 Subject: [PATCH 11/16] mod: ps to create gw --- .../attach-vpn-gateway/attachVPNGateway.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 index 7ec2c1ac..84ddb7d4 100644 --- a/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 +++ b/samples/manage/azure-sql-db-managed-instance/attach-vpn-gateway/attachVPNGateway.ps1 @@ -5,7 +5,7 @@ $subscriptionId = $parameters['subscriptionId'] $resourceGroupName = $parameters['resourceGroupName'] $virtualNetworkName = $parameters['virtualNetworkName'] $certificateNamePrefix = $parameters['certificateNamePrefix'] -$clientCertificatePassword = $parameters['clientCertificatePassword'] +$clientCertificatePassword = $parameters['clientCertificatePassword'] #used only when certificates are created using openssl if ($clientCertificatePassword -eq '' -or ($null -eq $clientCertificatePassword)) { $clientCertificatePassword = 'S0m3Str0nGP@ssw0rd' @@ -197,7 +197,10 @@ function CreateCerificateOpenSsl() { ipsec pki --gen --outform pem > "$($dn)Key.pem" ipsec pki --pub --in "$($dn)Key.pem" --outform pem > "$($dn)PubKey.pem" ipsec pki --issue --in "$($dn)PubKey.pem" --cacert caCert.pem --cakey caKey.pem --dn "CN=$($dn)" --san $dn --flag clientAuth --outform pem > "$($dn)Cert.pem" + openssl pkcs12 -in "$($dn)Cert.pem" -inkey "$($dn)Key.pem" -certfile caCert.pem -export -out "$($dn).p12" -password "pass:$($clientCertificatePassword)" + #openssl pkcs12 -in "$($dn).p12" -password "pass:$($clientCertificatePassword)" -nocerts -out "$($dn)PrivateKey.pem" -nodes + #openssl pkcs12 -in "$($dn).p12" -password "pass:$($clientCertificatePassword)" -nokeys -out "$($dn)PublicCert.pem" -nodes $publicRootCertData = openssl x509 -in caCert.pem -outform pem $publicRootCertData = $publicRootCertData -replace "-----BEGIN CERTIFICATE-----", "" @@ -239,6 +242,8 @@ If ($false -eq $subnets.Contains($gatewaySubnetName)) { } else { Write-Host "The subnet $gatewaySubnetName exists in the virtual network." -ForegroundColor Green + $gatewaySubnet = Get-AzVirtualNetworkSubnetConfig -Name $gatewaySubnetName -VirtualNetwork $virtualNetwork + $gatewaySubnetPrefix = $gatewaySubnet.AddressPrefix[0] } $vpnClientAddressPoolPrefix = CalculateVpnClientAddressPoolPrefix $gatewaySubnetPrefix From 5753521d729b38f61efd3fb7bc1c98a563b2d113 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Tue, 7 May 2019 09:03:52 +0200 Subject: [PATCH 12/16] Compare environment settings --- .../compare-properties.sql | 41 +++++++++++++++++++ .../get-properties.sql | 41 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql create mode 100644 samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql new file mode 100644 index 00000000..b9fcbdb5 --- /dev/null +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql @@ -0,0 +1,41 @@ +declare @source xml = ''; +declare @target xml = ''; + +with +src as( +select property = x.v.value('name[1]', 'nvarchar(300)'), + value = x.v.value('value[1]', 'nvarchar(300)') +from @source.nodes('//row') x(v) +UNION ALL +select property = 'DB-CONFIG:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @source.nodes('//db') x(v) +cross apply x.v.nodes('*') y(v) +UNION ALL +select property = 'TEMPDB:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @source.nodes('//tempdb') x(v) +cross apply x.v.nodes('*') y(v) +), +tgt as( +select property = x.v.value('name[1]', 'nvarchar(300)'), + value = x.v.value('value[1]', 'nvarchar(300)') +from @target.nodes('//row') x(v) +UNION ALL +select property = 'DB-CONFIG:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @target.nodes('//db') x(v) +cross apply x.v.nodes('*') y(v) +UNION ALL +select property = 'TEMPDB:'+y.v.value('local-name(.)', 'nvarchar(300)'), + value = y.v.value('.[1]', 'nvarchar(300)') +from @target.nodes('//tempdb') x(v) +cross apply x.v.nodes('*') y(v) +) +select property = isnull(src.property, tgt.property), + source = src.value, target = tgt.value +from src full outer join tgt on src.property = tgt.property +where (src.value <> tgt.value +or src.value is null and tgt.value is not null +or src.value is not null and tgt.value is null) +order by isnull(src.property, tgt.property) diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql new file mode 100644 index 00000000..a56ebf1f --- /dev/null +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql @@ -0,0 +1,41 @@ +declare @db_name sysname = 'master' + +begin +declare @result NVARCHAR(MAX); +set @result = (select compatibility_level, snapshot_isolation_state_desc, is_read_committed_snapshot_on, + is_auto_update_stats_on, is_auto_update_stats_async_on, delayed_durability_desc, + is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced +from sys.databases +where name = @db_name +for xml raw('db'), elements); +set @result += (select compatibility_level, snapshot_isolation_state_desc, is_read_committed_snapshot_on, + is_auto_update_stats_on, is_auto_update_stats_async_on, delayed_durability_desc, + is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced +from sys.databases +where name = 'tempdb' +for xml raw('tempdb'), elements); +set @result += ( +select name = CONCAT('DB-CONFIG:',name), value +from sys.database_scoped_configurations +for xml raw, elements ); +declare @tf table (TraceFlag smallint, status bit,global bit, session bit) +insert into @tf execute('DBCC TRACESTATUS(-1)'); +set @result += ( +select name=CONCAT('TF:',TraceFlag), value=status from @tf +where global=1 and session=0 +for xml raw, elements +); +set @result += ( +select name = CONCAT('CONFIG:',name), value from sys.configurations +where name in ('cost threshold for parallelism','cursor threshold','fill factor (%)' +,'index create memory (KB)','lightweight pooling' +,'locks','max degree of parallelism','max full-text crawl range','max text repl size (B)' +,'max worker threads','min memory per query (KB)','nested triggers' +,'network packet size (B)','optimize for ad hoc workloads' +,'priority boost','query governor cost limit','query wait (s)','recovery interval (min)' +,'set working set size','user connections') +for xml raw, elements +); +select cast(@result as xml); +end; + From 6178cc2f62e164c0bd7520200bc6367b872cd7d6 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Tue, 7 May 2019 09:42:39 +0200 Subject: [PATCH 13/16] Adding tempdb #files and refactoring --- .../compare-properties.sql | 15 +++++++++++---- .../get-properties.sql | 3 ++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql index b9fcbdb5..a0610e9e 100644 --- a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/compare-properties.sql @@ -1,5 +1,5 @@ declare @source xml = ''; -declare @target xml = ''; +declare @target xml = ''; with src as( @@ -31,11 +31,18 @@ select property = 'TEMPDB:'+y.v.value('local-name(.)', 'nvarchar(300)'), value = y.v.value('.[1]', 'nvarchar(300)') from @target.nodes('//tempdb') x(v) cross apply x.v.nodes('*') y(v) -) +), +diff as ( select property = isnull(src.property, tgt.property), - source = src.value, target = tgt.value + source = src.value, target = tgt.value, + is_missing = (case when src.value is null or tgt.value is null then 1 else 0 end) from src full outer join tgt on src.property = tgt.property where (src.value <> tgt.value or src.value is null and tgt.value is not null or src.value is not null and tgt.value is null) -order by isnull(src.property, tgt.property) +) +select * +from diff +where is_missing = 0 -- comment-out this line to compare missing properties. +order by property + diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql index a56ebf1f..ee277816 100644 --- a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql @@ -10,7 +10,8 @@ where name = @db_name for xml raw('db'), elements); set @result += (select compatibility_level, snapshot_isolation_state_desc, is_read_committed_snapshot_on, is_auto_update_stats_on, is_auto_update_stats_async_on, delayed_durability_desc, - is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced + is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced, + number_of_files = (select count(*) from master.sys.master_files where database_id = db_id('tempdb')) from sys.databases where name = 'tempdb' for xml raw('tempdb'), elements); From f767a69ff09b4d8d1ab88cf88f30d152abd07377 Mon Sep 17 00:00:00 2001 From: "Jovan Popovic (MSFT)" Date: Tue, 7 May 2019 12:20:20 +0200 Subject: [PATCH 14/16] Added recovery_model_desc, --- .../compare-environment-settings/get-properties.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql index ee277816..df0ecc81 100644 --- a/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql +++ b/samples/manage/azure-sql-db-managed-instance/compare-environment-settings/get-properties.sql @@ -2,7 +2,7 @@ declare @db_name sysname = 'master' begin declare @result NVARCHAR(MAX); -set @result = (select compatibility_level, snapshot_isolation_state_desc, is_read_committed_snapshot_on, +set @result = (select compatibility_level, recovery_model_desc, snapshot_isolation_state_desc, is_read_committed_snapshot_on, is_auto_update_stats_on, is_auto_update_stats_async_on, delayed_durability_desc, is_encrypted, is_auto_create_stats_incremental_on, is_arithabort_on, is_ansi_warnings_on, is_parameterization_forced from sys.databases From 673f55e56afbf325d798ffad2fecdb2130b79841 Mon Sep 17 00:00:00 2001 From: Mario Inchiosa <4316698+inchiosa@users.noreply.github.com> Date: Wed, 8 May 2019 11:01:05 -0700 Subject: [PATCH 15/16] Add instruction for CTP 2.5 --- .../machine-learning/spark/h2o/h2o-automl-powerplant.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb index 8f334205..48e45da1 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb +++ b/samples/features/sql-big-data-cluster/machine-learning/spark/h2o/h2o-automl-powerplant.ipynb @@ -203,7 +203,7 @@ }, { "cell_type": "markdown", - "source": "# Configuration settings for scaling to larger data\n\n## Number and size of nodes in our Kubernetes cluster\nWe can control the number and size of nodes in our Kubernetes cluster via the node-vm-size and node-count switches in our `aks create` command:\n\n`az aks create --name mycluster --resource-group myrg --generate-ssh-keys --node-vm-size Standard_DS14_v2 --node-count 3 --kubernetes-version 1.10.9`\n\nMore information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n\n## Number of Spark pods\nWe can control the number of Spark pods via the CLUSTER_STORAGE_POOL_REPLICAS environment variable used by `mssqlctl create cluster`:\n\nSET CLUSTER_STORAGE_POOL_REPLICAS=2\n\n## YARN scheduler memory and cores\nWe can control the YARN scheduler memory and cores via the following environment variable used by `mssqlctl create cluster`:\n\n- YARN_SCHEDULER_MAX_MEMORY\n- YARN_SCHEDULER_MAX_VCORES\n- YARN_NODEMANAGER_RESOURCE_MEMORY\n- YARN_NODEMANAGER_RESOURCE_VCORES\n\nFurther information regarding mssqlctl environtment variables is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#define-environment-variables).\n\n## Livy timeout\nThe Livy timeout sets a limit on the runtime of a cell in a PySpark3 Jupyter notebook. In SQL Server 2019 Big Data CTP 2.1, the Livy timeout defaults to 1 hour. In CTP 2.2, it defaults to 24 days. One can modify this as follows:\n\n- Log into the mssql-master-pool-0 pod using this command (requires permission to run kubectl):\n\n```\nkubectl exec -it mssql-master-pool-0 -n -- /bin/bash\n```\n- To set the Livy timeout to 24 days, run the following command or edit /livy/conf/livy.conf accordingly:\n\n```\necho 'livy.server.session.timeout = 24d' | cat >> /livy/conf/livy.conf \n```\n- Then restart the Livy server by running the following command:\n\n```\nsupervisorctl restart livy\n```", + "source": "# Configuration settings for scaling to larger data\n\n## Number and size of nodes in our Kubernetes cluster\nWe can control the number and size of nodes in our Kubernetes cluster via the node-vm-size and node-count switches in our `aks create` command:\n\n`az aks create --name mycluster --resource-group myrg --generate-ssh-keys --node-vm-size Standard_DS14_v2 --node-count 3 --kubernetes-version 1.10.9`\n\nMore information is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deploy-on-aks?view=sqlallproducts-allversions#create-a-kubernetes-cluster).\n\n## Number of Spark pods\nWe can control the number of Spark pods via the CLUSTER_STORAGE_POOL_REPLICAS environment variable used by `mssqlctl create cluster`:\n\nSET CLUSTER_STORAGE_POOL_REPLICAS=2\n\n## YARN scheduler memory and cores\nWe can control the YARN scheduler memory and cores via the following environment variable used by `mssqlctl create cluster`:\n\n- YARN_SCHEDULER_MAX_MEMORY\n- YARN_SCHEDULER_MAX_VCORES\n- YARN_NODEMANAGER_RESOURCE_MEMORY\n- YARN_NODEMANAGER_RESOURCE_VCORES\n\nFurther information regarding mssqlctl environtment variables is available [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#define-environment-variables).\n\nIn CTP 2.5 and later, these environment variables are replaced by similarly named properties in a JSON file. See [Custom configurations](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions#customconfig).\n\n## Livy timeout\nThe Livy timeout sets a limit on the runtime of a cell in a PySpark3 Jupyter notebook. In SQL Server 2019 Big Data CTP 2.1, the Livy timeout defaults to 1 hour. In CTP 2.2, it defaults to 24 days. One can modify this as follows:\n\n- Log into the mssql-master-pool-0 pod using this command (requires permission to run kubectl):\n\n```\nkubectl exec -it mssql-master-pool-0 -n -- /bin/bash\n```\n- To set the Livy timeout to 24 days, run the following command or edit /livy/conf/livy.conf accordingly:\n\n```\necho 'livy.server.session.timeout = 24d' | cat >> /livy/conf/livy.conf \n```\n- Then restart the Livy server by running the following command:\n\n```\nsupervisorctl restart livy\n```", "metadata": {} }, { From 0bc9dab395a0e2cb91f1921166615698893cacf0 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 13 May 2019 20:31:18 -0700 Subject: [PATCH 16/16] Fixed check for master key --- samples/features/sql-big-data-cluster/bootstrap-sample-db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 8cdd71f7..5a525be8 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -70,8 +70,8 @@ GO CREATE OR ALTER PROCEDURE #create_data_sources AS BEGIN - -- Create database master key (required for database scoped credentials used in the samples) - IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1) + -- Create database master key (required for database scoped credentials used in the samples) + IF NOT EXISTS(SELECT * FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##') CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!'; -- Create default data sources for SQL Big Data Cluster