From 5bee11e1d56e0ebb96adeeecca7c794c0cfe3edc Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 28 Aug 2019 15:25:49 -0700 Subject: [PATCH] Updated for RC changes & HA --- .../bootstrap-sample-db.cmd | 32 ++++++--- .../bootstrap-sample-db.sh | 31 ++++++--- .../bootstrap-sample-db.sql | 69 ++++++++++++------- 3 files changed, 89 insertions(+), 43 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index a994bb2f..300b9533 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -41,7 +41,10 @@ if NOT EXIST tpcxbb_1gb.bak ( set SQLCMDSERVER=%SQL_MASTER_INSTANCE% set SQLCMDUSER=sa set SQLCMDPASSWORD=%SQL_MASTER_SA_PASSWORD% -for /F "usebackq" %%v in (`sqlcmd -I -b -h-1 -Q "print @@SERVERNAME;"`) do SET MASTER_POD_NAME=%%v +for /F "usebackq tokens=1,2" %%v in (`sqlcmd -I -b -h-1 -W -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');"`) do ( + SET MASTER_POD_NAME=%%v + SET HADR_ENABLED=%%w +) REM Copy the backup file, restore the database, create necessary objects and data file echo Copying sales database backup file to SQL Master instance... @@ -57,6 +60,9 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( ) echo Copying %%f database backup file to SQL Master instance... %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit + + echo Removing database backup file... + %DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f" ) set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak @@ -67,39 +73,49 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( ) echo Copying %%f database backup file to SQL Master instance... %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit + + echo Removing database backup file... + %DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f" ) ) +REM If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly +REM Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically +if /i "%HADR_ENABLED%" EQU "1" ( + %DEBUG% start kubectl port-forward pods/%MASTER_POD_NAME% 1533:1533 -n %CLUSTER_NAMESPACE% + SET SQLCMDSERVER=127.0.0.1,1533 +) + echo Configuring sample database(s)... %DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit REM remove files copied into the pod: -echo Removing database backup files... +echo Removing database backup file... %DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak" for %%F in (web_clickstreams inventory customer) do ( if NOT EXIST %%F.csv ( echo Exporting %%F data... if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|) - %DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit + %DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit ) ) if NOT EXIST product_reviews.csv ( echo Exporting product_reviews data... - %DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit + %DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit ) REM Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... -%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit -%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit +%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit +%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit :: del /q web_clickstreams.* echo. echo Uploading product_reviews data to HDFS... -%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit -%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit +%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit +%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit :: del /q product_reviews.* REM %DEBUG% del /q *.out *.err *.csv diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 147a151b..7a7dc15f 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -51,11 +51,10 @@ then $DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak fi -MASTER_POD_NAME=$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "print @@SERVERNAME;") +read -r MASTER_POD_NAME HADR_ENABLED <<<$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');") echo Copying sales database backup file... $DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) -# $DEBUG rm tpcxbb_1gb.bak if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ] then @@ -68,6 +67,9 @@ then fi echo Copying $file database backup file to SQL Master instance... $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + + echo Removing database backup file... + $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file" done @@ -80,16 +82,27 @@ then fi echo Copying $file database backup file to SQL Master instance... $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + + echo Removing database backup file... + $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file" done fi +# If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly +# Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically +if [ "$HADR_ENABLED" == "1" ] +then + $DEBUG kubectl port-forward pods/$MASTER_POD_NAME 1533:1533 -n $CLUSTER_NAMESPACE & + SQL_MASTER_INSTANCE=127.0.0.1,1533 +fi + echo Configuring sample database... # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE" export SA_PASSWORD=$KNOX_PASSWORD $DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2) # remove files copied into the pod: -echo Removing database backup files... +echo Removing database backup file... $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak" for table in web_clickstreams inventory customer @@ -104,26 +117,26 @@ for table in web_clickstreams inventory customer # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe" if [ ! -f $table.csv ] then - $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3) + $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3) fi done if [ ! -f product_reviews.csv ] then echo Exporting product_reviews data... - $DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3) + $DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3) fi # Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... -$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4) -$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5) +$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4) +$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5) #$DEBUG rm -f web_clickstreams.* echo echo Uploading product_reviews data to HDFS... -$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6) -$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7) +$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6) +$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7) #$DEBUG rm -f product_reviews.* echo diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index d1edd9df..5aa651b0 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -18,7 +18,16 @@ BEGIN END; GO -CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255)) +-- Enable option to allow INSERT against external table defined on HADOOP data source +DECLARE @config_option nvarchar(100) = 'allow polybase export'; +IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1) +BEGIN + EXECUTE sp_configure @config_option, 1; + RECONFIGURE WITH OVERRIDE; +END; +GO + +CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255), @db_name nvarchar(128)) AS BEGIN DECLARE @restore_filelist_tmpl nvarchar(1000) = N'RESTORE FILELISTONLY FROM DISK = N''/var/opt/mssql/data/%F'''; @@ -53,7 +62,7 @@ BEGIN INSERT INTO @files EXECUTE(@restore_cmd); - SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', LEFT(@backup_file, CHARINDEX('.', @backup_file)-1)); + SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', @db_name); SET @restore_cur = CURSOR FAST_FORWARD FOR SELECT LogicalName, REVERSE(LEFT(REVERSE(PhysicalName), CHARINDEX('\', REVERSE(PhysicalName))-1)) FROM @files; OPEN @restore_cur; WHILE(1=1) @@ -84,50 +93,58 @@ BEGIN WITH (LOCATION = 'sqlhdfs://controller-svc/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-svc:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032' + ); END; GO --- Sample dbs: DECLARE @sample_dbs CURSOR, @proc nvarchar(255); SET @sample_dbs = CURSOR FAST_FORWARD FOR - SELECT file_or_directory_name - FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak') - WHERE DB_ID(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', '')) IS NULL; -DECLARE @file nvarchar(260); + SELECT file_or_directory_name, d.db_name + FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak') as f + CROSS APPLY (VALUES(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', ''))) as d(db_name) + WHERE DB_ID(d.db_name) IS NULL; +DECLARE @file nvarchar(260), @db_name nvarchar(128); OPEN @sample_dbs; WHILE(1=1) BEGIN - FETCH @sample_dbs INTO @file; + FETCH @sample_dbs INTO @file, @db_name; IF @@FETCH_STATUS < 0 BREAK; -- Restore the sample databases: - EXECUTE #restore_database @file; + EXECUTE #restore_database @file, @db_name; -- Get database name used in restore: - SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql'); + SET @db_name = QUOTENAME(@db_name); + SET @proc = CONCAT(@db_name, N'.sys.sp_executesql'); EXECUTE @proc N'#create_data_sources'; -- Set compatibility level to 150: EXECUTE @proc N'ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 150'; - -- Rename TPCx-BB database: - IF DB_ID('tpcxbb_1gb') IS NOT NULL - ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales; + -- Check for HADR & add database to containedag: + IF SERVERPROPERTY('IsHadrEnabled') = 1 + BEGIN + DECLARE @command nvarchar(1000); + IF EXISTS(SELECT * FROM sys.databases WHERE name = PARSENAME(@db_name,1) and recovery_model_desc = 'SIMPLE') + BEGIN + -- Set recovery to full + EXECUTE @proc N'ALTER DATABASE CURRENT SET RECOVERY FULL'; + + SET @command = CONCAT(N'BACKUP DATABASE ', @db_name, ' TO DISK = ''/dev/null'';' ); + EXEC(@command); + END; + + -- Add database to AG + SET @command = CONCAT(N'ALTER AVAILABILITY GROUP containedag ADD DATABASE ', @db_name); + EXEC(@command); + END; END; GO