From 6357693783c53a8f5ad4372098c1b5275a26c99d Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 27 Mar 2019 12:30:22 -0700 Subject: [PATCH] Support for CTP2.4 --- .../bootstrap-sample-db.cmd | 4 +- .../bootstrap-sample-db.sh | 55 ++++++++++++++++--- .../bootstrap-sample-db.sql | 6 +- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index 3ad06998..d74b8ae2 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -38,7 +38,7 @@ REM Copy the backup file, restore the database, create necessary objects and dat echo Copying sales database backup file to SQL Master instance... %DEBUG% kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit -if /i %AW_WWI_SAMPLES% EQU install_extra_samples ( +if /i %AW_WWI_SAMPLES% EQU --install-extra-samples ( if NOT EXIST AdventureWorks2016_EXT.bak ( echo Downloading AdventureWorks2016_EXT sample database backup file... %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2016_EXT.bak" -o AdventureWorks2016_EXT.bak @@ -113,6 +113,6 @@ goto :eof exit /b 1 :usage - echo USAGE: %0 ^ ^ ^ ^ [^] + echo USAGE: %0 ^ ^ ^ ^ [^] [--install-extra-samples] echo Default ports are assumed for SQL Master instance ^& Knox gateway. exit /b 0 \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 14cb9628..92fb198c 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -3,7 +3,7 @@ set -e set -o pipefail STARTUP_PATH=$(pwd) TMP_DIR_NAME=$(basename $0) -USAGE_MESSAGE="USAGE: $0 []" +USAGE_MESSAGE="USAGE: $0 [] [--install-extra-samples]" ERROR_MESSAGE="Bootstrap of the sample database failed. Output and error files are in directory [/tmp/$TMP_DIR_NAME]." # Print usage if mandatory parameters are missing @@ -19,6 +19,8 @@ SQL_MASTER_IP=$2 SQL_MASTER_SA_PASSWORD=$3 KNOX_IP=$4 KNOX_PASSWORD=$5 +AW_WWI_SAMPLES=$6 + # If Knox password is not supplied then default to SQL Master password KNOX_PASSWORD=${KNOX_PASSWORD:=$SQL_MASTER_SA_PASSWORD} @@ -35,12 +37,41 @@ pushd "/tmp" $DEBUG mkdir -p "$TMP_DIR_NAME" $DEBUG cd "$TMP_DIR_NAME" -echo Downloading sample database backup file... -$DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak +if [ ! -f tpcxbb_1gb.bak ] +then + echo Downloading sample database backup file... + $DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak +fi echo Copying database backup file... $DEBUG kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n $CLUSTER_NAMESPACE || (echo $ERROR_MESSAGE && exit 1) -$DEBUG rm tpcxbb_1gb.bak +# $DEBUG rm tpcxbb_1gb.bak + +if [ $AW_WWI_SAMPLES == --install-extra-samples ] +then + for file in AdventureWorks2016_EXT.bak AdventureWorksDW2016_EXT.bak + do + if [ ! -f $file ] + then + echo Downloading $file sample database backup file... + $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/$file" -o $file + fi + echo Copying $file database backup file to SQL Master instance... + $DEBUG kubectl cp $file mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n $CLUSTER_NAMESPACE || (echo $ERROR_MESSAGE && exit 1) + done + + + for file in WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak + do + if [ ! -f $file ] + then + echo Downloading $file sample database backup file... + $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/$file" -o $file + fi + echo Copying $file database backup file to SQL Master instance... + $DEBUG kubectl cp $file mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n $CLUSTER_NAMESPACE || (echo $ERROR_MESSAGE && exit 1) + done +fi echo Configuring sample database... # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE" @@ -57,23 +88,29 @@ for table in web_clickstreams inventory customer DELIMITER="|" fi # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe" - $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3) + if [ ! -f $table.csv ] + then + $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3) + fi done -echo Exporting product_reviews data... -$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3) +if [ ! -f product_reviews.csv ] +then + echo Exporting product_reviews data... + $DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3) +fi # Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... $DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4) $DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5) -$DEBUG rm -f web_clickstreams.* +#$DEBUG rm -f web_clickstreams.* echo echo Uploading product_reviews data to HDFS... $DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6) $DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7) -$DEBUG rm -f product_reviews.* +#$DEBUG rm -f product_reviews.* echo echo Bootstrap of the sample database completed successfully. diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 91d92980..50f0ddb2 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -11,9 +11,9 @@ GO -- Enable external scripts execution for R/Python/Java: DECLARE @config_option nvarchar(100) = 'external scripts enabled'; -IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option) +IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1) BEGIN - exec sp_configure @config_option, 1; + EXECUTE sp_configure @config_option, 1; RECONFIGURE WITH OVERRIDE; END; GO @@ -21,7 +21,7 @@ GO CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255)) AS BEGIN - DECLARE @restore_filelist_tmpl nvarchar(1000) = N'restore filelistonly FROM DISK = N''/var/opt/mssql/data/%F'''; + DECLARE @restore_filelist_tmpl nvarchar(1000) = N'RESTORE FILELISTONLY FROM DISK = N''/var/opt/mssql/data/%F'''; DECLARE @restore_database_tmpl nvarchar(1000) = N'RESTORE DATABASE [%D] FROM DISK = N''/var/opt/mssql/data/%F'' WITH FILE = 1'; DECLARE @move_tmpl nvarchar(1000) = N', MOVE N''%L'' TO N''/var/opt/mssql/data/%F'''; DECLARE @restore_cmd nvarchar(4000), @logical_name nvarchar(128), @filename nvarchar(260), @restore_cur CURSOR;