From 817ec963b017a95c48720e51dc7a3915c1b62a0e Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 17 Apr 2019 18:02:01 -0700 Subject: [PATCH] Updates to handle CTP 2.5 changes --- .../bootstrap-sample-db.cmd | 70 ++++++++++--------- .../bootstrap-sample-db.sh | 33 ++++++--- .../bootstrap-sample-db.sql | 24 +++++-- 3 files changed, 79 insertions(+), 48 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index ccc853a2..600c21d5 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -8,6 +8,8 @@ set SQL_MASTER_SA_PASSWORD=%3 set KNOX_IP=%4 set KNOX_PASSWORD=%5 set AW_WWI_SAMPLES=%6 +set SQL_MASTER_PORT=%7 +set KNOX_PORT=%8 set STARTUP_PATH=%~dp0 set TMP_DIR_NAME=%~nx0 @@ -17,16 +19,18 @@ if NOT DEFINED SQL_MASTER_SA_PASSWORD goto :usage if NOT DEFINED KNOX_IP goto :usage if NOT DEFINED KNOX_PASSWORD set KNOX_PASSWORD=%SQL_MASTER_SA_PASSWORD% if NOT DEFINED AW_WWI_SAMPLES set AW_WWI_SAMPLES=no +if NOT DEFINED SQL_MASTER_PORT set SQL_MASTER_PORT=31433 +if NOT DEFINED KNOX_PORT set KNOX_PORT=30443 -set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,31433 -set KNOX_ENDPOINT=%KNOX_IP%:30443 +set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,%SQL_MASTER_PORT% +set KNOX_ENDPOINT=%KNOX_IP%:%KNOX_PORT% for %%F in (sqlcmd.exe bcp.exe kubectl.exe curl.exe) do ( echo Verifying %%F is in path & CALL WHERE /Q %%F || GOTO exit ) pushd "%tmp%" -md %TMP_DIR_NAME% +md %TMP_DIR_NAME% >NUL cd %TMP_DIR_NAME% if NOT EXIST tpcxbb_1gb.bak ( @@ -34,47 +38,45 @@ if NOT EXIST tpcxbb_1gb.bak ( %DEBUG% curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak ) +set SQLCMDSERVER=%SQL_MASTER_INSTANCE% +set SQLCMDUSER=sa +set SQLCMDPASSWORD=%SQL_MASTER_SA_PASSWORD% +for /F "usebackq" %%v in (`sqlcmd -I -b -h-1 -Q "print RTRIM((CAST(SERVERPROPERTY('ProductLevel') as nvarchar(128))));"`) do SET CTP_VERSION=%%v +if /i "%CTP_VERSION%" EQU "CTP2.4" (set MASTER_POD_NAME=mssql-master-pool-0) else (set MASTER_POD_NAME=master-0) + REM Copy the backup file, restore the database, create necessary objects and data file echo Copying sales database backup file to SQL Master instance... -%DEBUG% kubectl cp tpcxbb_1gb.bak %CLUSTER_NAMESPACE%/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || goto exit +%DEBUG% kubectl cp tpcxbb_1gb.bak %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit REM Download and copy the sample backup files -if /i %AW_WWI_SAMPLES% EQU --install-extra-samples ( - if NOT EXIST AdventureWorks2016_EXT.bak ( - echo Downloading AdventureWorks2016_EXT sample database backup file... - %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2016_EXT.bak" -o AdventureWorks2016_EXT.bak +if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( + set FILES=AdventureWorks2016_EXT.bak AdventureWorksDW2016_EXT.bak + for %%f in (!FILES!) do ( + if NOT EXIST %%f ( + echo Downloading %%f sample database backup file... + %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/%%f" -o %%f + ) + echo Copying %%f database backup file to SQL Master instance... + %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit ) - echo Copying AdventureWorks2016_EXT database backup file to SQL Master instance... - %DEBUG% kubectl cp AdventureWorks2016_EXT.bak %CLUSTER_NAMESPACE%/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || goto exit - if NOT EXIST AdventureWorksDW2016_EXT.bak ( - echo Downloading AdventureWorksDW2016_EXT sample database backup file... - %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorksDW2016_EXT.bak" -o AdventureWorksDW2016_EXT.bak + set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak + for %%f in (!FILES!) do ( + if NOT EXIST %%f ( + echo Downloading %%f sample database backup file... + %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/%%f" -o %%f + ) + echo Copying %%f database backup file to SQL Master instance... + %DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:/var/opt/mssql/data -c mssql-server || goto exit ) - echo Copying AdventureWorksDW2016_EXT database backup file to SQL Master instance... - %DEBUG% kubectl cp AdventureWorksDW2016_EXT.bak %CLUSTER_NAMESPACE%/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || goto exit - - if NOT EXIST WideWorldImporters-Full.bak ( - echo Downloading WideWorldImporters sample database backup file... - %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImporters-Full.bak" -o WideWorldImporters-Full.bak - ) - echo Copying WideWorldImporters-Full database backup file to SQL Master instance... - %DEBUG% kubectl cp WideWorldImporters-Full.bak %CLUSTER_NAMESPACE%/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || goto exit - - if NOT EXIST WideWorldImportersDW-Full.bak ( - echo Downloading WideWorldImportersDW sample database backup file... - %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImportersDW-Full.bak" -o WideWorldImportersDW-Full.bak - ) - echo Copying WideWorldImportersDW-Full database backup file to SQL Master instance... - %DEBUG% kubectl cp WideWorldImportersDW-Full.bak %CLUSTER_NAMESPACE%/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || goto exit ) echo Configuring sample database(s)... -%DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit +%DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit REM remove files copied into the pod: echo Removing database backup files... -kubectl exec mssql-master-pool-0 -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" +%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" for %%F in (web_clickstreams inventory customer) do ( if NOT EXIST %%F.csv ( @@ -84,7 +86,6 @@ for %%F in (web_clickstreams inventory customer) do ( ) ) - if NOT EXIST product_reviews.csv ( echo Exporting product_reviews data... %DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit @@ -103,6 +104,7 @@ echo Uploading product_reviews data to HDFS... :: del /q product_reviews.* REM %DEBUG% del /q *.out *.err *.csv +echo . echo Bootstrap of the sample database completed successfully. echo You can now login using "root" and Knox password to get the unified experience in Azure Data Studio. echo Data files for Oracle setup are located at [%TMP%\%TMP_DIR_NAME%]. @@ -118,6 +120,6 @@ goto :eof exit /b 1 :usage - echo USAGE: %0 ^ ^ ^ ^ [^] [--install-extra-samples] - echo Default ports are assumed for SQL Master instance ^& Knox gateway. + echo USAGE: %0 ^ ^ ^ ^ [^] [--install-extra-samples] [SQL_MASTER_PORT] [KNOX_PORT] + echo Default ports are assumed for SQL Master instance ^& Knox gateway unless specified. exit /b 0 \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 9c23c9d0..aab74626 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -3,7 +3,7 @@ set -e set -o pipefail STARTUP_PATH=$(pwd) TMP_DIR_NAME=$(basename $0) -USAGE_MESSAGE="USAGE: $0 [] [--install-extra-samples]" +USAGE_MESSAGE="USAGE: $0 [] [--install-extra-samples] [SQL_MASTER_PORT] [KNOX_PORT]" ERROR_MESSAGE="Bootstrap of the sample database failed. Output and error files are in directory [/tmp/$TMP_DIR_NAME]." # Print usage if mandatory parameters are missing @@ -20,12 +20,20 @@ SQL_MASTER_SA_PASSWORD=$3 KNOX_IP=$4 KNOX_PASSWORD=$5 AW_WWI_SAMPLES=$6 +SQL_MASTER_PORT=$7 +KNOX_PORT=$8 # If Knox password is not supplied then default to SQL Master password KNOX_PASSWORD=${KNOX_PASSWORD:=$SQL_MASTER_SA_PASSWORD} -SQL_MASTER_INSTANCE=$SQL_MASTER_IP,31433 -KNOX_ENDPOINT=$KNOX_IP:30443 +# Skip if extra samples doesn't need to be installed +AW_WWI_SAMPLES=${AW_WWI_SAMPLES:=no} + +# Use default ports if not specified +SQL_MASTER_PORT=${SQL_MASTER_PORT:=31433} +KNOX_PORT=${KNOX_PORT:=30433} +SQL_MASTER_INSTANCE=$SQL_MASTER_IP,$SQL_MASTER_PORT +KNOX_ENDPOINT=$KNOX_IP:$KNOX_PORT for util in sqlcmd bcp kubectl curl do @@ -43,11 +51,20 @@ then $DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak fi +CTP_VERSION=$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "print RTRIM((CAST(SERVERPROPERTY('ProductLevel') as nvarchar(128))));") + +if [ "$CTP_VERSION" == "CTP2.4" ] +then + MASTER_POD_NAME=mssql-master-pool-0 +else + MASTER_POD_NAME=master-0 +fi + echo Copying database backup file... -$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) +$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) # $DEBUG rm tpcxbb_1gb.bak -if [ $AW_WWI_SAMPLES == --install-extra-samples ] +if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ] then for file in AdventureWorks2016_EXT.bak AdventureWorksDW2016_EXT.bak do @@ -57,7 +74,7 @@ then $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/$file" -o $file fi echo Copying $file database backup file to SQL Master instance... - $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) done @@ -69,7 +86,7 @@ then $DEBUG curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/$file" -o $file fi echo Copying $file database backup file to SQL Master instance... - $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/mssql-master-pool-0:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) + $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:/var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) done fi @@ -80,7 +97,7 @@ $DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b < "$S # remove files copied into the pod: echo Removing database backup files... -kubectl exec mssql-master-pool-0 -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" +kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/*.bak" for table in web_clickstreams inventory customer do diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 0a422bc1..982b4835 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -80,16 +80,28 @@ BEGIN WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' + ); END; GO