From 1da7395796d08c485105072fbf08ea1843631453 Mon Sep 17 00:00:00 2001 From: "Umachandar Jayachandran (UC)" Date: Fri, 24 Jan 2020 22:50:26 -0800 Subject: [PATCH] Support RTM behavior & AD auth. --- .../bootstrap-sample-db.cmd | 76 ++++++++----- .../bootstrap-sample-db.sh | 107 ++++++++++++------ .../bootstrap-sample-db.sql | 34 +++--- 3 files changed, 134 insertions(+), 83 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index 300b9533..d9e44a00 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -3,34 +3,40 @@ REM bootstrap sample database CMD script setlocal enableextensions setlocal enabledelayedexpansion set CLUSTER_NAMESPACE=%1 -set SQL_MASTER_IP=%2 -set SQL_MASTER_SA_PASSWORD=%3 -set KNOX_IP=%4 -set KNOX_PASSWORD=%5 -set AW_WWI_SAMPLES=%6 -set SQL_MASTER_PORT=%7 -set KNOX_PORT=%8 +set SQL_MASTER_ENDPOINT=%2 +set KNOX_ENDPOINT=%3 +set AW_WWI_SAMPLES=%4 +set SQL_MASTER_PORT=%5 +set KNOX_PORT=%6 set STARTUP_PATH=%~dp0 set TMP_DIR_NAME=%~nx0 if NOT DEFINED CLUSTER_NAMESPACE goto :usage -if NOT DEFINED SQL_MASTER_IP goto :usage -if NOT DEFINED SQL_MASTER_SA_PASSWORD goto :usage -if NOT DEFINED KNOX_IP goto :usage -if NOT DEFINED KNOX_PASSWORD set KNOX_PASSWORD=%SQL_MASTER_SA_PASSWORD% +if NOT DEFINED SQL_MASTER_ENDPOINT goto :usage +if NOT DEFINED KNOX_ENDPOINT goto :usage if NOT DEFINED AW_WWI_SAMPLES set AW_WWI_SAMPLES=no if NOT DEFINED SQL_MASTER_PORT set SQL_MASTER_PORT=31433 if NOT DEFINED KNOX_PORT set KNOX_PORT=30443 -set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,%SQL_MASTER_PORT% -set KNOX_ENDPOINT=%KNOX_IP%:%KNOX_PORT% +set SQL_MASTER_INSTANCE=%SQL_MASTER_ENDPOINT%,%SQL_MASTER_PORT% +set KNOX_ENDPOINT=%KNOX_ENDPOINT%:%KNOX_PORT% +set SQLCMDSERVER=%SQL_MASTER_INSTANCE% +if DEFINED AZDATA_USERNAME set SQLCMDUSER=%AZDATA_USERNAME% +if DEFINED AZDATA_PASSWORD set SQLCMDPASSWORD=%AZDATA_PASSWORD% +if DEFINED AZDATA_PASSWORD set set KNOX_PASSWORD=%AZDATA_PASSWORD% + +if NOT DEFINED SQLCMDUSER ( + set BCP_CREDENTIALS=-T +) else ( + set BCP_CREDENTIALS=-U%SQLCMDUSER% -P%SQLCMDPASSWORD% +) for %%F in (sqlcmd.exe bcp.exe kubectl.exe curl.exe) do ( echo Verifying %%F is in path & CALL WHERE /Q %%F || GOTO exit ) pushd "%tmp%" -md %TMP_DIR_NAME% >NUL +md %TMP_DIR_NAME% >NUL 2>NUL cd %TMP_DIR_NAME% if NOT EXIST tpcxbb_1gb.bak ( @@ -38,13 +44,11 @@ if NOT EXIST tpcxbb_1gb.bak ( %DEBUG% curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak ) -set SQLCMDSERVER=%SQL_MASTER_INSTANCE% -set SQLCMDUSER=sa -set SQLCMDPASSWORD=%SQL_MASTER_SA_PASSWORD% for /F "usebackq tokens=1,2" %%v in (`sqlcmd -I -b -h-1 -W -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');"`) do ( SET MASTER_POD_NAME=%%v SET HADR_ENABLED=%%w ) +if NOT DEFINED MASTER_POD_NAME goto exit REM Copy the backup file, restore the database, create necessary objects and data file echo Copying sales database backup file to SQL Master instance... @@ -82,12 +86,12 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" ( REM If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly REM Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically if /i "%HADR_ENABLED%" EQU "1" ( - %DEBUG% start kubectl port-forward pods/%MASTER_POD_NAME% 1533:1533 -n %CLUSTER_NAMESPACE% + %DEBUG% start "bootstrap-kubectl" kubectl port-forward svc/master-svc 1533:1533 -n %CLUSTER_NAMESPACE% SET SQLCMDSERVER=127.0.0.1,1533 ) echo Configuring sample database(s)... -%DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit +%DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b || goto exit REM remove files copied into the pod: echo Removing database backup file... @@ -97,31 +101,43 @@ for %%F in (web_clickstreams inventory customer) do ( if NOT EXIST %%F.csv ( echo Exporting %%F data... if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|) - %DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit + %DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQLCMDSERVER% %BCP_CREDENTIALS% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit ) ) if NOT EXIST product_reviews.csv ( echo Exporting product_reviews data... - %DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit + %DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQLCMDSERVER% %BCP_CREDENTIALS% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit ) +REM Kill kubectl process if started +taskkill /F /T /FI "WINDOWTITLE EQ bootstrap-kubectl" /IM kubectl.exe >NUL 2>NUL + REM Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... -%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit -%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit +if DEFINED KNOX_PASSWORD ( + %DEBUG% curl -s -S -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit + %DEBUG% curl -s -S -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit +) else ( + %DEBUG% curl -s -S -L -k -u : --negotiate -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit + %DEBUG% curl -s -S -L -k -u : --negotiate -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit +) :: del /q web_clickstreams.* echo. echo Uploading product_reviews data to HDFS... -%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit -%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit +if DEFINED KNOX_PASSWORD ( + %DEBUG% curl -s -S -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit + %DEBUG% curl -s -S -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit +) else ( + %DEBUG% curl -s -S -L -k -u : --negotiate -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit + %DEBUG% curl -s -S -L -k -u : --negotiate -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit +) :: del /q product_reviews.* REM %DEBUG% del /q *.out *.err *.csv echo . echo Bootstrap of the sample database completed successfully. -echo You can now login using "root" and Knox password to get the unified experience in Azure Data Studio. echo Data files for Oracle setup are located at [%TMP%\%TMP_DIR_NAME%]. popd @@ -130,11 +146,15 @@ exit /b 0 goto :eof :exit + REM Kill kubectl process if started + taskkill /F /T /FI "WINDOWTITLE EQ bootstrap-kubectl" /IM kubectl.exe >NUL 2>NUL echo Bootstrap of the sample database failed. echo Output and error files are in directory [%TMP%\%TMP_DIR_NAME%]. exit /b 1 :usage - echo USAGE: %0 ^ ^ ^ ^ [^] [--install-extra-samples] [SQL_MASTER_PORT] [KNOX_PORT] - echo Default ports are assumed for SQL Master instance ^& Knox gateway unless specified. + echo USAGE: %0 ^ ^ ^ [--install-extra-samples] [SQL_MASTER_PORT] [KNOX_PORT] + echo To use basic authentication please set AZDATA_USERNAME and AZDATA_PASSWORD environment variables. + echo To use integrated authentication provide the DNS names for the endpoints. + echo Port can be specified separately if using non-default values." exit /b 0 diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 7c0aaf86..910a03b6 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -4,28 +4,25 @@ set -o pipefail STARTUP_PATH=$(pwd) TMP_DIR_NAME=$(basename $0) -USAGE_MESSAGE="USAGE: $0 [] [--install-extra-samples] [SQL_MASTER_PORT] [KNOX_PORT]" +USAGE_MESSAGE="USAGE: $0 [--install-extra-samples] [SQL_MASTER_PORT] [KNOX_PORT] +To use basic authentication please set AZDATA_USERNAME and AZDATA_PASSWORD environment variables. +To use integrated authentication provide the DNS names for the endpoints. +Port can be specified separately if using non-default values." ERROR_MESSAGE="Bootstrap of the sample database failed. Output and error files are in directory [/tmp/$TMP_DIR_NAME]." # Print usage if mandatory parameters are missing : "${1:?$USAGE_MESSAGE}" : "${2:?$USAGE_MESSAGE}" : "${3:?$USAGE_MESSAGE}" -: "${4:?$USAGE_MESSAGE}" : "${DEBUG=}" # Save the input parameters CLUSTER_NAMESPACE=$1 -SQL_MASTER_IP=$2 -SQL_MASTER_SA_PASSWORD=$3 -KNOX_IP=$4 -KNOX_PASSWORD=$5 -AW_WWI_SAMPLES=$6 -SQL_MASTER_PORT=$7 -KNOX_PORT=$8 - -# If Knox password is not supplied then default to SQL Master password -KNOX_PASSWORD=${KNOX_PASSWORD:=$SQL_MASTER_SA_PASSWORD} +SQL_MASTER_ENDPOINT=$2 +KNOX_ENDPOINT=$3 +AW_WWI_SAMPLES=$4 +SQL_MASTER_PORT=$5 +KNOX_PORT=$6 # Skip if extra samples doesn't need to be installed AW_WWI_SAMPLES=${AW_WWI_SAMPLES:=no} @@ -33,8 +30,21 @@ AW_WWI_SAMPLES=${AW_WWI_SAMPLES:=no} # Use default ports if not specified SQL_MASTER_PORT=${SQL_MASTER_PORT:=31433} KNOX_PORT=${KNOX_PORT:=30443} -SQL_MASTER_INSTANCE=$SQL_MASTER_IP,$SQL_MASTER_PORT -KNOX_ENDPOINT=$KNOX_IP:$KNOX_PORT +SQL_MASTER_INSTANCE=$SQL_MASTER_ENDPOINT,$SQL_MASTER_PORT +KNOX_ENDPOINT=$KNOX_ENDPOINT:$KNOX_PORT + +# Set username/password variables +export SQLCMDSERVER=$SQL_MASTER_INSTANCE +export SQLCMDUSER=$AZDATA_USERNAME +export SQLCMDPASSWORD=$AZDATA_PASSWORD +KNOX_PASSWORD=$AZDATA_PASSWORD + +if [ -z SQLCMDUSER ] +then + BCP_CREDENTIALS="-T" +else + BCP_CREDENTIALS="-U$SQLCMDUSER -P$SQLCMDPASSWORD" +fi for util in sqlcmd bcp kubectl curl do @@ -42,7 +52,7 @@ for util in sqlcmd bcp kubectl curl done # Copy the backup file, restore the database, create necessary objects and data file -pushd "/tmp" +pushd "/tmp" > /dev/null $DEBUG mkdir -p "$TMP_DIR_NAME" $DEBUG cd "$TMP_DIR_NAME" @@ -52,7 +62,12 @@ then $DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak fi -read -r MASTER_POD_NAME HADR_ENABLED <<<$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');") || (echo $ERROR_MESSAGE && exit 1) +read -r MASTER_POD_NAME HADR_ENABLED <<<$(sqlcmd -I -b -h-1 -W -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');") +if [ -z $MASTER_POD_NAME ] +then + echo $ERROR_MESSAGE + exit 1 +fi echo Copying sales database backup file... $DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) @@ -68,9 +83,6 @@ then fi echo Copying $file database backup file to SQL Master instance... $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) - - echo Removing database backup file... - $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file" done @@ -83,9 +95,6 @@ then fi echo Copying $file database backup file to SQL Master instance... $DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1) - - echo Removing database backup file... - $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file" done fi @@ -94,18 +103,30 @@ fi if [ "$HADR_ENABLED" == "1" ] then $DEBUG kubectl port-forward pods/$MASTER_POD_NAME 1533:1533 -n $CLUSTER_NAMESPACE & - SQL_MASTER_INSTANCE=127.0.0.1,1533 + PROC_ID=$! + SQLCMDSERVER=127.0.0.1,1533 fi echo Configuring sample database... -# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE" -export SA_PASSWORD=$KNOX_PASSWORD -$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2) +$DEBUG sqlcmd -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && kill -9 $PROC_ID > /dev/null && exit 2) # remove files copied into the pod: -echo Removing database backup file... +echo "Removing database backup file(s)..." $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak" +if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ] +then + for file in AdventureWorks2016_EXT.bak AdventureWorksDW2016_EXT.bak + do + $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file" + done + + for file in WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak + do + $DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file" + done +fi + for table in web_clickstreams inventory customer do echo Exporting $table data... @@ -118,31 +139,45 @@ for table in web_clickstreams inventory customer # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe" if [ ! -f $table.csv ] then - $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3) + $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQLCMDSERVER $BCP_CREDENTIALS -c -t"$DELIMITER" -e "$table.err" > "$table.out" || (echo $ERROR_MESSAGE && kill -9 $PROC_ID > /dev/null && exit 3) fi done if [ ! -f product_reviews.csv ] then echo Exporting product_reviews data... - $DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3) + $DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQLCMDSERVER $BCP_CREDENTIALS -c -t, -e "product_reviews.err" > "$table.out" || (echo $ERROR_MESSAGE && kill -9 $PROC_ID > /dev/null && exit 3) +fi + +if [[ $PROC_ID ]] +then + kill -9 $PROC_ID > /dev/null fi # Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... -$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4) -$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5) +if [[ $KNOX_PASSWORD ]] +then + $DEBUG curl -s -S -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" 1>/dev/null || (echo $ERROR_MESSAGE && exit 4) + $DEBUG curl -s -S -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" 1>/dev/null || (echo $ERROR_MESSAGE && exit 5) +else + $DEBUG curl -s -S -L -k -u : --negotiate -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" 1>/dev/null || (echo $ERROR_MESSAGE && exit 4) + $DEBUG curl -s -S -L -k -u : --negotiate -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" 1>/dev/null || (echo $ERROR_MESSAGE && exit 5) +fi #$DEBUG rm -f web_clickstreams.* -echo echo Uploading product_reviews data to HDFS... -$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6) -$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7) +if [[ $KNOX_PASSWORD ]] +then + $DEBUG curl -s -S -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" 1>/dev/null || (echo $ERROR_MESSAGE && exit 6) + $DEBUG curl -s -S -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" 1>/dev/null || (echo $ERROR_MESSAGE && exit 7) +else + $DEBUG curl -s -S -L -k -u : --negotiate -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" 1>/dev/null || (echo $ERROR_MESSAGE && exit 6) + $DEBUG curl -s -S -L -k -u : --negotiate -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" 1>/dev/null || (echo $ERROR_MESSAGE && exit 7) +fi #$DEBUG rm -f product_reviews.* -echo echo Bootstrap of the sample database completed successfully. -echo You can now login using "root" and Knox password to get the unified experience in Azure Data Studio. echo Data files for Oracle setup are located at [/tmp/$TMP_DIR_NAME]. # $DEBUG rm -f *.out *.err *.csv diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 2e1f0ed3..0e9d7a9a 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -1,14 +1,5 @@ USE master; GO --- Create login root that is part of sysadmin. You can then login as root to get the integrated --- login experience in Azure Data Studio -IF SUSER_SID('root') IS NULL -BEGIN - CREATE LOGIN root WITH PASSWORD = '$(SA_PASSWORD)'; - ALTER SERVER ROLE sysadmin ADD MEMBER root; -END; -GO - -- Enable external scripts execution for R/Python/Java: DECLARE @config_option nvarchar(100) = 'external scripts enabled'; IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1) @@ -119,10 +110,8 @@ BEGIN -- Restore the sample databases: EXECUTE #restore_database @file, @db_name; - -- Get database name used in restore: - SET @db_name = QUOTENAME(@db_name); - SET @proc = CONCAT(@db_name, N'.sys.sp_executesql'); - + -- Create context for database: + SET @proc = CONCAT(QUOTENAME(@db_name), N'.sys.sp_executesql'); EXECUTE @proc N'#create_data_sources'; -- Set compatibility level to 150: @@ -132,17 +121,24 @@ BEGIN IF SERVERPROPERTY('IsHadrEnabled') = 1 BEGIN DECLARE @command nvarchar(1000); - IF EXISTS(SELECT * FROM sys.databases WHERE name = PARSENAME(@db_name,1) and recovery_model_desc = 'SIMPLE') - BEGIN + IF EXISTS(SELECT * FROM sys.databases WHERE name = @db_name and recovery_model_desc <> 'FULL') -- Set recovery to full EXECUTE @proc N'ALTER DATABASE CURRENT SET RECOVERY FULL'; - SET @command = CONCAT(N'BACKUP DATABASE ', @db_name, ' TO DISK = ''NUL'';' ); - EXEC(@command); - END; + -- Workaround to avoid hk snapshot backup error (happens with wwidw & awdw dbs) + BEGIN TRY + -- Perform database & log backup to file to start a new chain + BACKUP DATABASE @db_name TO DISK = 'NUL' WITH INIT, FORMAT; + END TRY + BEGIN CATCH + -- Retry backup again + BACKUP DATABASE @db_name TO DISK = 'NUL' WITH INIT, FORMAT; + END CATCH; + + BACKUP LOG @db_name TO DISK = 'NUL' WITH INIT, FORMAT; -- Add database to AG - SET @command = CONCAT(N'ALTER AVAILABILITY GROUP containedag ADD DATABASE ', @db_name); + SET @command = CONCAT(N'ALTER AVAILABILITY GROUP containedag ADD DATABASE ', QUOTENAME(@db_name)); EXEC(@command); END; END;