github/sql-server-samples
1
0
Fork 0
mirror of https://github.com/Microsoft/sql-server-samples.git synced 2025-12-08 14:58:54 +00:00
Code Issues Releases Wiki Activity

Updated for RC changes & HA

Browse Source
This commit is contained in:
Umachandar Jayachandran
2019-08-28 15:25:49 -07:00
parent 1897cca4de
commit 5bee11e1d5
3 changed files with 89 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd
+24 -8
View File
@@ -41,7 +41,10 @@ if NOT EXIST tpcxbb_1gb.bak (
set SQLCMDSERVER=%SQL_MASTER_INSTANCE%
set SQLCMDUSER=sa
set SQLCMDPASSWORD=%SQL_MASTER_SA_PASSWORD%
for /F "usebackq" %%v in (`sqlcmd -I -b -h-1 -Q "print @@SERVERNAME;"`) do SET MASTER_POD_NAME=%%v
for /F "usebackq tokens=1,2" %%v in (`sqlcmd -I -b -h-1 -W -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');"`) do (
SET MASTER_POD_NAME=%%v
SET HADR_ENABLED=%%w
)
REM Copy the backup file, restore the database, create necessary objects and data file
echo Copying sales database backup file to SQL Master instance...
@@ -57,6 +60,9 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" (
)
echo Copying %%f database backup file to SQL Master instance...
%DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit
echo Removing database backup file...
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f"
)
set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak
@@ -67,39 +73,49 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" (
)
echo Copying %%f database backup file to SQL Master instance...
%DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit
echo Removing database backup file...
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f"
)
)
REM If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly
REM Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically
if /i "%HADR_ENABLED%" EQU "1" (
%DEBUG% start kubectl port-forward pods/%MASTER_POD_NAME% 1533:1533 -n %CLUSTER_NAMESPACE%
SET SQLCMDSERVER=127.0.0.1,1533
)
echo Configuring sample database(s)...
%DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit
REM remove files copied into the pod:
echo Removing database backup files...
echo Removing database backup file...
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak"
for %%F in (web_clickstreams inventory customer) do (
if NOT EXIST %%F.csv (
echo Exporting %%F data...
if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|)
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
)
)
if NOT EXIST product_reviews.csv (
echo Exporting product_reviews data...
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
)
REM Copy the data file to HDFS
echo Uploading web_clickstreams data to HDFS...
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
:: del /q web_clickstreams.*
echo.
echo Uploading product_reviews data to HDFS...
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
:: del /q product_reviews.*
REM %DEBUG% del /q *.out *.err *.csv
samples/features/sql-big-data-cluster/bootstrap-sample-db.sh
+22 -9
View File
@@ -51,11 +51,10 @@ then
$DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak
fi
MASTER_POD_NAME=$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "print @@SERVERNAME;")
read -r MASTER_POD_NAME HADR_ENABLED <<<$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');")
echo Copying sales database backup file...
$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
# $DEBUG rm tpcxbb_1gb.bak
if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ]
then
@@ -68,6 +67,9 @@ then
fi
echo Copying $file database backup file to SQL Master instance...
$DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
echo Removing database backup file...
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file"
done
@@ -80,16 +82,27 @@ then
fi
echo Copying $file database backup file to SQL Master instance...
$DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
echo Removing database backup file...
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file"
done
fi
# If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly
# Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically
if [ "$HADR_ENABLED" == "1" ]
then
$DEBUG kubectl port-forward pods/$MASTER_POD_NAME 1533:1533 -n $CLUSTER_NAMESPACE &
SQL_MASTER_INSTANCE=127.0.0.1,1533
fi
echo Configuring sample database...
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE"
export SA_PASSWORD=$KNOX_PASSWORD
$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2)
# remove files copied into the pod:
echo Removing database backup files...
echo Removing database backup file...
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak"
for table in web_clickstreams inventory customer
@@ -104,26 +117,26 @@ for table in web_clickstreams inventory customer
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe"
if [ ! -f $table.csv ]
then
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3)
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3)
fi
done
if [ ! -f product_reviews.csv ]
then
echo Exporting product_reviews data...
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3)
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3)
fi
# Copy the data file to HDFS
echo Uploading web_clickstreams data to HDFS...
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
#$DEBUG rm -f web_clickstreams.*
echo
echo Uploading product_reviews data to HDFS...
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
#$DEBUG rm -f product_reviews.*
echo
samples/features/sql-big-data-cluster/bootstrap-sample-db.sql
+43 -26
View File
@@ -18,7 +18,16 @@ BEGIN
END;
GO
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255))
-- Enable option to allow INSERT against external table defined on HADOOP data source
DECLARE @config_option nvarchar(100) = 'allow polybase export';
IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1)
BEGIN
EXECUTE sp_configure @config_option, 1;
RECONFIGURE WITH OVERRIDE;
END;
GO
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255), @db_name nvarchar(128))
AS
BEGIN
DECLARE @restore_filelist_tmpl nvarchar(1000) = N'RESTORE FILELISTONLY FROM DISK = N''/var/opt/mssql/data/%F''';
@@ -53,7 +62,7 @@ BEGIN
INSERT INTO @files
EXECUTE(@restore_cmd);
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', LEFT(@backup_file, CHARINDEX('.', @backup_file)-1));
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', @db_name);
SET @restore_cur = CURSOR FAST_FORWARD FOR SELECT LogicalName, REVERSE(LEFT(REVERSE(PhysicalName), CHARINDEX('\', REVERSE(PhysicalName))-1)) FROM @files;
OPEN @restore_cur;
WHILE(1=1)
@@ -84,50 +93,58 @@ BEGIN
WITH (LOCATION = 'sqlhdfs://controller-svc/default');
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://nmnode-0-svc:9000/',
RESOURCE_MANAGER_LOCATION='master-svc:8032'
);
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://nmnode-0-svc:9000/',
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
);
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://nmnode-0-svc:9000/',
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
);
END;
GO
--- Sample dbs:
DECLARE @sample_dbs CURSOR, @proc nvarchar(255);
SET @sample_dbs = CURSOR FAST_FORWARD FOR
SELECT file_or_directory_name
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak')
WHERE DB_ID(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', '')) IS NULL;
DECLARE @file nvarchar(260);
SELECT file_or_directory_name, d.db_name
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak') as f
CROSS APPLY (VALUES(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', ''))) as d(db_name)
WHERE DB_ID(d.db_name) IS NULL;
DECLARE @file nvarchar(260), @db_name nvarchar(128);
OPEN @sample_dbs;
WHILE(1=1)
BEGIN
FETCH @sample_dbs INTO @file;
FETCH @sample_dbs INTO @file, @db_name;
IF @@FETCH_STATUS < 0 BREAK;
-- Restore the sample databases:
EXECUTE #restore_database @file;
EXECUTE #restore_database @file, @db_name;
-- Get database name used in restore:
SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql');
SET @db_name = QUOTENAME(@db_name);
SET @proc = CONCAT(@db_name, N'.sys.sp_executesql');
EXECUTE @proc N'#create_data_sources';
-- Set compatibility level to 150:
EXECUTE @proc N'ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 150';
-- Rename TPCx-BB database:
IF DB_ID('tpcxbb_1gb') IS NOT NULL
ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales;
-- Check for HADR & add database to containedag:
IF SERVERPROPERTY('IsHadrEnabled') = 1
BEGIN
DECLARE @command nvarchar(1000);
IF EXISTS(SELECT * FROM sys.databases WHERE name = PARSENAME(@db_name,1) and recovery_model_desc = 'SIMPLE')
BEGIN
-- Set recovery to full
EXECUTE @proc N'ALTER DATABASE CURRENT SET RECOVERY FULL';
SET @command = CONCAT(N'BACKUP DATABASE ', @db_name, ' TO DISK = ''/dev/null'';' );
EXEC(@command);
END;
-- Add database to AG
SET @command = CONCAT(N'ALTER AVAILABILITY GROUP containedag ADD DATABASE ', @db_name);
EXEC(@command);
END;
END;
GO