diff --git a/samples/features/sql-big-data-cluster/README.md b/samples/features/sql-big-data-cluster/README.md index f65ce8e2..5331001b 100644 --- a/samples/features/sql-big-data-cluster/README.md +++ b/samples/features/sql-big-data-cluster/README.md @@ -6,6 +6,16 @@ Installation instructions for SQL Server 2019 big data clusters can be found [he **Before you begin**, load the sample data into your big data cluster. For instructions, see [Load sample data into a SQL Server 2019 big data cluster](https://docs.microsoft.com/en-us/sql/big-data-cluster/tutorial-load-sample-data). +## Executing the sample scripts +The scripts should be executed in a specific order to test the various features. Execute the scripts from each folder in below order: + +1. __[spark](spark/)__ +1. __[data-virtualization/storage-pool](data-virtualization/storage-pool)__ +1. __[data-virtualization/oracle](data-virtualization/oracle)__ +1. __[data-pool](data-pool/)__ +1. __[machine-learning/r](machine-learning/r)__ +1. __[machine-learning/python](machine-learning/python)__ + ## __[data-pool](data-pool/)__ SQL Server 2019 big data cluster contains a data pool which consists of many SQL Server instances to store data & query in a scale-out manner. diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index 2c62d088..d57917b0 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -36,7 +36,7 @@ echo Copying database backup file... del tpcxbb_1gb.bak >NUL echo Configuring sample database... -%DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b || goto exit +%DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit for %%F in (web_clickstreams inventory customer) do ( echo Exporting %%F data... @@ -61,6 +61,7 @@ del /q product_reviews.* REM %DEBUG% del /q *.out *.err *.csv echo Bootstrap of the sample database completed successfully. +echo You can now login using "root" and Knox password to get the unified experience in Azure Data Studio. echo Data files for Oracle setup are located at [%TMP%\%TMP_DIR_NAME%]. popd diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index 1b41ca0b..434f0b12 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -44,7 +44,7 @@ $DEBUG rm tpcxbb_1gb.bak echo Configuring sample database... # WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE" -$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b < "$STARTUP_PATH/bootstrap-sample-db.sql" > "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2) +$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -v SA_PASSWORD="$KNOX_PASSWORD" < "$STARTUP_PATH/bootstrap-sample-db.sql" > "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2) for table in web_clickstreams inventory customer do @@ -76,6 +76,7 @@ $DEBUG rm -f product_reviews.* echo echo Bootstrap of the sample database completed successfully. +echo You can now login using "root" and Knox password to get the unified experience in Azure Data Studio. echo Data files for Oracle setup are located at [/tmp/$TMP_DIR_NAME]. # $DEBUG rm -f *.out *.err *.csv diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 8fc721d2..06b820c6 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -1,5 +1,14 @@ USE master; -GO +GO +-- Create login root that is part of sysadmin. You can then login as root to get the integrated +-- login experience in Azure Data Studio +IF SUSER_SID('root') IS NULL +BEGIN + CREATE LOGIN root WITH PASSWORD = '$(SA_PASSWORD)'; + ALTER SERVER ROLE sysadmin ADD MEMBER root; +END; +GO + -- Enable external scripts execution for R/Python/Java: exec sp_configure 'external scripts enabled', 1; RECONFIGURE WITH OVERRIDE; diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql index e2843491..3ef3ac11 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql @@ -64,5 +64,8 @@ SELECT TOP (100) GROUP BY w.wcs_user_sk; GO +-- Cleanup +/* DROP EXTERNAL TABLE [dbo].[web_clickstream_clicks_data_pool]; -GO \ No newline at end of file +GO +*/ \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/python/README.md b/samples/features/sql-big-data-cluster/machine-learning/sql/python/README.md index 2af9a001..3f4f73b3 100644 --- a/samples/features/sql-big-data-cluster/machine-learning/sql/python/README.md +++ b/samples/features/sql-big-data-cluster/machine-learning/sql/python/README.md @@ -14,6 +14,12 @@ In this example, we are building a machine learning model using Python. The scri In this example, we are leveraging the new partitioning support (SQL Server 2019) in sp_execute_external_script to partition the input data and run the Python script per partition. So we will modify the training script to train model per group of users based on credit rating. The Python script will produce N models for the same input data set. +[book-click-prediction-mml-py.sql](book-click-prediction-mml-py.sql/) + +**Applies to:** SQL Server 2017+, SQL Server 2019 big data cluster + +In this example, we are building a machine learning model using Python. The script uses a logistic regression algorithm from microsoftml package to train and score the model. + [book-click-prediction-sklearn-py.sql](book-click-prediction-sklearn-py.sql/) **Applies to:** SQL Server 2017+, SQL Server 2019 big data cluster diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql new file mode 100644 index 00000000..0852b41c --- /dev/null +++ b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-mml-py.sql @@ -0,0 +1,134 @@ +USE sales +GO + +-- Create the training stored procedure +CREATE OR ALTER PROCEDURE [dbo].[train_book_category_visitor_python_mml] +(@model_name varchar(100)) +AS +BEGIN + DECLARE @model varbinary(max) + , @model_native varbinary(max) + , @input_query nvarchar(max) + , @train_script nvarchar(max) + +-- Set the input query for training. We will use 80% of the data. + SET @input_query = N' +SELECT TOP(80) PERCENT SIGN(q.clicks_in_category) AS book_category + , q.college_education + , q.male + , q.clicks_in_1 + , q.clicks_in_2 + , q.clicks_in_3 + , q.clicks_in_4 + , q.clicks_in_5 + , q.clicks_in_6 + , q.clicks_in_7 + , q.clicks_in_8 + , q.clicks_in_9 + FROM web_clickstreams_book_clicks as q +'; + -- Training R script that uses rxLogit function from RevoScaleR package (Microsoft R Server) to generate model to predict book_category click(s). + SET @train_script = N' +# build classification model to predict book_category +from microsoftml import rx_logistic_regression +from revoscalepy import rx_serialize_model +import pickle + +logitObj = rx_logistic_regression(formula = """ + book_category ~ college_education + male + + clicks_in_1 + clicks_in_2 + clicks_in_3 + clicks_in_4 + clicks_in_5 + + clicks_in_6 + clicks_in_7 + clicks_in_8 + clicks_in_9 +""", data = indata); + +model = pickle.dumps(logitObj) +'; + + -- Generate sales model using R scirpt with the book clicks stats for each user + EXECUTE sp_execute_external_script + @language = N'Python' + , @script = @train_script + , @input_data_1 = @input_query + , @input_data_1_name = N'indata' + , @params = N'@input_query nvarchar(max), @model varbinary(max) OUTPUT' + , @input_query = @input_query + , @model = @model OUTPUT; + + -- Save the trained models to predict user clicks on book category in the website + DELETE FROM sales_models WHERE model_name = @model_name; + INSERT INTO sales_models (model_name, model) VALUES(@model_name, @model); +END; +GO + +-- Step #1 +-- Train the book category prediction model: +DECLARE @model_name varchar(100) = 'category_model (Python MML)'; +EXECUTE dbo.train_book_category_visitor_python_mml @model_name; +SELECT * FROM sales_models WHERE model_name = @model_name; +GO + +-- Step #2a +-- Predict the book category clicks for new users based on their pattern of +-- visiting various categories in the web site +CREATE OR ALTER PROCEDURE [dbo].[predict_book_category_visitor_python_mml] +(@model_name varchar(100), @top_percent int = 20) +AS +BEGIN + DECLARE @model varbinary(max) = (SELECT model FROM sales_models WHERE model_name = @model_name) + , @input_query nvarchar(max) + , @predict_script nvarchar(max); + + -- Set the input query for scoring. We will use 20% of the data by default + SET @input_query = N' +SELECT TOP(@top_count_value) PERCENT SIGN(q.clicks_in_category) AS book_category + , q.college_education + , q.male + , q.clicks_in_1 + , q.clicks_in_2 + , q.clicks_in_3 + , q.clicks_in_4 + , q.clicks_in_5 + , q.clicks_in_6 + , q.clicks_in_7 + , q.clicks_in_8 + , q.clicks_in_9 + FROM web_clickstreams_book_clicks as q +'; + + -- Scoring script that uses sklearn logistic regression model to predict book_category click(s) + SET @predict_script = N' +from microsoftml import rx_predict +import pandas as pd +import pickle + +logit_model = pickle.loads(model) + +feature_cols = ["college_education", "male", "clicks_in_1", "clicks_in_2","clicks_in_3","clicks_in_4","clicks_in_5","clicks_in_6","clicks_in_7","clicks_in_8","clicks_in_9"] + +predictions = rx_predict(logit_model, indata[feature_cols]) + +predictions_df = pd.DataFrame(predictions, columns = ["PredictedLabel"]) +outdata = pd.concat([predictions_df, indata], axis = 1, copy = False) +'; + + -- Predict the book category click based on the sklearn model + EXECUTE sp_execute_external_script + @language = N'Python' + , @script = @predict_script + , @input_data_1 = @input_query + , @input_data_1_name = N'indata' + , @output_data_1_name = N'outdata' + , @params = N'@model varbinary(max), @top_count_value int' + , @model = @model + , @top_count_value = @top_percent + WITH RESULT SETS ((book_category_prediction bit, book_category_actual bit, college_education varchar(30), male bit, + clicks_in_1 int, clicks_in_2 int, clicks_in_3 int, clicks_in_4 int, clicks_in_5 int, + clicks_in_6 int, clicks_in_7 int, clicks_in_8 int, clicks_in_9 int)); +END +GO + +-- Step #2b +-- Predict the book category clicks for new users based on their pattern of +-- visiting various categories in the web site +DECLARE @model_name varchar(100) = 'category_model (Python MML)'; +EXECUTE dbo.predict_book_category_visitor_python_mml @model_name, 1 /* Score only on 1 PERCENT for testing purpose. */; +GO