mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Initial samples for SQL Server 2019 big data cluster
Demonstrates various functionality in big data cluster.
This commit is contained in:
Umachandar Jayachandran
@@ -0,0 +1,44 @@
|
||||
# SQL Server big data clusters
|
||||
|
||||
## Pre-requisites
|
||||
1. Kubernetes cluster configuration & Kubectl command-line utility
|
||||
2. Curl utility
|
||||
3. Sqlcmd utility
|
||||
4. Bcp utility
|
||||
5. Azure Data Studio or SQL Server Management Studio
|
||||
6. SQL Server 2019 big data cluster
|
||||
|
||||
Installation instructions for SQL Server 2019 big data cluster can be found [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sql-server-2017).
|
||||
|
||||
## Samples Setup
|
||||
|
||||
**Before you begin**, download the sample database [backup file](https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak) and save it locally. Run the CMD script called *bootstrap-sample-db.cmd* or the shell script *bootstrap-sample-db.sh* depending on your platform. This script will restore the database on the SQL Master instance, execute the *bootstrap-sample-db.sql* script, create the database objects needed, export the web_clickstreams & inventory tables to CSV file, and upload the web_clickstreams CSV file to HDFS inside the SQL Server 2019 big data cluster.
|
||||
|
||||
__[data-pool](data-pool/)__
|
||||
|
||||
### Data ingestion using Spark
|
||||
Connect to the master instance in your SQL Server big data cluster and the SQL Server big data cluster endpoint, and follow the steps in *data-pool/data-ingestion-spark.sql*.
|
||||
|
||||
### Data ingestion using sql
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *data-pool/data-ingestion-sql.sql*.
|
||||
|
||||
__[data-virtualization](data-virtualization/)__
|
||||
|
||||
### External table over HDFS
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *data-virtualization/external-table-hdfs.sql*.
|
||||
|
||||
### External table over Oracle
|
||||
To execute this sample script, you will need following:
|
||||
1. Oracle instance and credentials
|
||||
1. Create inventory table in Oracle using [data-virtualization/inventory-oracle.sql](data-virtualization/inventory-oracle.sql/) script
|
||||
1. Import the inventory.csv file generated by the bootstrap-sample-db script to a table in Oracle
|
||||
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *data-virtualization/external-table-oracle.sql*.
|
||||
|
||||
__[machine-learning](machine-learning/)__
|
||||
|
||||
### SQL Server ML Services on master instance
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *machine-learning/sql/book-category-r-ml.sql*.
|
||||
|
||||
### Spark ML
|
||||
Connect to the SQL Server big data cluster endpoint, and run the notebook files *machine-learning/spark/1-data-prep.ipynb* and *machine-learning/spark/2-build-ml-model.ipynb* cell by cell.
|
||||
@@ -0,0 +1,61 @@
|
||||
@echo off
|
||||
REM CLICKSTREAM FILES
|
||||
setlocal enableextensions
|
||||
set CLUSTER_NAMESPACE=%1
|
||||
set SQL_MASTER_IP=%2
|
||||
set SQL_MASTER_SA_PASSWORD=%3
|
||||
set BACKUP_FILE_PATH=%~4
|
||||
set KNOX_IP=%5
|
||||
set KNOX_PASSWORD=%6
|
||||
set STARTUP_PATH=%~dp0
|
||||
|
||||
if NOT DEFINED CLUSTER_NAMESPACE goto :usage
|
||||
if NOT DEFINED SQL_MASTER_IP goto :usage
|
||||
if NOT DEFINED SQL_MASTER_SA_PASSWORD goto :usage
|
||||
if NOT DEFINED BACKUP_FILE_PATH goto :usage
|
||||
if NOT DEFINED KNOX_IP goto :usage
|
||||
if NOT DEFINED KNOX_PASSWORD set KNOX_PASSWORD=%SQL_MASTER_SA_PASSWORD%
|
||||
|
||||
set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,31433
|
||||
set KNOX_ENDPOINT=%KNOX_IP%:30443
|
||||
|
||||
echo Verifying sqlcmd.exe is in path & CALL WHERE /Q sqlcmd.exe || GOTO exit
|
||||
echo Verifying bcp.exe is in path & CALL WHERE /Q bcp.exe || GOTO exit
|
||||
echo Verifying kubectl.exe is in path & CALL WHERE /Q kubectl.exe || echo HINT: Install the kubernetes-cli - https://kubernetes.io/docs/tasks/tools/install-kubectl && GOTO exit
|
||||
echo Verifying curl.exe is in path & CALL WHERE /Q curl.exe || echo HINT: Install curl - https://curl.haxx.se/download.html && GOTO exit
|
||||
|
||||
REM Copy the backup file, restore the database, create necessary objects and data file
|
||||
echo Copying database backup file...
|
||||
pushd "%BACKUP_FILE_PATH%"
|
||||
%DEBUG% kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
|
||||
popd
|
||||
|
||||
echo Configuring sample database...
|
||||
%DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "%STARTUP_PATH%bootstrap.out" -I -b || goto exit
|
||||
|
||||
for %%F in (web_clickstreams inventory) do (
|
||||
echo Exporting %%F data...
|
||||
%DEBUG% bcp sales.dbo.%%F out "%STARTUP_PATH%%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "%STARTUP_PATH%%%F.out" -e "%STARTUP_PATH%%%F.err" || goto exit
|
||||
)
|
||||
|
||||
REM Copy the data file to HDFS
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
pushd "%STARTUP_PATH%"
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
|
||||
|
||||
:: del /q *.out *.err *.csv
|
||||
popd
|
||||
|
||||
endlocal
|
||||
exit /b 0
|
||||
goto :eof
|
||||
|
||||
:exit
|
||||
echo Bootstrap of the sample database failed.
|
||||
exit /b %ERRORLEVEL%
|
||||
|
||||
:usage
|
||||
echo USAGE: %0 ^<CLUSTER_NAMESPACE^> ^<SQL_MASTER_IP^> ^<SQL_MASTER_SA_PASSWORD^> ^<BACKUP_FILE_PATH^> ^<KNOX_IP^> [^<KNOX_PASSWORD^>]
|
||||
echo Default ports are assumed for SQL Master instance ^& Knox gateway.
|
||||
exit /b 0
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -o pipefail
|
||||
USAGE_MESSAGE="USAGE: $0 <CLUSTER_NAMESPACE> <SQL_MASTER_IP> <SQL_MASTER_SA_PASSWORD> <BACKUP_FILE_PATH> <KNOX_IP> [<KNOX_PASSWORD>]"
|
||||
ERROR_MESSAGE="Bootstrap of the sample database failed."
|
||||
|
||||
# Print usage if mandatory parameters are missing
|
||||
: "${1:?$USAGE_MESSAGE}"
|
||||
: "${2:?$USAGE_MESSAGE}"
|
||||
: "${3:?$USAGE_MESSAGE}"
|
||||
: "${4:?$USAGE_MESSAGE}"
|
||||
: "${5:?$USAGE_MESSAGE}"
|
||||
: "${DEBUG=}"
|
||||
|
||||
# Save the input parameters
|
||||
CLUSTER_NAMESPACE=$1
|
||||
SQL_MASTER_IP=$2
|
||||
SQL_MASTER_SA_PASSWORD=$3
|
||||
BACKUP_FILE_PATH=$4
|
||||
KNOX_IP=$5
|
||||
KNOX_PASSWORD=$6
|
||||
# If Knox password is not supplied then default to SQL Master password
|
||||
KNOX_PASSWORD=${KNOX_PASSWORD:=$SQL_MASTER_SA_PASSWORD}
|
||||
|
||||
SQL_MASTER_INSTANCE=$SQL_MASTER_IP,31433
|
||||
KNOX_ENDPOINT=$KNOX_IP:30443
|
||||
|
||||
# Copy the backup file, restore the database, create necessary objects and data file
|
||||
echo Copying database backup file...
|
||||
pushd "$BACKUP_FILE_PATH"
|
||||
$DEBUG kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n $CLUSTER_NAMESPACE || (echo $ERROR_MESSAGE && exit 1)
|
||||
popd
|
||||
|
||||
echo Configuring sample database...
|
||||
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE"
|
||||
$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -i "bootstrap-sample-db.sql" -o "bootstrap.out" -I -b || (echo $ERROR_MESSAGE && exit 2)
|
||||
|
||||
for table in web_clickstreams inventory
|
||||
do
|
||||
echo Exporting $table data...
|
||||
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe"
|
||||
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -o "$table.out" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3)
|
||||
done
|
||||
|
||||
# Copy the data file to HDFS
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
|
||||
|
||||
# rm -f *.out *.err *.csv
|
||||
exit
|
||||
@@ -0,0 +1,74 @@
|
||||
USE master;
|
||||
GO
|
||||
-- Enable external scripts execution for R/Python/Java:
|
||||
exec sp_configure 'external scripts enabled', 1;
|
||||
RECONFIGURE WITH OVERRIDE;
|
||||
GO
|
||||
|
||||
IF DB_ID('sales') IS NULL
|
||||
RESTORE DATABASE sales
|
||||
FROM DISK=N'/var/opt/mssql/data/tpcxbb_1gb.bak'
|
||||
WITH
|
||||
MOVE N'tpcxbb_1gb' TO N'/var/opt/mssql/data/sales.mdf',
|
||||
MOVE N'tpcxbb_1gb_log' TO N'/var/opt/mssql/data/sales.ldf';
|
||||
GO
|
||||
|
||||
USE sales;
|
||||
GO
|
||||
-- Create default data sources for SQL Big Data Cluster
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool')
|
||||
CREATE EXTERNAL DATA SOURCE SqlDataPool
|
||||
WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default');
|
||||
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
|
||||
CREATE EXTERNAL DATA SOURCE SqlStoragePool
|
||||
WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080');
|
||||
GO
|
||||
|
||||
-- Create view used for ML services training stored procedure
|
||||
CREATE OR ALTER VIEW [dbo].[web_clickstreams_book_clicks]
|
||||
AS
|
||||
SELECT
|
||||
q.clicks_in_category,
|
||||
CASE WHEN cd.cd_education_status IN ('Advanced Degree', 'College', '4 yr Degree', '2 yr Degree') THEN 1 ELSE 0 END AS college_education,
|
||||
CASE WHEN cd.cd_gender = 'M' THEN 1 ELSE 0 END AS male,
|
||||
q.clicks_in_1,
|
||||
q.clicks_in_2,
|
||||
q.clicks_in_3,
|
||||
q.clicks_in_4,
|
||||
q.clicks_in_5,
|
||||
q.clicks_in_6,
|
||||
q.clicks_in_7,
|
||||
q.clicks_in_8,
|
||||
q.clicks_in_9
|
||||
FROM(
|
||||
SELECT
|
||||
w.wcs_user_sk,
|
||||
SUM( CASE WHEN i.i_category = 'Books' THEN 1 ELSE 0 END) AS clicks_in_category,
|
||||
SUM( CASE WHEN i.i_category_id = 1 THEN 1 ELSE 0 END) AS clicks_in_1,
|
||||
SUM( CASE WHEN i.i_category_id = 2 THEN 1 ELSE 0 END) AS clicks_in_2,
|
||||
SUM( CASE WHEN i.i_category_id = 3 THEN 1 ELSE 0 END) AS clicks_in_3,
|
||||
SUM( CASE WHEN i.i_category_id = 4 THEN 1 ELSE 0 END) AS clicks_in_4,
|
||||
SUM( CASE WHEN i.i_category_id = 5 THEN 1 ELSE 0 END) AS clicks_in_5,
|
||||
SUM( CASE WHEN i.i_category_id = 6 THEN 1 ELSE 0 END) AS clicks_in_6,
|
||||
SUM( CASE WHEN i.i_category_id = 7 THEN 1 ELSE 0 END) AS clicks_in_7,
|
||||
SUM( CASE WHEN i.i_category_id = 8 THEN 1 ELSE 0 END) AS clicks_in_8,
|
||||
SUM( CASE WHEN i.i_category_id = 9 THEN 1 ELSE 0 END) AS clicks_in_9
|
||||
FROM web_clickstreams as w
|
||||
INNER JOIN item as i ON (w.wcs_item_sk = i_item_sk
|
||||
AND w.wcs_user_sk IS NOT NULL)
|
||||
GROUP BY w.wcs_user_sk
|
||||
) AS q
|
||||
INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk
|
||||
INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk;
|
||||
GO
|
||||
|
||||
-- Create table for storing the machine learning models
|
||||
CREATE TABLE sales_models (
|
||||
model_name varchar(100) NOT NULL PRIMARY KEY,
|
||||
model varbinary(max) NOT NULL,
|
||||
model_native varbinary(max) NOT NULL,
|
||||
created_by nvarchar(300) NOT NULL DEFAULT(SYSTEM_USER),
|
||||
create_time datetime2 NOT NULL DEFAULT(SYSDATETIME())
|
||||
);
|
||||
GO
|
||||
@@ -0,0 +1,48 @@
|
||||
# Data ingestion using Spark streaming
|
||||
|
||||
SQL Server Big Data clusters provide scale-out compute and storage to improve the performance of analyzing any data. Data from a variety of sources can be ingested and distributed across data pool instances for analysis. In this example, you are going to use Spark to read and transform data from HDFS and cache it in a data pool. Querying the external table created over this aggregated data stored in data pools will be much more efficient than going to the raw data always.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Using Azure Data Studio, connect to the HDFS/Spark gateway (SQL Server big data cluster connection type).
|
||||
|
||||
1. Connect to SQL Server Master instance using Azure Data Studio.
|
||||
|
||||
1. Execute the SQL script [data-ingestion-spark.sql](data-ingestion-spark.sql).
|
||||
|
||||
1. Create and submit a Spark job that ingests data from HDFS into the external table.
|
||||
|
||||
Submitting a Spark job will start a Spark streaming session using spark-submit.
|
||||
|
||||
The arguments to the jar file are:
|
||||
|
||||
1. server name - sql server to connect to read the table schema
|
||||
2. port number
|
||||
3. username - sql server username for master instance
|
||||
4. password - sql server password for master instance
|
||||
5. database name
|
||||
6. external table name
|
||||
7. Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
||||
8. Input format. This can be "csv", "parquet", "json".
|
||||
9. enable checkpoint: true or false
|
||||
|
||||
Submit a Spark job with the below parameters. You can use the Spark submit experience from Azure Data Studio (right click on big data cluster endpoint -> Submit Spark Job):
|
||||
|
||||
ARGUMENTS:
|
||||
|
||||
**job name:** yourJobName
|
||||
|
||||
**switch** from "Local" to "HDFS"
|
||||
|
||||
**Path to jar** (copy/paste this):
|
||||
|
||||
/jar/mssql-spark-lib-assembly-1.0.jar
|
||||
|
||||
**Main class:**
|
||||
FileStreaming
|
||||
|
||||
**Parameters (copy/paste this; make sure you replace the password!):**
|
||||
|
||||
mssql-master-pool-0.service-master-pool 1433 sa passwordHere sales web_clickstreams_spark_results hdfs:///clickstream_data csv false
|
||||
|
||||
6. Query the external table we created earlier using the SELECT queries in the script to see data coming from the streaming job and landing in the table.
|
||||
@@ -0,0 +1,54 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create external table in a data pool in SQL Server 2019 big data cluster.
|
||||
-- The SqlDataPool data source is a special data source that is available in
|
||||
-- any new database in SQL Master instance. This is used to reference the
|
||||
-- data pool in a SQL Server 2019 big data cluster.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_spark_results]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlDataPool,
|
||||
DISTRIBUTION = ROUND_ROBIN
|
||||
);
|
||||
|
||||
-- Data can be ingested into the external table from a spark job.
|
||||
--
|
||||
-- Submit spark job with below parameters. You can use the Spark submit experience from Azure Data Studio.
|
||||
-- Right click on server name in a SQL Server big data cluster connection and click "Submit Spark Job".
|
||||
--
|
||||
-- Specify following values in the Job submission dialog box:
|
||||
---- job name: <yourJobName>
|
||||
---- switch from "Local" to "HDFS"
|
||||
---- Main class: "FileStreaming"
|
||||
---- Path to jar: /jar/mssql-spark-lib-assembly-1.0.jar
|
||||
---- Arguments:
|
||||
---- mssql-master-pool-0.service-master-pool 1433 sa %PASSWORD% sales web_clickstreams_spark_results hdfs:///clickstream_data csv false
|
||||
|
||||
-- The arguments to jar file are
|
||||
-- 1: server name - sql server to connect to read the table schema
|
||||
-- 2: port number
|
||||
-- 3: username - sql server username for master instance
|
||||
-- 4: password - sql server password for master instance
|
||||
-- 5: database name
|
||||
-- 6: external table name
|
||||
-- 7: Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
||||
-- 8: Input format. This can be "csv", "parquet", "json".
|
||||
-- 9: enable checkpoint: true or false
|
||||
--
|
||||
|
||||
-- After the Spark streaming job has been sucessfully submitted, you can run below query to view the results.
|
||||
--
|
||||
-- Wait until some rows are available.
|
||||
WHILE (1=1)
|
||||
IF EXISTS(SELECT * FROM [web_clickstreams_spark_results])
|
||||
BREAK;
|
||||
|
||||
SELECT count(*) FROM [web_clickstreams_spark_results];
|
||||
SELECT TOP 10 * FROM [web_clickstreams_spark_results];
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_spark_results];
|
||||
GO
|
||||
@@ -0,0 +1,9 @@
|
||||
# Data ingestion using SQL stored procedure
|
||||
|
||||
SQL Server Big Data clusters provide scale-out compute and storage to improve the performance of analyzing any data. Data from a variety of sources can be ingested and distributed across data pool instances for analysis. In this example, we will insert data from a SQL query into an external table stored in a data pool and query it.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the .sql script [data-ingestion-sql.sql](data-ingestion-sql.sql).
|
||||
@@ -0,0 +1,58 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create external table in a data pool in SQL Server 2019 big data cluster.
|
||||
-- The SqlDataPool data source is a special data source that is available in
|
||||
-- any new database in SQL Master instance. This is used to reference the
|
||||
-- data pool in a SQL Server 2019 big data cluster.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_dp]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlDataPool,
|
||||
DISTRIBUTION = ROUND_ROBIN
|
||||
);
|
||||
GO
|
||||
-- Currently the create external table operation is asynchronous and there is no
|
||||
-- way to determine completion of the operation. To prevent failures of the insert
|
||||
-- into the external table, wait for few minutes.
|
||||
WAITFOR DELAY '00:02:00';
|
||||
GO
|
||||
-- Insert results of a SELECT statement into the external table created on the data pool
|
||||
--
|
||||
DECLARE @db_name SYSNAME = 'sales'
|
||||
DECLARE @schema_name SYSNAME = 'dbo'
|
||||
DECLARE @table_name SYSNAME = 'web_clickstreams_dp'
|
||||
DECLARE @query SYSNAME = 'SELECT TOP(1000) * FROM sales.dbo.web_clickstreams WHERE wcs_user_sk IS NOT NULL'
|
||||
|
||||
EXEC model..sp_data_pool_table_insert_data @db_name, @schema_name, @table_name, @query
|
||||
GO
|
||||
|
||||
-- Query data inserted from sp_data_pool_table_insert_data
|
||||
--
|
||||
SELECT count(*) FROM [dbo].[web_clickstreams_dp]
|
||||
SELECT TOP 10 * FROM [dbo].[web_clickstreams_dp]
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT TOP (100)
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[web_clickstreams_dp]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_dp];
|
||||
GO
|
||||
@@ -0,0 +1,11 @@
|
||||
# Query data in HDFS from SQL Server master
|
||||
|
||||
In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes.
|
||||
|
||||
In this example, you are going to create an external table in the SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster. Then you will join the data in the external table with high value data in SQL Master instance.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the [external-table-hdfs.sql](external-table-hdfs.sql).
|
||||
@@ -0,0 +1,52 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create file format for CSV file with appropriate properties.
|
||||
--
|
||||
CREATE EXTERNAL FILE FORMAT csv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = ',',
|
||||
STRING_DELIMITER = '"',
|
||||
FIRST_ROW = 2,
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
-- is a special data source that is available in any new database in
|
||||
-- SQL Master instance.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_hdfs]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/clickstream_data',
|
||||
FILE_FORMAT = csv_file
|
||||
);
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[web_clickstreams_hdfs]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs];
|
||||
GO
|
||||
@@ -0,0 +1,12 @@
|
||||
# Query data in Oracle from SQL Server master
|
||||
|
||||
Create external table over an Oracle database
|
||||
by leveraging SQL Server Polybase technology. SQL Server Big Data clusters can query external data sources without importing the data in SQL Server. SQL Server 2019 introduces new connectors to data sources like Oracle, MongoDB and Teradata. In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server.
|
||||
|
||||
**Before you begin**, you need to have an Oracle instance and credentials. Execute the SQL script [inventory-ora.sql](inventory-ora.sql/) in Oracle to create the table and import the "inventory.csv" file created by the bootstrap sample database.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [external-table-oracle.sql](external-table-oracle.sql/).
|
||||
@@ -0,0 +1,44 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create database scoped credential to connect to Oracle server
|
||||
-- Provide appropriate credentials to Oracle server in below statement.
|
||||
-- If you are using SQL Server Management Studio then you can replace the parameters using
|
||||
-- the Query menu, and "Specify Values for Template Parameters" option.
|
||||
CREATE DATABASE SCOPED CREDENTIAL [OracleCredential]
|
||||
WITH IDENTITY = '<oracle_user,nvarchar(100),SYSTEM>', SECRET = '<oracle_user_password,nvarchar(100),manager>';
|
||||
|
||||
-- Create external data source that points to Oracle server
|
||||
--
|
||||
CREATE EXTERNAL DATA SOURCE [OracleSalesSrvr]
|
||||
WITH (LOCATION = 'oracle://<oracle_server,nvarchar(100)>',CREDENTIAL = [OracleCredential]);
|
||||
|
||||
-- Create external table over inventory table on Oracle server
|
||||
-- NOTE: Table names and column names will use ANSI SQL quoted identifier while querying against Oracle.
|
||||
-- As a result, the names are case-sensitive so specify the name in the external table definition
|
||||
-- that matches the exact case of the table and column names in the Oracle metadata.
|
||||
CREATE EXTERNAL TABLE [inventory_ora]
|
||||
([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL,
|
||||
[inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0))
|
||||
WITH (DATA_SOURCE=[OracleSalesSrvr],
|
||||
LOCATION='<oracle_service_name,nvarchar(30),xe>.<oracle_schema,nvarchar(128),HR>.<oracle_table,nvarchar(128),INVENTORY>');
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity
|
||||
FROM [inventory_ora] as i
|
||||
JOIN item as it
|
||||
ON it.i_item_sk = i.inv_item
|
||||
JOIN warehouse as w
|
||||
ON w.w_warehouse_sk = i.inv_warehouse
|
||||
WHERE it.i_category = 'Books' and i.inv_item BETWEEN 1 and 18000 --> get items within specific range
|
||||
GROUP BY w.w_warehouse_name, i.inv_item;
|
||||
GO
|
||||
|
||||
-- Cleanup
|
||||
--
|
||||
DROP EXTERNAL TABLE [inventory_ora];
|
||||
DROP EXTERNAL DATA SOURCE [OracleSalesSrvr] ;
|
||||
DROP DATABASE SCOPED CREDENTIAL [OracleCredential];
|
||||
GO
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Inventory table over which the SQL Server external table will be defined
|
||||
CREATE TABLE "INVENTORY"
|
||||
(
|
||||
"INV_DATE" NUMBER(10,0) NOT NULL,
|
||||
"INV_ITEM" NUMBER(10,0) NOT NULL,
|
||||
"INV_WAREHOUSE" NUMBER(10,0) NOT NULL,
|
||||
"INV_QUANTITY_ON_HAND" NUMBER(10,0)
|
||||
);
|
||||
|
||||
CREATE INDEX INV_ITEM ON HR.INVENTORY(INV_ITEM);
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pyspark3kernel",
|
||||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Load data to a dataframe\n- Download AdultCensusIncome.csv from [here](https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv).\n- Create a diretory named /spark_ml. Upload AdultCensusIncome.csv to /spark_ml",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "import os\nimport pandas as pd\n\ndatafile = \"/spark_ml/AdultCensusIncome.csv\"\n\n# Read and Load data\n# Create a Spark dataframe out of the csv file.\ndata_all = spark.read.format('csv').options(header='true', inferSchema='true', ignoreLeadingWhiteSpace='true', ignoreTrailingWhiteSpace='true').load(datafile)\nprint(\"({}, {})\".format(data_all.count(), len(data_all.columns)))\n\n#Replace \"-\" with \"_\" in column names\ncolumns_new = [col.replace(\"-\", \"_\") for col in data_all.columns]\ndata_all = data_all.toDF(*columns_new)\ndata_all.printSchema() #human-readable format\n\ndf = pd.DataFrame(data_all.take(10))\nprint(df.to_string())\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "(32561, 15)\nroot\n |-- age: integer (nullable = true)\n |-- workclass: string (nullable = true)\n |-- fnlwgt: integer (nullable = true)\n |-- education: string (nullable = true)\n |-- education_num: integer (nullable = true)\n |-- marital_status: string (nullable = true)\n |-- occupation: string (nullable = true)\n |-- relationship: string (nullable = true)\n |-- race: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- capital_gain: integer (nullable = true)\n |-- capital_loss: integer (nullable = true)\n |-- hours_per_week: integer (nullable = true)\n |-- native_country: string (nullable = true)\n |-- income: string (nullable = true)\n\n 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14\n0 39 State-gov 77516 Bachelors 13 Never-married Adm-clerical Not-in-family White Male 2174 0 40 United-States <=50K\n1 50 Self-emp-not-inc 83311 Bachelors 13 Married-civ-spouse Exec-managerial Husband White Male 0 0 13 United-States <=50K\n2 38 Private 215646 HS-grad 9 Divorced Handlers-cleaners Not-in-family White Male 0 0 40 United-States <=50K\n3 53 Private 234721 11th 7 Married-civ-spouse Handlers-cleaners Husband Black Male 0 0 40 United-States <=50K\n4 28 Private 338409 Bachelors 13 Married-civ-spouse Prof-specialty Wife Black Female 0 0 40 Cuba <=50K\n5 37 Private 284582 Masters 14 Married-civ-spouse Exec-managerial Wife White Female 0 0 40 United-States <=50K\n6 49 Private 160187 9th 5 Married-spouse-absent Other-service Not-in-family Black Female 0 0 16 Jamaica <=50K\n7 52 Self-emp-not-inc 209642 HS-grad 9 Married-civ-spouse Exec-managerial Husband White Male 0 0 45 United-States >50K\n8 31 Private 45781 Masters 14 Never-married Prof-specialty Not-in-family White Female 14084 0 50 United-States >50K\n9 42 Private 159449 Bachelors 13 Married-civ-spouse Exec-managerial Husband White Male 5178 0 40 United-States >50K",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Data Preparation\n- Choose the feature that we want to use to build the model\n- Split the data set as training and test\n- Write traning and data set as AdultCensusIncomeTrain and AdultCensusIncomeTest to /spark_ml directory\n\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Choose feature columns and the label column.\nlabel = \"income\"\nxvars = [\"age\", \"hours_per_week\"] #all numeric\n\nprint(\"label = {}\".format(label))\nprint(\"features = {}\".format(xvars))\n\nselect_cols = xvars\nselect_cols.append(label)\ndata = data_all.select(select_cols)\n\n# Split data into train and test.\ntrain, test = data.randomSplit([0.75, 0.25], seed=123)\n\nprint(\"train ({}, {})\".format(train.count(), len(train.columns)))\nprint(\"test ({}, {})\".format(test.count(), len(test.columns)))\n\n\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "label = income\nfeatures = ['age', 'hours_per_week']\ntrain (24469, 3)\ntest (8092, 3)",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Data Persistence\n- Save the training and test set as ORC data for persistance\n- Will use the persisted data to build model\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Write the train and test data sets to intermediate storage\n# Write the train and test data sets to intermediate storage\ntrain_data_path = \"/spark_ml/AdultCensusIncomeTrain\"\ntest_data_path = \"/spark_ml/AdultCensusIncomeTest\"\n\ntrain.write.mode('overwrite').orc(train_data_path)\ntest.write.mode('overwrite').orc(test_data_path)\nprint(\"train and test datasets saved to {} and {}\".format(train_data_path, test_data_path))",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "train and test datasets saved to /spark_ml/AdultCensusIncomeTrain and /spark_ml/AdultCensusIncomeTest",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pyspark3kernel",
|
||||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Building - Import the training and test data\r\n\r\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "import os\nimport pprint\nimport numpy as np\nimport os\nimport pprint\nimport numpy as np\nimport pandas as pd\n\nfrom pyspark.ml import Pipeline, PipelineModel\nfrom pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler\nfrom pyspark.ml.classification import LogisticRegression\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator\nfrom pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n\ntrain_data_path = \"/spark_ml/AdultCensusIncomeTrain\"\ntest_data_path = \"/spark_ml/AdultCensusIncomeTest\"\n\ntrain = spark.read.orc(train_data_path)\ntest = spark.read.orc(test_data_path)\n\nprint(\"train: ({}, {})\".format(train.count(), len(train.columns)))\nprint(\"test: ({}, {})\".format(test.count(), len(test.columns)))\n\ntrain.printSchema()\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "train: (24469, 3)\ntest: (8092, 3)\nroot\n |-- age: integer (nullable = true)\n |-- hours_per_week: integer (nullable = true)\n |-- income: string (nullable = true)",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model building - Encode features and Build Model",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "label = \"income\"\nreg = 0.1\nprint(\"Regularization Rate is {}.\".format(reg))\n\n# create a new Logistic Regression model.\nlr = LogisticRegression(regParam=reg)\n\ndtypes = dict(train.dtypes)\ndtypes.pop(label)\n\nsi_xvars = []\nohe_xvars = []\nfeatureCols = []\nfor idx,key in enumerate(dtypes):\n if dtypes[key] == \"string\":\n featureCol = \"-\".join([key, \"encoded\"])\n featureCols.append(featureCol)\n \n tmpCol = \"-\".join([key, \"tmp\"])\n # string-index and one-hot encode the string column\n #https://spark.apache.org/docs/2.3.0/api/java/org/apache/spark/ml/feature/StringIndexer.html\n #handleInvalid: Param for how to handle invalid data (unseen labels or NULL values). \n #Options are 'skip' (filter out rows with invalid data), 'error' (throw an error), \n #or 'keep' (put invalid data in a special additional bucket, at index numLabels). Default: \"error\"\n si_xvars.append(StringIndexer(inputCol=key, outputCol=tmpCol, handleInvalid=\"skip\")) #, handleInvalid=\"keep\"\n ohe_xvars.append(OneHotEncoder(inputCol=tmpCol, outputCol=featureCol))\n else:\n featureCols.append(key)\n\n# string-index the label column into a column named \"label\"\nsi_label = StringIndexer(inputCol=label, outputCol='label')\n\n# assemble the encoded feature columns in to a column named \"features\"\nassembler = VectorAssembler(inputCols=featureCols, outputCol=\"features\")\n\n# put together the pipeline\nstages = []\nstages.extend(si_xvars)\nstages.extend(ohe_xvars)\nstages.append(si_label)\nstages.append(assembler)\nstages.append(lr)\npipe = Pipeline(stages=stages)\n\n# train the model\nmodel = pipe.fit(train)\nprint(model)\nmodel.stages\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "Regularization Rate is 0.1.\nPipelineModel_49cfbacdb54dd44bcca2\n[StringIndexer_4e5ab09117dc68a07eae, VectorAssembler_43b7be097576e3659c49, LogisticRegression_42b491b66df1978b6ebc]",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Building - Select the best model",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "\nregs = np.arange(0.0, 1.0, 0.2)\n\nparamGrid = ParamGridBuilder().addGrid(lr.regParam, regs).build()\ncv = CrossValidator(estimator=pipe, evaluator=BinaryClassificationEvaluator(), estimatorParamMaps=paramGrid)\n\ncvModel = cv.fit(train)\n\nmodel = cvModel.bestModel",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Evaluation",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# make prediction\npred = model.transform(test)\nprint(pd.DataFrame(pred.take(10)).to_string())\n\n# evaluate. note only 2 metrics are supported out of the box by Spark ML.\nbce = BinaryClassificationEvaluator(rawPredictionCol='rawPrediction')\nau_roc = bce.setMetricName('areaUnderROC').evaluate(pred)\nau_prc = bce.setMetricName('areaUnderPR').evaluate(pred)\n\nprint(\"Area under ROC: {}\".format(au_roc))\nprint(\"Area Under PR: {}\".format(au_prc))",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": " 0 1 2 3 4 5 6 7\n0 17 4 <=50K 0.0 [4.0, 17.0] [3.984203061099825, -3.984203061099825] [0.9817326384088789, 0.018267361591121044] 0.0\n1 17 5 <=50K 0.0 [5.0, 17.0] [3.935897389723122, -3.935897389723122] [0.9808458778128771, 0.019154122187122896] 0.0\n2 17 5 <=50K 0.0 [5.0, 17.0] [3.935897389723122, -3.935897389723122] [0.9808458778128771, 0.019154122187122896] 0.0\n3 17 6 <=50K 0.0 [6.0, 17.0] [3.8875917183464184, -3.8875917183464184] [0.9799169513950979, 0.020083048604902023] 0.0\n4 17 6 <=50K 0.0 [6.0, 17.0] [3.8875917183464184, -3.8875917183464184] [0.9799169513950979, 0.020083048604902023] 0.0\n5 17 8 <=50K 0.0 [8.0, 17.0] [3.7909803755930116, -3.7909803755930116] [0.9779248519533819, 0.022075148046618136] 0.0\n6 17 8 <=50K 0.0 [8.0, 17.0] [3.7909803755930116, -3.7909803755930116] [0.9779248519533819, 0.022075148046618136] 0.0\n7 17 9 <=50K 0.0 [9.0, 17.0] [3.7426747042163084, -3.7426747042163084] [0.9768576056063788, 0.02314239439362117] 0.0\n8 17 9 <=50K 0.0 [9.0, 17.0] [3.7426747042163084, -3.7426747042163084] [0.9768576056063788, 0.02314239439362117] 0.0\n9 17 10 <=50K 0.0 [10.0, 17.0] [3.694369032839605, -3.694369032839605] [0.9757400421084974, 0.024259957891502638] 0.0\nArea under ROC: 0.7364507807436806\nArea Under PR: 0.3950675919086818",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Persistence",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "##NOTE: by default the model is saved to and loaded from path\n\nmodel_name = \"AdultCensus.mml\"\nmodel_fs = \"/spark_ml/\" + model_name\n\nmodel.write().overwrite().save(model_fs)\nprint(\"saved model to {}\".format(model_fs))\n\n\n# load the model file (from dbfs)\nmodel2 = PipelineModel.load(model_fs)\nassert str(model2) == str(model)\nprint(\"loaded model from {}\".format(model_fs))",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "saved model to /spark_ml/AdultCensus.mml\nloaded model from /spark_ml/AdultCensus.mml",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# Machine learning with Spark on SQL Server 2019 big data cluster
|
||||
|
||||
The new built-in notebooks in Azure Data Studio enables data scientists and data engineers to run Python, R, or Scala code against the cluster. This is a great way to explore the data and build machine learning models. Notebooks facilitate collaboration between teammates working on a shared data set.
|
||||
|
||||
This sample builds a machine learning model using AdultCensusIncome.csv available [here](https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv).
|
||||
|
||||
|
||||
## Instructions
|
||||
|
||||
In this example, you are going to run sample notebooks that build a machine learning model over a public data set.
|
||||
|
||||
Follow the steps below to get up and running with the sample.
|
||||
|
||||
## Upload the data for analysis
|
||||
|
||||
1. From Azure Data Studio, connect to the SQL Server big data cluster endpoint. Information about how you connect from Azure Data Studio can be found [here](https://docs.microsoft.com/en-us/sql/azure-data-studio/sql-server-2019-extension?view=sql-server-ver15).
|
||||
|
||||
2. Download the data from https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv and save AdultCensusIncome.csv in a folder called spark_ml in HDFS.
|
||||
|
||||
## Run notebook for data preparation
|
||||
As a first step we'll load the data, do some basic cleanup on that data, choose the features that we want to build the machine learning model with. Finally we'll split the data set as training and test sets.
|
||||
|
||||
1. Download and save the notebook file [1-data-prep.ipynb](1-data-prep.ipynb/) locally.
|
||||
|
||||
1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook.
|
||||
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint.
|
||||
|
||||
1. Run each cell in the Notebook sequentially using Azure Data Studio. Expect the first cell to take 20 sec to finish.
|
||||
|
||||
1. The training and test sets created would be stored as /spark_ml/AdultCensusIncomeTrain and /spark_ml/AdultCensusIncomeTest
|
||||
|
||||
## Run notebook to create a machine learning model and use it to predict
|
||||
We'll now create the machine learning model, use the model to predict results on the test set and then save the created model to a file.
|
||||
|
||||
1. Download and save the notebook (ipynb) file [2-build-ml-model.ipynb] (2-build-ml-model.ipynb/)
|
||||
|
||||
1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook.
|
||||
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint.
|
||||
|
||||
1. Run each cell in the Notebook sequentially using Azure Data Studio.
|
||||
|
||||
1. The machine learning model would be persisted as /spark_ml/AdultCensus.mml.
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,10 @@
|
||||
# SQL Server Machine Learning Services on master
|
||||
|
||||
In this example, we are building a machine learning model using R and a logistic regression algorithm for a recommendation engine on an online store. Based on existing users' click pattern online and their interest in other categories and demographics, we are training a machine learning model. This model will then be used to predict if the visitor is interested in a given item category using the T-SQL PREDICT function.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [book-click-prediction-r.sql](book-click-prediction-r.sql/).
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# SQL Server big data clusters
|
||||
|
||||
The new built-in notebooks in Azure Data Studio enables data scientists and data engineers to run Python, R, or Scala code against the cluster.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Download and save the notebook file [spark-sql.ipynb](spark-sql.ipynb/) locally.
|
||||
|
||||
1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook.
|
||||
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint.
|
||||
|
||||
1. Run each cell in the Notebook sequentially using Azure Data Studio.
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pyspark3kernel",
|
||||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Spark sample showing read/write methods\nIn this sample notebook, we will read CSV file from HDFS, write it as parquet file and save a Hive table definition. We will also run some Spark SQL commands using the Hive table.\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Read the CSV into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data/web_clickstreams.csv').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "root\n |-- wcs_click_date_sk: integer (nullable = true)\n |-- wcs_click_time_sk: integer (nullable = true)\n |-- wcs_sales_sk: integer (nullable = true)\n |-- wcs_item_sk: integer (nullable = true)\n |-- wcs_web_page_sk: integer (nullable = true)\n |-- wcs_user_sk: integer (nullable = true)\n\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 36890| 40052| null| 4379| 34| null|\n| 36890| 41285| null| 6245| 34| null|\n| 36890| 23115| null| 13852| 34| null|\n| 36890| 17702| null| 15975| 34| null|\n| 36890| 62676| null| 2119| 34| null|\n| 36890| 34267| null| 10273| 34| null|\n| 36890| 8502| null| 17790| 34| null|\n| 36890| 54340| null| 3453| 34| null|\n| 36890| 54370| null| 6372| 34| null|\n| 36890| 6578| null| 17203| 34| null|\n| 36890| 75088| null| 4891| 34| null|\n| 36890| 23922| null| 11332| 34| null|\n| 36890| 28761| null| 4484| 34| null|\n| 36890| 21444| null| 5582| 34| null|\n| 36890| 58917| null| 8833| 34| null|\n| 36890| 27578| null| 8599| 34| null|\n| 36890| 8059| null| 6720| 34| null|\n| 36890| 43008| null| 17175| 34| null|\n| 36890| 4378| null| 10644| 34| null|\n| 36890| 55403| null| 8139| 34| null|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "hdfs:///user/hive/warehouse",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT * FROM web_clickstreams LIMIT 100\")\r\nsqlDF.show()\r\n\r\nsqlDF = spark.sql(\"SELECT wcs_user_sk, COUNT(*)\\\r\n FROM web_clickstreams\\\r\n WHERE wcs_user_sk IS NOT NULL\\\r\n GROUP BY wcs_user_sk\\\r\n ORDER BY COUNT(*) DESC LIMIT 100\")\r\nsqlDF.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 37506| 7933| null| 1384| 2| 39437|\n| 37506| 56044| null| 14689| 2| 26419|\n| 37506| 52706| null| 8541| 2| 44016|\n| 37506| 67325| null| 16129| 2| 83371|\n| 37506| 84857| null| 1869| 2| 13090|\n| 37506| 49599| null| 2994| 2| 8940|\n| 37506| 78150| null| 11392| 2| 65633|\n| 37506| 38720| null| 14366| 2| 22281|\n| 37506| 79915| null| 11102| 2| 81755|\n| 37506| 67253| null| 5380| 2| 46868|\n| 37506| 6507| null| 6813| 2| 49363|\n| 37506| 18280| null| 1458| 2| 49363|\n| 37506| 72258| null| 2869| 2| 67756|\n| 37506| 8045| null| 615| 2| 86035|\n| 37506| 86164| null| 7000| 2| 94821|\n| 37506| 29724| null| 2767| 2| 94821|\n| 37506| 55471| null| 3584| 2| 62792|\n| 37506| 677| null| 1720| 2| 27212|\n| 37506| 66638| null| 9898| 2| 20370|\n| 37506| 48515| null| 9394| 2| 17157|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user