mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Merge branch 'master' of https://github.com/Microsoft/sql-server-samples
This commit is contained in:
Jovan Popovic
@@ -3,8 +3,8 @@ GEM
|
||||
specs:
|
||||
addressable (2.4.0)
|
||||
colorator (0.1)
|
||||
ffi (1.9.14-x64-mingw32)
|
||||
jekyll (3.1.6)
|
||||
ffi (1.9.24-x64-mingw32)
|
||||
jekyll (3.6.3)
|
||||
colorator (~> 0.1)
|
||||
jekyll-sass-converter (~> 1.0)
|
||||
jekyll-watch (~> 1.1)
|
||||
|
||||
+1
@@ -172,6 +172,7 @@ EXEC DataLoadSimulation.DailyProcessToCreateHistory
|
||||
GO
|
||||
|
||||
:r .\pds400-ins-unkown-orderline.sql
|
||||
:r .\pds410-update-archive-tables.sql
|
||||
|
||||
/*
|
||||
There is one other stored procedure you may find useful:
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
-- NOTE: This script should be moved to MakeTemporalChanges procedure, but currently it doesn't work there.
|
||||
-- jovanpop creating a separate file here.
|
||||
-- @TODO: Investigate how to move it there.
|
||||
|
||||
PRINT N'Updating StockItems history...'
|
||||
GO
|
||||
EXEC DataloadSimulation.DeactivatetemporalTablesBeforeDataLoad;
|
||||
GO
|
||||
UPDATE Warehouse.StockItems_Archive
|
||||
SET UnitPrice = s.UnitPrice * (1 - .05 *(DATEDIFF(DAY, sa.ValidFrom, GETDATE())/365 )),
|
||||
RecommendedRetailPrice = s.RecommendedRetailPrice * (1 - .03 *(DATEDIFF(DAY, sa.ValidFrom, GETDATE())/365 )),
|
||||
TaxRate = s.TaxRate * (1 + .02 *(DATEDIFF(DAY, sa.ValidFrom, GETDATE())/365 )),
|
||||
QuantityPerOuter = CEILING(s.QuantityPerOuter * (1 + .05 *(DATEDIFF(DAY, sa.ValidFrom, GETDATE())/365 ))),
|
||||
LeadTimeDays = CEILING(s.LeadTimeDays * (1 + .03 *(DATEDIFF(DAY, sa.ValidFrom, GETDATE())/365 ))),
|
||||
TypicalWeightPerUnit = CEILING(s.TypicalWeightPerUnit * (1 + .02 *(DATEDIFF(DAY, sa.ValidFrom, GETDATE())/365 )))
|
||||
FROM Warehouse.StockItems_Archive sa
|
||||
JOIN Warehouse.StockItems s
|
||||
ON sa.StockItemID = s.StockItemID;
|
||||
GO
|
||||
EXEC DataloadSimulation.ReActivatetemporalTablesAfterDataLoad;
|
||||
GO
|
||||
@@ -844,5 +844,6 @@
|
||||
<None Include="PostDeploymentScripts\pds105-ins-dls-ficticiousnamepool.sql" />
|
||||
<None Include="PostDeploymentScripts\pds106-ins-dls-areacode.sql" />
|
||||
<None Include="PostDeploymentScripts\pds400-ins-unkown-orderline.sql" />
|
||||
<None Include="PostDeploymentScripts\pds410-update-archive-tables.sql" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,44 @@
|
||||
# SQL Server big data clusters
|
||||
|
||||
## Pre-requisites
|
||||
1. Kubernetes cluster configuration & Kubectl command-line utility
|
||||
2. Curl utility
|
||||
3. Sqlcmd utility
|
||||
4. Bcp utility
|
||||
5. Azure Data Studio or SQL Server Management Studio
|
||||
6. SQL Server 2019 big data cluster
|
||||
|
||||
Installation instructions for SQL Server 2019 big data cluster can be found [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sql-server-2017).
|
||||
|
||||
## Samples Setup
|
||||
|
||||
**Before you begin**, download the sample database [backup file](https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak) and save it locally. Run the CMD script called *bootstrap-sample-db.cmd* or the shell script *bootstrap-sample-db.sh* depending on your platform. This script will restore the database on the SQL Master instance, execute the *bootstrap-sample-db.sql* script, create the database objects needed, export the web_clickstreams & inventory tables to CSV file, and upload the web_clickstreams CSV file to HDFS inside the SQL Server 2019 big data cluster.
|
||||
|
||||
__[data-pool](data-pool/)__
|
||||
|
||||
### Data ingestion using Spark
|
||||
Connect to the master instance in your SQL Server big data cluster and the SQL Server big data cluster endpoint, and follow the steps in *data-pool/data-ingestion-spark.sql*.
|
||||
|
||||
### Data ingestion using sql
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *data-pool/data-ingestion-sql.sql*.
|
||||
|
||||
__[data-virtualization](data-virtualization/)__
|
||||
|
||||
### External table over HDFS
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *data-virtualization/external-table-hdfs.sql*.
|
||||
|
||||
### External table over Oracle
|
||||
To execute this sample script, you will need following:
|
||||
1. Oracle instance and credentials
|
||||
1. Create inventory table in Oracle using [data-virtualization/inventory-oracle.sql](data-virtualization/inventory-oracle.sql/) script
|
||||
1. Import the inventory.csv file generated by the bootstrap-sample-db script to a table in Oracle
|
||||
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *data-virtualization/external-table-oracle.sql*.
|
||||
|
||||
__[machine-learning](machine-learning/)__
|
||||
|
||||
### SQL Server ML Services on master instance
|
||||
Connect to the master instance in your SQL Server big data cluster and execute the steps in *machine-learning/sql/book-category-r-ml.sql*.
|
||||
|
||||
### Spark ML
|
||||
Connect to the SQL Server big data cluster endpoint, and run the notebook files *machine-learning/spark/1-data-prep.ipynb* and *machine-learning/spark/2-build-ml-model.ipynb* cell by cell.
|
||||
@@ -0,0 +1,61 @@
|
||||
@echo off
|
||||
REM CLICKSTREAM FILES
|
||||
setlocal enableextensions
|
||||
set CLUSTER_NAMESPACE=%1
|
||||
set SQL_MASTER_IP=%2
|
||||
set SQL_MASTER_SA_PASSWORD=%3
|
||||
set BACKUP_FILE_PATH=%~4
|
||||
set KNOX_IP=%5
|
||||
set KNOX_PASSWORD=%6
|
||||
set STARTUP_PATH=%~dp0
|
||||
|
||||
if NOT DEFINED CLUSTER_NAMESPACE goto :usage
|
||||
if NOT DEFINED SQL_MASTER_IP goto :usage
|
||||
if NOT DEFINED SQL_MASTER_SA_PASSWORD goto :usage
|
||||
if NOT DEFINED BACKUP_FILE_PATH goto :usage
|
||||
if NOT DEFINED KNOX_IP goto :usage
|
||||
if NOT DEFINED KNOX_PASSWORD set KNOX_PASSWORD=%SQL_MASTER_SA_PASSWORD%
|
||||
|
||||
set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,31433
|
||||
set KNOX_ENDPOINT=%KNOX_IP%:30443
|
||||
|
||||
echo Verifying sqlcmd.exe is in path & CALL WHERE /Q sqlcmd.exe || GOTO exit
|
||||
echo Verifying bcp.exe is in path & CALL WHERE /Q bcp.exe || GOTO exit
|
||||
echo Verifying kubectl.exe is in path & CALL WHERE /Q kubectl.exe || echo HINT: Install the kubernetes-cli - https://kubernetes.io/docs/tasks/tools/install-kubectl && GOTO exit
|
||||
echo Verifying curl.exe is in path & CALL WHERE /Q curl.exe || echo HINT: Install curl - https://curl.haxx.se/download.html && GOTO exit
|
||||
|
||||
REM Copy the backup file, restore the database, create necessary objects and data file
|
||||
echo Copying database backup file...
|
||||
pushd "%BACKUP_FILE_PATH%"
|
||||
%DEBUG% kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
|
||||
popd
|
||||
|
||||
echo Configuring sample database...
|
||||
%DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "%STARTUP_PATH%bootstrap.out" -I -b || goto exit
|
||||
|
||||
for %%F in (web_clickstreams inventory) do (
|
||||
echo Exporting %%F data...
|
||||
%DEBUG% bcp sales.dbo.%%F out "%STARTUP_PATH%%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "%STARTUP_PATH%%%F.out" -e "%STARTUP_PATH%%%F.err" || goto exit
|
||||
)
|
||||
|
||||
REM Copy the data file to HDFS
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
pushd "%STARTUP_PATH%"
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
|
||||
|
||||
:: del /q *.out *.err *.csv
|
||||
popd
|
||||
|
||||
endlocal
|
||||
exit /b 0
|
||||
goto :eof
|
||||
|
||||
:exit
|
||||
echo Bootstrap of the sample database failed.
|
||||
exit /b %ERRORLEVEL%
|
||||
|
||||
:usage
|
||||
echo USAGE: %0 ^<CLUSTER_NAMESPACE^> ^<SQL_MASTER_IP^> ^<SQL_MASTER_SA_PASSWORD^> ^<BACKUP_FILE_PATH^> ^<KNOX_IP^> [^<KNOX_PASSWORD^>]
|
||||
echo Default ports are assumed for SQL Master instance ^& Knox gateway.
|
||||
exit /b 0
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -o pipefail
|
||||
USAGE_MESSAGE="USAGE: $0 <CLUSTER_NAMESPACE> <SQL_MASTER_IP> <SQL_MASTER_SA_PASSWORD> <BACKUP_FILE_PATH> <KNOX_IP> [<KNOX_PASSWORD>]"
|
||||
ERROR_MESSAGE="Bootstrap of the sample database failed."
|
||||
|
||||
# Print usage if mandatory parameters are missing
|
||||
: "${1:?$USAGE_MESSAGE}"
|
||||
: "${2:?$USAGE_MESSAGE}"
|
||||
: "${3:?$USAGE_MESSAGE}"
|
||||
: "${4:?$USAGE_MESSAGE}"
|
||||
: "${5:?$USAGE_MESSAGE}"
|
||||
: "${DEBUG=}"
|
||||
|
||||
# Save the input parameters
|
||||
CLUSTER_NAMESPACE=$1
|
||||
SQL_MASTER_IP=$2
|
||||
SQL_MASTER_SA_PASSWORD=$3
|
||||
BACKUP_FILE_PATH=$4
|
||||
KNOX_IP=$5
|
||||
KNOX_PASSWORD=$6
|
||||
# If Knox password is not supplied then default to SQL Master password
|
||||
KNOX_PASSWORD=${KNOX_PASSWORD:=$SQL_MASTER_SA_PASSWORD}
|
||||
|
||||
SQL_MASTER_INSTANCE=$SQL_MASTER_IP,31433
|
||||
KNOX_ENDPOINT=$KNOX_IP:30443
|
||||
|
||||
# Copy the backup file, restore the database, create necessary objects and data file
|
||||
echo Copying database backup file...
|
||||
pushd "$BACKUP_FILE_PATH"
|
||||
$DEBUG kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n $CLUSTER_NAMESPACE || (echo $ERROR_MESSAGE && exit 1)
|
||||
popd
|
||||
|
||||
echo Configuring sample database...
|
||||
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE"
|
||||
$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -i "bootstrap-sample-db.sql" -o "bootstrap.out" -I -b || (echo $ERROR_MESSAGE && exit 2)
|
||||
|
||||
for table in web_clickstreams inventory
|
||||
do
|
||||
echo Exporting $table data...
|
||||
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe"
|
||||
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -o "$table.out" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3)
|
||||
done
|
||||
|
||||
# Copy the data file to HDFS
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
|
||||
|
||||
# rm -f *.out *.err *.csv
|
||||
exit
|
||||
@@ -0,0 +1,74 @@
|
||||
USE master;
|
||||
GO
|
||||
-- Enable external scripts execution for R/Python/Java:
|
||||
exec sp_configure 'external scripts enabled', 1;
|
||||
RECONFIGURE WITH OVERRIDE;
|
||||
GO
|
||||
|
||||
IF DB_ID('sales') IS NULL
|
||||
RESTORE DATABASE sales
|
||||
FROM DISK=N'/var/opt/mssql/data/tpcxbb_1gb.bak'
|
||||
WITH
|
||||
MOVE N'tpcxbb_1gb' TO N'/var/opt/mssql/data/sales.mdf',
|
||||
MOVE N'tpcxbb_1gb_log' TO N'/var/opt/mssql/data/sales.ldf';
|
||||
GO
|
||||
|
||||
USE sales;
|
||||
GO
|
||||
-- Create default data sources for SQL Big Data Cluster
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool')
|
||||
CREATE EXTERNAL DATA SOURCE SqlDataPool
|
||||
WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default');
|
||||
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
|
||||
CREATE EXTERNAL DATA SOURCE SqlStoragePool
|
||||
WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080');
|
||||
GO
|
||||
|
||||
-- Create view used for ML services training stored procedure
|
||||
CREATE OR ALTER VIEW [dbo].[web_clickstreams_book_clicks]
|
||||
AS
|
||||
SELECT
|
||||
q.clicks_in_category,
|
||||
CASE WHEN cd.cd_education_status IN ('Advanced Degree', 'College', '4 yr Degree', '2 yr Degree') THEN 1 ELSE 0 END AS college_education,
|
||||
CASE WHEN cd.cd_gender = 'M' THEN 1 ELSE 0 END AS male,
|
||||
q.clicks_in_1,
|
||||
q.clicks_in_2,
|
||||
q.clicks_in_3,
|
||||
q.clicks_in_4,
|
||||
q.clicks_in_5,
|
||||
q.clicks_in_6,
|
||||
q.clicks_in_7,
|
||||
q.clicks_in_8,
|
||||
q.clicks_in_9
|
||||
FROM(
|
||||
SELECT
|
||||
w.wcs_user_sk,
|
||||
SUM( CASE WHEN i.i_category = 'Books' THEN 1 ELSE 0 END) AS clicks_in_category,
|
||||
SUM( CASE WHEN i.i_category_id = 1 THEN 1 ELSE 0 END) AS clicks_in_1,
|
||||
SUM( CASE WHEN i.i_category_id = 2 THEN 1 ELSE 0 END) AS clicks_in_2,
|
||||
SUM( CASE WHEN i.i_category_id = 3 THEN 1 ELSE 0 END) AS clicks_in_3,
|
||||
SUM( CASE WHEN i.i_category_id = 4 THEN 1 ELSE 0 END) AS clicks_in_4,
|
||||
SUM( CASE WHEN i.i_category_id = 5 THEN 1 ELSE 0 END) AS clicks_in_5,
|
||||
SUM( CASE WHEN i.i_category_id = 6 THEN 1 ELSE 0 END) AS clicks_in_6,
|
||||
SUM( CASE WHEN i.i_category_id = 7 THEN 1 ELSE 0 END) AS clicks_in_7,
|
||||
SUM( CASE WHEN i.i_category_id = 8 THEN 1 ELSE 0 END) AS clicks_in_8,
|
||||
SUM( CASE WHEN i.i_category_id = 9 THEN 1 ELSE 0 END) AS clicks_in_9
|
||||
FROM web_clickstreams as w
|
||||
INNER JOIN item as i ON (w.wcs_item_sk = i_item_sk
|
||||
AND w.wcs_user_sk IS NOT NULL)
|
||||
GROUP BY w.wcs_user_sk
|
||||
) AS q
|
||||
INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk
|
||||
INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk;
|
||||
GO
|
||||
|
||||
-- Create table for storing the machine learning models
|
||||
CREATE TABLE sales_models (
|
||||
model_name varchar(100) NOT NULL PRIMARY KEY,
|
||||
model varbinary(max) NOT NULL,
|
||||
model_native varbinary(max) NOT NULL,
|
||||
created_by nvarchar(300) NOT NULL DEFAULT(SYSTEM_USER),
|
||||
create_time datetime2 NOT NULL DEFAULT(SYSDATETIME())
|
||||
);
|
||||
GO
|
||||
@@ -0,0 +1,62 @@
|
||||
# Data pools in SQL Server 2019 big data cluster
|
||||
|
||||
SQL Server Big Data clusters provide scale-out compute and storage to improve the performance of analyzing any data. Data from a variety of sources can be ingested and distributed across data pool instances for analysis. In this example, we will insert data from a SQL query into an external table stored in a data pool and query it.
|
||||
|
||||
## Data ingestion using SQL stored procedure
|
||||
|
||||
SQL Server Big Data clusters provide scale-out compute and storage to improve the performance of analyzing any data. Data from a variety of sources can be ingested and distributed across data pool instances for analysis. In this example, we will insert data from a SQL query into an external table stored in a data pool and query it.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the .sql script [data-ingestion-sql.sql](data-ingestion-sql.sql).
|
||||
|
||||
## Data ingestion using Spark streaming
|
||||
|
||||
In this example, you are going to use Spark to read and transform data from HDFS and cache it in a data pool. Querying the external table created over this aggregated data stored in data pools will be much more efficient than going to the raw data always.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Using Azure Data Studio, connect to the HDFS/Spark gateway (SQL Server big data cluster connection type).
|
||||
|
||||
1. Connect to SQL Server Master instance using Azure Data Studio.
|
||||
|
||||
1. Execute the SQL script [data-ingestion-spark.sql](data-ingestion-spark.sql).
|
||||
|
||||
1. Create and submit a Spark job that ingests data from HDFS into the external table.
|
||||
|
||||
Submitting a Spark job will start a Spark streaming session using spark-submit.
|
||||
|
||||
The arguments to the jar file are:
|
||||
|
||||
1. server name - sql server to connect to read the table schema
|
||||
2. port number
|
||||
3. username - sql server username for master instance
|
||||
4. password - sql server password for master instance
|
||||
5. database name
|
||||
6. external table name
|
||||
7. Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
||||
8. Input format. This can be "csv", "parquet", "json".
|
||||
9. enable checkpoint: true or false
|
||||
|
||||
Submit a Spark job with the below parameters. You can use the Spark submit experience from Azure Data Studio (right click on big data cluster endpoint -> Submit Spark Job):
|
||||
|
||||
ARGUMENTS:
|
||||
|
||||
**job name:** yourJobName
|
||||
|
||||
**switch** from "Local" to "HDFS"
|
||||
|
||||
**Path to jar** (copy/paste this):
|
||||
|
||||
/jar/mssql-spark-lib-assembly-1.0.jar
|
||||
|
||||
**Main class:**
|
||||
FileStreaming
|
||||
|
||||
**Parameters (copy/paste this; make sure you replace the password!):**
|
||||
|
||||
mssql-master-pool-0.service-master-pool 1433 sa passwordHere sales web_clickstreams_spark_results hdfs:///clickstream_data csv false
|
||||
|
||||
6. Query the external table we created earlier using the SELECT queries in the script to see data coming from the streaming job and landing in the table.
|
||||
@@ -0,0 +1,54 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create external table in a data pool in SQL Server 2019 big data cluster.
|
||||
-- The SqlDataPool data source is a special data source that is available in
|
||||
-- any new database in SQL Master instance. This is used to reference the
|
||||
-- data pool in a SQL Server 2019 big data cluster.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_spark_results]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlDataPool,
|
||||
DISTRIBUTION = ROUND_ROBIN
|
||||
);
|
||||
|
||||
-- Data can be ingested into the external table from a spark job.
|
||||
--
|
||||
-- Submit spark job with below parameters. You can use the Spark submit experience from Azure Data Studio.
|
||||
-- Right click on server name in a SQL Server big data cluster connection and click "Submit Spark Job".
|
||||
--
|
||||
-- Specify following values in the Job submission dialog box:
|
||||
---- job name: <yourJobName>
|
||||
---- switch from "Local" to "HDFS"
|
||||
---- Main class: "FileStreaming"
|
||||
---- Path to jar: /jar/mssql-spark-lib-assembly-1.0.jar
|
||||
---- Arguments:
|
||||
---- mssql-master-pool-0.service-master-pool 1433 sa %PASSWORD% sales web_clickstreams_spark_results hdfs:///clickstream_data csv false
|
||||
|
||||
-- The arguments to jar file are
|
||||
-- 1: server name - sql server to connect to read the table schema
|
||||
-- 2: port number
|
||||
-- 3: username - sql server username for master instance
|
||||
-- 4: password - sql server password for master instance
|
||||
-- 5: database name
|
||||
-- 6: external table name
|
||||
-- 7: Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
||||
-- 8: Input format. This can be "csv", "parquet", "json".
|
||||
-- 9: enable checkpoint: true or false
|
||||
--
|
||||
|
||||
-- After the Spark streaming job has been sucessfully submitted, you can run below query to view the results.
|
||||
--
|
||||
-- Wait until some rows are available.
|
||||
WHILE (1=1)
|
||||
IF EXISTS(SELECT * FROM [web_clickstreams_spark_results])
|
||||
BREAK;
|
||||
|
||||
SELECT count(*) FROM [web_clickstreams_spark_results];
|
||||
SELECT TOP 10 * FROM [web_clickstreams_spark_results];
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_spark_results];
|
||||
GO
|
||||
@@ -0,0 +1,58 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create external table in a data pool in SQL Server 2019 big data cluster.
|
||||
-- The SqlDataPool data source is a special data source that is available in
|
||||
-- any new database in SQL Master instance. This is used to reference the
|
||||
-- data pool in a SQL Server 2019 big data cluster.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_dp]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlDataPool,
|
||||
DISTRIBUTION = ROUND_ROBIN
|
||||
);
|
||||
GO
|
||||
-- Currently the create external table operation is asynchronous and there is no
|
||||
-- way to determine completion of the operation. To prevent failures of the insert
|
||||
-- into the external table, wait for few minutes.
|
||||
WAITFOR DELAY '00:02:00';
|
||||
GO
|
||||
-- Insert results of a SELECT statement into the external table created on the data pool
|
||||
--
|
||||
DECLARE @db_name SYSNAME = 'sales'
|
||||
DECLARE @schema_name SYSNAME = 'dbo'
|
||||
DECLARE @table_name SYSNAME = 'web_clickstreams_dp'
|
||||
DECLARE @query SYSNAME = 'SELECT TOP(1000) * FROM sales.dbo.web_clickstreams WHERE wcs_user_sk IS NOT NULL'
|
||||
|
||||
EXEC model..sp_data_pool_table_insert_data @db_name, @schema_name, @table_name, @query
|
||||
GO
|
||||
|
||||
-- Query data inserted from sp_data_pool_table_insert_data
|
||||
--
|
||||
SELECT count(*) FROM [dbo].[web_clickstreams_dp]
|
||||
SELECT TOP 10 * FROM [dbo].[web_clickstreams_dp]
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT TOP (100)
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[web_clickstreams_dp]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_dp];
|
||||
GO
|
||||
@@ -0,0 +1,27 @@
|
||||
# Data virtualization in SQL Server 2019 big data cluster
|
||||
|
||||
In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes. SQL Server 2019 introduces new ODBC connectors to data sources like SQL Server, Oracle, MongoDB and Teradata.
|
||||
|
||||
## Query data in HDFS from SQL Server master
|
||||
|
||||
In this example, you are going to create an external table in the SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster. Then you will join the data in the external table with high value data in SQL Master instance.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the [external-table-hdfs-csv.sql](external-table-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
|
||||
1. Before you use execute the *external-table-hdfs-parquet.sql* script, make sure you run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file. Execute the [external-table-hdfs-parquet.sql](external-table-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS.
|
||||
|
||||
## Query data in Oracle from SQL Server master
|
||||
|
||||
In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server.
|
||||
|
||||
**Before you begin**, you need to have an Oracle instance and credentials. Execute the SQL script [inventory-ora.sql](inventory-ora.sql/) in Oracle to create the table and import the "inventory.csv" file created by the bootstrap sample database.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [external-table-oracle.sql](external-table-oracle.sql/).
|
||||
@@ -0,0 +1,52 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create file format for CSV file with appropriate properties.
|
||||
--
|
||||
CREATE EXTERNAL FILE FORMAT csv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = ',',
|
||||
STRING_DELIMITER = '"',
|
||||
FIRST_ROW = 2,
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
-- is a special data source that is available in any new database in
|
||||
-- SQL Master instance.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_hdfs_csv]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/clickstream_data',
|
||||
FILE_FORMAT = csv_file
|
||||
);
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[web_clickstreams_hdfs_csv]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_csv];
|
||||
GO
|
||||
+47
-41
@@ -1,41 +1,47 @@
|
||||
--
|
||||
-- BEFORE RUNNING THIS SCRIPT, UPDATE IT WITH YOUR OWN STRING FOR THE EXTERNAL TABLE NAME (search for "<yourTableName>")
|
||||
--
|
||||
|
||||
PRINT 'STEP 1: Connect to SQL Server Master instance'
|
||||
|
||||
USE sales
|
||||
GO
|
||||
|
||||
PRINT 'STEP 2: Create external table over CSV file'
|
||||
CREATE EXTERNAL TABLE [<yourTableName>]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/clickstream_data',
|
||||
FILE_FORMAT = csv_file
|
||||
);
|
||||
|
||||
-- Join external table with local tables
|
||||
-- Execution time: ~10 secs
|
||||
PRINT 'STEP 3: Join external table with high value data in SQL Master'
|
||||
SELECT TOP (100)
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[<yourTableName>]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
|
||||
PRINT 'STEP 4: Cleanup...drop external table'
|
||||
DROP EXTERNAL TABLE [dbo].[<yourTableName>]
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create file format for parquet file with appropriate properties.
|
||||
--
|
||||
CREATE EXTERNAL FILE FORMAT parquet_file
|
||||
WITH (
|
||||
FORMAT_TYPE = PARQUET
|
||||
);
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
-- is a special data source that is available in any new database in
|
||||
-- SQL Master instance.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [web_clickstreams_hdfs_parquet]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/user/hive/warehouse/web_clickstreams',
|
||||
FILE_FORMAT = parquet_file
|
||||
);
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[web_clickstreams_hdfs_parquet]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_parquet];
|
||||
GO
|
||||
@@ -0,0 +1,44 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create database scoped credential to connect to Oracle server
|
||||
-- Provide appropriate credentials to Oracle server in below statement.
|
||||
-- If you are using SQL Server Management Studio then you can replace the parameters using
|
||||
-- the Query menu, and "Specify Values for Template Parameters" option.
|
||||
CREATE DATABASE SCOPED CREDENTIAL [OracleCredential]
|
||||
WITH IDENTITY = '<oracle_user,nvarchar(100),SYSTEM>', SECRET = '<oracle_user_password,nvarchar(100),manager>';
|
||||
|
||||
-- Create external data source that points to Oracle server
|
||||
--
|
||||
CREATE EXTERNAL DATA SOURCE [OracleSalesSrvr]
|
||||
WITH (LOCATION = 'oracle://<oracle_server,nvarchar(100)>',CREDENTIAL = [OracleCredential]);
|
||||
|
||||
-- Create external table over inventory table on Oracle server
|
||||
-- NOTE: Table names and column names will use ANSI SQL quoted identifier while querying against Oracle.
|
||||
-- As a result, the names are case-sensitive so specify the name in the external table definition
|
||||
-- that matches the exact case of the table and column names in the Oracle metadata.
|
||||
CREATE EXTERNAL TABLE [inventory_ora]
|
||||
([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL,
|
||||
[inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0))
|
||||
WITH (DATA_SOURCE=[OracleSalesSrvr],
|
||||
LOCATION='<oracle_service_name,nvarchar(30),xe>.<oracle_schema,nvarchar(128),HR>.<oracle_table,nvarchar(128),INVENTORY>');
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity
|
||||
FROM [inventory_ora] as i
|
||||
JOIN item as it
|
||||
ON it.i_item_sk = i.inv_item
|
||||
JOIN warehouse as w
|
||||
ON w.w_warehouse_sk = i.inv_warehouse
|
||||
WHERE it.i_category = 'Books' and i.inv_item BETWEEN 1 and 18000 --> get items within specific range
|
||||
GROUP BY w.w_warehouse_name, i.inv_item;
|
||||
GO
|
||||
|
||||
-- Cleanup
|
||||
--
|
||||
DROP EXTERNAL TABLE [inventory_ora];
|
||||
DROP EXTERNAL DATA SOURCE [OracleSalesSrvr] ;
|
||||
DROP DATABASE SCOPED CREDENTIAL [OracleCredential];
|
||||
GO
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Inventory table over which the SQL Server external table will be defined
|
||||
CREATE TABLE "INVENTORY"
|
||||
(
|
||||
"INV_DATE" NUMBER(10,0) NOT NULL,
|
||||
"INV_ITEM" NUMBER(10,0) NOT NULL,
|
||||
"INV_WAREHOUSE" NUMBER(10,0) NOT NULL,
|
||||
"INV_QUANTITY_ON_HAND" NUMBER(10,0)
|
||||
);
|
||||
|
||||
CREATE INDEX INV_ITEM ON HR.INVENTORY(INV_ITEM);
|
||||
@@ -0,0 +1,56 @@
|
||||
# Machine learning in SQL Server 2019 big data cluster
|
||||
|
||||
## SQL Server Machine Learning Services on SQL Master instance
|
||||
|
||||
In this example, we are building a machine learning model using R and a logistic regression algorithm for a recommendation engine on an online store. Based on existing users' click pattern online and their interest in other categories and demographics, we are training a machine learning model. This model will then be used to predict if the visitor is interested in a given item category using the T-SQL PREDICT function.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [sql/book-click-prediction-r.sql](sql/book-click-prediction-r.sql/).
|
||||
|
||||
## Machine learning using Spark
|
||||
|
||||
The new built-in notebooks in Azure Data Studio enables data scientists and data engineers to run Python, R, or Scala code against the cluster. This is a great way to explore the data and build machine learning models. Notebooks facilitate collaboration between teammates working on a shared data set.
|
||||
|
||||
This sample builds a machine learning model using AdultCensusIncome.csv available [here](https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv).
|
||||
|
||||
|
||||
### Instructions
|
||||
|
||||
In this example, you are going to run sample notebooks that build a machine learning model over a public data set.
|
||||
|
||||
Follow the steps below to get up and running with the sample.
|
||||
|
||||
#### Upload the data for analysis
|
||||
|
||||
1. From Azure Data Studio, connect to the SQL Server big data cluster endpoint. Information about how you connect from Azure Data Studio can be found [here](https://docs.microsoft.com/en-us/sql/azure-data-studio/sql-server-2019-extension?view=sql-server-ver15).
|
||||
|
||||
2. Download the data from https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv and save AdultCensusIncome.csv in a folder called spark_ml in HDFS.
|
||||
|
||||
#### Run notebook for data preparation
|
||||
As a first step we'll load the data, do some basic cleanup on that data, choose the features that we want to build the machine learning model with. Finally we'll split the data set as training and test sets.
|
||||
|
||||
1. Download and save the notebook file [spark/1-data-prep.ipynb](spark/1-data-prep.ipynb/) locally.
|
||||
|
||||
1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook.
|
||||
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint.
|
||||
|
||||
1. Run each cell in the Notebook sequentially using Azure Data Studio. Expect the first cell to take 20 sec to finish.
|
||||
|
||||
1. The training and test sets created would be stored as /spark_ml/AdultCensusIncomeTrain and /spark_ml/AdultCensusIncomeTest
|
||||
|
||||
#### Run notebook to create a machine learning model and use it to predict
|
||||
We'll now create the machine learning model, use the model to predict results on the test set and then save the created model to a file.
|
||||
|
||||
1. Download and save the notebook (ipynb) file [spark\2-build-ml-model.ipynb](spark/2-build-ml-model.ipynb/)
|
||||
|
||||
1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook.
|
||||
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint.
|
||||
|
||||
1. Run each cell in the Notebook sequentially using Azure Data Studio.
|
||||
|
||||
1. The machine learning model would be persisted as /spark_ml/AdultCensus.mml.
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pyspark3kernel",
|
||||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Load data to a dataframe\n- Download AdultCensusIncome.csv from [here](https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv).\n- Create a diretory named /spark_ml. Upload AdultCensusIncome.csv to /spark_ml",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "import os\nimport pandas as pd\n\ndatafile = \"/spark_ml/AdultCensusIncome.csv\"\n\n# Read and Load data\n# Create a Spark dataframe out of the csv file.\ndata_all = spark.read.format('csv').options(header='true', inferSchema='true', ignoreLeadingWhiteSpace='true', ignoreTrailingWhiteSpace='true').load(datafile)\nprint(\"({}, {})\".format(data_all.count(), len(data_all.columns)))\n\n#Replace \"-\" with \"_\" in column names\ncolumns_new = [col.replace(\"-\", \"_\") for col in data_all.columns]\ndata_all = data_all.toDF(*columns_new)\ndata_all.printSchema() #human-readable format\n\ndf = pd.DataFrame(data_all.take(10))\nprint(df.to_string())\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "(32561, 15)\nroot\n |-- age: integer (nullable = true)\n |-- workclass: string (nullable = true)\n |-- fnlwgt: integer (nullable = true)\n |-- education: string (nullable = true)\n |-- education_num: integer (nullable = true)\n |-- marital_status: string (nullable = true)\n |-- occupation: string (nullable = true)\n |-- relationship: string (nullable = true)\n |-- race: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- capital_gain: integer (nullable = true)\n |-- capital_loss: integer (nullable = true)\n |-- hours_per_week: integer (nullable = true)\n |-- native_country: string (nullable = true)\n |-- income: string (nullable = true)\n\n 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14\n0 39 State-gov 77516 Bachelors 13 Never-married Adm-clerical Not-in-family White Male 2174 0 40 United-States <=50K\n1 50 Self-emp-not-inc 83311 Bachelors 13 Married-civ-spouse Exec-managerial Husband White Male 0 0 13 United-States <=50K\n2 38 Private 215646 HS-grad 9 Divorced Handlers-cleaners Not-in-family White Male 0 0 40 United-States <=50K\n3 53 Private 234721 11th 7 Married-civ-spouse Handlers-cleaners Husband Black Male 0 0 40 United-States <=50K\n4 28 Private 338409 Bachelors 13 Married-civ-spouse Prof-specialty Wife Black Female 0 0 40 Cuba <=50K\n5 37 Private 284582 Masters 14 Married-civ-spouse Exec-managerial Wife White Female 0 0 40 United-States <=50K\n6 49 Private 160187 9th 5 Married-spouse-absent Other-service Not-in-family Black Female 0 0 16 Jamaica <=50K\n7 52 Self-emp-not-inc 209642 HS-grad 9 Married-civ-spouse Exec-managerial Husband White Male 0 0 45 United-States >50K\n8 31 Private 45781 Masters 14 Never-married Prof-specialty Not-in-family White Female 14084 0 50 United-States >50K\n9 42 Private 159449 Bachelors 13 Married-civ-spouse Exec-managerial Husband White Male 5178 0 40 United-States >50K",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Data Preparation\n- Choose the feature that we want to use to build the model\n- Split the data set as training and test\n- Write traning and data set as AdultCensusIncomeTrain and AdultCensusIncomeTest to /spark_ml directory\n\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Choose feature columns and the label column.\nlabel = \"income\"\nxvars = [\"age\", \"hours_per_week\"] #all numeric\n\nprint(\"label = {}\".format(label))\nprint(\"features = {}\".format(xvars))\n\nselect_cols = xvars\nselect_cols.append(label)\ndata = data_all.select(select_cols)\n\n# Split data into train and test.\ntrain, test = data.randomSplit([0.75, 0.25], seed=123)\n\nprint(\"train ({}, {})\".format(train.count(), len(train.columns)))\nprint(\"test ({}, {})\".format(test.count(), len(test.columns)))\n\n\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "label = income\nfeatures = ['age', 'hours_per_week']\ntrain (24469, 3)\ntest (8092, 3)",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Data Persistence\n- Save the training and test set as ORC data for persistance\n- Will use the persisted data to build model\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Write the train and test data sets to intermediate storage\n# Write the train and test data sets to intermediate storage\ntrain_data_path = \"/spark_ml/AdultCensusIncomeTrain\"\ntest_data_path = \"/spark_ml/AdultCensusIncomeTest\"\n\ntrain.write.mode('overwrite').orc(train_data_path)\ntest.write.mode('overwrite').orc(test_data_path)\nprint(\"train and test datasets saved to {} and {}\".format(train_data_path, test_data_path))",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "train and test datasets saved to /spark_ml/AdultCensusIncomeTrain and /spark_ml/AdultCensusIncomeTest",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pyspark3kernel",
|
||||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Building - Import the training and test data\r\n\r\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "import os\nimport pprint\nimport numpy as np\nimport os\nimport pprint\nimport numpy as np\nimport pandas as pd\n\nfrom pyspark.ml import Pipeline, PipelineModel\nfrom pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler\nfrom pyspark.ml.classification import LogisticRegression\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator\nfrom pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n\ntrain_data_path = \"/spark_ml/AdultCensusIncomeTrain\"\ntest_data_path = \"/spark_ml/AdultCensusIncomeTest\"\n\ntrain = spark.read.orc(train_data_path)\ntest = spark.read.orc(test_data_path)\n\nprint(\"train: ({}, {})\".format(train.count(), len(train.columns)))\nprint(\"test: ({}, {})\".format(test.count(), len(test.columns)))\n\ntrain.printSchema()\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "train: (24469, 3)\ntest: (8092, 3)\nroot\n |-- age: integer (nullable = true)\n |-- hours_per_week: integer (nullable = true)\n |-- income: string (nullable = true)",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model building - Encode features and Build Model",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "label = \"income\"\nreg = 0.1\nprint(\"Regularization Rate is {}.\".format(reg))\n\n# create a new Logistic Regression model.\nlr = LogisticRegression(regParam=reg)\n\ndtypes = dict(train.dtypes)\ndtypes.pop(label)\n\nsi_xvars = []\nohe_xvars = []\nfeatureCols = []\nfor idx,key in enumerate(dtypes):\n if dtypes[key] == \"string\":\n featureCol = \"-\".join([key, \"encoded\"])\n featureCols.append(featureCol)\n \n tmpCol = \"-\".join([key, \"tmp\"])\n # string-index and one-hot encode the string column\n #https://spark.apache.org/docs/2.3.0/api/java/org/apache/spark/ml/feature/StringIndexer.html\n #handleInvalid: Param for how to handle invalid data (unseen labels or NULL values). \n #Options are 'skip' (filter out rows with invalid data), 'error' (throw an error), \n #or 'keep' (put invalid data in a special additional bucket, at index numLabels). Default: \"error\"\n si_xvars.append(StringIndexer(inputCol=key, outputCol=tmpCol, handleInvalid=\"skip\")) #, handleInvalid=\"keep\"\n ohe_xvars.append(OneHotEncoder(inputCol=tmpCol, outputCol=featureCol))\n else:\n featureCols.append(key)\n\n# string-index the label column into a column named \"label\"\nsi_label = StringIndexer(inputCol=label, outputCol='label')\n\n# assemble the encoded feature columns in to a column named \"features\"\nassembler = VectorAssembler(inputCols=featureCols, outputCol=\"features\")\n\n# put together the pipeline\nstages = []\nstages.extend(si_xvars)\nstages.extend(ohe_xvars)\nstages.append(si_label)\nstages.append(assembler)\nstages.append(lr)\npipe = Pipeline(stages=stages)\n\n# train the model\nmodel = pipe.fit(train)\nprint(model)\nmodel.stages\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "Regularization Rate is 0.1.\nPipelineModel_49cfbacdb54dd44bcca2\n[StringIndexer_4e5ab09117dc68a07eae, VectorAssembler_43b7be097576e3659c49, LogisticRegression_42b491b66df1978b6ebc]",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Building - Select the best model",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "\nregs = np.arange(0.0, 1.0, 0.2)\n\nparamGrid = ParamGridBuilder().addGrid(lr.regParam, regs).build()\ncv = CrossValidator(estimator=pipe, evaluator=BinaryClassificationEvaluator(), estimatorParamMaps=paramGrid)\n\ncvModel = cv.fit(train)\n\nmodel = cvModel.bestModel",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Evaluation",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# make prediction\npred = model.transform(test)\nprint(pd.DataFrame(pred.take(10)).to_string())\n\n# evaluate. note only 2 metrics are supported out of the box by Spark ML.\nbce = BinaryClassificationEvaluator(rawPredictionCol='rawPrediction')\nau_roc = bce.setMetricName('areaUnderROC').evaluate(pred)\nau_prc = bce.setMetricName('areaUnderPR').evaluate(pred)\n\nprint(\"Area under ROC: {}\".format(au_roc))\nprint(\"Area Under PR: {}\".format(au_prc))",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": " 0 1 2 3 4 5 6 7\n0 17 4 <=50K 0.0 [4.0, 17.0] [3.984203061099825, -3.984203061099825] [0.9817326384088789, 0.018267361591121044] 0.0\n1 17 5 <=50K 0.0 [5.0, 17.0] [3.935897389723122, -3.935897389723122] [0.9808458778128771, 0.019154122187122896] 0.0\n2 17 5 <=50K 0.0 [5.0, 17.0] [3.935897389723122, -3.935897389723122] [0.9808458778128771, 0.019154122187122896] 0.0\n3 17 6 <=50K 0.0 [6.0, 17.0] [3.8875917183464184, -3.8875917183464184] [0.9799169513950979, 0.020083048604902023] 0.0\n4 17 6 <=50K 0.0 [6.0, 17.0] [3.8875917183464184, -3.8875917183464184] [0.9799169513950979, 0.020083048604902023] 0.0\n5 17 8 <=50K 0.0 [8.0, 17.0] [3.7909803755930116, -3.7909803755930116] [0.9779248519533819, 0.022075148046618136] 0.0\n6 17 8 <=50K 0.0 [8.0, 17.0] [3.7909803755930116, -3.7909803755930116] [0.9779248519533819, 0.022075148046618136] 0.0\n7 17 9 <=50K 0.0 [9.0, 17.0] [3.7426747042163084, -3.7426747042163084] [0.9768576056063788, 0.02314239439362117] 0.0\n8 17 9 <=50K 0.0 [9.0, 17.0] [3.7426747042163084, -3.7426747042163084] [0.9768576056063788, 0.02314239439362117] 0.0\n9 17 10 <=50K 0.0 [10.0, 17.0] [3.694369032839605, -3.694369032839605] [0.9757400421084974, 0.024259957891502638] 0.0\nArea under ROC: 0.7364507807436806\nArea Under PR: 0.3950675919086818",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Model Persistence",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "##NOTE: by default the model is saved to and loaded from path\n\nmodel_name = \"AdultCensus.mml\"\nmodel_fs = \"/spark_ml/\" + model_name\n\nmodel.write().overwrite().save(model_fs)\nprint(\"saved model to {}\".format(model_fs))\n\n\n# load the model file (from dbfs)\nmodel2 = PipelineModel.load(model_fs)\nassert str(model2) == str(model)\nprint(\"loaded model from {}\".format(model_fs))",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "saved model to /spark_ml/AdultCensus.mml\nloaded model from /spark_ml/AdultCensus.mml",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,13 @@
|
||||
# SQL Server big data clusters
|
||||
|
||||
The new built-in notebooks in Azure Data Studio enables data scientists and data engineers to run Python, R, or Scala code against the cluster.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Download and save the notebook file [spark-sql.ipynb](spark-sql.ipynb/) locally.
|
||||
|
||||
1. Open the notebook file in Azure Data Studio (right click on the SQL Server big data cluster server name-> **Manage**-> Open Notebook.
|
||||
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. Set the “Kernel” to **PySpark3** and “Attach to” needs to be the IP address of your big data cluster endpoint.
|
||||
|
||||
1. Run each cell in the Notebook sequentially using Azure Data Studio.
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pyspark3kernel",
|
||||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": "# Spark sample showing read/write methods\nIn this sample notebook, we will read CSV file from HDFS, write it as parquet file and save a Hive table definition. We will also run some Spark SQL commands using the Hive table.\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Read the CSV into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data/web_clickstreams.csv').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "root\n |-- wcs_click_date_sk: integer (nullable = true)\n |-- wcs_click_time_sk: integer (nullable = true)\n |-- wcs_sales_sk: integer (nullable = true)\n |-- wcs_item_sk: integer (nullable = true)\n |-- wcs_web_page_sk: integer (nullable = true)\n |-- wcs_user_sk: integer (nullable = true)\n\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 36890| 40052| null| 4379| 34| null|\n| 36890| 41285| null| 6245| 34| null|\n| 36890| 23115| null| 13852| 34| null|\n| 36890| 17702| null| 15975| 34| null|\n| 36890| 62676| null| 2119| 34| null|\n| 36890| 34267| null| 10273| 34| null|\n| 36890| 8502| null| 17790| 34| null|\n| 36890| 54340| null| 3453| 34| null|\n| 36890| 54370| null| 6372| 34| null|\n| 36890| 6578| null| 17203| 34| null|\n| 36890| 75088| null| 4891| 34| null|\n| 36890| 23922| null| 11332| 34| null|\n| 36890| 28761| null| 4484| 34| null|\n| 36890| 21444| null| 5582| 34| null|\n| 36890| 58917| null| 8833| 34| null|\n| 36890| 27578| null| 8599| 34| null|\n| 36890| 8059| null| 6720| 34| null|\n| 36890| 43008| null| 17175| 34| null|\n| 36890| 4378| null| 10644| 34| null|\n| 36890| 55403| null| 8139| 34| null|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "hdfs:///user/hive/warehouse",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT * FROM web_clickstreams LIMIT 100\")\r\nsqlDF.show()\r\n\r\nsqlDF = spark.sql(\"SELECT wcs_user_sk, COUNT(*)\\\r\n FROM web_clickstreams\\\r\n WHERE wcs_user_sk IS NOT NULL\\\r\n GROUP BY wcs_user_sk\\\r\n ORDER BY COUNT(*) DESC LIMIT 100\")\r\nsqlDF.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 37506| 7933| null| 1384| 2| 39437|\n| 37506| 56044| null| 14689| 2| 26419|\n| 37506| 52706| null| 8541| 2| 44016|\n| 37506| 67325| null| 16129| 2| 83371|\n| 37506| 84857| null| 1869| 2| 13090|\n| 37506| 49599| null| 2994| 2| 8940|\n| 37506| 78150| null| 11392| 2| 65633|\n| 37506| 38720| null| 14366| 2| 22281|\n| 37506| 79915| null| 11102| 2| 81755|\n| 37506| 67253| null| 5380| 2| 46868|\n| 37506| 6507| null| 6813| 2| 49363|\n| 37506| 18280| null| 1458| 2| 49363|\n| 37506| 72258| null| 2869| 2| 67756|\n| 37506| 8045| null| 615| 2| 86035|\n| 37506| 86164| null| 7000| 2| 94821|\n| 37506| 29724| null| 2767| 2| 94821|\n| 37506| 55471| null| 3584| 2| 62792|\n| 37506| 677| null| 1720| 2| 27212|\n| 37506| 66638| null| 9898| 2| 20370|\n| 37506| 48515| null| 9394| 2| 17157|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
{
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"name": "pysparkkernel",
|
||||
"display_name": "PySpark"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "pyspark",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 2
|
||||
},
|
||||
"pygments_lexer": "python2"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
"nbformat": 4,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "df = spark.read.csv('/diabetes_data/custom_diabetes_dataset.csv', header=True, sep=',', inferSchema=True)\ndf.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "df.createOrReplaceTempView(\"diabetes\")",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "%%sql\nselect age, avg(insulin) as insulin from diabetes where diabetes = 1 group by age, insulin order by age desc",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "from pyspark.ml.feature import VectorAssembler\n\ntrain = VectorAssembler(inputCols = [\"pregnancies\", \"plasma glucose\", \"blood pressure\", \"triceps skin thickness\", \"insulin\", \"bmi\", \"diabetes pedigree\", \"age\", \"diabetes\"], outputCol = \"features\").transform(df)\ntrain1=train.withColumnRenamed(\"diabetes\", \"label\")\ntrain1.printSchema()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "from pyspark.ml import *\nfrom pyspark.ml.feature import *\nfrom pyspark.ml.classification import *\nfrom pyspark.ml.tuning import *\nfrom pyspark.ml.evaluation import *\nfrom pyspark.ml.clustering import KMeans\n\nkmeans = KMeans().setK(2).setSeed(1)\nmodel = kmeans.fit(train)\ntransformed = model.transform(train)\ntransformed.sample(False, fraction = 0.5).show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "transformed.groupBy(\"prediction\").avg(\"bmi\").show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "transformed.groupBy(\"prediction\").avg(\"pregnancies\").show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
# WRK3010 Powering AI by integrating SQL Server with big data and other data sources
|
||||
|
||||
In this workshop you will get hands on integrating data in SQL Server with big data to power your AI and analytics. SQL Server 2019 enables you to easily to integrate SQL Server with different types of data sources including big data. Integrating data sources like this improves the velocity, veracity, volume, and variety of the data that you are feeding into AI. You will learn how you can use Machine Learning Services directly in SQL Server to train, store, and operationalize your models. You’ll get a chance to use some of the new features of SQL Server 2019 like big data clusters!
|
||||
|
||||
## Setup
|
||||
Scenarios in this lab are using a SQL Server big data cluster that is already provisioned for you on top of a Kubernetes cluster running in an HP Enterprise datacenter (Thanks for the partnership HPE!).
|
||||
|
||||
For interacting with the cluster and run through the data scenarios below, you will use Azure Data Studio and the newly released SQL Server 2019 preview extension. Azure Data Studio and the SQL Server 2019 preview extension are already installed on your VM.
|
||||
|
||||
The data virtualization scenario uses an Oracle server that is already provisioned.
|
||||
|
||||
*Sales* database is already restored on the SQL Server master instance for you to use as sample database. Other sample scripts and notebooks are stored in HDFS, in the SQL Server big data cluster.
|
||||
|
||||
>**!!! IMPORTANT !!!**
|
||||
For the examples below, you will have to save the *.sql* scripts and the notebook file on your local VM, open the local copy, connect to the SQL Server Master or Knox/HDFS gateway and run the script/notebook step by step.
|
||||
|
||||
Before starting the workshop, validate you can connect to all SQL Server big data cluster endpoints. Passwords will be provided by your proctor.
|
||||
- SQL Server Master – using Azure Data Studio -> New Connection -> Connection type “Microsoft SQL Server” -> Host: 15.226.40.8,31433 -> User: sa/Password: xxxxxxx
|
||||
- HDFS/Spark gateway – using Azure Data Studio -> New Connection -> Connection type “SQL Server big data cluster” -> Host: 15.226.40.8 -> User: root/Password: xxxxxxxx
|
||||
|
||||
## 1. Data ingestion using Spark streaming
|
||||
SQL Server Big Data clusters provide scale-out compute and storage to improve the performance of analyzing any data. Data from a variety of sources can be ingested and distributed across data pool instances for analysis.
|
||||
In this example, you are going to use Spark to read and transform data from HDFS and cache it in data pools. Querying the external table created over this aggregated data stored in data pools will be much more efficient.
|
||||
|
||||
### Instructions
|
||||
Using Azure Data Studio, connect to HDFS/SPARK gateway, save locally on your VM the *data_ingestion_using_spark_streaming.sql* script located in HDFS under *sql_scripts* folder. Open the local copy of your script and follow instructions in the script to:
|
||||
|
||||
1. Connect to SQL Server Master (*sales* database) using Azure Data Studio
|
||||
2. Create an external table using the SQL script. Make sure you rename the table throughout the script to something unique.
|
||||
3. Create and submit a Spark job that ingests data from HDFS into the external table
|
||||
|
||||
|
||||
This object is used for starting spark streaming session using spark-submit
|
||||
|
||||
The arguments to jar file are:
|
||||
|
||||
1. server name - sql server to connect to read the table schema
|
||||
2. port number
|
||||
3. username - sql server username for master instance
|
||||
4. password - sql server password for master instance
|
||||
5. database name
|
||||
6. external table name
|
||||
7. Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
||||
8. Input format. This can be "csv", "parquet", "json".
|
||||
9. enable checkpoint: true or false
|
||||
|
||||
Submit spark job with below parameters. You can use the Spark submit experience from Azure Data Studio (right click on big data cluster server name-> Submit Spark Job):
|
||||
|
||||
ARGUMENTS:
|
||||
|
||||
**job name:** yourJobName
|
||||
|
||||
**switch** from "Local" to "HDFS"
|
||||
|
||||
**Path to jar** (copy/paste this):
|
||||
|
||||
/jar/mssql-spark-lib-assembly-1.0.jar
|
||||
|
||||
**Main class:**
|
||||
FileStreaming
|
||||
|
||||
**Parameters (copy/paste this; make sure you replace the password and table name!):**
|
||||
|
||||
|
||||
mssql-master-pool-0.service-master-pool 1433 sa passwordHere sales yourTableNameHere hdfs:///clickstream_data csv false
|
||||
|
||||
4. Query external table using the SELECT queries in the to see data coming from the streaming job
|
||||
|
||||
## 2. Data ingestion using SQL stored proc
|
||||
This scenario is similar to the above Spark sample. If you are more familiar with using TSQL, you can use your preferred query language to achieve the same performance enhancements leveraging data pools in SQL Server Big Data clusters.
|
||||
|
||||
### Instructions
|
||||
Using Azure Data Studio, connect to HDFS/SPARK gateway, save locally on your VM a local copy of the *data_ingestion_using_sql_store_proc.sql* script located in HDFS under *sql_scripts* folder. Follow instructions in the script to:
|
||||
|
||||
1. Connect to SQL Server Master (*sales* database) using Azure Data Studio
|
||||
2. Create external table
|
||||
3. Call sp_data_pool_table_insert_data to insert data from web_clickstreams table into the external table
|
||||
4. Query external table
|
||||
5. Cleanup
|
||||
|
||||
## 3. Query HDFS data using SQL Server Master
|
||||
In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes.
|
||||
In this example, you are going to create an external table in SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster. Then you will join the data in the external table with high value data in SQL Master instance.
|
||||
|
||||
### Instructions
|
||||
Using Azure Data Studio, connect to HDFS/SPARK gateway, save locally on your VM a local copy of the *data_virtualization_HDFS.sql* script located in HDFS under *sql_scripts* folder. Follow instructions in the script to:
|
||||
|
||||
1. Connect to SQL Server Master (*sales* database) using Azure Data Studio
|
||||
1. Create external table
|
||||
1. Run query to join data in external table with high value data
|
||||
1. Cleanup
|
||||
|
||||
## 4. Create external table over Oracle database
|
||||
By leveraging SQL Server Polybase technologies, SQL Server Big Data clusters can query external data sources without importing the data in SQL Server. SQL Server 2019 preview introduces new connectors to data sources like Oracle, MongoDB or Teradata. In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server.
|
||||
|
||||
### Instructions
|
||||
|
||||
*Option# 1*
|
||||
|
||||
1. Using Azure Data Studio, connect to SQL Server Master *sales* database-> Right click on database name-> Create external table
|
||||
|
||||
|
||||
|
||||
2. In the “Select a data source” dialog, choose “Oracle” as external data source type, then click “Next”:
|
||||
|
||||
|
||||
|
||||
3. In the next step, create a database master key for database *sales*. If the database already has a master key, the input is greyed out and you just click “Next”.
|
||||
4. In the “Create a connection to your data source” dialog, you are configuring the external data source, including the name (you can use any **_unique_** name for the external data source), the server/database name of the Oracle data source, as well as the credentials to access authenticate to it (you can use any **_unique_** name for the credential). You are going to use a pre-provisioned Oracle server: **APS40-10.oltp.sql.cass.hp.com** (database: **XE**; username: **SYSTEM**; password: **Admin123**).
|
||||
|
||||
|
||||
|
||||
5. In the “Map your data source objects to your external table” dialog, select the HR.INVENTORY table (you must mark the checkbox next to the table name _and_ select the table name so the table name is highlighted as below) and map its columns and types to columns and types in the SQL server external table:
|
||||
|
||||
> !! IMPORTANT !! Make sure you use a **unique** table name for the external table name.
|
||||
>
|
||||
|
||||
|
||||
6. In the final summary dialog, click “Create” to complete the external table creation.
|
||||
7. Query external table
|
||||
8. Connect to HDFS/SPARK gateway, save locally and open the local copy of the *query_external_table_over_Oracle.sql* script located in HDFS under *sql_scripts* folder. Follow the instructions in the script to run a query that joins the inventory data from the external table with the high value data in the SQL Server Master *sales* database.
|
||||
9. Run the cleanup step from the above script to remove the database objects you created for this example.
|
||||
|
||||
*Option# 2*
|
||||
|
||||
Same scenario can be achieved using TSQL script. Connect to HDFS/SPARK gateway, “Preview”, save locally and open the local copy of the *data_virtualization_oracle.sql* script located in HDFS under *sql_scripts* folder. Follow instructions in the script to:
|
||||
|
||||
1. Connect to SQL Server Master (*sales* database) using Azure Data Studio
|
||||
1. Create an external data source and an external table in sales database that points to inventory table on Oracle server
|
||||
1. Query external table
|
||||
1. Connect to HDFS/SPARK gateway, “Preview”, save locally and open the local copy of the *query_external_table_over_Oracle.sql* script located in HDFS under *sql_scripts* folder. Follow the instructions in the script to run a query that joins the inventory data from the external table with the high value data in the SQL Server Master sales database.
|
||||
1. Run the cleanup step from the above script to remove the database objects you created for this example.
|
||||
|
||||
## 5. Run Notebooks to query data in HDFS
|
||||
The new built-in notebooks in Azure Data Studio are enabling data scientists and engineers to write Python, R, or Scala code before submitting the code as Spark jobs and viewing the results inline. Notebooks facilitate collaboration between teammates working on a data analysis project together.
|
||||
|
||||
### Instructions
|
||||
In this example, you are going to run a sample notebook that analyzes the data over a diabetes dataset publicly available, and try to infer the different patterns that influence the outcome of diabetes.
|
||||
|
||||
1. Connect to HDFS/Spark gateway and locate *Cluster_Diabetes_Demo.ipynb* file under *notebooks* folder in HDFS. Save it locally on your VM: right click on the file name, then "Save".
|
||||
1. Open the notebook saved locally (right click on the Knox/HDFS gateway server name-> **Manage**-> Open Notebook
|
||||
1. Wait for the “Kernel” and the target context (“Attach to”) to be populated. “Kernel” should be **PySpark (NOT PySpark3)** and “Attach to” is **15.226.40.8**.
|
||||
1. Run each cell from the Notebook sequentially using Azure Data Studio. It will take about 20 seconds to run the first cell.
|
||||
## 6. Build a ML model and predict in SQL server Master instance
|
||||
Machine Learning services are running in the SQL Server Master instance of the big data cluster, which enables you to run R and Python scripts using the stored procedure “sp_execute_external_script”.
|
||||
|
||||
### Instructions
|
||||
In this example, we are building a machine learning model using logistic regression for a recommendation engine on an online store. Based on existing users' click pattern online and their interest in other categories and demographics, we are training a machine learning model. This model will be used to predict if the visitor is interested in a given item category.
|
||||
|
||||
Connect to connection type "SQL Server Big Data Cluster", go to the HDFS folder *sql_scripts*, right click on *ml_training_and_scoring.sql* script and *save* locally.
|
||||
|
||||
Connect to SQL Server Master instance (*sales* database) and run the script step by step:
|
||||
|
||||
1. Replace "<model_name>" with the unique name for your model. Now run Step 1 in the script to train your model and verify that your model was saved in the table sales_models.
|
||||
2. Replace "<model_name>" with the unique name for your model. Run Step 2 to predict the book category clicks for new users based on their pattern of visiting various categories in the web site.
|
||||
|
||||
|
||||
YOU COMPLETED THE WORKSHOP! CONGRATULATIONS!!!!
|
||||
@@ -1,769 +0,0 @@
|
||||
pregnancies,plasma glucose,blood pressure,triceps skin thickness,insulin,bmi,diabetes pedigree,age,diabetes
|
||||
6,148,72,35,0,33.6,0.627,50,1
|
||||
1,85,66,29,0,26.6,0.351,31,0
|
||||
8,183,64,0,0,23.3,0.672,32,1
|
||||
1,89,66,23,94,28.1,0.167,21,0
|
||||
0,137,40,35,168,43.1,2.288,33,1
|
||||
5,116,74,0,0,25.6,0.201,30,0
|
||||
3,78,50,32,88,31,0.248,26,1
|
||||
10,115,0,0,0,35.3,0.134,29,0
|
||||
2,197,70,45,543,30.5,0.158,53,1
|
||||
8,125,96,0,0,0,0.232,54,1
|
||||
4,110,92,0,0,37.6,0.191,30,0
|
||||
10,168,74,0,0,38,0.537,34,1
|
||||
10,139,80,0,0,27.1,1.441,57,0
|
||||
1,189,60,23,846,30.1,0.398,59,1
|
||||
5,166,72,19,175,25.8,0.587,51,1
|
||||
7,100,0,0,0,30,0.484,32,1
|
||||
0,118,84,47,230,45.8,0.551,31,1
|
||||
7,107,74,0,0,29.6,0.254,31,1
|
||||
1,103,30,38,83,43.3,0.183,33,0
|
||||
1,115,70,30,96,34.6,0.529,32,1
|
||||
3,126,88,41,235,39.3,0.704,27,0
|
||||
8,99,84,0,0,35.4,0.388,50,0
|
||||
7,196,90,0,0,39.8,0.451,41,1
|
||||
9,119,80,35,0,29,0.263,29,1
|
||||
11,143,94,33,146,36.6,0.254,51,1
|
||||
10,125,70,26,115,31.1,0.205,41,1
|
||||
7,147,76,0,0,39.4,0.257,43,1
|
||||
1,97,66,15,140,23.2,0.487,22,0
|
||||
13,145,82,19,110,22.2,0.245,57,0
|
||||
5,117,92,0,0,34.1,0.337,38,0
|
||||
5,109,75,26,0,36,0.546,60,0
|
||||
3,158,76,36,245,31.6,0.851,28,1
|
||||
3,88,58,11,54,24.8,0.267,22,0
|
||||
6,92,92,0,0,19.9,0.188,28,0
|
||||
10,122,78,31,0,27.6,0.512,45,0
|
||||
4,103,60,33,192,24,0.966,33,0
|
||||
11,138,76,0,0,33.2,0.42,35,0
|
||||
9,102,76,37,0,32.9,0.665,46,1
|
||||
2,90,68,42,0,38.2,0.503,27,1
|
||||
4,111,72,47,207,37.1,1.39,56,1
|
||||
3,180,64,25,70,34,0.271,26,0
|
||||
7,133,84,0,0,40.2,0.696,37,0
|
||||
7,106,92,18,0,22.7,0.235,48,0
|
||||
9,171,110,24,240,45.4,0.721,54,1
|
||||
7,159,64,0,0,27.4,0.294,40,0
|
||||
0,180,66,39,0,42,1.893,25,1
|
||||
1,146,56,0,0,29.7,0.564,29,0
|
||||
2,71,70,27,0,28,0.586,22,0
|
||||
7,103,66,32,0,39.1,0.344,31,1
|
||||
7,105,0,0,0,0,0.305,24,0
|
||||
1,103,80,11,82,19.4,0.491,22,0
|
||||
1,101,50,15,36,24.2,0.526,26,0
|
||||
5,88,66,21,23,24.4,0.342,30,0
|
||||
8,176,90,34,300,33.7,0.467,58,1
|
||||
7,150,66,42,342,34.7,0.718,42,0
|
||||
1,73,50,10,0,23,0.248,21,0
|
||||
7,187,68,39,304,37.7,0.254,41,1
|
||||
0,100,88,60,110,46.8,0.962,31,0
|
||||
0,146,82,0,0,40.5,1.781,44,0
|
||||
0,105,64,41,142,41.5,0.173,22,0
|
||||
2,84,0,0,0,0,0.304,21,0
|
||||
8,133,72,0,0,32.9,0.27,39,1
|
||||
5,44,62,0,0,25,0.587,36,0
|
||||
2,141,58,34,128,25.4,0.699,24,0
|
||||
7,114,66,0,0,32.8,0.258,42,1
|
||||
5,99,74,27,0,29,0.203,32,0
|
||||
0,109,88,30,0,32.5,0.855,38,1
|
||||
2,109,92,0,0,42.7,0.845,54,0
|
||||
1,95,66,13,38,19.6,0.334,25,0
|
||||
4,146,85,27,100,28.9,0.189,27,0
|
||||
2,100,66,20,90,32.9,0.867,28,1
|
||||
5,139,64,35,140,28.6,0.411,26,0
|
||||
13,126,90,0,0,43.4,0.583,42,1
|
||||
4,129,86,20,270,35.1,0.231,23,0
|
||||
1,79,75,30,0,32,0.396,22,0
|
||||
1,0,48,20,0,24.7,0.14,22,0
|
||||
7,62,78,0,0,32.6,0.391,41,0
|
||||
5,95,72,33,0,37.7,0.37,27,0
|
||||
0,131,0,0,0,43.2,0.27,26,1
|
||||
2,112,66,22,0,25,0.307,24,0
|
||||
3,113,44,13,0,22.4,0.14,22,0
|
||||
2,74,0,0,0,0,0.102,22,0
|
||||
7,83,78,26,71,29.3,0.767,36,0
|
||||
0,101,65,28,0,24.6,0.237,22,0
|
||||
5,137,108,0,0,48.8,0.227,37,1
|
||||
2,110,74,29,125,32.4,0.698,27,0
|
||||
13,106,72,54,0,36.6,0.178,45,0
|
||||
2,100,68,25,71,38.5,0.324,26,0
|
||||
15,136,70,32,110,37.1,0.153,43,1
|
||||
1,107,68,19,0,26.5,0.165,24,0
|
||||
1,80,55,0,0,19.1,0.258,21,0
|
||||
4,123,80,15,176,32,0.443,34,0
|
||||
7,81,78,40,48,46.7,0.261,42,0
|
||||
4,134,72,0,0,23.8,0.277,60,1
|
||||
2,142,82,18,64,24.7,0.761,21,0
|
||||
6,144,72,27,228,33.9,0.255,40,0
|
||||
2,92,62,28,0,31.6,0.13,24,0
|
||||
1,71,48,18,76,20.4,0.323,22,0
|
||||
6,93,50,30,64,28.7,0.356,23,0
|
||||
1,122,90,51,220,49.7,0.325,31,1
|
||||
1,163,72,0,0,39,1.222,33,1
|
||||
1,151,60,0,0,26.1,0.179,22,0
|
||||
0,125,96,0,0,22.5,0.262,21,0
|
||||
1,81,72,18,40,26.6,0.283,24,0
|
||||
2,85,65,0,0,39.6,0.93,27,0
|
||||
1,126,56,29,152,28.7,0.801,21,0
|
||||
1,96,122,0,0,22.4,0.207,27,0
|
||||
4,144,58,28,140,29.5,0.287,37,0
|
||||
3,83,58,31,18,34.3,0.336,25,0
|
||||
0,95,85,25,36,37.4,0.247,24,1
|
||||
3,171,72,33,135,33.3,0.199,24,1
|
||||
8,155,62,26,495,34,0.543,46,1
|
||||
1,89,76,34,37,31.2,0.192,23,0
|
||||
4,76,62,0,0,34,0.391,25,0
|
||||
7,160,54,32,175,30.5,0.588,39,1
|
||||
4,146,92,0,0,31.2,0.539,61,1
|
||||
5,124,74,0,0,34,0.22,38,1
|
||||
5,78,48,0,0,33.7,0.654,25,0
|
||||
4,97,60,23,0,28.2,0.443,22,0
|
||||
4,99,76,15,51,23.2,0.223,21,0
|
||||
0,162,76,56,100,53.2,0.759,25,1
|
||||
6,111,64,39,0,34.2,0.26,24,0
|
||||
2,107,74,30,100,33.6,0.404,23,0
|
||||
5,132,80,0,0,26.8,0.186,69,0
|
||||
0,113,76,0,0,33.3,0.278,23,1
|
||||
1,88,30,42,99,55,0.496,26,1
|
||||
3,120,70,30,135,42.9,0.452,30,0
|
||||
1,118,58,36,94,33.3,0.261,23,0
|
||||
1,117,88,24,145,34.5,0.403,40,1
|
||||
0,105,84,0,0,27.9,0.741,62,1
|
||||
4,173,70,14,168,29.7,0.361,33,1
|
||||
9,122,56,0,0,33.3,1.114,33,1
|
||||
3,170,64,37,225,34.5,0.356,30,1
|
||||
8,84,74,31,0,38.3,0.457,39,0
|
||||
2,96,68,13,49,21.1,0.647,26,0
|
||||
2,125,60,20,140,33.8,0.088,31,0
|
||||
0,100,70,26,50,30.8,0.597,21,0
|
||||
0,93,60,25,92,28.7,0.532,22,0
|
||||
0,129,80,0,0,31.2,0.703,29,0
|
||||
5,105,72,29,325,36.9,0.159,28,0
|
||||
3,128,78,0,0,21.1,0.268,55,0
|
||||
5,106,82,30,0,39.5,0.286,38,0
|
||||
2,108,52,26,63,32.5,0.318,22,0
|
||||
10,108,66,0,0,32.4,0.272,42,1
|
||||
4,154,62,31,284,32.8,0.237,23,0
|
||||
0,102,75,23,0,0,0.572,21,0
|
||||
9,57,80,37,0,32.8,0.096,41,0
|
||||
2,106,64,35,119,30.5,1.4,34,0
|
||||
5,147,78,0,0,33.7,0.218,65,0
|
||||
2,90,70,17,0,27.3,0.085,22,0
|
||||
1,136,74,50,204,37.4,0.399,24,0
|
||||
4,114,65,0,0,21.9,0.432,37,0
|
||||
9,156,86,28,155,34.3,1.189,42,1
|
||||
1,153,82,42,485,40.6,0.687,23,0
|
||||
8,188,78,0,0,47.9,0.137,43,1
|
||||
7,152,88,44,0,50,0.337,36,1
|
||||
2,99,52,15,94,24.6,0.637,21,0
|
||||
1,109,56,21,135,25.2,0.833,23,0
|
||||
2,88,74,19,53,29,0.229,22,0
|
||||
17,163,72,41,114,40.9,0.817,47,1
|
||||
4,151,90,38,0,29.7,0.294,36,0
|
||||
7,102,74,40,105,37.2,0.204,45,0
|
||||
0,114,80,34,285,44.2,0.167,27,0
|
||||
2,100,64,23,0,29.7,0.368,21,0
|
||||
0,131,88,0,0,31.6,0.743,32,1
|
||||
6,104,74,18,156,29.9,0.722,41,1
|
||||
3,148,66,25,0,32.5,0.256,22,0
|
||||
4,120,68,0,0,29.6,0.709,34,0
|
||||
4,110,66,0,0,31.9,0.471,29,0
|
||||
3,111,90,12,78,28.4,0.495,29,0
|
||||
6,102,82,0,0,30.8,0.18,36,1
|
||||
6,134,70,23,130,35.4,0.542,29,1
|
||||
2,87,0,23,0,28.9,0.773,25,0
|
||||
1,79,60,42,48,43.5,0.678,23,0
|
||||
2,75,64,24,55,29.7,0.37,33,0
|
||||
8,179,72,42,130,32.7,0.719,36,1
|
||||
6,85,78,0,0,31.2,0.382,42,0
|
||||
0,129,110,46,130,67.1,0.319,26,1
|
||||
5,143,78,0,0,45,0.19,47,0
|
||||
5,130,82,0,0,39.1,0.956,37,1
|
||||
6,87,80,0,0,23.2,0.084,32,0
|
||||
0,119,64,18,92,34.9,0.725,23,0
|
||||
1,0,74,20,23,27.7,0.299,21,0
|
||||
5,73,60,0,0,26.8,0.268,27,0
|
||||
4,141,74,0,0,27.6,0.244,40,0
|
||||
7,194,68,28,0,35.9,0.745,41,1
|
||||
8,181,68,36,495,30.1,0.615,60,1
|
||||
1,128,98,41,58,32,1.321,33,1
|
||||
8,109,76,39,114,27.9,0.64,31,1
|
||||
5,139,80,35,160,31.6,0.361,25,1
|
||||
3,111,62,0,0,22.6,0.142,21,0
|
||||
9,123,70,44,94,33.1,0.374,40,0
|
||||
7,159,66,0,0,30.4,0.383,36,1
|
||||
11,135,0,0,0,52.3,0.578,40,1
|
||||
8,85,55,20,0,24.4,0.136,42,0
|
||||
5,158,84,41,210,39.4,0.395,29,1
|
||||
1,105,58,0,0,24.3,0.187,21,0
|
||||
3,107,62,13,48,22.9,0.678,23,1
|
||||
4,109,64,44,99,34.8,0.905,26,1
|
||||
4,148,60,27,318,30.9,0.15,29,1
|
||||
0,113,80,16,0,31,0.874,21,0
|
||||
1,138,82,0,0,40.1,0.236,28,0
|
||||
0,108,68,20,0,27.3,0.787,32,0
|
||||
2,99,70,16,44,20.4,0.235,27,0
|
||||
6,103,72,32,190,37.7,0.324,55,0
|
||||
5,111,72,28,0,23.9,0.407,27,0
|
||||
8,196,76,29,280,37.5,0.605,57,1
|
||||
5,162,104,0,0,37.7,0.151,52,1
|
||||
1,96,64,27,87,33.2,0.289,21,0
|
||||
7,184,84,33,0,35.5,0.355,41,1
|
||||
2,81,60,22,0,27.7,0.29,25,0
|
||||
0,147,85,54,0,42.8,0.375,24,0
|
||||
7,179,95,31,0,34.2,0.164,60,0
|
||||
0,140,65,26,130,42.6,0.431,24,1
|
||||
9,112,82,32,175,34.2,0.26,36,1
|
||||
12,151,70,40,271,41.8,0.742,38,1
|
||||
5,109,62,41,129,35.8,0.514,25,1
|
||||
6,125,68,30,120,30,0.464,32,0
|
||||
5,85,74,22,0,29,1.224,32,1
|
||||
5,112,66,0,0,37.8,0.261,41,1
|
||||
0,177,60,29,478,34.6,1.072,21,1
|
||||
2,158,90,0,0,31.6,0.805,66,1
|
||||
7,119,0,0,0,25.2,0.209,37,0
|
||||
7,142,60,33,190,28.8,0.687,61,0
|
||||
1,100,66,15,56,23.6,0.666,26,0
|
||||
1,87,78,27,32,34.6,0.101,22,0
|
||||
0,101,76,0,0,35.7,0.198,26,0
|
||||
3,162,52,38,0,37.2,0.652,24,1
|
||||
4,197,70,39,744,36.7,2.329,31,0
|
||||
0,117,80,31,53,45.2,0.089,24,0
|
||||
4,142,86,0,0,44,0.645,22,1
|
||||
6,134,80,37,370,46.2,0.238,46,1
|
||||
1,79,80,25,37,25.4,0.583,22,0
|
||||
4,122,68,0,0,35,0.394,29,0
|
||||
3,74,68,28,45,29.7,0.293,23,0
|
||||
4,171,72,0,0,43.6,0.479,26,1
|
||||
7,181,84,21,192,35.9,0.586,51,1
|
||||
0,179,90,27,0,44.1,0.686,23,1
|
||||
9,164,84,21,0,30.8,0.831,32,1
|
||||
0,104,76,0,0,18.4,0.582,27,0
|
||||
1,91,64,24,0,29.2,0.192,21,0
|
||||
4,91,70,32,88,33.1,0.446,22,0
|
||||
3,139,54,0,0,25.6,0.402,22,1
|
||||
6,119,50,22,176,27.1,1.318,33,1
|
||||
2,146,76,35,194,38.2,0.329,29,0
|
||||
9,184,85,15,0,30,1.213,49,1
|
||||
10,122,68,0,0,31.2,0.258,41,0
|
||||
0,165,90,33,680,52.3,0.427,23,0
|
||||
9,124,70,33,402,35.4,0.282,34,0
|
||||
1,111,86,19,0,30.1,0.143,23,0
|
||||
9,106,52,0,0,31.2,0.38,42,0
|
||||
2,129,84,0,0,28,0.284,27,0
|
||||
2,90,80,14,55,24.4,0.249,24,0
|
||||
0,86,68,32,0,35.8,0.238,25,0
|
||||
12,92,62,7,258,27.6,0.926,44,1
|
||||
1,113,64,35,0,33.6,0.543,21,1
|
||||
3,111,56,39,0,30.1,0.557,30,0
|
||||
2,114,68,22,0,28.7,0.092,25,0
|
||||
1,193,50,16,375,25.9,0.655,24,0
|
||||
11,155,76,28,150,33.3,1.353,51,1
|
||||
3,191,68,15,130,30.9,0.299,34,0
|
||||
3,141,0,0,0,30,0.761,27,1
|
||||
4,95,70,32,0,32.1,0.612,24,0
|
||||
3,142,80,15,0,32.4,0.2,63,0
|
||||
4,123,62,0,0,32,0.226,35,1
|
||||
5,96,74,18,67,33.6,0.997,43,0
|
||||
0,138,0,0,0,36.3,0.933,25,1
|
||||
2,128,64,42,0,40,1.101,24,0
|
||||
0,102,52,0,0,25.1,0.078,21,0
|
||||
2,146,0,0,0,27.5,0.24,28,1
|
||||
10,101,86,37,0,45.6,1.136,38,1
|
||||
2,108,62,32,56,25.2,0.128,21,0
|
||||
3,122,78,0,0,23,0.254,40,0
|
||||
1,71,78,50,45,33.2,0.422,21,0
|
||||
13,106,70,0,0,34.2,0.251,52,0
|
||||
2,100,70,52,57,40.5,0.677,25,0
|
||||
7,106,60,24,0,26.5,0.296,29,1
|
||||
0,104,64,23,116,27.8,0.454,23,0
|
||||
5,114,74,0,0,24.9,0.744,57,0
|
||||
2,108,62,10,278,25.3,0.881,22,0
|
||||
0,146,70,0,0,37.9,0.334,28,1
|
||||
10,129,76,28,122,35.9,0.28,39,0
|
||||
7,133,88,15,155,32.4,0.262,37,0
|
||||
7,161,86,0,0,30.4,0.165,47,1
|
||||
2,108,80,0,0,27,0.259,52,1
|
||||
7,136,74,26,135,26,0.647,51,0
|
||||
5,155,84,44,545,38.7,0.619,34,0
|
||||
1,119,86,39,220,45.6,0.808,29,1
|
||||
4,96,56,17,49,20.8,0.34,26,0
|
||||
5,108,72,43,75,36.1,0.263,33,0
|
||||
0,78,88,29,40,36.9,0.434,21,0
|
||||
0,107,62,30,74,36.6,0.757,25,1
|
||||
2,128,78,37,182,43.3,1.224,31,1
|
||||
1,128,48,45,194,40.5,0.613,24,1
|
||||
0,161,50,0,0,21.9,0.254,65,0
|
||||
6,151,62,31,120,35.5,0.692,28,0
|
||||
2,146,70,38,360,28,0.337,29,1
|
||||
0,126,84,29,215,30.7,0.52,24,0
|
||||
14,100,78,25,184,36.6,0.412,46,1
|
||||
8,112,72,0,0,23.6,0.84,58,0
|
||||
0,167,0,0,0,32.3,0.839,30,1
|
||||
2,144,58,33,135,31.6,0.422,25,1
|
||||
5,77,82,41,42,35.8,0.156,35,0
|
||||
5,115,98,0,0,52.9,0.209,28,1
|
||||
3,150,76,0,0,21,0.207,37,0
|
||||
2,120,76,37,105,39.7,0.215,29,0
|
||||
10,161,68,23,132,25.5,0.326,47,1
|
||||
0,137,68,14,148,24.8,0.143,21,0
|
||||
0,128,68,19,180,30.5,1.391,25,1
|
||||
2,124,68,28,205,32.9,0.875,30,1
|
||||
6,80,66,30,0,26.2,0.313,41,0
|
||||
0,106,70,37,148,39.4,0.605,22,0
|
||||
2,155,74,17,96,26.6,0.433,27,1
|
||||
3,113,50,10,85,29.5,0.626,25,0
|
||||
7,109,80,31,0,35.9,1.127,43,1
|
||||
2,112,68,22,94,34.1,0.315,26,0
|
||||
3,99,80,11,64,19.3,0.284,30,0
|
||||
3,182,74,0,0,30.5,0.345,29,1
|
||||
3,115,66,39,140,38.1,0.15,28,0
|
||||
6,194,78,0,0,23.5,0.129,59,1
|
||||
4,129,60,12,231,27.5,0.527,31,0
|
||||
3,112,74,30,0,31.6,0.197,25,1
|
||||
0,124,70,20,0,27.4,0.254,36,1
|
||||
13,152,90,33,29,26.8,0.731,43,1
|
||||
2,112,75,32,0,35.7,0.148,21,0
|
||||
1,157,72,21,168,25.6,0.123,24,0
|
||||
1,122,64,32,156,35.1,0.692,30,1
|
||||
10,179,70,0,0,35.1,0.2,37,0
|
||||
2,102,86,36,120,45.5,0.127,23,1
|
||||
6,105,70,32,68,30.8,0.122,37,0
|
||||
8,118,72,19,0,23.1,1.476,46,0
|
||||
2,87,58,16,52,32.7,0.166,25,0
|
||||
1,180,0,0,0,43.3,0.282,41,1
|
||||
12,106,80,0,0,23.6,0.137,44,0
|
||||
1,95,60,18,58,23.9,0.26,22,0
|
||||
0,165,76,43,255,47.9,0.259,26,0
|
||||
0,117,0,0,0,33.8,0.932,44,0
|
||||
5,115,76,0,0,31.2,0.343,44,1
|
||||
9,152,78,34,171,34.2,0.893,33,1
|
||||
7,178,84,0,0,39.9,0.331,41,1
|
||||
1,130,70,13,105,25.9,0.472,22,0
|
||||
1,95,74,21,73,25.9,0.673,36,0
|
||||
1,0,68,35,0,32,0.389,22,0
|
||||
5,122,86,0,0,34.7,0.29,33,0
|
||||
8,95,72,0,0,36.8,0.485,57,0
|
||||
8,126,88,36,108,38.5,0.349,49,0
|
||||
1,139,46,19,83,28.7,0.654,22,0
|
||||
3,116,0,0,0,23.5,0.187,23,0
|
||||
3,99,62,19,74,21.8,0.279,26,0
|
||||
5,0,80,32,0,41,0.346,37,1
|
||||
4,92,80,0,0,42.2,0.237,29,0
|
||||
4,137,84,0,0,31.2,0.252,30,0
|
||||
3,61,82,28,0,34.4,0.243,46,0
|
||||
1,90,62,12,43,27.2,0.58,24,0
|
||||
3,90,78,0,0,42.7,0.559,21,0
|
||||
9,165,88,0,0,30.4,0.302,49,1
|
||||
1,125,50,40,167,33.3,0.962,28,1
|
||||
13,129,0,30,0,39.9,0.569,44,1
|
||||
12,88,74,40,54,35.3,0.378,48,0
|
||||
1,196,76,36,249,36.5,0.875,29,1
|
||||
5,189,64,33,325,31.2,0.583,29,1
|
||||
5,158,70,0,0,29.8,0.207,63,0
|
||||
5,103,108,37,0,39.2,0.305,65,0
|
||||
4,146,78,0,0,38.5,0.52,67,1
|
||||
4,147,74,25,293,34.9,0.385,30,0
|
||||
5,99,54,28,83,34,0.499,30,0
|
||||
6,124,72,0,0,27.6,0.368,29,1
|
||||
0,101,64,17,0,21,0.252,21,0
|
||||
3,81,86,16,66,27.5,0.306,22,0
|
||||
1,133,102,28,140,32.8,0.234,45,1
|
||||
3,173,82,48,465,38.4,2.137,25,1
|
||||
0,118,64,23,89,0,1.731,21,0
|
||||
0,84,64,22,66,35.8,0.545,21,0
|
||||
2,105,58,40,94,34.9,0.225,25,0
|
||||
2,122,52,43,158,36.2,0.816,28,0
|
||||
12,140,82,43,325,39.2,0.528,58,1
|
||||
0,98,82,15,84,25.2,0.299,22,0
|
||||
1,87,60,37,75,37.2,0.509,22,0
|
||||
4,156,75,0,0,48.3,0.238,32,1
|
||||
0,93,100,39,72,43.4,1.021,35,0
|
||||
1,107,72,30,82,30.8,0.821,24,0
|
||||
0,105,68,22,0,20,0.236,22,0
|
||||
1,109,60,8,182,25.4,0.947,21,0
|
||||
1,90,62,18,59,25.1,1.268,25,0
|
||||
1,125,70,24,110,24.3,0.221,25,0
|
||||
1,119,54,13,50,22.3,0.205,24,0
|
||||
5,116,74,29,0,32.3,0.66,35,1
|
||||
8,105,100,36,0,43.3,0.239,45,1
|
||||
5,144,82,26,285,32,0.452,58,1
|
||||
3,100,68,23,81,31.6,0.949,28,0
|
||||
1,100,66,29,196,32,0.444,42,0
|
||||
5,166,76,0,0,45.7,0.34,27,1
|
||||
1,131,64,14,415,23.7,0.389,21,0
|
||||
4,116,72,12,87,22.1,0.463,37,0
|
||||
4,158,78,0,0,32.9,0.803,31,1
|
||||
2,127,58,24,275,27.7,1.6,25,0
|
||||
3,96,56,34,115,24.7,0.944,39,0
|
||||
0,131,66,40,0,34.3,0.196,22,1
|
||||
3,82,70,0,0,21.1,0.389,25,0
|
||||
3,193,70,31,0,34.9,0.241,25,1
|
||||
4,95,64,0,0,32,0.161,31,1
|
||||
6,137,61,0,0,24.2,0.151,55,0
|
||||
5,136,84,41,88,35,0.286,35,1
|
||||
9,72,78,25,0,31.6,0.28,38,0
|
||||
5,168,64,0,0,32.9,0.135,41,1
|
||||
2,123,48,32,165,42.1,0.52,26,0
|
||||
4,115,72,0,0,28.9,0.376,46,1
|
||||
0,101,62,0,0,21.9,0.336,25,0
|
||||
8,197,74,0,0,25.9,1.191,39,1
|
||||
1,172,68,49,579,42.4,0.702,28,1
|
||||
6,102,90,39,0,35.7,0.674,28,0
|
||||
1,112,72,30,176,34.4,0.528,25,0
|
||||
1,143,84,23,310,42.4,1.076,22,0
|
||||
1,143,74,22,61,26.2,0.256,21,0
|
||||
0,138,60,35,167,34.6,0.534,21,1
|
||||
3,173,84,33,474,35.7,0.258,22,1
|
||||
1,97,68,21,0,27.2,1.095,22,0
|
||||
4,144,82,32,0,38.5,0.554,37,1
|
||||
1,83,68,0,0,18.2,0.624,27,0
|
||||
3,129,64,29,115,26.4,0.219,28,1
|
||||
1,119,88,41,170,45.3,0.507,26,0
|
||||
2,94,68,18,76,26,0.561,21,0
|
||||
0,102,64,46,78,40.6,0.496,21,0
|
||||
2,115,64,22,0,30.8,0.421,21,0
|
||||
8,151,78,32,210,42.9,0.516,36,1
|
||||
4,184,78,39,277,37,0.264,31,1
|
||||
0,94,0,0,0,0,0.256,25,0
|
||||
1,181,64,30,180,34.1,0.328,38,1
|
||||
0,135,94,46,145,40.6,0.284,26,0
|
||||
1,95,82,25,180,35,0.233,43,1
|
||||
2,99,0,0,0,22.2,0.108,23,0
|
||||
3,89,74,16,85,30.4,0.551,38,0
|
||||
1,80,74,11,60,30,0.527,22,0
|
||||
2,139,75,0,0,25.6,0.167,29,0
|
||||
1,90,68,8,0,24.5,1.138,36,0
|
||||
0,141,0,0,0,42.4,0.205,29,1
|
||||
12,140,85,33,0,37.4,0.244,41,0
|
||||
5,147,75,0,0,29.9,0.434,28,0
|
||||
1,97,70,15,0,18.2,0.147,21,0
|
||||
6,107,88,0,0,36.8,0.727,31,0
|
||||
0,189,104,25,0,34.3,0.435,41,1
|
||||
2,83,66,23,50,32.2,0.497,22,0
|
||||
4,117,64,27,120,33.2,0.23,24,0
|
||||
8,108,70,0,0,30.5,0.955,33,1
|
||||
4,117,62,12,0,29.7,0.38,30,1
|
||||
0,180,78,63,14,59.4,2.42,25,1
|
||||
1,100,72,12,70,25.3,0.658,28,0
|
||||
0,95,80,45,92,36.5,0.33,26,0
|
||||
0,104,64,37,64,33.6,0.51,22,1
|
||||
0,120,74,18,63,30.5,0.285,26,0
|
||||
1,82,64,13,95,21.2,0.415,23,0
|
||||
2,134,70,0,0,28.9,0.542,23,1
|
||||
0,91,68,32,210,39.9,0.381,25,0
|
||||
2,119,0,0,0,19.6,0.832,72,0
|
||||
2,100,54,28,105,37.8,0.498,24,0
|
||||
14,175,62,30,0,33.6,0.212,38,1
|
||||
1,135,54,0,0,26.7,0.687,62,0
|
||||
5,86,68,28,71,30.2,0.364,24,0
|
||||
10,148,84,48,237,37.6,1.001,51,1
|
||||
9,134,74,33,60,25.9,0.46,81,0
|
||||
9,120,72,22,56,20.8,0.733,48,0
|
||||
1,71,62,0,0,21.8,0.416,26,0
|
||||
8,74,70,40,49,35.3,0.705,39,0
|
||||
5,88,78,30,0,27.6,0.258,37,0
|
||||
10,115,98,0,0,24,1.022,34,0
|
||||
0,124,56,13,105,21.8,0.452,21,0
|
||||
0,74,52,10,36,27.8,0.269,22,0
|
||||
0,97,64,36,100,36.8,0.6,25,0
|
||||
8,120,0,0,0,30,0.183,38,1
|
||||
6,154,78,41,140,46.1,0.571,27,0
|
||||
1,144,82,40,0,41.3,0.607,28,0
|
||||
0,137,70,38,0,33.2,0.17,22,0
|
||||
0,119,66,27,0,38.8,0.259,22,0
|
||||
7,136,90,0,0,29.9,0.21,50,0
|
||||
4,114,64,0,0,28.9,0.126,24,0
|
||||
0,137,84,27,0,27.3,0.231,59,0
|
||||
2,105,80,45,191,33.7,0.711,29,1
|
||||
7,114,76,17,110,23.8,0.466,31,0
|
||||
8,126,74,38,75,25.9,0.162,39,0
|
||||
4,132,86,31,0,28,0.419,63,0
|
||||
3,158,70,30,328,35.5,0.344,35,1
|
||||
0,123,88,37,0,35.2,0.197,29,0
|
||||
4,85,58,22,49,27.8,0.306,28,0
|
||||
0,84,82,31,125,38.2,0.233,23,0
|
||||
0,145,0,0,0,44.2,0.63,31,1
|
||||
0,135,68,42,250,42.3,0.365,24,1
|
||||
1,139,62,41,480,40.7,0.536,21,0
|
||||
0,173,78,32,265,46.5,1.159,58,0
|
||||
4,99,72,17,0,25.6,0.294,28,0
|
||||
8,194,80,0,0,26.1,0.551,67,0
|
||||
2,83,65,28,66,36.8,0.629,24,0
|
||||
2,89,90,30,0,33.5,0.292,42,0
|
||||
4,99,68,38,0,32.8,0.145,33,0
|
||||
4,125,70,18,122,28.9,1.144,45,1
|
||||
3,80,0,0,0,0,0.174,22,0
|
||||
6,166,74,0,0,26.6,0.304,66,0
|
||||
5,110,68,0,0,26,0.292,30,0
|
||||
2,81,72,15,76,30.1,0.547,25,0
|
||||
7,195,70,33,145,25.1,0.163,55,1
|
||||
6,154,74,32,193,29.3,0.839,39,0
|
||||
2,117,90,19,71,25.2,0.313,21,0
|
||||
3,84,72,32,0,37.2,0.267,28,0
|
||||
6,0,68,41,0,39,0.727,41,1
|
||||
7,94,64,25,79,33.3,0.738,41,0
|
||||
3,96,78,39,0,37.3,0.238,40,0
|
||||
10,75,82,0,0,33.3,0.263,38,0
|
||||
0,180,90,26,90,36.5,0.314,35,1
|
||||
1,130,60,23,170,28.6,0.692,21,0
|
||||
2,84,50,23,76,30.4,0.968,21,0
|
||||
8,120,78,0,0,25,0.409,64,0
|
||||
12,84,72,31,0,29.7,0.297,46,1
|
||||
0,139,62,17,210,22.1,0.207,21,0
|
||||
9,91,68,0,0,24.2,0.2,58,0
|
||||
2,91,62,0,0,27.3,0.525,22,0
|
||||
3,99,54,19,86,25.6,0.154,24,0
|
||||
3,163,70,18,105,31.6,0.268,28,1
|
||||
9,145,88,34,165,30.3,0.771,53,1
|
||||
7,125,86,0,0,37.6,0.304,51,0
|
||||
13,76,60,0,0,32.8,0.18,41,0
|
||||
6,129,90,7,326,19.6,0.582,60,0
|
||||
2,68,70,32,66,25,0.187,25,0
|
||||
3,124,80,33,130,33.2,0.305,26,0
|
||||
6,114,0,0,0,0,0.189,26,0
|
||||
9,130,70,0,0,34.2,0.652,45,1
|
||||
3,125,58,0,0,31.6,0.151,24,0
|
||||
3,87,60,18,0,21.8,0.444,21,0
|
||||
1,97,64,19,82,18.2,0.299,21,0
|
||||
3,116,74,15,105,26.3,0.107,24,0
|
||||
0,117,66,31,188,30.8,0.493,22,0
|
||||
0,111,65,0,0,24.6,0.66,31,0
|
||||
2,122,60,18,106,29.8,0.717,22,0
|
||||
0,107,76,0,0,45.3,0.686,24,0
|
||||
1,86,66,52,65,41.3,0.917,29,0
|
||||
6,91,0,0,0,29.8,0.501,31,0
|
||||
1,77,56,30,56,33.3,1.251,24,0
|
||||
4,132,0,0,0,32.9,0.302,23,1
|
||||
0,105,90,0,0,29.6,0.197,46,0
|
||||
0,57,60,0,0,21.7,0.735,67,0
|
||||
0,127,80,37,210,36.3,0.804,23,0
|
||||
3,129,92,49,155,36.4,0.968,32,1
|
||||
8,100,74,40,215,39.4,0.661,43,1
|
||||
3,128,72,25,190,32.4,0.549,27,1
|
||||
10,90,85,32,0,34.9,0.825,56,1
|
||||
4,84,90,23,56,39.5,0.159,25,0
|
||||
1,88,78,29,76,32,0.365,29,0
|
||||
8,186,90,35,225,34.5,0.423,37,1
|
||||
5,187,76,27,207,43.6,1.034,53,1
|
||||
4,131,68,21,166,33.1,0.16,28,0
|
||||
1,164,82,43,67,32.8,0.341,50,0
|
||||
4,189,110,31,0,28.5,0.68,37,0
|
||||
1,116,70,28,0,27.4,0.204,21,0
|
||||
3,84,68,30,106,31.9,0.591,25,0
|
||||
6,114,88,0,0,27.8,0.247,66,0
|
||||
1,88,62,24,44,29.9,0.422,23,0
|
||||
1,84,64,23,115,36.9,0.471,28,0
|
||||
7,124,70,33,215,25.5,0.161,37,0
|
||||
1,97,70,40,0,38.1,0.218,30,0
|
||||
8,110,76,0,0,27.8,0.237,58,0
|
||||
11,103,68,40,0,46.2,0.126,42,0
|
||||
11,85,74,0,0,30.1,0.3,35,0
|
||||
6,125,76,0,0,33.8,0.121,54,1
|
||||
0,198,66,32,274,41.3,0.502,28,1
|
||||
1,87,68,34,77,37.6,0.401,24,0
|
||||
6,99,60,19,54,26.9,0.497,32,0
|
||||
0,91,80,0,0,32.4,0.601,27,0
|
||||
2,95,54,14,88,26.1,0.748,22,0
|
||||
1,99,72,30,18,38.6,0.412,21,0
|
||||
6,92,62,32,126,32,0.085,46,0
|
||||
4,154,72,29,126,31.3,0.338,37,0
|
||||
0,121,66,30,165,34.3,0.203,33,1
|
||||
3,78,70,0,0,32.5,0.27,39,0
|
||||
2,130,96,0,0,22.6,0.268,21,0
|
||||
3,111,58,31,44,29.5,0.43,22,0
|
||||
2,98,60,17,120,34.7,0.198,22,0
|
||||
1,143,86,30,330,30.1,0.892,23,0
|
||||
1,119,44,47,63,35.5,0.28,25,0
|
||||
6,108,44,20,130,24,0.813,35,0
|
||||
2,118,80,0,0,42.9,0.693,21,1
|
||||
10,133,68,0,0,27,0.245,36,0
|
||||
2,197,70,99,0,34.7,0.575,62,1
|
||||
0,151,90,46,0,42.1,0.371,21,1
|
||||
6,109,60,27,0,25,0.206,27,0
|
||||
12,121,78,17,0,26.5,0.259,62,0
|
||||
8,100,76,0,0,38.7,0.19,42,0
|
||||
8,124,76,24,600,28.7,0.687,52,1
|
||||
1,93,56,11,0,22.5,0.417,22,0
|
||||
8,143,66,0,0,34.9,0.129,41,1
|
||||
6,103,66,0,0,24.3,0.249,29,0
|
||||
3,176,86,27,156,33.3,1.154,52,1
|
||||
0,73,0,0,0,21.1,0.342,25,0
|
||||
11,111,84,40,0,46.8,0.925,45,1
|
||||
2,112,78,50,140,39.4,0.175,24,0
|
||||
3,132,80,0,0,34.4,0.402,44,1
|
||||
2,82,52,22,115,28.5,1.699,25,0
|
||||
6,123,72,45,230,33.6,0.733,34,0
|
||||
0,188,82,14,185,32,0.682,22,1
|
||||
0,67,76,0,0,45.3,0.194,46,0
|
||||
1,89,24,19,25,27.8,0.559,21,0
|
||||
1,173,74,0,0,36.8,0.088,38,1
|
||||
1,109,38,18,120,23.1,0.407,26,0
|
||||
1,108,88,19,0,27.1,0.4,24,0
|
||||
6,96,0,0,0,23.7,0.19,28,0
|
||||
1,124,74,36,0,27.8,0.1,30,0
|
||||
7,150,78,29,126,35.2,0.692,54,1
|
||||
4,183,0,0,0,28.4,0.212,36,1
|
||||
1,124,60,32,0,35.8,0.514,21,0
|
||||
1,181,78,42,293,40,1.258,22,1
|
||||
1,92,62,25,41,19.5,0.482,25,0
|
||||
0,152,82,39,272,41.5,0.27,27,0
|
||||
1,111,62,13,182,24,0.138,23,0
|
||||
3,106,54,21,158,30.9,0.292,24,0
|
||||
3,174,58,22,194,32.9,0.593,36,1
|
||||
7,168,88,42,321,38.2,0.787,40,1
|
||||
6,105,80,28,0,32.5,0.878,26,0
|
||||
11,138,74,26,144,36.1,0.557,50,1
|
||||
3,106,72,0,0,25.8,0.207,27,0
|
||||
6,117,96,0,0,28.7,0.157,30,0
|
||||
2,68,62,13,15,20.1,0.257,23,0
|
||||
9,112,82,24,0,28.2,1.282,50,1
|
||||
0,119,0,0,0,32.4,0.141,24,1
|
||||
2,112,86,42,160,38.4,0.246,28,0
|
||||
2,92,76,20,0,24.2,1.698,28,0
|
||||
6,183,94,0,0,40.8,1.461,45,0
|
||||
0,94,70,27,115,43.5,0.347,21,0
|
||||
2,108,64,0,0,30.8,0.158,21,0
|
||||
4,90,88,47,54,37.7,0.362,29,0
|
||||
0,125,68,0,0,24.7,0.206,21,0
|
||||
0,132,78,0,0,32.4,0.393,21,0
|
||||
5,128,80,0,0,34.6,0.144,45,0
|
||||
4,94,65,22,0,24.7,0.148,21,0
|
||||
7,114,64,0,0,27.4,0.732,34,1
|
||||
0,102,78,40,90,34.5,0.238,24,0
|
||||
2,111,60,0,0,26.2,0.343,23,0
|
||||
1,128,82,17,183,27.5,0.115,22,0
|
||||
10,92,62,0,0,25.9,0.167,31,0
|
||||
13,104,72,0,0,31.2,0.465,38,1
|
||||
5,104,74,0,0,28.8,0.153,48,0
|
||||
2,94,76,18,66,31.6,0.649,23,0
|
||||
7,97,76,32,91,40.9,0.871,32,1
|
||||
1,100,74,12,46,19.5,0.149,28,0
|
||||
0,102,86,17,105,29.3,0.695,27,0
|
||||
4,128,70,0,0,34.3,0.303,24,0
|
||||
6,147,80,0,0,29.5,0.178,50,1
|
||||
4,90,0,0,0,28,0.61,31,0
|
||||
3,103,72,30,152,27.6,0.73,27,0
|
||||
2,157,74,35,440,39.4,0.134,30,0
|
||||
1,167,74,17,144,23.4,0.447,33,1
|
||||
0,179,50,36,159,37.8,0.455,22,1
|
||||
11,136,84,35,130,28.3,0.26,42,1
|
||||
0,107,60,25,0,26.4,0.133,23,0
|
||||
1,91,54,25,100,25.2,0.234,23,0
|
||||
1,117,60,23,106,33.8,0.466,27,0
|
||||
5,123,74,40,77,34.1,0.269,28,0
|
||||
2,120,54,0,0,26.8,0.455,27,0
|
||||
1,106,70,28,135,34.2,0.142,22,0
|
||||
2,155,52,27,540,38.7,0.24,25,1
|
||||
2,101,58,35,90,21.8,0.155,22,0
|
||||
1,120,80,48,200,38.9,1.162,41,0
|
||||
11,127,106,0,0,39,0.19,51,0
|
||||
3,80,82,31,70,34.2,1.292,27,1
|
||||
10,162,84,0,0,27.7,0.182,54,0
|
||||
1,199,76,43,0,42.9,1.394,22,1
|
||||
8,167,106,46,231,37.6,0.165,43,1
|
||||
9,145,80,46,130,37.9,0.637,40,1
|
||||
6,115,60,39,0,33.7,0.245,40,1
|
||||
1,112,80,45,132,34.8,0.217,24,0
|
||||
4,145,82,18,0,32.5,0.235,70,1
|
||||
10,111,70,27,0,27.5,0.141,40,1
|
||||
6,98,58,33,190,34,0.43,43,0
|
||||
9,154,78,30,100,30.9,0.164,45,0
|
||||
6,165,68,26,168,33.6,0.631,49,0
|
||||
1,99,58,10,0,25.4,0.551,21,0
|
||||
10,68,106,23,49,35.5,0.285,47,0
|
||||
3,123,100,35,240,57.3,0.88,22,0
|
||||
8,91,82,0,0,35.6,0.587,68,0
|
||||
6,195,70,0,0,30.9,0.328,31,1
|
||||
9,156,86,0,0,24.8,0.23,53,1
|
||||
0,93,60,0,0,35.3,0.263,25,0
|
||||
3,121,52,0,0,36,0.127,25,1
|
||||
2,101,58,17,265,24.2,0.614,23,0
|
||||
2,56,56,28,45,24.2,0.332,22,0
|
||||
0,162,76,36,0,49.6,0.364,26,1
|
||||
0,95,64,39,105,44.6,0.366,22,0
|
||||
4,125,80,0,0,32.3,0.536,27,1
|
||||
5,136,82,0,0,0,0.64,69,0
|
||||
2,129,74,26,205,33.2,0.591,25,0
|
||||
3,130,64,0,0,23.1,0.314,22,0
|
||||
1,107,50,19,0,28.3,0.181,29,0
|
||||
1,140,74,26,180,24.1,0.828,23,0
|
||||
1,144,82,46,180,46.1,0.335,46,1
|
||||
8,107,80,0,0,24.6,0.856,34,0
|
||||
13,158,114,0,0,42.3,0.257,44,1
|
||||
2,121,70,32,95,39.1,0.886,23,0
|
||||
7,129,68,49,125,38.5,0.439,43,1
|
||||
2,90,60,0,0,23.5,0.191,25,0
|
||||
7,142,90,24,480,30.4,0.128,43,1
|
||||
3,169,74,19,125,29.9,0.268,31,1
|
||||
0,99,0,0,0,25,0.253,22,0
|
||||
4,127,88,11,155,34.5,0.598,28,0
|
||||
4,118,70,0,0,44.5,0.904,26,0
|
||||
2,122,76,27,200,35.9,0.483,26,0
|
||||
6,125,78,31,0,27.6,0.565,49,1
|
||||
1,168,88,29,0,35,0.905,52,1
|
||||
2,129,0,0,0,38.5,0.304,41,0
|
||||
4,110,76,20,100,28.4,0.118,27,0
|
||||
6,80,80,36,0,39.8,0.177,28,0
|
||||
10,115,0,0,0,0,0.261,30,1
|
||||
2,127,46,21,335,34.4,0.176,22,0
|
||||
9,164,78,0,0,32.8,0.148,45,1
|
||||
2,93,64,32,160,38,0.674,23,1
|
||||
3,158,64,13,387,31.2,0.295,24,0
|
||||
5,126,78,27,22,29.6,0.439,40,0
|
||||
10,129,62,36,0,41.2,0.441,38,1
|
||||
0,134,58,20,291,26.4,0.352,21,0
|
||||
3,102,74,0,0,29.5,0.121,32,0
|
||||
7,187,50,33,392,33.9,0.826,34,1
|
||||
3,173,78,39,185,33.8,0.97,31,1
|
||||
10,94,72,18,0,23.1,0.595,56,0
|
||||
1,108,60,46,178,35.5,0.415,24,0
|
||||
5,97,76,27,0,35.6,0.378,52,1
|
||||
4,83,86,19,0,29.3,0.317,34,0
|
||||
1,114,66,36,200,38.1,0.289,21,0
|
||||
1,149,68,29,127,29.3,0.349,42,1
|
||||
5,117,86,30,105,39.1,0.251,42,0
|
||||
1,111,94,0,0,32.8,0.265,45,0
|
||||
4,112,78,40,0,39.4,0.236,38,0
|
||||
1,116,78,29,180,36.1,0.496,25,0
|
||||
0,141,84,26,0,32.4,0.433,22,0
|
||||
2,175,88,0,0,22.9,0.326,22,0
|
||||
2,92,52,0,0,30.1,0.141,22,0
|
||||
3,130,78,23,79,28.4,0.323,34,1
|
||||
8,120,86,0,0,28.4,0.259,22,1
|
||||
2,174,88,37,120,44.5,0.646,24,1
|
||||
2,106,56,27,165,29,0.426,22,0
|
||||
2,105,75,0,0,23.3,0.56,53,0
|
||||
4,95,60,32,0,35.4,0.284,28,0
|
||||
0,126,86,27,120,27.4,0.515,21,0
|
||||
8,65,72,23,0,32,0.6,42,0
|
||||
2,99,60,17,160,36.6,0.453,21,0
|
||||
1,102,74,0,0,39.5,0.293,42,1
|
||||
11,120,80,37,150,42.3,0.785,48,1
|
||||
3,102,44,20,94,30.8,0.4,26,0
|
||||
1,109,58,18,116,28.5,0.219,22,0
|
||||
9,140,94,0,0,32.7,0.734,45,1
|
||||
13,153,88,37,140,40.6,1.174,39,0
|
||||
12,100,84,33,105,30,0.488,46,0
|
||||
1,147,94,41,0,49.3,0.358,27,1
|
||||
1,81,74,41,57,46.3,1.096,32,0
|
||||
3,187,70,22,200,36.4,0.408,36,1
|
||||
6,162,62,0,0,24.3,0.178,50,1
|
||||
4,136,70,0,0,31.2,1.182,22,1
|
||||
1,121,78,39,74,39,0.261,28,0
|
||||
3,108,62,24,0,26,0.223,25,0
|
||||
0,181,88,44,510,43.3,0.222,26,1
|
||||
8,154,78,32,0,32.4,0.443,45,1
|
||||
1,128,88,39,110,36.5,1.057,37,1
|
||||
7,137,90,41,0,32,0.391,39,0
|
||||
0,123,72,0,0,36.3,0.258,52,1
|
||||
1,106,76,0,0,37.5,0.197,26,0
|
||||
6,190,92,0,0,35.5,0.278,66,1
|
||||
2,88,58,26,16,28.4,0.766,22,0
|
||||
9,170,74,31,0,44,0.403,43,1
|
||||
9,89,62,0,0,22.5,0.142,33,0
|
||||
10,101,76,48,180,32.9,0.171,63,0
|
||||
2,122,70,27,0,36.8,0.34,27,0
|
||||
5,121,72,23,112,26.2,0.245,30,0
|
||||
1,126,60,0,0,30.1,0.349,47,1
|
||||
1,93,70,31,0,30.4,0.315,23,0
|
||||
|
-46
@@ -1,46 +0,0 @@
|
||||
--
|
||||
-- BEFORE RUNNING THIS SCRIPT, UPDATE IT WITH YOUR OWN STRING FOR THE EXTERNAL TABLE NAME (search for "<yourTableNameHere>")
|
||||
--
|
||||
USE Sales
|
||||
GO
|
||||
|
||||
PRINT 'STEP 1: Create external table'
|
||||
CREATE EXTERNAL TABLE [<yourTableNameHere>]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlDataPool,
|
||||
DISTRIBUTION = ROUND_ROBIN
|
||||
)
|
||||
|
||||
PRINT 'STEP 2: Populate external table using Spark job'
|
||||
-- This object is used for starting spark streaming session using spark-submit
|
||||
--
|
||||
-- The arguments to jar file are
|
||||
-- 1: server name - sql server to connect to read the table schema
|
||||
-- 2: port number
|
||||
-- 3: username - sql server username for master instance
|
||||
-- 4: password - sql server password for master instance
|
||||
-- 5: database name
|
||||
-- 6: external table name
|
||||
-- 7: Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
||||
-- 8: Input format. This can be "csv", "parquet", "json".
|
||||
-- 9: enable checkpoint: true or false
|
||||
--
|
||||
-- Submit spark job with below parameters. You can use the Spark submit experience from Azure Data Studio (right click on server name-> Submit Spark Job):
|
||||
-- ARGUMENTS:
|
||||
---- job name: <yourJobName>
|
||||
---- switch from "Local" to "HDFS"
|
||||
---- Main class: "FileStreaming"
|
||||
---- Path to jar: /jar/mssql-spark-lib-assembly-1.0.jar
|
||||
---- Arguments (UPDATE WITH THE NAME OF YOUR EXTERNAL TABLE):
|
||||
---- mssql-master-pool-0.service-master-pool 1433 sa Orland0!gnite sales yourTableNameHere hdfs:///clickstream_data csv false
|
||||
|
||||
PRINT 'STEP 3: Check data in external table'
|
||||
SELECT count(*) FROM [dbo].[<yourTableNameHere>]
|
||||
SELECT TOP 10 * FROM [dbo].[<yourTableNameHere>]
|
||||
GO
|
||||
|
||||
PRINT 'STEP 4: Cleanup...drop external table'
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[<yourTableNameHere>]
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
--
|
||||
-- BEFORE RUNNING THIS SCRIPT, UPDATE IT WITH YOUR OWN STRING FOR THE EXTERNAL TABLE NAME (search for "<yourTableNameHere>")
|
||||
--
|
||||
|
||||
PRINT 'STEP 1: Connect to SQL Server Master instance'
|
||||
|
||||
USE Sales
|
||||
GO
|
||||
|
||||
PRINT 'STEP 2: Create external table'
|
||||
CREATE EXTERNAL TABLE [<yourTableNameHere>]]
|
||||
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlDataPool,
|
||||
DISTRIBUTION = ROUND_ROBIN
|
||||
)
|
||||
|
||||
PRINT 'STEP 3: Populate external table using sql stored proc'
|
||||
DECLARE @db_name SYSNAME = 'sales'
|
||||
DECLARE @schema_name SYSNAME = 'dbo'
|
||||
DECLARE @table_name SYSNAME = '<yourTableNameHere>' --UPDATE WITH THE NAME OF YOUR EXTERNAL TABLE
|
||||
DECLARE @query SYSNAME = 'SELECT TOP(1000) * FROM sales.dbo.web_clickstreams'
|
||||
|
||||
EXEC model..sp_data_pool_table_insert_data @db_name, @schema_name, @table_name, @query
|
||||
|
||||
PRINT 'STEP 4: Check data in external table'
|
||||
SELECT count(*) FROM [dbo].[<yourTableNameHere>]
|
||||
SELECT TOP 10 * FROM [dbo].[<yourTableNameHere>]
|
||||
|
||||
SELECT
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS clicks_in_category,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS clicks_in_1,
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS clicks_in_2,
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS clicks_in_3,
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS clicks_in_4,
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS clicks_in_5,
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS clicks_in_6,
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS clicks_in_7,
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS clicks_in_8,
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS clicks_in_9
|
||||
FROM [dbo].[<yourTableNameHere>]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
PRINT 'STEP 5: Cleanup...drop external table'
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[<yourTableNameHere>]
|
||||
@@ -1,41 +0,0 @@
|
||||
USE Sales
|
||||
GO
|
||||
|
||||
--Create database scoped credential to connect to Oracle server
|
||||
--
|
||||
CREATE DATABASE SCOPED CREDENTIAL [<yourCredentialNameHere>] WITH IDENTITY = 'SYSTEM', SECRET = 'Admin123';
|
||||
GO
|
||||
|
||||
--Create external data source that points to Oracle server
|
||||
--
|
||||
CREATE EXTERNAL DATA SOURCE [<yourDataSourceNameHere>]
|
||||
WITH (LOCATION = 'oracle://APS40-10.oltp.sql.cass.hp.com',CREDENTIAL = [demo_credential]);
|
||||
|
||||
--Create external table over inventory table on Oracle server
|
||||
--
|
||||
CREATE EXTERNAL TABLE [<yourTableNameHere>]
|
||||
([inv_date] DECIMAL(10,0) NOT NULL,[inv_item] DECIMAL(10,0) NOT NULL,
|
||||
[inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0))
|
||||
WITH (DATA_SOURCE=[<yourDataSourceNameHere>], LOCATION='xe.HR.INVENTORY');
|
||||
|
||||
DROP EXTERNAL TABLE ORACLE_INVENTORY
|
||||
DROP EXTERNAL DATA SOURCE ORACLE_INVENTORY
|
||||
|
||||
-- Query external table with local tables
|
||||
-- Execution time: ~54 secs
|
||||
SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity
|
||||
FROM [<yourTableNameHere>]] as i
|
||||
JOIN item as it
|
||||
ON it.i_item_sk = i.inv_item
|
||||
JOIN warehouse as w
|
||||
ON w.w_warehouse_sk = i.inv_warehouse
|
||||
WHERE it.i_category = 'Books'
|
||||
GROUP BY w.w_warehouse_name, i.inv_item;
|
||||
GO
|
||||
|
||||
--Cleanup
|
||||
--
|
||||
DROP EXTERNAL TABLE [<yourTableName>];
|
||||
DROP EXTERNAL DATA SOURCE [<yourDataSourceNameHere>] ;
|
||||
DROP DATABASE SCOPED CREDENTIAL [<yourCredentialNameHere>];
|
||||
GO
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 32 KiB |
Binary file not shown.
-60
@@ -1,60 +0,0 @@
|
||||
/*
|
||||
For all items whose price was changed on a given date, compute the percentage change in inventory between the 30day period BEFORE the price change
|
||||
and the 30day period AFTER the change. Group this information by warehouse.
|
||||
*/
|
||||
|
||||
--Replace <yourTableNameHere> with your external table name that you created pointing to Oracle table
|
||||
|
||||
DECLARE @q22_date VARCHAR(10);
|
||||
DECLARE @q22_current_price_min DECIMAL(18, 6);
|
||||
DECLARE @q22_current_price_max DECIMAL(18, 6);
|
||||
|
||||
SET @q22_date = '2001-05-08';
|
||||
SET @q22_current_price_min = 0.98;
|
||||
SET @q22_current_price_max = 1.5;
|
||||
|
||||
SELECT TOP (100) *
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
w_warehouse_name,
|
||||
i_item_id,
|
||||
SUM
|
||||
(
|
||||
CASE WHEN DATEDIFF(dd, d_date, CAST(@q22_date AS DATETIME)) >= 0
|
||||
THEN inv_quantity_on_hand
|
||||
ELSE 0 END
|
||||
) AS inv_after,
|
||||
SUM
|
||||
(
|
||||
CASE WHEN DATEDIFF(dd, d_date, CAST(@q22_date AS DATETIME)) < 0
|
||||
THEN inv_quantity_on_hand
|
||||
ELSE 0 END
|
||||
) AS inv_before
|
||||
FROM
|
||||
<yourTableNameHere> inv, --UPDATE WITH THE NAME OF YOUR EXTERNAL TABLE
|
||||
item i,
|
||||
warehouse w,
|
||||
date_dim d
|
||||
WHERE
|
||||
i_current_price BETWEEN @q22_current_price_min AND @q22_current_price_max
|
||||
AND i_item_sk = inv_item
|
||||
AND inv_warehouse = w_warehouse_sk
|
||||
AND inv_date = d_date_sk
|
||||
AND DATEDIFF(dd, d_date, @q22_date) >= -30
|
||||
AND DATEDIFF(dd, d_date, @q22_date) <= 30
|
||||
GROUP BY w_warehouse_name, i_item_id
|
||||
) T
|
||||
WHERE
|
||||
inv_before > 0
|
||||
-- CAST is required, otherwise the division is computed as an integer
|
||||
AND CAST(inv_after AS DECIMAL) / CAST(inv_before AS DECIMAL) >= 2.0 / 3.0
|
||||
AND CAST(inv_after AS DECIMAL) / CAST(inv_before AS DECIMAL) <= 3.0 / 2.0
|
||||
ORDER BY w_warehouse_name, i_item_id
|
||||
;
|
||||
|
||||
--Cleanup
|
||||
--
|
||||
DROP EXTERNAL TABLE [<yourTableNameHere>]
|
||||
DROP EXTERNAL DATA SOURCE [<yourDataSourceNameHere>]
|
||||
DROP DATABASE SCOPED CREDENTIAL [<yourCredentialNameHere>]
|
||||
@@ -1,136 +0,0 @@
|
||||
USE master;
|
||||
GO
|
||||
|
||||
RESTORE DATABASE sales
|
||||
FROM DISK=N'/var/opt/mssql/data/tpcxbb_1gb_latest_sql17.bak'
|
||||
WITH
|
||||
MOVE N'tpcxbb_1gb_test' TO N'/var/opt/mssql/data/sales.mdf',
|
||||
MOVE N'tpcxbb_1gb_test_log' TO N'/var/opt/mssql/data/sales.ldf';
|
||||
GO
|
||||
|
||||
USE sales;
|
||||
GO
|
||||
CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'Orland0!gnite';
|
||||
|
||||
CREATE EXTERNAL DATA SOURCE SqlDataPool
|
||||
WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default');
|
||||
|
||||
CREATE EXTERNAL DATA SOURCE SqlStoragePool
|
||||
WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080');
|
||||
|
||||
CREATE EXTERNAL FILE FORMAT csv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = ',',
|
||||
STRING_DELIMITER = '"',
|
||||
FIRST_ROW = 2,
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
|
||||
CREATE EXTERNAL FILE FORMAT parquet_file
|
||||
WITH (
|
||||
FORMAT_TYPE = PARQUET
|
||||
);
|
||||
|
||||
GO
|
||||
CREATE VIEW [dbo].[web_clickstreams_book_clicks]
|
||||
AS
|
||||
SELECT
|
||||
q.clicks_in_category,
|
||||
CASE WHEN cd.cd_education_status IN ('Advanced Degree', 'College', '4 yr Degree', '2 yr Degree') THEN 1 ELSE 0 END AS college_education,
|
||||
CASE WHEN cd.cd_gender = 'M' THEN 1 ELSE 0 END AS male,
|
||||
q.clicks_in_1,
|
||||
q.clicks_in_2,
|
||||
q.clicks_in_3,
|
||||
q.clicks_in_4,
|
||||
q.clicks_in_5,
|
||||
q.clicks_in_6,
|
||||
q.clicks_in_7,
|
||||
q.clicks_in_8,
|
||||
q.clicks_in_9
|
||||
FROM(
|
||||
SELECT
|
||||
w.wcs_user_sk,
|
||||
SUM( CASE WHEN i.i_category = 'Books' THEN 1 ELSE 0 END) AS clicks_in_category,
|
||||
SUM( CASE WHEN i.i_category_id = 1 THEN 1 ELSE 0 END) AS clicks_in_1,
|
||||
SUM( CASE WHEN i.i_category_id = 2 THEN 1 ELSE 0 END) AS clicks_in_2,
|
||||
SUM( CASE WHEN i.i_category_id = 3 THEN 1 ELSE 0 END) AS clicks_in_3,
|
||||
SUM( CASE WHEN i.i_category_id = 4 THEN 1 ELSE 0 END) AS clicks_in_4,
|
||||
SUM( CASE WHEN i.i_category_id = 5 THEN 1 ELSE 0 END) AS clicks_in_5,
|
||||
SUM( CASE WHEN i.i_category_id = 6 THEN 1 ELSE 0 END) AS clicks_in_6,
|
||||
SUM( CASE WHEN i.i_category_id = 7 THEN 1 ELSE 0 END) AS clicks_in_7,
|
||||
SUM( CASE WHEN i.i_category_id = 8 THEN 1 ELSE 0 END) AS clicks_in_8,
|
||||
SUM( CASE WHEN i.i_category_id = 9 THEN 1 ELSE 0 END) AS clicks_in_9
|
||||
FROM web_clickstreams as w
|
||||
INNER JOIN item as i ON (w.wcs_item_sk = i_item_sk
|
||||
AND w.wcs_user_sk IS NOT NULL)
|
||||
GROUP BY w.wcs_user_sk
|
||||
) AS q
|
||||
INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk
|
||||
INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk;
|
||||
GO
|
||||
|
||||
CREATE TABLE sales_models (
|
||||
model_name varchar(100) primary key clustered,
|
||||
model varbinary(max),
|
||||
model_native varbinary(max),
|
||||
create_time datetime2 DEFAULT(SYSDATETIME())
|
||||
);
|
||||
GO
|
||||
|
||||
CREATE OR ALTER PROCEDURE [dbo].[train_book_category_visitor]
|
||||
(@model_name varchar(100))
|
||||
AS
|
||||
BEGIN
|
||||
DECLARE @start_time datetime2 = SYSDATETIME()
|
||||
, @model varbinary(max)
|
||||
, @model_native varbinary(max)
|
||||
, @input_query nvarchar(max)
|
||||
, @train_script nvarchar(max)
|
||||
|
||||
-- Set the input query for training. We will use 80% of the data.
|
||||
SET @input_query = N'
|
||||
SELECT TOP(80) PERCENT SIGN(q.clicks_in_category) AS book_category
|
||||
, q.college_education
|
||||
, q.male
|
||||
, q.clicks_in_1
|
||||
, q.clicks_in_2
|
||||
, q.clicks_in_3
|
||||
, q.clicks_in_4
|
||||
, q.clicks_in_5
|
||||
, q.clicks_in_6
|
||||
, q.clicks_in_7
|
||||
, q.clicks_in_8
|
||||
, q.clicks_in_9
|
||||
FROM web_clickstreams_book_clicks as q
|
||||
';
|
||||
-- Training R script that uses rxLogit function from RevoScaleR package (Microsoft R Server) to generate model to predict book_category click(s).
|
||||
SET @train_script = N'
|
||||
# build classification model to predict book_category
|
||||
logitObj <- rxLogit(book_category ~ college_education + male +
|
||||
clicks_in_1 + clicks_in_2 + clicks_in_3 + clicks_in_4 + clicks_in_5 +
|
||||
clicks_in_6 + clicks_in_7 + clicks_in_8 + clicks_in_9 , data = indata)
|
||||
|
||||
# First, serialize a model and put it into a database table
|
||||
modelbin <- as.raw(serialize(logitObj, NULL));
|
||||
|
||||
model_native <- rxSerializeModel(logitObj, realtimeScoringOnly = TRUE)
|
||||
';
|
||||
|
||||
-- Generate sales model using R scirpt with the book clicks stats for each user
|
||||
EXECUTE sp_execute_external_script
|
||||
@language = N'R'
|
||||
, @script = @train_script
|
||||
, @input_data_1 = @input_query
|
||||
, @input_data_1_name = N'indata'
|
||||
, @params = N'@input_query nvarchar(max), @modelbin varbinary(max) OUTPUT, @model_native varbinary(max) OUTPUT'
|
||||
, @input_query = @input_query
|
||||
, @modelbin = @model OUTPUT
|
||||
, @model_native = @model_native OUTPUT;
|
||||
|
||||
-- Save the trained models to predict user clicks on book category in the website
|
||||
DELETE FROM sales_models WHERE model_name = @model_name;
|
||||
INSERT INTO sales_models (model_name, model, model_native) VALUES(@model_name, @model, @model_native);
|
||||
END;
|
||||
GO
|
||||
@@ -1,500 +0,0 @@
|
||||
wcs_click_date_sk,wcs_click_time_sk,wcs_sales_sk,wcs_item_sk,wcs_web_page_sk,wcs_user_sk
|
||||
38569,4250,,7840,18,
|
||||
38569,85106,,11130,18,
|
||||
38569,52655,,3716,18,
|
||||
38569,70934,,13243,18,
|
||||
38569,40166,,5389,18,
|
||||
38570,73271,,3331,18,
|
||||
38570,24651,,10049,18,
|
||||
38570,23805,,921,18,
|
||||
38570,66458,,4407,18,
|
||||
38570,65912,,11494,18,
|
||||
38570,80073,,1833,18,
|
||||
38570,9522,,17636,18,
|
||||
38570,22779,,9063,18,
|
||||
38570,43276,,8696,18,
|
||||
38570,18021,,3231,18,
|
||||
38570,37963,,17050,18,
|
||||
38570,49125,,9044,18,
|
||||
38570,69824,,3255,18,
|
||||
38570,10028,,1355,18,
|
||||
38570,6749,,16880,18,
|
||||
38570,11109,,8200,18,
|
||||
38570,26098,,8862,18,
|
||||
38570,67236,,322,18,
|
||||
38570,73267,,16552,18,
|
||||
38570,13880,,8774,18,
|
||||
38570,6876,,8901,18,
|
||||
38571,7522,,6096,18,
|
||||
38571,68166,,4562,18,
|
||||
38571,46447,,10029,18,
|
||||
38571,73196,,4953,18,
|
||||
38571,11100,,7955,18,
|
||||
38571,43257,,5902,18,
|
||||
38571,30485,,17700,18,
|
||||
38571,74897,,9614,18,
|
||||
38571,70216,,11894,18,
|
||||
38571,78084,,16521,18,
|
||||
38571,78060,,14707,18,
|
||||
38571,38882,,197,18,
|
||||
38571,45785,,1238,18,
|
||||
38571,34447,,15758,18,
|
||||
38572,6785,,14458,18,
|
||||
38572,54768,,656,18,
|
||||
38572,52120,,3976,18,
|
||||
38572,9755,,12483,18,
|
||||
38572,22885,,5818,18,
|
||||
38572,71948,,17455,18,
|
||||
38572,40789,,14944,18,
|
||||
38572,32523,,12935,18,
|
||||
38572,3987,,6591,18,
|
||||
38572,70836,,7604,18,
|
||||
38572,37252,,7716,18,
|
||||
38572,7251,,12287,18,
|
||||
38572,13776,,1215,18,
|
||||
38572,21272,,3965,18,
|
||||
38573,5542,,4934,18,
|
||||
38573,42940,,1222,18,
|
||||
38573,86380,,4466,18,
|
||||
38573,63611,,14164,18,
|
||||
38573,78273,,17345,18,
|
||||
38573,28306,,12573,18,
|
||||
38573,40781,,15144,18,
|
||||
38573,14237,,10141,18,
|
||||
38573,37348,,9041,18,
|
||||
38573,68988,,2593,18,
|
||||
38573,65047,,267,18,
|
||||
38573,84512,,806,18,
|
||||
38573,71987,,3436,18,
|
||||
38573,40063,,7593,18,
|
||||
38573,16101,,11291,18,
|
||||
38573,64439,,15939,18,
|
||||
38573,78635,,785,18,
|
||||
38573,81811,,14581,18,
|
||||
38573,68243,,7937,18,
|
||||
38573,9201,,12472,18,
|
||||
38573,57819,,2620,18,
|
||||
38573,52961,,11203,18,
|
||||
38573,8482,,15404,18,
|
||||
38573,32887,,11922,18,
|
||||
38573,7094,,14820,18,
|
||||
38573,79728,,12718,18,
|
||||
38573,72771,,15878,18,
|
||||
38573,38611,,4694,18,
|
||||
38573,30542,,10161,18,
|
||||
38573,83641,,3764,18,
|
||||
38573,25807,,12884,18,
|
||||
38574,3941,,15752,18,
|
||||
38574,19108,,11883,18,
|
||||
38574,33942,,10788,18,
|
||||
38574,69711,,3763,18,
|
||||
38574,72428,,4126,18,
|
||||
38574,551,,14683,18,
|
||||
38574,81306,,16240,18,
|
||||
38574,25350,,9416,18,
|
||||
38574,6203,,14814,18,
|
||||
38574,58724,,14706,18,
|
||||
38574,1506,,15654,18,
|
||||
38574,47787,,3823,18,
|
||||
38574,13383,,9789,18,
|
||||
38574,85322,,16246,18,
|
||||
38574,48614,,7037,18,
|
||||
38574,34654,,15814,18,
|
||||
38574,30336,,7678,18,
|
||||
38574,26695,,14729,18,
|
||||
38574,12275,,12145,18,
|
||||
38574,56898,,11745,18,
|
||||
38575,42608,,3114,18,
|
||||
38575,47488,,8060,18,
|
||||
38575,26214,,17150,18,
|
||||
38575,45588,,13126,18,
|
||||
38575,31045,,14711,18,
|
||||
38575,10941,,14717,18,
|
||||
38575,26668,,11525,18,
|
||||
38575,63025,,12964,18,
|
||||
38575,26264,,4925,18,
|
||||
38575,64929,,13618,18,
|
||||
38575,41552,,11210,18,
|
||||
38575,3466,,3841,18,
|
||||
38575,23178,,9323,18,
|
||||
38575,71201,,13210,18,
|
||||
38575,60590,,4562,18,
|
||||
38575,1265,,17123,18,
|
||||
38575,64935,,12085,18,
|
||||
38575,77088,,1044,18,
|
||||
38575,3848,,2132,18,
|
||||
38575,5468,,1690,18,
|
||||
38575,35399,,9772,18,
|
||||
38576,45921,,11459,18,
|
||||
38576,14505,,14933,18,
|
||||
38576,47469,,14196,18,
|
||||
38576,81085,,13953,18,
|
||||
38576,13505,,5400,18,
|
||||
38576,72264,,2468,18,
|
||||
38576,22781,,6807,18,
|
||||
38576,32914,,12777,18,
|
||||
38576,83295,,1347,18,
|
||||
38576,7895,,2218,18,
|
||||
38576,23964,,12063,18,
|
||||
38576,57349,,8984,18,
|
||||
38576,71762,,1062,18,
|
||||
38576,60590,,1503,18,
|
||||
38576,38714,,1969,18,
|
||||
38576,67355,,11850,18,
|
||||
38576,74178,,4570,18,
|
||||
38576,62682,,12434,18,
|
||||
38576,52673,,9934,18,
|
||||
38576,6508,,8621,18,
|
||||
38576,24839,,5319,18,
|
||||
38576,44285,,15069,18,
|
||||
38576,37848,,10352,18,
|
||||
38576,45200,,14083,18,
|
||||
38576,66998,,4483,18,
|
||||
38576,67849,,7235,18,
|
||||
38577,12949,,14421,18,
|
||||
38577,46725,,3486,18,
|
||||
38577,40291,,5470,18,
|
||||
38577,28390,,7865,18,
|
||||
38577,52150,,4449,18,
|
||||
38577,23827,,4545,18,
|
||||
38577,39919,,6824,18,
|
||||
38577,71175,,3414,18,
|
||||
38577,57788,,7440,18,
|
||||
38577,25113,,14131,18,
|
||||
38577,49503,,14947,18,
|
||||
38577,47827,,14703,18,
|
||||
38577,35895,,15551,18,
|
||||
38577,6009,,7026,18,
|
||||
38577,71294,,14609,18,
|
||||
38577,82471,,3202,18,
|
||||
38577,45245,,2351,18,
|
||||
38577,22174,,6224,18,
|
||||
38577,38891,,17555,18,
|
||||
38577,25725,,4498,18,
|
||||
38577,6268,,10208,18,
|
||||
38577,3237,,16050,18,
|
||||
38577,80669,,16291,18,
|
||||
38577,1824,,12462,18,
|
||||
38577,15858,,14097,18,
|
||||
38577,18149,,17380,18,
|
||||
38577,46344,,6309,18,
|
||||
38577,24420,,14210,18,
|
||||
38577,6972,,15207,18,
|
||||
38577,78632,,709,18,
|
||||
38577,8548,,5498,18,
|
||||
38577,1452,,2077,18,
|
||||
38577,869,,3087,18,
|
||||
38577,2009,,17360,18,
|
||||
38577,66111,,11952,18,
|
||||
38577,82801,,13014,18,
|
||||
38578,30702,,1364,18,
|
||||
38578,55011,,4608,18,
|
||||
38578,492,,16704,18,
|
||||
38578,32672,,5254,18,
|
||||
38578,28393,,13034,18,
|
||||
38578,77008,,10819,18,
|
||||
38578,78656,,5748,18,
|
||||
38578,51429,,17448,18,
|
||||
38578,23476,,5390,18,
|
||||
38578,53055,,13583,18,
|
||||
38578,10458,,1095,18,
|
||||
38578,18781,,14345,18,
|
||||
38578,47396,,12463,18,
|
||||
38578,25849,,323,18,
|
||||
38578,39235,,13558,18,
|
||||
38578,34733,,9033,18,
|
||||
38578,30988,,945,18,
|
||||
38578,26349,,1644,18,
|
||||
38578,23363,,5502,18,
|
||||
38578,21179,,13463,18,
|
||||
38578,24534,,5550,18,
|
||||
38578,32800,,8894,18,
|
||||
38579,9542,,13822,18,
|
||||
38579,35181,,8664,18,
|
||||
38579,48362,,3992,18,
|
||||
38579,41498,,10327,18,
|
||||
38579,80695,,3208,18,
|
||||
38579,13640,,6173,18,
|
||||
38579,81205,,1219,18,
|
||||
38579,4789,,1018,18,
|
||||
38579,65427,,8260,18,
|
||||
38579,41882,,7383,18,
|
||||
38579,34653,,7273,18,
|
||||
38579,56081,,10705,18,
|
||||
38579,74611,,13268,18,
|
||||
38579,30352,,5934,18,
|
||||
38579,40293,,2274,18,
|
||||
38579,9908,,11146,18,
|
||||
38579,82394,,16077,18,
|
||||
38579,83904,,14000,18,
|
||||
38579,78199,,7184,18,
|
||||
38579,55886,,6936,18,
|
||||
38579,67118,,11097,18,
|
||||
38579,31124,,861,18,
|
||||
38579,55138,,11377,18,
|
||||
38579,2400,,2960,18,
|
||||
38580,63986,,12913,18,
|
||||
38580,2882,,8997,18,
|
||||
38580,36397,,4380,18,
|
||||
38580,8009,,11088,18,
|
||||
38580,70122,,12902,18,
|
||||
38580,10509,,7592,18,
|
||||
38580,20123,,4488,18,
|
||||
38580,40057,,16296,18,
|
||||
38580,43136,,5671,18,
|
||||
38580,36311,,8049,18,
|
||||
38580,11373,,3722,18,
|
||||
38580,58980,,7369,18,
|
||||
38580,41803,,663,18,
|
||||
38580,77406,,13454,18,
|
||||
38580,84772,,12117,18,
|
||||
38580,13478,,6871,18,
|
||||
38580,43365,,9443,18,
|
||||
38580,15024,,7148,18,
|
||||
38580,38683,,2392,18,
|
||||
38580,30697,,15049,18,
|
||||
38580,23663,,12545,18,
|
||||
38580,72976,,4081,18,
|
||||
38580,40762,,5195,18,
|
||||
38580,63839,,15102,18,
|
||||
38580,75628,,16879,18,
|
||||
38580,4654,,1815,18,
|
||||
38580,39722,,9890,18,
|
||||
38580,84128,,11633,18,
|
||||
38581,28758,,16759,18,
|
||||
38581,66245,,4823,18,
|
||||
38581,65973,,5118,18,
|
||||
38581,58227,,7793,18,
|
||||
38581,37179,,9453,18,
|
||||
38581,76530,,12540,18,
|
||||
38581,8758,,3014,18,
|
||||
38581,81302,,5301,18,
|
||||
38581,45791,,3793,18,
|
||||
38581,11103,,2754,18,
|
||||
38581,32590,,8103,18,
|
||||
38581,48095,,8038,18,
|
||||
38581,67015,,2236,18,
|
||||
38581,42,,4702,18,
|
||||
38581,65905,,7444,18,
|
||||
38581,30525,,9507,18,
|
||||
38581,23020,,14100,18,
|
||||
38581,77402,,6036,18,
|
||||
38581,19419,,5893,18,
|
||||
38581,62362,,5270,18,
|
||||
38581,27868,,16665,18,
|
||||
38581,37728,,5124,18,
|
||||
38581,61040,,4922,18,
|
||||
38581,67241,,15188,18,
|
||||
38581,60301,,17023,18,
|
||||
38581,9158,,4794,18,
|
||||
38581,76503,,16760,18,
|
||||
38582,52715,,9354,18,
|
||||
38582,14731,,9601,18,
|
||||
38582,2332,,12152,18,
|
||||
38582,55326,,2825,18,
|
||||
38582,1325,,13434,18,
|
||||
38582,41117,,7275,18,
|
||||
38582,9850,,3672,18,
|
||||
38582,45971,,11003,18,
|
||||
38582,72289,,3929,18,
|
||||
38582,65896,,2733,18,
|
||||
38582,79243,,3486,18,
|
||||
38582,51879,,6453,18,
|
||||
38582,73191,,4978,18,
|
||||
38582,62984,,17802,18,
|
||||
38582,43224,,12694,18,
|
||||
38582,54211,,13225,18,
|
||||
38582,29359,,4781,18,
|
||||
38582,77082,,11189,18,
|
||||
38582,61889,,15666,18,
|
||||
38582,52178,,14918,18,
|
||||
38582,80180,,5008,18,
|
||||
38582,44126,,389,18,
|
||||
38582,20830,,14899,18,
|
||||
38582,50118,,4055,18,
|
||||
38582,18602,,2123,18,
|
||||
38582,52415,,11273,18,
|
||||
38582,7587,,16400,18,
|
||||
38583,48537,,7426,18,
|
||||
38583,49777,,16561,18,
|
||||
38583,52225,,15935,18,
|
||||
38583,23701,,3563,18,
|
||||
38583,8684,,1473,18,
|
||||
38583,55734,,8901,18,
|
||||
38583,22178,,13311,18,
|
||||
38583,6737,,14466,18,
|
||||
38583,71815,,15922,18,
|
||||
38583,40947,,11874,18,
|
||||
38583,14834,,15599,18,
|
||||
38583,57195,,6408,18,
|
||||
38583,39302,,11310,18,
|
||||
38583,343,,15011,18,
|
||||
38583,66539,,5457,18,
|
||||
38583,16825,,12586,18,
|
||||
38583,55627,,6309,18,
|
||||
38583,4193,,7928,18,
|
||||
38583,57864,,5717,18,
|
||||
38583,70185,,4265,18,
|
||||
38583,3826,,6928,18,
|
||||
38584,34176,,7493,18,
|
||||
38584,42370,,1042,18,
|
||||
38584,40641,,7951,18,
|
||||
38584,55356,,6499,18,
|
||||
38584,53740,,17798,18,
|
||||
38584,42373,,11965,18,
|
||||
38584,60657,,10185,18,
|
||||
38584,13440,,15478,18,
|
||||
38584,78804,,10026,18,
|
||||
38584,86159,,245,18,
|
||||
38584,79364,,5722,18,
|
||||
38584,86173,,4495,18,
|
||||
38584,82380,,17036,18,
|
||||
38584,37842,,10401,18,
|
||||
38584,12063,,5248,18,
|
||||
38584,68179,,6310,18,
|
||||
38584,76441,,2755,18,
|
||||
38584,8460,,605,18,
|
||||
38584,51736,,10991,18,
|
||||
38584,54535,,14499,18,
|
||||
38584,55646,,16968,18,
|
||||
38584,76280,,1982,18,
|
||||
38584,85237,,5316,18,
|
||||
38584,34979,,12207,18,
|
||||
38584,36367,,8592,18,
|
||||
38584,3823,,8634,18,
|
||||
38585,54051,,9176,18,
|
||||
38585,46219,,16813,18,
|
||||
38585,5783,,2678,18,
|
||||
38585,26574,,1387,18,
|
||||
38585,73914,,3747,18,
|
||||
38585,4921,,7197,18,
|
||||
38585,54784,,7686,18,
|
||||
38585,7886,,13511,18,
|
||||
38585,9839,,5627,18,
|
||||
38585,11799,,15230,18,
|
||||
38585,37200,,12640,18,
|
||||
38585,22169,,15175,18,
|
||||
38585,59324,,15794,18,
|
||||
38585,871,,9616,18,
|
||||
38585,37607,,5294,18,
|
||||
38585,11608,,14964,18,
|
||||
38585,29286,,13835,18,
|
||||
38585,70164,,5415,18,
|
||||
38585,26267,,11922,18,
|
||||
38585,57805,,3493,18,
|
||||
38585,77195,,3525,18,
|
||||
38585,73900,,2846,18,
|
||||
38586,54143,,3919,18,
|
||||
38586,6242,,11626,18,
|
||||
38586,79590,,6815,18,
|
||||
38586,38027,,851,18,
|
||||
38586,34195,,16634,18,
|
||||
38586,4594,,5348,18,
|
||||
38586,49367,,12363,18,
|
||||
38586,41749,,6917,18,
|
||||
38586,57532,,3567,18,
|
||||
38586,75450,,13961,18,
|
||||
38586,42154,,7813,18,
|
||||
38586,51173,,16047,18,
|
||||
38586,22777,,97,18,
|
||||
38586,13084,,16406,18,
|
||||
38586,67291,,5314,18,
|
||||
38586,59902,,15106,18,
|
||||
38586,46891,,5702,18,
|
||||
38586,10298,,9110,18,
|
||||
38586,45112,,14406,18,
|
||||
38586,70284,,4399,18,
|
||||
38586,41884,,16347,18,
|
||||
38586,55550,,10840,18,
|
||||
38586,62564,,14074,18,
|
||||
38586,38709,,9710,18,
|
||||
38586,20345,,7332,18,
|
||||
38586,22951,,2394,18,
|
||||
38586,80448,,7315,18,
|
||||
38587,52235,,13939,18,
|
||||
38587,40439,,3723,18,
|
||||
38587,35425,,9949,18,
|
||||
38587,34854,,13147,18,
|
||||
38587,28919,,15240,18,
|
||||
38587,14794,,8954,18,
|
||||
38587,77222,,6852,18,
|
||||
38587,45315,,10160,18,
|
||||
38587,32071,,1459,18,
|
||||
38587,30806,,179,18,
|
||||
38587,58135,,12252,18,
|
||||
38587,21044,,12031,18,
|
||||
38587,67725,,11699,18,
|
||||
38587,79535,,7694,18,
|
||||
38587,27160,,2041,18,
|
||||
38587,19548,,5786,18,
|
||||
38587,46321,,3630,18,
|
||||
38587,80145,,2669,18,
|
||||
38587,42847,,2806,18,
|
||||
38587,55245,,4513,18,
|
||||
38587,10012,,14647,18,
|
||||
38587,25823,,7672,18,
|
||||
38587,71733,,13606,18,
|
||||
38587,82147,,13991,18,
|
||||
38587,83267,,7865,18,
|
||||
38587,12199,,2101,18,
|
||||
38587,40681,,401,18,
|
||||
38587,32602,,11383,18,
|
||||
38587,71148,,5836,18,
|
||||
38587,68762,,17181,18,
|
||||
38587,32583,,8568,18,
|
||||
38588,9024,,3521,18,
|
||||
38588,11661,,9807,18,
|
||||
38588,33800,,11531,18,
|
||||
38588,61052,,10310,18,
|
||||
38588,22395,,1889,18,
|
||||
38588,37038,,17465,18,
|
||||
38588,82734,,12463,18,
|
||||
38588,31600,,14463,18,
|
||||
38588,7456,,3812,18,
|
||||
38588,44626,,15989,18,
|
||||
38588,62143,,5784,18,
|
||||
38588,54037,,9806,18,
|
||||
38588,51024,,5896,18,
|
||||
38588,31683,,10019,18,
|
||||
38588,20780,,473,18,
|
||||
38588,25395,,616,18,
|
||||
38588,50964,,4002,18,
|
||||
38588,12523,,4489,18,
|
||||
38588,76817,,3984,18,
|
||||
38588,5486,,16266,18,
|
||||
38588,17248,,10970,18,
|
||||
38589,34796,,3642,18,
|
||||
38589,84285,,11908,18,
|
||||
38589,6311,,9602,18,
|
||||
38589,14712,,16768,18,
|
||||
38589,75596,,17637,18,
|
||||
38589,85189,,1012,18,
|
||||
38589,12086,,261,18,
|
||||
38589,5499,,7172,18,
|
||||
38589,74565,,2688,18,
|
||||
38589,63568,,5492,18,
|
||||
38589,63758,,17407,18,
|
||||
38589,79838,,7783,18,
|
||||
38589,8382,,15048,18,
|
||||
38589,15301,,2833,18,
|
||||
38589,28304,,12548,18,
|
||||
38589,71491,,14753,18,
|
||||
38589,46238,,4430,18,
|
||||
38589,84169,,17477,18,
|
||||
38589,53804,,12782,18,
|
||||
38589,9861,,7367,18,
|
||||
38589,81433,,2806,18,
|
||||
38589,63032,,9512,18,
|
||||
38589,20368,,13352,18,
|
||||
38589,40592,,16200,18,
|
||||
38590,21306,,2378,18,
|
||||
38590,22304,,12596,18,
|
||||
38590,42299,,15225,18,
|
||||
38590,33704,,2419,18,
|
||||
38590,25550,,7913,18,
|
||||
38590,24897,,12874,18,
|
||||
38590,33862,,4442,18,
|
||||
38590,30913,,17443,18,
|
||||
38590,65221,,15879,18,
|
||||
38590,58025,,8725,18,
|
||||
38590,29051,,9868,18,
|
||||
|
Reference in New Issue
Block a user