From 9b297668a42713b0d7afa72685c5d78614f078d9 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 15 Oct 2018 11:31:20 -0700 Subject: [PATCH] Minor changes to make script idempotent --- .../bootstrap-sample-db.sql | 15 +++-- .../data-pool/data-ingestion-spark.sql | 15 +++-- .../data-pool/data-ingestion-sql.sql | 66 +++++++++++-------- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index f5cdf5dc..b3b4dba1 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -64,11 +64,12 @@ AS GO -- Create table for storing the machine learning models -CREATE TABLE sales_models ( - model_name varchar(100) NOT NULL PRIMARY KEY, - model varbinary(max) NOT NULL, - model_native varbinary(max) NOT NULL, - created_by nvarchar(300) NOT NULL DEFAULT(SYSTEM_USER), - create_time datetime2 NOT NULL DEFAULT(SYSDATETIME()) -); +IF NOT EXISTS(SELECT * FROM sys.tables WHERE name = 'sales_models') + CREATE TABLE sales_models ( + model_name varchar(100) NOT NULL PRIMARY KEY, + model varbinary(max) NOT NULL, + model_native varbinary(max) NOT NULL, + created_by nvarchar(300) NOT NULL DEFAULT(SYSTEM_USER), + create_time datetime2 NOT NULL DEFAULT(SYSDATETIME()) + ); GO diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql index 33b25a7d..d0e15990 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-spark.sql @@ -6,13 +6,14 @@ GO -- any new database in SQL Master instance. This is used to reference the -- data pool in a SQL Server 2019 big data cluster. -- -CREATE EXTERNAL TABLE [web_clickstreams_spark_results] -("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT) -WITH -( - DATA_SOURCE = SqlDataPool, - DISTRIBUTION = ROUND_ROBIN -); +IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstreams_spark_results') + CREATE EXTERNAL TABLE [web_clickstreams_spark_results] + ("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT) + WITH + ( + DATA_SOURCE = SqlDataPool, + DISTRIBUTION = ROUND_ROBIN + ); -- Data can be ingested into the external table from a spark job. -- diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql index 8d0d1a61..09c77076 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql @@ -6,53 +6,61 @@ GO -- any new database in SQL Master instance. This is used to reference the -- data pool in a SQL Server 2019 big data cluster. -- -CREATE EXTERNAL TABLE [web_clickstreams_dp] -("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT) -WITH -( - DATA_SOURCE = SqlDataPool, - DISTRIBUTION = ROUND_ROBIN -); +IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstream_clicks_data_pool') + CREATE EXTERNAL TABLE [web_clickstream_clicks_data_pool] + ("wcs_user_sk" BIGINT , "i_category_id" BIGINT , "clicks" BIGINT) + WITH + ( + DATA_SOURCE = SqlDataPool, + DISTRIBUTION = ROUND_ROBIN + ); GO -- Currently the create external table operation is asynchronous and there is no -- way to determine completion of the operation. To prevent failures of the insert -- into the external table, wait for few minutes. WAITFOR DELAY '00:02:00'; GO --- Insert results of a SELECT statement into the external table created on the data pool +-- Insert results of a SELECT statement into the external table created on the data pool. +-- Store summary results for quick access instead of going to the source tables always. -- DECLARE @db_name SYSNAME = 'sales' DECLARE @schema_name SYSNAME = 'dbo' -DECLARE @table_name SYSNAME = 'web_clickstreams_dp' -DECLARE @query SYSNAME = 'SELECT TOP(1000) * FROM sales.dbo.web_clickstreams WHERE wcs_user_sk IS NOT NULL' - +DECLARE @table_name SYSNAME = 'web_clickstream_clicks_data_pool' +DECLARE @query NVARCHAR(MAX) = ' +SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks + FROM sales.dbo.web_clickstreams + INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk + AND wcs_user_sk IS NOT NULL) + GROUP BY wcs_user_sk, i_category_id +HAVING COUNT_BIG(*) > 100; +' EXEC model..sp_data_pool_table_insert_data @db_name, @schema_name, @table_name, @query GO -- Query data inserted from sp_data_pool_table_insert_data -- -SELECT count(*) FROM [dbo].[web_clickstreams_dp] -SELECT TOP 10 * FROM [dbo].[web_clickstreams_dp] +SELECT count(*) FROM [dbo].[web_clickstream_clicks_data_pool] +SELECT TOP 10 * FROM [dbo].[web_clickstream_clicks_data_pool] -- Join external table with local tables -- SELECT TOP (100) - wcs_user_sk, - SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks, - SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen], - SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music], - SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books], - SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories], - SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics], - SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement], - SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games], - SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV], - SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors] - FROM [dbo].[web_clickstreams_dp] - INNER JOIN item it ON (wcs_item_sk = i_item_sk - AND wcs_user_sk IS NOT NULL) -GROUP BY wcs_user_sk; + w.wcs_user_sk, + SUM( CASE WHEN i.i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks, + SUM( CASE WHEN w.i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen], + SUM( CASE WHEN w.i_category_id = 2 THEN 1 ELSE 0 END) AS [Music], + SUM( CASE WHEN w.i_category_id = 3 THEN 1 ELSE 0 END) AS [Books], + SUM( CASE WHEN w.i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories], + SUM( CASE WHEN w.i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics], + SUM( CASE WHEN w.i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement], + SUM( CASE WHEN w.i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games], + SUM( CASE WHEN w.i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV], + SUM( CASE WHEN w.i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors] + FROM [dbo].[web_clickstream_clicks_data_pool] as w + INNER JOIN (SELECT DISTINCT i_category_id, i_category FROM item) as i + ON i.i_category_id = w.i_category_id +GROUP BY w.wcs_user_sk; GO -DROP EXTERNAL TABLE [dbo].[web_clickstreams_dp]; +DROP EXTERNAL TABLE [dbo].[web_clickstream_clicks_data_pool]; GO \ No newline at end of file