Added data source DDLs

so script can be self-contained.
2025-12-08 14:58:54 +00:00 · 2019-04-12 13:25:42 -07:00
parent 960f46962b
commit d2f2834644
9 changed files with 84 additions and 24 deletions
@@ -1,6 +1,12 @@
 USE sales
 GO

+-- Create external data source for Data Pool inside a SQL big data cluster
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool')
+	CREATE EXTERNAL DATA SOURCE SqlDataPool
+	WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default');
+
 -- Create external table in a data pool in SQL Server 2019 big data cluster.
 -- The SqlDataPool data source is a special data source that is available in 
 -- any new database in SQL Master instance. This is used to reference the
@@ -25,30 +31,13 @@ GO
 -- Insert results of a SELECT statement into the external table created on the data pool.
 -- Store summary results for quick access instead of going to the source tables always.
 --
-IF SERVERPROPERTY('ProductLevel') = 'CTP2.3'
-BEGIN
-  DECLARE @db_name SYSNAME = 'sales'
-  DECLARE @schema_name SYSNAME = 'dbo'
-  DECLARE @table_name SYSNAME = 'web_clickstream_clicks_data_pool'
-  DECLARE @query NVARCHAR(MAX) = '
-  SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks
-    FROM sales.dbo.web_clickstreams
-  INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk
-                          AND wcs_user_sk IS NOT NULL)
-  GROUP BY wcs_user_sk, i_category_id
-  HAVING COUNT_BIG(*) > 100;
-  '
-  EXEC model..sp_data_pool_table_insert_data @db_name, @schema_name, @table_name, @query
-END;
-
-IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
-  INSERT INTO web_clickstream_clicks_data_pool
-  SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks
-    FROM sales.dbo.web_clickstreams_hdfs_parquet
-  INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk
-                          AND wcs_user_sk IS NOT NULL)
-  GROUP BY wcs_user_sk, i_category_id
-  HAVING COUNT_BIG(*) > 100;
+INSERT INTO web_clickstream_clicks_data_pool
+SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks
+  FROM sales.dbo.web_clickstreams_hdfs_parquet
+ INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk
+                        AND wcs_user_sk IS NOT NULL)
+ GROUP BY wcs_user_sk, i_category_id
+HAVING COUNT_BIG(*) > 100;
 GO

 -- Query data inserted into the data pool table
@@ -1,6 +1,8 @@
 USE sales
 GO

+-- Enable option to allow INSERT against external table defined on HADOOP data source
+--
 DECLARE @config_option nvarchar(100) = 'allow polybase export';
 IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1)
 BEGIN
@@ -9,6 +11,19 @@ BEGIN
 END;
 GO

+-- Create data source for HDFS inside SQL big data cluster using the HADOOP type.
+-- The HADOOP data source type was introduced in SQL Server 2016 to query data in
+-- Hadoop clusters and relies on Java Hadoop client libraries and Map/Reduce for query
+-- execution.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
+    CREATE EXTERNAL DATA SOURCE HadoopData
+    WITH(
+            TYPE=HADOOP,
+            LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
+            RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
+    );
+
 -- Create file format for RCFILE with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'RCFILE')
@@ -1,6 +1,19 @@
 USE sales
 GO

+-- Create data source for HDFS inside SQL big data cluster using the HADOOP type.
+-- The HADOOP data source type was introduced in SQL Server 2016 to query data in
+-- Hadoop clusters and relies on Java Hadoop client libraries and Map/Reduce for query
+-- execution.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
+    CREATE EXTERNAL DATA SOURCE HadoopData
+    WITH(
+            TYPE=HADOOP,
+            LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
+            RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
+    );
+
 -- Create file format for orc file with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file')
@@ -1,6 +1,19 @@
 USE sales
 GO

+-- Create data source for HDFS inside SQL big data cluster using the HADOOP type.
+-- The HADOOP data source type was introduced in SQL Server 2016 to query data in
+-- Hadoop clusters and relies on Java Hadoop client libraries and Map/Reduce for query
+-- execution.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
+    CREATE EXTERNAL DATA SOURCE HadoopData
+    WITH(
+            TYPE=HADOOP,
+            LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
+            RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
+    );
+
 -- Create file format for orc file with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file')
@@ -1,6 +1,12 @@
 USE sales
 GO

+-- Create external data source for HDFS inside SQ: big data cluster.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
+    CREATE EXTERNAL DATA SOURCE SqlStoragePool
+    WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
+
 -- Create file format for CSV separated file with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file')
@@ -1,6 +1,12 @@
 USE sales
 GO

+-- Create external data source for HDFS inside SQ: big data cluster.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
+    CREATE EXTERNAL DATA SOURCE SqlStoragePool
+    WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
+
 -- Create file format for parquet file with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file')
@@ -1,6 +1,12 @@
 USE sales
 GO

+-- Create external data source for HDFS inside SQ: big data cluster.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
+    CREATE EXTERNAL DATA SOURCE SqlStoragePool
+    WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
+
 -- Create file format for tab separated file with appropriate properties.
 --
 CREATE EXTERNAL FILE FORMAT tsv_file
@@ -1,6 +1,12 @@
 USE sales
 GO

+-- Create external data source for HDFS inside SQ: big data cluster.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
+    CREATE EXTERNAL DATA SOURCE SqlStoragePool
+    WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
+
 -- Create file format for CSV file with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file')
@@ -1,6 +1,12 @@
 USE sales
 GO

+-- Create external data source for HDFS inside SQ: big data cluster.
+--
+IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
+    CREATE EXTERNAL DATA SOURCE SqlStoragePool
+    WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
+
 -- Create file format for parquet file with appropriate properties.
 --
 IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file')