diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql index e2a9a27b..3073f88b 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql @@ -1,6 +1,12 @@ USE sales GO +-- Create external data source for Data Pool inside a SQL big data cluster +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + -- Create external table in a data pool in SQL Server 2019 big data cluster. -- The SqlDataPool data source is a special data source that is available in -- any new database in SQL Master instance. This is used to reference the @@ -25,30 +31,13 @@ GO -- Insert results of a SELECT statement into the external table created on the data pool. -- Store summary results for quick access instead of going to the source tables always. -- -IF SERVERPROPERTY('ProductLevel') = 'CTP2.3' -BEGIN - DECLARE @db_name SYSNAME = 'sales' - DECLARE @schema_name SYSNAME = 'dbo' - DECLARE @table_name SYSNAME = 'web_clickstream_clicks_data_pool' - DECLARE @query NVARCHAR(MAX) = ' - SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks - FROM sales.dbo.web_clickstreams - INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk - AND wcs_user_sk IS NOT NULL) - GROUP BY wcs_user_sk, i_category_id - HAVING COUNT_BIG(*) > 100; - ' - EXEC model..sp_data_pool_table_insert_data @db_name, @schema_name, @table_name, @query -END; - -IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - INSERT INTO web_clickstream_clicks_data_pool - SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks - FROM sales.dbo.web_clickstreams_hdfs_parquet - INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk - AND wcs_user_sk IS NOT NULL) - GROUP BY wcs_user_sk, i_category_id - HAVING COUNT_BIG(*) > 100; +INSERT INTO web_clickstream_clicks_data_pool +SELECT wcs_user_sk, i_category_id, COUNT_BIG(*) as clicks + FROM sales.dbo.web_clickstreams_hdfs_parquet + INNER JOIN sales.dbo.item it ON (wcs_item_sk = i_item_sk + AND wcs_user_sk IS NOT NULL) + GROUP BY wcs_user_sk, i_category_id +HAVING COUNT_BIG(*) > 100; GO -- Query data inserted into the data pool table diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql index 57106cd9..53a59471 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql @@ -1,6 +1,8 @@ USE sales GO +-- Enable option to allow INSERT against external table defined on HADOOP data source +-- DECLARE @config_option nvarchar(100) = 'allow polybase export'; IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1) BEGIN @@ -9,6 +11,19 @@ BEGIN END; GO +-- Create data source for HDFS inside SQL big data cluster using the HADOOP type. +-- The HADOOP data source type was introduced in SQL Server 2016 to query data in +-- Hadoop clusters and relies on Java Hadoop client libraries and Map/Reduce for query +-- execution. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + -- Create file format for RCFILE with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'RCFILE') diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql index 7054271f..2840793e 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql @@ -1,6 +1,19 @@ USE sales GO +-- Create data source for HDFS inside SQL big data cluster using the HADOOP type. +-- The HADOOP data source type was introduced in SQL Server 2016 to query data in +-- Hadoop clusters and relies on Java Hadoop client libraries and Map/Reduce for query +-- execution. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + -- Create file format for orc file with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file') diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql index 39a3beb2..5e5b7935 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql @@ -1,6 +1,19 @@ USE sales GO +-- Create data source for HDFS inside SQL big data cluster using the HADOOP type. +-- The HADOOP data source type was introduced in SQL Server 2016 to query data in +-- Hadoop clusters and relies on Java Hadoop client libraries and Map/Reduce for query +-- execution. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + -- Create file format for orc file with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file') diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql index 5221de48..9750e30b 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql @@ -1,6 +1,12 @@ USE sales GO +-- Create external data source for HDFS inside SQ: big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + -- Create file format for CSV separated file with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file') diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql index 1d2e318f..ef6eb86d 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql @@ -1,6 +1,12 @@ USE sales GO +-- Create external data source for HDFS inside SQ: big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + -- Create file format for parquet file with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file') diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql index e00e1cf1..c7e8194a 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql @@ -1,6 +1,12 @@ USE sales GO +-- Create external data source for HDFS inside SQ: big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + -- Create file format for tab separated file with appropriate properties. -- CREATE EXTERNAL FILE FORMAT tsv_file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql index a36319f2..02509971 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql @@ -1,6 +1,12 @@ USE sales GO +-- Create external data source for HDFS inside SQ: big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + -- Create file format for CSV file with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file') diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index fd0a0a77..71f2dfda 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -1,6 +1,12 @@ USE sales GO +-- Create external data source for HDFS inside SQ: big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + -- Create file format for parquet file with appropriate properties. -- IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file')