From 9b611a3930b31d48abe9332c4026ca6eea664476 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 17 Apr 2019 18:22:44 -0700 Subject: [PATCH] Updated for CTP2.5 changes --- .../data-virtualization/hadoop/README.md | 6 ++---- .../hadoop/inventory-export-hdfs-rcfile.sql | 20 +++++++++++++------ .../hadoop/product-reviews-hdfs-orc.sql | 20 +++++++++++++------ .../hadoop/web-clickstreams-hdfs-orc.sql | 20 +++++++++++++------ .../storage-pool/README.md | 6 ++---- .../storage-pool/product-reviews-hdfs-csv.sql | 10 +++++++--- .../product-reviews-hdfs-parquet.sql | 10 +++++++--- .../storage-pool/product-reviews-hdfs-tsv.sql | 10 +++++++--- .../web-clickstreams-hdfs-csv.sql | 10 +++++++--- .../web-clickstreams-hdfs-parquet.sql | 10 +++++++--- 10 files changed, 81 insertions(+), 41 deletions(-) diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md index e572b966..3a0734ea 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md @@ -10,12 +10,10 @@ In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node ### Instructions -1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type. - -1. Run the [../../spark/spark-sql.ipynb](../../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s). - 1. Connect to SQL Server Master instance. +1. Run the [../../spark/dataloading/transform-csv-files.ipynb](../../spark/dataloading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s). + 1. Execute the [web-clickstreams-hdfs-orc.sql](web-clickstreams-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS. 1. Execute the [product-reviews-hdfs-orc.sql](product-reviews-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS. diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql index 53a59471..93936138 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql @@ -17,12 +17,20 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' + ); -- Create file format for RCFILE with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql index 2840793e..40b7bb06 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql @@ -7,12 +7,20 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' + ); -- Create file format for orc file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql index 5e5b7935..18422fcf 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql @@ -7,12 +7,20 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032' + ); -- Create file format for orc file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md index 29c9971a..d35a9a58 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md @@ -10,12 +10,10 @@ In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node ### Instructions -1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type. - -1. Run the [../../spark/spark-sql.ipynb](../../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s). - 1. Connect to SQL Server Master instance. +1. Run the [../../spark/dataloading/transform-csv-files.ipynb](../../spark/dataloading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s). + 1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS. 1. Execute the [web-clickstreams-parquet.sql](web-clickstreams-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS. diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql index 9750e30b..c8797fed 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql @@ -1,11 +1,15 @@ USE sales GO --- Create external data source for HDFS inside SQ: big data cluster. +-- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); -- Create file format for CSV separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql index ef6eb86d..5c8545dc 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql @@ -1,11 +1,15 @@ USE sales GO --- Create external data source for HDFS inside SQ: big data cluster. +-- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); -- Create file format for parquet file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql index c7e8194a..6e0175d9 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql @@ -1,11 +1,15 @@ USE sales GO --- Create external data source for HDFS inside SQ: big data cluster. +-- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); -- Create file format for tab separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql index 02509971..6f683a84 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql @@ -1,11 +1,15 @@ USE sales GO --- Create external data source for HDFS inside SQ: big data cluster. +-- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); -- Create file format for CSV file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index 71f2dfda..1164a26e 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -1,11 +1,15 @@ USE sales GO --- Create external data source for HDFS inside SQ: big data cluster. +-- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); -- Create file format for parquet file with appropriate properties. --