github/sql-server-samples
1
0
Fork 0
mirror of https://github.com/Microsoft/sql-server-samples.git synced 2025-12-08 14:58:54 +00:00
Code Issues Releases Wiki Activity

Updated for CTP2.5 changes

Browse Source
This commit is contained in:
Umachandar Jayachandran
2019-04-17 18:22:44 -07:00
parent 5783811029
commit 9b611a3930
10 changed files with 81 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md
+2 -4
View File
@@ -10,12 +10,10 @@ In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node
### Instructions
1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type.
1. Run the [../../spark/spark-sql.ipynb](../../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s).
1. Connect to SQL Server Master instance.
1. Run the [../../spark/dataloading/transform-csv-files.ipynb](../../spark/dataloading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s).
1. Execute the [web-clickstreams-hdfs-orc.sql](web-clickstreams-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS.
1. Execute the [product-reviews-hdfs-orc.sql](product-reviews-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS.
samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql
+14 -6
View File
@@ -17,12 +17,20 @@ GO
-- execution.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/',
RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032'
);
-- Create file format for RCFILE with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql
+14 -6
View File
@@ -7,12 +7,20 @@ GO
-- execution.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/',
RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032'
);
-- Create file format for orc file with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql
+14 -6
View File
@@ -7,12 +7,20 @@ GO
-- execution.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://nmnode-0-0.nmnode-0-svc:9000/',
RESOURCE_MANAGER_LOCATION='master-0.master-svc:8032'
);
-- Create file format for orc file with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/storage-pool/README.md
+2 -4
View File
@@ -10,12 +10,10 @@ In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node
### Instructions
1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type.
1. Run the [../../spark/spark-sql.ipynb](../../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s).
1. Connect to SQL Server Master instance.
1. Run the [../../spark/dataloading/transform-csv-files.ipynb](../../spark/dataloading/transform-csv-files.ipynb/) notebook to generate the sample parquet file(s).
1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
1. Execute the [web-clickstreams-parquet.sql](web-clickstreams-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS.
samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql
+7 -3
View File
@@ -1,11 +1,15 @@
USE sales
GO
-- Create external data source for HDFS inside SQ: big data cluster.
-- Create external data source for HDFS inside SQL big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070');
-- Create file format for CSV separated file with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql
+7 -3
View File
@@ -1,11 +1,15 @@
USE sales
GO
-- Create external data source for HDFS inside SQ: big data cluster.
-- Create external data source for HDFS inside SQL big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070');
-- Create file format for parquet file with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql
+7 -3
View File
@@ -1,11 +1,15 @@
USE sales
GO
-- Create external data source for HDFS inside SQ: big data cluster.
-- Create external data source for HDFS inside SQL big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070');
-- Create file format for tab separated file with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql
+7 -3
View File
@@ -1,11 +1,15 @@
USE sales
GO
-- Create external data source for HDFS inside SQ: big data cluster.
-- Create external data source for HDFS inside SQL big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070');
-- Create file format for CSV file with appropriate properties.
--
samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql
+7 -3
View File
@@ -1,11 +1,15 @@
USE sales
GO
-- Create external data source for HDFS inside SQ: big data cluster.
-- Create external data source for HDFS inside SQL big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
IF SERVERPROPERTY('ProductLevel') = 'CTP2.4'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5'
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070');
-- Create file format for parquet file with appropriate properties.
--