From 40e1ff4362706b7716f6176927b68962864924e2 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Tue, 21 May 2019 09:39:38 -0700 Subject: [PATCH] Data source changes for CTP 3.0 --- .../bootstrap-sample-db.sql | 36 +++++++++---------- .../data-pool/data-ingestion-sql.sql | 8 +++-- .../hadoop/inventory-export-hdfs-rcfile.sql | 20 ++++------- .../hadoop/product-reviews-hdfs-orc.sql | 20 ++++------- .../hadoop/web-clickstreams-hdfs-orc.sql | 20 ++++------- .../storage-pool/product-reviews-hdfs-csv.sql | 8 ++--- .../product-reviews-hdfs-parquet.sql | 8 ++--- .../storage-pool/product-reviews-hdfs-tsv.sql | 8 ++--- .../web-clickstreams-hdfs-csv.sql | 8 ++--- .../web-clickstreams-hdfs-parquet.sql | 8 ++--- 10 files changed, 60 insertions(+), 84 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 5a525be8..64368706 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -76,32 +76,28 @@ BEGIN -- Create default data sources for SQL Big Data Cluster IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070'); + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); END; GO diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql index 3073f88b..5d457be5 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql @@ -4,8 +4,12 @@ GO -- Create external data source for Data Pool inside a SQL big data cluster -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); -- Create external table in a data pool in SQL Server 2019 big data cluster. -- The SqlDataPool data source is a special data source that is available in diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql index a934f203..41d5a2d3 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql @@ -17,20 +17,12 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); -- Create file format for RCFILE with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql index 7f9b223b..983d3eef 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql @@ -7,20 +7,12 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); -- Create file format for orc file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql index 90917c3b..bd9a10ce 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql @@ -7,20 +7,12 @@ GO -- execution. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', - RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' - ); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE HadoopData - WITH( - TYPE=HADOOP, - LOCATION='hdfs://nmnode-0-svc:9000/', - RESOURCE_MANAGER_LOCATION='master-svc:8032' - ); + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://nmnode-0-svc:9000/', + RESOURCE_MANAGER_LOCATION='master-svc:8032' + ); -- Create file format for orc file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql index f6b87442..5ae3193a 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070'); + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for CSV separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql index a0eba2d5..a83299cd 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070'); + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for parquet file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql index 8b7da015..df333057 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070'); + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for tab separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql index d0b3cfde..b07ca68b 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070'); + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for CSV file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index 7fb2fcdc..ed61d1fb 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.4' + IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-master-pool:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' + WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070'); + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); -- Create file format for parquet file with appropriate properties. --