From b9253c617f3610acb883ac970af15fe379ebb239 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Mon, 24 Jun 2019 15:39:15 -0700 Subject: [PATCH] CTP 3.1 changes for data source --- .../bootstrap-sample-db.sql | 16 ++++++++-------- .../data-pool/data-ingestion-sql.sql | 19 +++++++------------ .../oracle/customer-oracle.sql | 18 +++++++++--------- .../storage-pool/product-reviews-hdfs-csv.sql | 12 ++++++------ .../product-reviews-hdfs-parquet.sql | 12 ++++++------ .../storage-pool/product-reviews-hdfs-tsv.sql | 12 ++++++------ .../web-clickstreams-hdfs-csv.sql | 12 ++++++------ .../web-clickstreams-hdfs-parquet.sql | 12 ++++++------ 8 files changed, 54 insertions(+), 59 deletions(-) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 64368706..e1ef8f07 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -76,20 +76,20 @@ BEGIN -- Create default data sources for SQL Big Data Cluster IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlDataPool WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' CREATE EXTERNAL DATA SOURCE SqlStoragePool WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc/default'); IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') CREATE EXTERNAL DATA SOURCE HadoopData diff --git a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql index 5d457be5..29e3ff6e 100644 --- a/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql +++ b/samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql @@ -4,12 +4,12 @@ GO -- Create external data source for Data Pool inside a SQL big data cluster -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://controller-svc/default'); -- Create external table in a data pool in SQL Server 2019 big data cluster. -- The SqlDataPool data source is a special data source that is available in @@ -26,12 +26,6 @@ IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstream_cl ); GO --- Currently the create external table operation is asynchronous and there is no --- way to determine completion of the operation. To prevent failures of the insert --- into the external table, wait for few minutes. -WAITFOR DELAY '00:02:00'; -GO - -- Insert results of a SELECT statement into the external table created on the data pool. -- Store summary results for quick access instead of going to the source tables always. -- @@ -72,5 +66,6 @@ GO -- Cleanup /* DROP EXTERNAL TABLE [dbo].[web_clickstream_clicks_data_pool]; +DROP EXTERNAL DATA SOURCE SqlDataPool; GO */ \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql index 6f308f88..c2f5b3c9 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql @@ -22,23 +22,23 @@ IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'OracleSalesS CREATE EXTERNAL TABLE [dbo].[customer_ora] ( [C_CUSTOMER_SK] DECIMAL(10,0), - [C_CUSTOMER_ID] NVARCHAR(16) COLLATE SQL_Latin1_General_CP1_CI_AS, + [C_CUSTOMER_ID] NVARCHAR(16) COLLATE Latin1_General_100_BIN2_UTF8, [C_CURRENT_CDEMO_SK] DECIMAL(10,0), [C_CURRENT_HDEMO_SK] DECIMAL(10,0), [C_CURRENT_ADDR_SK] DECIMAL(10,0), [C_FIRST_SHIPTO_DATE_SK] DECIMAL(10,0), [C_FIRST_SALES_DATE_SK] DECIMAL(8,0), - [C_SALUTATION] NVARCHAR(10) COLLATE SQL_Latin1_General_CP1_CI_AS, - [C_FIRST_NAME] NVARCHAR(20) COLLATE SQL_Latin1_General_CP1_CI_AS, - [C_LAST_NAME] NVARCHAR(30) COLLATE SQL_Latin1_General_CP1_CI_AS, - [C_PREFERRED_CUST_FLAG] NVARCHAR COLLATE SQL_Latin1_General_CP1_CI_AS, + [C_SALUTATION] NVARCHAR(10) COLLATE Latin1_General_100_BIN2_UTF8, + [C_FIRST_NAME] NVARCHAR(20) COLLATE Latin1_General_100_BIN2_UTF8, + [C_LAST_NAME] NVARCHAR(30) COLLATE Latin1_General_100_BIN2_UTF8, + [C_PREFERRED_CUST_FLAG] NVARCHAR COLLATE Latin1_General_100_BIN2_UTF8, [C_BIRTH_DAY] DECIMAL(8,0), [C_BIRTH_MONTH] DECIMAL(8,0), [C_BIRTH_YEAR] DECIMAL(8,0), - [C_BIRTH_COUNTRY] NVARCHAR(20) COLLATE SQL_Latin1_General_CP1_CI_AS, - [C_LOGIN] NVARCHAR(13) COLLATE SQL_Latin1_General_CP1_CI_AS, - [C_EMAIL_ADDRESS] NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS, - [C_LAST_REVIEW_DATE] NVARCHAR(20) COLLATE SQL_Latin1_General_CP1_CI_AS + [C_BIRTH_COUNTRY] NVARCHAR(20) COLLATE Latin1_General_100_BIN2_UTF8, + [C_LOGIN] NVARCHAR(13) COLLATE Latin1_General_100_BIN2_UTF8, + [C_EMAIL_ADDRESS] NVARCHAR(50) COLLATE Latin1_General_100_BIN2_UTF8, + [C_LAST_REVIEW_DATE] NVARCHAR(20) COLLATE Latin1_General_100_BIN2_UTF8 ) WITH (DATA_SOURCE=[OracleSalesSrvr], LOCATION='.SALES.CUSTOMER'); diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql index 5ae3193a..fa8dd511 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-csv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc/default'); -- Create file format for CSV separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql index a83299cd..a3d54cde 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-parquet.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc/default'); -- Create file format for parquet file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql index df333057..50a22854 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/product-reviews-hdfs-tsv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc/default'); -- Create file format for tab separated file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql index b07ca68b..1ac73000 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-csv.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc/default'); -- Create file format for CSV file with appropriate properties. -- diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index ed61d1fb..08afb537 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -4,12 +4,12 @@ GO -- Create external data source for HDFS inside SQL big data cluster. -- IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - IF SERVERPROPERTY('ProductLevel') = 'CTP2.5' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://nmnode-0-0.nmnode-0-svc:50070'); - ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + IF SERVERPROPERTY('ProductLevel') = 'CTP3.0' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc:8080/default'); + ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1' + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://controller-svc/default'); -- Create file format for parquet file with appropriate properties. --