From c27f06a91d6c5a349542abb8119d649071370483 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Thu, 11 Oct 2018 16:59:20 -0700 Subject: [PATCH] Added sample for reading from parquet files --- .../data-virtualization/README.md | 4 +- ...e-hdfs.sql => external-table-hdfs-csv.sql} | 6 +-- .../external-table-hdfs-parquet.sql | 47 +++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) rename samples/features/sql-big-data-cluster/data-virtualization/{external-table-hdfs.sql => external-table-hdfs-csv.sql} (92%) create mode 100644 samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql diff --git a/samples/features/sql-big-data-cluster/data-virtualization/README.md b/samples/features/sql-big-data-cluster/data-virtualization/README.md index 6daf213d..a1aa4233 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/README.md @@ -10,7 +10,9 @@ In this example, you are going to create an external table in the SQL Server Mas 1. Connect to SQL Server Master instance. -1. Execute the [external-table-hdfs.sql](external-table-hdfs.sql). +1. Execute the [external-table-hdfs-csv.sql](external-table-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS. + +1. Before you use execute the *external-table-hdfs-parquet.sql* script, make sure you run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file. Execute the [external-table-hdfs-parquet.sql](external-table-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS. ## Query data in Oracle from SQL Server master diff --git a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs.sql b/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-csv.sql similarity index 92% rename from samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs.sql rename to samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-csv.sql index 1d03e209..7596b89a 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-csv.sql @@ -18,7 +18,7 @@ WITH ( -- is a special data source that is available in any new database in -- SQL Master instance. -- -CREATE EXTERNAL TABLE [web_clickstreams_hdfs] +CREATE EXTERNAL TABLE [web_clickstreams_hdfs_csv] ("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT) WITH ( @@ -42,11 +42,11 @@ SELECT SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games], SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV], SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors] - FROM [dbo].[web_clickstreams_hdfs] + FROM [dbo].[web_clickstreams_hdfs_csv] INNER JOIN item it ON (wcs_item_sk = i_item_sk AND wcs_user_sk IS NOT NULL) GROUP BY wcs_user_sk; GO -DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs]; +DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_csv]; GO \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql new file mode 100644 index 00000000..c4214620 --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql @@ -0,0 +1,47 @@ +USE sales +GO + +-- Create file format for parquet file with appropriate properties. +-- +CREATE EXTERNAL FILE FORMAT parquet_file +WITH ( + FORMAT_TYPE = PARQUET +); + +-- Create external table over HDFS data source (SqlStoragePool) in +-- SQL Server 2019 big data cluster. The SqlStoragePool data source +-- is a special data source that is available in any new database in +-- SQL Master instance. +-- +CREATE EXTERNAL TABLE [web_clickstreams_hdfs_parquet] +("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT) +WITH +( + DATA_SOURCE = SqlStoragePool, + LOCATION = '/user/hive/warehouse/web_clickstreams', + FILE_FORMAT = parquet_file +); +GO + +-- Join external table with local tables +-- +SELECT + wcs_user_sk, + SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks, + SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen], + SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music], + SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books], + SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories], + SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics], + SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement], + SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games], + SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV], + SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors] + FROM [dbo].[web_clickstreams_hdfs_parquet] + INNER JOIN item it ON (wcs_item_sk = i_item_sk + AND wcs_user_sk IS NOT NULL) +GROUP BY wcs_user_sk; +GO + +DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_parquet]; +GO \ No newline at end of file