diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index d57917b0..3ad06998 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -7,6 +7,7 @@ set SQL_MASTER_IP=%2 set SQL_MASTER_SA_PASSWORD=%3 set KNOX_IP=%4 set KNOX_PASSWORD=%5 +set AW_WWI_SAMPLES=%6 set STARTUP_PATH=%~dp0 set TMP_DIR_NAME=%~nx0 @@ -15,6 +16,7 @@ if NOT DEFINED SQL_MASTER_IP goto :usage if NOT DEFINED SQL_MASTER_SA_PASSWORD goto :usage if NOT DEFINED KNOX_IP goto :usage if NOT DEFINED KNOX_PASSWORD set KNOX_PASSWORD=%SQL_MASTER_SA_PASSWORD% +if NOT DEFINED AW_WWI_SAMPLES set AW_WWI_SAMPLES=no set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,31433 set KNOX_ENDPOINT=%KNOX_IP%:30443 @@ -26,38 +28,74 @@ for %%F in (sqlcmd.exe bcp.exe kubectl.exe curl.exe) do ( pushd "%tmp%" md %TMP_DIR_NAME% cd %TMP_DIR_NAME% -echo Downloading sample database backup file... -%DEBUG% curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak + +if NOT EXIST tpcxbb_1gb.bak ( + echo Downloading sample database backup file... + %DEBUG% curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak +) REM Copy the backup file, restore the database, create necessary objects and data file -echo Copying database backup file... +echo Copying sales database backup file to SQL Master instance... %DEBUG% kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit -del tpcxbb_1gb.bak >NUL +if /i %AW_WWI_SAMPLES% EQU install_extra_samples ( + if NOT EXIST AdventureWorks2016_EXT.bak ( + echo Downloading AdventureWorks2016_EXT sample database backup file... + %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2016_EXT.bak" -o AdventureWorks2016_EXT.bak + ) + echo Copying AdventureWorks2016_EXT database backup file to SQL Master instance... + %DEBUG% kubectl cp AdventureWorks2016_EXT.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit -echo Configuring sample database... + if NOT EXIST AdventureWorksDW2016_EXT.bak ( + echo Downloading AdventureWorksDW2016_EXT sample database backup file... + %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorksDW2016_EXT.bak" -o AdventureWorksDW2016_EXT.bak + ) + echo Copying AdventureWorksDW2016_EXT database backup file to SQL Master instance... + %DEBUG% kubectl cp AdventureWorksDW2016_EXT.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit + + if NOT EXIST WideWorldImporters-Full.bak ( + echo Downloading WideWorldImporters sample database backup file... + %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImporters-Full.bak" -o WideWorldImporters-Full.bak + ) + echo Copying WideWorldImporters-Full database backup file to SQL Master instance... + %DEBUG% kubectl cp WideWorldImporters-Full.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit + + if NOT EXIST WideWorldImportersDW-Full.bak ( + echo Downloading WideWorldImportersDW sample database backup file... + %DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImportersDW-Full.bak" -o WideWorldImportersDW-Full.bak + ) + echo Copying WideWorldImportersDW-Full database backup file to SQL Master instance... + %DEBUG% kubectl cp WideWorldImportersDW-Full.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit +) + +echo Configuring sample database(s)... %DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit for %%F in (web_clickstreams inventory customer) do ( - echo Exporting %%F data... - if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|) - %DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit + if NOT EXIST %%F.csv ( + echo Exporting %%F data... + if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|) + %DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit + ) ) -echo Exporting product_reviews data... -%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit + +if NOT EXIST product_reviews.csv ( + echo Exporting product_reviews data... + %DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit +) REM Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... %DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit %DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit -del /q web_clickstreams.* +:: del /q web_clickstreams.* echo. echo Uploading product_reviews data to HDFS... %DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit %DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit -del /q product_reviews.* +:: del /q product_reviews.* REM %DEBUG% del /q *.out *.err *.csv echo Bootstrap of the sample database completed successfully. diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql index 2cb8159e..91d92980 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sql @@ -18,29 +18,107 @@ BEGIN END; GO -IF DB_ID('sales') IS NULL - RESTORE DATABASE sales - FROM DISK=N'/var/opt/mssql/data/tpcxbb_1gb.bak' - WITH - MOVE N'tpcxbb_1gb' TO N'/var/opt/mssql/data/sales.mdf', - MOVE N'tpcxbb_1gb_log' TO N'/var/opt/mssql/data/sales.ldf'; +CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255)) +AS +BEGIN + DECLARE @restore_filelist_tmpl nvarchar(1000) = N'restore filelistonly FROM DISK = N''/var/opt/mssql/data/%F'''; + DECLARE @restore_database_tmpl nvarchar(1000) = N'RESTORE DATABASE [%D] FROM DISK = N''/var/opt/mssql/data/%F'' WITH FILE = 1'; + DECLARE @move_tmpl nvarchar(1000) = N', MOVE N''%L'' TO N''/var/opt/mssql/data/%F'''; + DECLARE @restore_cmd nvarchar(4000), @logical_name nvarchar(128), @filename nvarchar(260), @restore_cur CURSOR; + DECLARE @files TABLE ( + [LogicalName] NVARCHAR(128), + [PhysicalName] NVARCHAR(260), + [Type] CHAR(1), + [FileGroupName] NVARCHAR(128), + [Size] NUMERIC(20,0), + [MaxSize] NUMERIC(20,0), + [FileID] BIGINT, + [CreateLSN] NUMERIC(25,0), + [DropLSN] NUMERIC(25,0), + [UniqueID] UNIQUEIDENTIFIER, + [ReadOnlyLSN] NUMERIC(25,0), + [ReadWriteLSN] NUMERIC(25,0), + [BackupSizeInBytes] BIGINT, + [SourceBlockSize] INT, + [FileGroupID] INT, + [LogGroupGUID] UNIQUEIDENTIFIER, + [DifferentialBaseLSN] NUMERIC(25,0), + [DifferentialBaseGUID] UNIQUEIDENTIFIER, + [IsReadOnly] BIT, + [IsPresent] BIT, + [TDEThumbprint] VARBINARY(32), + [SnapshotUrl] NVARCHAR(260) + ) + SET @restore_cmd = REPLACE(@restore_filelist_tmpl, '%F', @backup_file); + INSERT INTO @files + EXECUTE(@restore_cmd); + + SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', LEFT(@backup_file, CHARINDEX('.', @backup_file)-1)); + SET @restore_cur = CURSOR FAST_FORWARD FOR SELECT LogicalName, REVERSE(LEFT(REVERSE(PhysicalName), CHARINDEX('\', REVERSE(PhysicalName))-1)) FROM @files; + OPEN @restore_cur; + WHILE(1=1) + BEGIN + FETCH FROM @restore_cur INTO @logical_name, @filename; + IF @@FETCH_STATUS < 0 BREAK; + + SET @restore_cmd += REPLACE(REPLACE(@move_tmpl, '%L', @logical_name), '%F', @filename); + END; + EXECUTE(@restore_cmd); +END; +GO + +CREATE OR ALTER PROCEDURE #create_data_sources +AS +BEGIN + -- Create database master key (required for database scoped credentials used in the samples) + IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1) + CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!'; + + -- Create default data sources for SQL Big Data Cluster + IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') + CREATE EXTERNAL DATA SOURCE SqlDataPool + WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); + + IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') + CREATE EXTERNAL DATA SOURCE SqlStoragePool + WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080'); + + IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData') + CREATE EXTERNAL DATA SOURCE HadoopData + WITH( + TYPE=HADOOP, + LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/', + RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032' + ); +END; +GO + +--- Sample dbs: +DECLARE @sample_dbs CURSOR, @proc nvarchar(255); +SET @sample_dbs = CURSOR FAST_FORWARD FOR + SELECT file_or_directory_name + FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak') + WHERE DB_ID(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', '')) IS NULL; +DECLARE @file nvarchar(260); +OPEN @sample_dbs; +WHILE(1=1) +BEGIN + FETCH @sample_dbs INTO @file; + IF @@FETCH_STATUS < 0 BREAK; + + EXECUTE #restore_database @file; + SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql'); + + EXECUTE @proc N'#create_data_sources'; + + -- Rename TPCx-BB database: + IF DB_ID('tpcxbb_1gb') IS NOT NULL + ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales; +END; GO USE sales; GO --- Create database master key (required for database scoped credentials used in the samples) -IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1) - CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!'; - --- Create default data sources for SQL Big Data Cluster -IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool') - CREATE EXTERNAL DATA SOURCE SqlDataPool - WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default'); - -IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool') - CREATE EXTERNAL DATA SOURCE SqlStoragePool - WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080'); -GO -- Create view used for ML services training and scoring stored procedures CREATE OR ALTER VIEW [dbo].[web_clickstreams_book_clicks] diff --git a/samples/features/sql-big-data-cluster/data-virtualization/README.md b/samples/features/sql-big-data-cluster/data-virtualization/README.md index d9df9ae0..f99fed74 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/README.md @@ -6,7 +6,8 @@ In **SQL Server 2019 big data clusters**, the SQL Server engine has gained the a **Applies to: SQL Server 2019 big data cluster** -In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. The [storage-pool](storage-pool) folder contains SQL scripts that demonstrate how to query data residing in HDFS data inside a big data cluster. +In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. The [storage-pool](storage-pool) folder contains SQL scripts that demonstrate how to query data residing in HDFS data inside a big data cluster. The [hadoop](hadoop) folder contains SQL scripts that demonstrate how to query data residing in HDFS data using the HADOOP data source for +operations that are not yet supported with storage pool (ex: export data to HDFS). ## Query data in Oracle from SQL Server master diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md new file mode 100644 index 00000000..e572b966 --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md @@ -0,0 +1,23 @@ +# Data virtualization in SQL Server 2019 big data cluster + +In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes. Using the PolyBase v1 HADOOP data source, you can manipulate ORC or RCFILE files inside the big data cluster. + +## Query data in HDFS from SQL Server master using HADOOP data source + +**Applies to:** SQL Server 2019 big data cluster + +In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. In this example, you are going to create an external table in the SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster using the HADOOP data source. You will then join the data in the external table with high value data in SQL Master instance. Or export data to HDFS from SQL Master instance. + +### Instructions + +1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type. + +1. Run the [../../spark/spark-sql.ipynb](../../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s). + +1. Connect to SQL Server Master instance. + +1. Execute the [web-clickstreams-hdfs-orc.sql](web-clickstreams-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS. + +1. Execute the [product-reviews-hdfs-orc.sql](product-reviews-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS. + +1. Execute the [inventory-hdfs-rcfile.sql](inventory-hdfs-rcfile.sql). This script demonstrates how to export data from SQL Server into HDFS using PolyBase v1 syntax. This script will export data from SQL Server into RCFILE format. \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql new file mode 100644 index 00000000..7577ca2d --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql @@ -0,0 +1,51 @@ +USE sales +GO + +exec sp_configure 'allow polybase export', 1; +RECONFIGURE WITH OVERRIDE; +GO + +-- Create file format for RCFILE with appropriate properties. +-- +IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'RCFILE') + CREATE EXTERNAL FILE FORMAT rcfile + WITH ( + FORMAT_TYPE = RCFILE, + SERDE_METHOD = 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe', + DATA_COMPRESSION = 'org.apache.hadoop.io.compress.DefaultCodec' + ); + + +-- Create external table over HDFS data source using HADOOP type in +-- SQL Server 2019 big data cluster. The HADOOP data source is existing +-- PolyBase v1 syntax available by specifying location to HDFS namenode in +-- SQL Server big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'inventory_hdfs_rcfile') + CREATE EXTERNAL TABLE [inventory_hdfs_rcfile] + ("inv_date_sk" BIGINT, "inv_item_sk" BIGINT, "inv_warehouse_sk" BIGINT, "inv_quantity_on_hand" BIGINT) + WITH + ( + DATA_SOURCE = HadoopData, + LOCATION = '/inventory_rcfile', + FILE_FORMAT = rcfile + ); +GO + +-- Export SQL Server table to HDFS +-- +INSERT INTO inventory_hdfs_rcfile +SELECT "inv_date_sk", "inv_item_sk", "inv_warehouse_sk", "inv_quantity_on_hand" + FROM inventory; +GO + +-- Query the exported data using external table +-- +SELECT COUNT(*) FROm inventory_hdfs_rcfile; +GO + +-- Cleanup external tables +-- +/* +DROP EXTERNAL TABLE inventory_hdfs_rcfile +*/ diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql new file mode 100644 index 00000000..7054271f --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql @@ -0,0 +1,43 @@ +USE sales +GO + +-- Create file format for orc file with appropriate properties. +-- +IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file') + CREATE EXTERNAL FILE FORMAT orc_file + WITH ( + FORMAT_TYPE = ORC, + DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec' + ); + + +-- Create external table over HDFS data source using HADOOP type in +-- SQL Server 2019 big data cluster. The HADOOP data source is existing +-- PolyBase v1 syntax available by specifying location to HDFS namenode in +-- SQL Server big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'product_reviews_hdfs_orc') + CREATE EXTERNAL TABLE [product_reviews_hdfs_orc] + ("pr_review_sk" BIGINT , "pr_review_content" varchar(8000)) + WITH + ( + DATA_SOURCE = HadoopData, + LOCATION = '/user/hive/warehouse/product_reviews_orc', + FILE_FORMAT = orc_file + ); +GO + +-- Join external table with local tables +-- +SELECT + p.pr_review_sk, pc.pr_review_content + FROM product_reviews as p + JOIN (SELECT TOP(10) * FROM product_reviews_hdfs_orc) AS pc + ON pc.pr_review_sk = p.pr_review_sk; +GO + +-- Cleanup +/* +DROP EXTERNAL TABLE [dbo].[product_reviews_hdfs_orc]; +GO +*/ diff --git a/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql new file mode 100644 index 00000000..39a3beb2 --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql @@ -0,0 +1,54 @@ +USE sales +GO + +-- Create file format for orc file with appropriate properties. +-- +IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file') + CREATE EXTERNAL FILE FORMAT orc_file + WITH ( + FORMAT_TYPE = ORC, + DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec' + ); + + +-- Create external table over HDFS data source using HADOOP type in +-- SQL Server 2019 big data cluster. The HADOOP data source is existing +-- PolyBase v1 syntax available by specifying location to HDFS namenode in +-- SQL Server big data cluster. +-- +IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstreams_hdfs_orc') + CREATE EXTERNAL TABLE [web_clickstreams_hdfs_orc] + ("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT) + WITH + ( + DATA_SOURCE = HadoopData, + LOCATION = '/user/hive/warehouse/web_clickstreams_orc', + FILE_FORMAT = orc_file + ); +GO + +-- Join external table with local tables +-- +SELECT + wcs_user_sk, + SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks, + SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen], + SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music], + SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books], + SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories], + SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics], + SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement], + SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games], + SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV], + SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors] + FROM [dbo].[web_clickstreams_hdfs_orc] + INNER JOIN item it ON (wcs_item_sk = i_item_sk + AND wcs_user_sk IS NOT NULL) +GROUP BY wcs_user_sk; +GO + +-- Cleanup +/* +DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_orc]; +GO +*/ diff --git a/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb b/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb index 1a87b8c9..e1a2d9dc 100644 --- a/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb +++ b/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb @@ -25,9 +25,7 @@ { "cell_type": "code", "source": "# Read the clickstream CSV file(s) into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -35,14 +33,12 @@ "output_type": "stream" } ], - "execution_count": 1 + "execution_count": 3 }, { "cell_type": "code", - "source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory where the parquet files will be stored\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\n", - "metadata": { - "language": "python" - }, + "source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory where the parquet files will be stored\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet & orc file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\nresults.write.format(\"orc\").mode(\"overwrite\").saveAsTable(\"web_clickstreams_orc\")", + "metadata": {}, "outputs": [ { "name": "stdout", @@ -50,29 +46,25 @@ "output_type": "stream" } ], - "execution_count": 1 + "execution_count": 4 }, { "cell_type": "code", "source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT * FROM web_clickstreams LIMIT 100\")\r\nsqlDF.show()\r\n\r\nsqlDF = spark.sql(\"SELECT wcs_user_sk, COUNT(*)\\\r\n FROM web_clickstreams\\\r\n WHERE wcs_user_sk IS NOT NULL\\\r\n GROUP BY wcs_user_sk\\\r\n ORDER BY COUNT(*) DESC LIMIT 100\")\r\nsqlDF.show()", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [ { "name": "stdout", - "text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 36890| 40052| null| 4379| 34| null|\n| 36890| 41285| null| 6245| 34| null|\n| 36890| 23115| null| 13852| 34| null|\n| 36890| 17702| null| 15975| 34| null|\n| 36890| 62676| null| 2119| 34| null|\n| 36890| 34267| null| 10273| 34| null|\n| 36890| 8502| null| 17790| 34| null|\n| 36890| 54340| null| 3453| 34| null|\n| 36890| 54370| null| 6372| 34| null|\n| 36890| 6578| null| 17203| 34| null|\n| 36890| 75088| null| 4891| 34| null|\n| 36890| 23922| null| 11332| 34| null|\n| 36890| 28761| null| 4484| 34| null|\n| 36890| 21444| null| 5582| 34| null|\n| 36890| 58917| null| 8833| 34| null|\n| 36890| 27578| null| 8599| 34| null|\n| 36890| 8059| null| 6720| 34| null|\n| 36890| 43008| null| 17175| 34| null|\n| 36890| 4378| null| 10644| 34| null|\n| 36890| 55403| null| 8139| 34| null|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows", + "text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 37506| 7933| null| 1384| 2| 39437|\n| 37506| 56044| null| 14689| 2| 26419|\n| 37506| 52706| null| 8541| 2| 44016|\n| 37506| 67325| null| 16129| 2| 83371|\n| 37506| 84857| null| 1869| 2| 13090|\n| 37506| 49599| null| 2994| 2| 8940|\n| 37506| 78150| null| 11392| 2| 65633|\n| 37506| 38720| null| 14366| 2| 22281|\n| 37506| 79915| null| 11102| 2| 81755|\n| 37506| 67253| null| 5380| 2| 46868|\n| 37506| 6507| null| 6813| 2| 49363|\n| 37506| 18280| null| 1458| 2| 49363|\n| 37506| 72258| null| 2869| 2| 67756|\n| 37506| 8045| null| 615| 2| 86035|\n| 37506| 86164| null| 7000| 2| 94821|\n| 37506| 29724| null| 2767| 2| 94821|\n| 37506| 55471| null| 3584| 2| 62792|\n| 37506| 677| null| 1720| 2| 27212|\n| 37506| 66638| null| 9898| 2| 20370|\n| 37506| 48515| null| 9394| 2| 17157|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows", "output_type": "stream" } ], - "execution_count": 1 + "execution_count": 5 }, { "cell_type": "code", "source": "# Read the product reviews CSV files into a spark data frame, print schema & top rows\r\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/product_review_data').toDF(\r\n \"pr_review_sk\", \"pr_review_content\"\r\n )\r\nresults.printSchema()\r\nresults.show()", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -80,31 +72,27 @@ "output_type": "stream" } ], - "execution_count": 1 + "execution_count": 6 }, { "cell_type": "code", - "source": "# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\n", - "metadata": { - "language": "python" - }, + "source": "# Save results as parquet, and orc formats and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\nresults.write.format(\"orc\").mode(\"overwrite\").saveAsTable(\"product_reviews_orc\")\r\n", + "metadata": {}, "outputs": [], - "execution_count": 1 + "execution_count": 7 }, { "cell_type": "code", "source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT pr_review_sk, CHAR_LENGTH(pr_review_content) as len FROM product_reviews LIMIT 100\")\r\nsqlDF.show()", - "metadata": { - "language": "python" - }, + "metadata": {}, "outputs": [ { "name": "stdout", - "text": "+------------+----+\n|pr_review_sk| len|\n+------------+----+\n| 14868| 985|\n| 14869|1601|\n| 14875|1221|\n| 14880| 665|\n| 14886| 91|\n| 14894| 697|\n| 14899| 356|\n| 14903|2361|\n| 14908| 872|\n| 14909| 74|\n| 14917| 908|\n| 14918| 50|\n| 14919| 256|\n| 14921| 723|\n| 14925| 313|\n| 14931|1304|\n| 14939|1023|\n| 14949| 552|\n| 14954|2144|\n| 14955| 123|\n+------------+----+\nonly showing top 20 rows", + "text": "+------------+----+\n|pr_review_sk| len|\n+------------+----+\n| 26035| 876|\n| 26037| 109|\n| 26038| 478|\n| 26041| 106|\n| 26043| 332|\n| 26044| 487|\n| 26045| 428|\n| 26048| 87|\n| 26049| 118|\n| 26051|2906|\n| 26053| 464|\n| 26054| 212|\n| 26059| 191|\n| 26060| 207|\n| 26061| 515|\n| 26063| 59|\n| 26069| 487|\n| 26070| 160|\n| 26071| 380|\n| 26072| 234|\n+------------+----+\nonly showing top 20 rows", "output_type": "stream" } ], - "execution_count": 1 + "execution_count": 8 } ] -} \ No newline at end of file +}} \ No newline at end of file