From 10199d1793901654e23fe91091ff756d94bbb3f9 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 31 Oct 2018 13:58:17 -0700 Subject: [PATCH] Added new external table sample & refactored some existing scripts --- .../bootstrap-sample-db.cmd | 8 ++- .../bootstrap-sample-db.sh | 7 +++ .../data-virtualization/README.md | 14 +++-- .../external-table-oracle.sql | 44 ---------------- .../data-virtualization/inventory-oracle.sql | 52 +++++++++++++++---- .../oracle-setup/README.md | 11 ++++ .../oracle-setup/inventory.sql | 10 ++++ .../oracle-setup/sales-user.sql | 9 ++++ .../product-reviews-hdfs-csv.sql | 42 +++++++++++++++ ...-csv.sql => web-clickstreams-hdfs-csv.sql} | 18 +++---- ....sql => web-clickstreams-hdfs-parquet.sql} | 9 ++-- .../spark/spark-sql.ipynb | 43 ++++++++++++++- 12 files changed, 194 insertions(+), 73 deletions(-) delete mode 100644 samples/features/sql-big-data-cluster/data-virtualization/external-table-oracle.sql create mode 100644 samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/README.md create mode 100644 samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/inventory.sql create mode 100644 samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/sales-user.sql create mode 100644 samples/features/sql-big-data-cluster/data-virtualization/product-reviews-hdfs-csv.sql rename samples/features/sql-big-data-cluster/data-virtualization/{external-table-hdfs-csv.sql => web-clickstreams-hdfs-csv.sql} (84%) rename samples/features/sql-big-data-cluster/data-virtualization/{external-table-hdfs-parquet.sql => web-clickstreams-hdfs-parquet.sql} (90%) diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd index fa123a3b..107a4a5b 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd @@ -38,12 +38,18 @@ for %%F in (web_clickstreams inventory) do ( %DEBUG% bcp sales.dbo.%%F out "%STARTUP_PATH%%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "%STARTUP_PATH%%%F.out" -e "%STARTUP_PATH%%%F.err" || goto exit ) +echo Exporting product_reviews data... +%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), '\"', '') from sales.dbo.product_reviews" queryout "%STARTUP_PATH%product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "%STARTUP_PATH%product_reviews.out" -e "%STARTUP_PATH%product_reviews.err" || goto exit + REM Copy the data file to HDFS -echo Uploading web_clickstreams data to HDFS... pushd "%STARTUP_PATH%" +echo Uploading web_clickstreams data to HDFS... %DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit %DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit +echo Uploading product_reviews data to HDFS... +%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit +%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit :: del /q *.out *.err *.csv popd diff --git a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh index c869f5b1..7ad160cc 100644 --- a/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh +++ b/samples/features/sql-big-data-cluster/bootstrap-sample-db.sh @@ -42,10 +42,17 @@ for table in web_clickstreams inventory $DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "$table.err" || (echo $ERROR_MESSAGE && exit 3) done +echo Exporting product_reviews data... +$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), '\"', '') from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3) + # Copy the data file to HDFS echo Uploading web_clickstreams data to HDFS... $DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4) $DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5) +echo Uploading product_reviews data to HDFS... +$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6) +$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7) + # rm -f *.out *.err *.csv exit \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/README.md b/samples/features/sql-big-data-cluster/data-virtualization/README.md index a1aa4233..cd030cfb 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/README.md +++ b/samples/features/sql-big-data-cluster/data-virtualization/README.md @@ -8,20 +8,26 @@ In this example, you are going to create an external table in the SQL Server Mas ### Instructions +1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type. + +1. Run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s). + 1. Connect to SQL Server Master instance. -1. Execute the [external-table-hdfs-csv.sql](external-table-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS. +1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS. -1. Before you use execute the *external-table-hdfs-parquet.sql* script, make sure you run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file. Execute the [external-table-hdfs-parquet.sql](external-table-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS. +1. Execute the [web-clickstreams-parquet.sql](web-clickstreams-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS. + +1. Execute the [product-reviews-hdfs-csv.sql](product-reviews-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS. ## Query data in Oracle from SQL Server master In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server. -**Before you begin**, you need to have an Oracle instance and credentials. Execute the SQL script [inventory-ora.sql](inventory-ora.sql/) in Oracle to create the table and import the "inventory.csv" file created by the bootstrap sample database. +**Before you begin**, you need to have an Oracle instance and credentials. Follow the instruction in the [oracle-setup\README.md](oracle-setup\README.md). ### Instructions 1. Connect to SQL Server Master instance. -1. Execute the SQL [external-table-oracle.sql](external-table-oracle.sql/). +1. Execute the SQL [inventory-oracle.sql](inventory-oracle.sql/). diff --git a/samples/features/sql-big-data-cluster/data-virtualization/external-table-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/external-table-oracle.sql deleted file mode 100644 index 61a91176..00000000 --- a/samples/features/sql-big-data-cluster/data-virtualization/external-table-oracle.sql +++ /dev/null @@ -1,44 +0,0 @@ -USE sales -GO - --- Create database scoped credential to connect to Oracle server --- Provide appropriate credentials to Oracle server in below statement. --- If you are using SQL Server Management Studio then you can replace the parameters using --- the Query menu, and "Specify Values for Template Parameters" option. -CREATE DATABASE SCOPED CREDENTIAL [OracleCredential] -WITH IDENTITY = '', SECRET = ''; - --- Create external data source that points to Oracle server --- -CREATE EXTERNAL DATA SOURCE [OracleSalesSrvr] -WITH (LOCATION = 'oracle://',CREDENTIAL = [OracleCredential]); - --- Create external table over inventory table on Oracle server --- NOTE: Table names and column names will use ANSI SQL quoted identifier while querying against Oracle. --- As a result, the names are case-sensitive so specify the name in the external table definition --- that matches the exact case of the table and column names in the Oracle metadata. -CREATE EXTERNAL TABLE [inventory_ora] - ([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL, - [inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0)) -WITH (DATA_SOURCE=[OracleSalesSrvr], - LOCATION='..'); -GO - --- Join external table with local tables --- -SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity - FROM [inventory_ora] as i - JOIN item as it - ON it.i_item_sk = i.inv_item - JOIN warehouse as w - ON w.w_warehouse_sk = i.inv_warehouse - WHERE it.i_category = 'Books' and i.inv_item BETWEEN 1 and 18000 --> get items within specific range - GROUP BY w.w_warehouse_name, i.inv_item; -GO - --- Cleanup --- -DROP EXTERNAL TABLE [inventory_ora]; -DROP EXTERNAL DATA SOURCE [OracleSalesSrvr] ; -DROP DATABASE SCOPED CREDENTIAL [OracleCredential]; -GO \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/inventory-oracle.sql b/samples/features/sql-big-data-cluster/data-virtualization/inventory-oracle.sql index 1fd70572..d18020f6 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/inventory-oracle.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/inventory-oracle.sql @@ -1,10 +1,44 @@ --- Inventory table over which the SQL Server external table will be defined -CREATE TABLE "INVENTORY" -( - "INV_DATE" NUMBER(10,0) NOT NULL, - "INV_ITEM" NUMBER(10,0) NOT NULL, - "INV_WAREHOUSE" NUMBER(10,0) NOT NULL, - "INV_QUANTITY_ON_HAND" NUMBER(10,0) -); +USE sales +GO -CREATE INDEX INV_ITEM ON HR.INVENTORY(INV_ITEM); +-- Create database scoped credential to connect to Oracle server +-- Provide appropriate credentials to Oracle server in below statement. +-- If you are using SQL Server Management Studio then you can replace the parameters using +-- the Query menu, and "Specify Values for Template Parameters" option. +CREATE DATABASE SCOPED CREDENTIAL [OracleCredential] +WITH IDENTITY = '', SECRET = ''; + +-- Create external data source that points to Oracle server +-- +CREATE EXTERNAL DATA SOURCE [OracleSalesSrvr] +WITH (LOCATION = 'oracle://',CREDENTIAL = [OracleCredential]); + +-- Create external table over inventory table on Oracle server +-- NOTE: Table names and column names will use ANSI SQL quoted identifier while querying against Oracle. +-- As a result, the names are case-sensitive so specify the name in the external table definition +-- that matches the exact case of the table and column names in the Oracle metadata. +CREATE EXTERNAL TABLE [inventory_ora] + ([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL, + [inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0)) +WITH (DATA_SOURCE=[OracleSalesSrvr], + LOCATION='.SALES.INVENTORY'); +GO + +-- Join external table with local tables +-- +SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity + FROM [inventory_ora] as i + JOIN item as it + ON it.i_item_sk = i.inv_item + JOIN warehouse as w + ON w.w_warehouse_sk = i.inv_warehouse + WHERE it.i_category = 'Books' and i.inv_item BETWEEN 1 and 18000 --> get items within specific range + GROUP BY w.w_warehouse_name, i.inv_item; +GO + +-- Cleanup +-- +DROP EXTERNAL TABLE [inventory_ora]; +DROP EXTERNAL DATA SOURCE [OracleSalesSrvr] ; +DROP DATABASE SCOPED CREDENTIAL [OracleCredential]; +GO \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/README.md b/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/README.md new file mode 100644 index 00000000..b15d7012 --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/README.md @@ -0,0 +1,11 @@ +# Oracle setup + +This folder contains scripts that can be executed on Oracle server to create the necessary objects for data virtualization in SQL Server 2019 big data cluster. + +# Instructions + +1. Connect to Oracle instance. + +1. Execute the [sales-user.sql](sales-user.sql). This script creates the sample user. If there is name conflict please change the script user/credentials. + +1. Execute the [inventory.sql](inventory.sql). This script creates the inventory table. \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/inventory.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/inventory.sql new file mode 100644 index 00000000..b3873ef3 --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/inventory.sql @@ -0,0 +1,10 @@ +-- Inventory table over which the SQL Server external table will be defined +CREATE TABLE "SALES"."INVENTORY" +( + "INV_DATE" NUMBER(10,0) NOT NULL, + "INV_ITEM" NUMBER(10,0) NOT NULL, + "INV_WAREHOUSE" NUMBER(10,0) NOT NULL, + "INV_QUANTITY_ON_HAND" NUMBER(10,0) +); + +CREATE INDEX INV_ITEM ON "SALES"."INVENTORY"("INV_ITEM"); diff --git a/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/sales-user.sql b/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/sales-user.sql new file mode 100644 index 00000000..e928c09a --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/oracle-setup/sales-user.sql @@ -0,0 +1,9 @@ +CREATE USER "SALES" IDENTIFIED BY "sql19tw0oracle" +DEFAULT TABLESPACE "USERS" +TEMPORARY TABLESPACE "TEMP" +-- QUOTAS +QUOTA UNLIMITED ON "USERS"; + +-- ROLES +GRANT "CONNECT" TO "SALES" ; +GRANT "RESOURCE" TO "SALES" ; diff --git a/samples/features/sql-big-data-cluster/data-virtualization/product-reviews-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/product-reviews-hdfs-csv.sql new file mode 100644 index 00000000..0df6094f --- /dev/null +++ b/samples/features/sql-big-data-cluster/data-virtualization/product-reviews-hdfs-csv.sql @@ -0,0 +1,42 @@ +USE sales +GO + +-- Create file format for CSV separated file with appropriate properties. +-- +IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file') + CREATE EXTERNAL FILE FORMAT csv_file + WITH ( + FORMAT_TYPE = DELIMITEDTEXT, + FORMAT_OPTIONS( + FIELD_TERMINATOR = ',', + STRING_DELIMITER = '"', + USE_TYPE_DEFAULT = TRUE) + ); + + +-- Create external table over HDFS data source (SqlStoragePool) in +-- SQL Server 2019 big data cluster. The SqlStoragePool data source +-- is a special data source that is available in any new database in +-- SQL Master instance. +-- +CREATE EXTERNAL TABLE [product_reviews_hdfs_csv] +("pr_review_sk" BIGINT , "pr_review_content" varchar(8000)) +WITH +( + DATA_SOURCE = SqlStoragePool, + LOCATION = '/product_review_data', + FILE_FORMAT = csv_file +); +GO + +-- Join external table with local tables +-- +SELECT + p.pr_review_sk, pc.pr_review_content + FROM product_reviews as p + JOIN (SELECT TOP(10) * FROM product_reviews_hdfs_csv) AS pc + ON pc.pr_review_sk = p.pr_review_sk; +GO + +DROP EXTERNAL TABLE [dbo].[product_reviews_hdfs_csv]; +GO \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-csv.sql b/samples/features/sql-big-data-cluster/data-virtualization/web-clickstreams-hdfs-csv.sql similarity index 84% rename from samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-csv.sql rename to samples/features/sql-big-data-cluster/data-virtualization/web-clickstreams-hdfs-csv.sql index 7596b89a..09da5050 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-csv.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/web-clickstreams-hdfs-csv.sql @@ -3,15 +3,15 @@ GO -- Create file format for CSV file with appropriate properties. -- -CREATE EXTERNAL FILE FORMAT csv_file -WITH ( - FORMAT_TYPE = DELIMITEDTEXT, - FORMAT_OPTIONS( - FIELD_TERMINATOR = ',', - STRING_DELIMITER = '"', - FIRST_ROW = 2, - USE_TYPE_DEFAULT = TRUE) -); +IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file') + CREATE EXTERNAL FILE FORMAT csv_file + WITH ( + FORMAT_TYPE = DELIMITEDTEXT, + FORMAT_OPTIONS( + FIELD_TERMINATOR = ',', + STRING_DELIMITER = '"', + USE_TYPE_DEFAULT = TRUE) + ); -- Create external table over HDFS data source (SqlStoragePool) in -- SQL Server 2019 big data cluster. The SqlStoragePool data source diff --git a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/web-clickstreams-hdfs-parquet.sql similarity index 90% rename from samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql rename to samples/features/sql-big-data-cluster/data-virtualization/web-clickstreams-hdfs-parquet.sql index c4214620..73d67ee6 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/external-table-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/web-clickstreams-hdfs-parquet.sql @@ -3,10 +3,11 @@ GO -- Create file format for parquet file with appropriate properties. -- -CREATE EXTERNAL FILE FORMAT parquet_file -WITH ( - FORMAT_TYPE = PARQUET -); +IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file') + CREATE EXTERNAL FILE FORMAT parquet_file + WITH ( + FORMAT_TYPE = PARQUET + ); -- Create external table over HDFS data source (SqlStoragePool) in -- SQL Server 2019 big data cluster. The SqlStoragePool data source diff --git a/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb b/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb index da24a190..cfc9422d 100644 --- a/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb +++ b/samples/features/sql-big-data-cluster/spark/spark-sql.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "source": "# Read the CSV into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data/web_clickstreams.csv').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()", + "source": "# Read the clickstream CSV file(s) into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()", "metadata": { "language": "python" }, @@ -61,7 +61,46 @@ "outputs": [ { "name": "stdout", - "text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 37506| 7933| null| 1384| 2| 39437|\n| 37506| 56044| null| 14689| 2| 26419|\n| 37506| 52706| null| 8541| 2| 44016|\n| 37506| 67325| null| 16129| 2| 83371|\n| 37506| 84857| null| 1869| 2| 13090|\n| 37506| 49599| null| 2994| 2| 8940|\n| 37506| 78150| null| 11392| 2| 65633|\n| 37506| 38720| null| 14366| 2| 22281|\n| 37506| 79915| null| 11102| 2| 81755|\n| 37506| 67253| null| 5380| 2| 46868|\n| 37506| 6507| null| 6813| 2| 49363|\n| 37506| 18280| null| 1458| 2| 49363|\n| 37506| 72258| null| 2869| 2| 67756|\n| 37506| 8045| null| 615| 2| 86035|\n| 37506| 86164| null| 7000| 2| 94821|\n| 37506| 29724| null| 2767| 2| 94821|\n| 37506| 55471| null| 3584| 2| 62792|\n| 37506| 677| null| 1720| 2| 27212|\n| 37506| 66638| null| 9898| 2| 20370|\n| 37506| 48515| null| 9394| 2| 17157|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows", + "text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 38250| 4172| 67067| 11504| 40| 16819|\n| 38251| 28919| 67090| 13782| 40| 11283|\n| 38251| 77021| 67096| 4330| 40| 60107|\n| 38251| 29023| 67109| 15796| 40| 31730|\n| 38251| 54047| 67110| 9739| 40| 50449|\n| 38251| 85733| 67117| 6843| 40| 39327|\n| 38252| 53176| 67141| 12525| 40| 37913|\n| 38252| 15873| 67153| 13008| 40| 92546|\n| 38252| 39147| 67167| 5208| 40| 74534|\n| 38252| 79540| 67171| 11552| 40| 94065|\n| 38252| 35200| 67175| 9622| 40| 80502|\n| 38253| 26068| 67191| 8585| 40| 43314|\n| 38253| 63065| 67195| 17486| 40| 63793|\n| 38253| 9687| 67214| 9856| 40| 92780|\n| 38253| 18373| 67219| 406| 40| 38319|\n| 38254| 80201| 67229| 13610| 40| 15342|\n| 38254| 40058| 67239| 13594| 40| 41879|\n| 38254| 79136| 67243| 1933| 40| 42095|\n| 38254| 14684| 67244| 14267| 40| 39119|\n| 38254| 36369| 67248| 641| 40| 82237|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows", + "output_type": "stream" + } + ], + "execution_count": 1 + }, + { + "cell_type": "code", + "source": "# Read the product reviews CSV files into a spark data frame, print schema & top rows\r\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/product_review_data').toDF(\r\n \"pr_review_sk\", \"pr_review_content\"\r\n )\r\nresults.printSchema()\r\nresults.show()", + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "text": "root\n |-- pr_review_sk: integer (nullable = true)\n |-- pr_review_content: string (nullable = true)\n\n+------------+--------------------+\n|pr_review_sk| pr_review_content|\n+------------+--------------------+\n| 72621|Works fine. Easy ...|\n| 89334|great product to ...|\n| 89335|Next time will go...|\n| 84259|Great Gift Great ...|\n| 84398|After trip to Par...|\n| 66434|Simply the best t...|\n| 66501|This is the exact...|\n| 66587|Not super magnet;...|\n| 66680|Installed as bath...|\n| 66694|Our home was buil...|\n| 84489|Hi ;We are runnin...|\n| 79052|Terra cotta is th...|\n| 73034|One of my fingern...|\n| 73298|We installed thes...|\n| 66810|needed silicone c...|\n| 66912|Great Gift Great ...|\n| 67028|Laguiole knives a...|\n| 89770|Good sound timers...|\n| 84679|AWESOME FEEDBACK ...|\n| 84953|love the retro gl...|\n+------------+--------------------+\nonly showing top 20 rows", + "output_type": "stream" + } + ], + "execution_count": 1 + }, + { + "cell_type": "code", + "source": "# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\n", + "metadata": { + "language": "python" + }, + "outputs": [], + "execution_count": 1 + }, + { + "cell_type": "code", + "source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT pr_review_sk, CHAR_LENGTH(pr_review_content) as len FROM product_reviews LIMIT 100\")\r\nsqlDF.show()", + "metadata": { + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "text": "+------------+----+\n|pr_review_sk| len|\n+------------+----+\n| 14868| 985|\n| 14869|1601|\n| 14875|1221|\n| 14880| 665|\n| 14886| 91|\n| 14894| 697|\n| 14899| 356|\n| 14903|2361|\n| 14908| 872|\n| 14909| 74|\n| 14917| 908|\n| 14918| 50|\n| 14919| 256|\n| 14921| 723|\n| 14925| 313|\n| 14931|1304|\n| 14939|1023|\n| 14949| 552|\n| 14954|2144|\n| 14955| 123|\n+------------+----+\nonly showing top 20 rows", "output_type": "stream" } ],