mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Added new external table sample & refactored some existing scripts
This commit is contained in:
Umachandar Jayachandran
@@ -38,12 +38,18 @@ for %%F in (web_clickstreams inventory) do (
|
||||
%DEBUG% bcp sales.dbo.%%F out "%STARTUP_PATH%%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "%STARTUP_PATH%%%F.out" -e "%STARTUP_PATH%%%F.err" || goto exit
|
||||
)
|
||||
|
||||
echo Exporting product_reviews data...
|
||||
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), '\"', '') from sales.dbo.product_reviews" queryout "%STARTUP_PATH%product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "%STARTUP_PATH%product_reviews.out" -e "%STARTUP_PATH%product_reviews.err" || goto exit
|
||||
|
||||
REM Copy the data file to HDFS
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
pushd "%STARTUP_PATH%"
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
|
||||
|
||||
echo Uploading product_reviews data to HDFS...
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
|
||||
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
|
||||
:: del /q *.out *.err *.csv
|
||||
popd
|
||||
|
||||
|
||||
@@ -42,10 +42,17 @@ for table in web_clickstreams inventory
|
||||
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "$table.err" || (echo $ERROR_MESSAGE && exit 3)
|
||||
done
|
||||
|
||||
echo Exporting product_reviews data...
|
||||
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), '\"', '') from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3)
|
||||
|
||||
# Copy the data file to HDFS
|
||||
echo Uploading web_clickstreams data to HDFS...
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
|
||||
|
||||
echo Uploading product_reviews data to HDFS...
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
|
||||
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
|
||||
|
||||
# rm -f *.out *.err *.csv
|
||||
exit
|
||||
@@ -8,20 +8,26 @@ In this example, you are going to create an external table in the SQL Server Mas
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type.
|
||||
|
||||
1. Run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s).
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the [external-table-hdfs-csv.sql](external-table-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
|
||||
1. Before you use execute the *external-table-hdfs-parquet.sql* script, make sure you run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file. Execute the [external-table-hdfs-parquet.sql](external-table-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS.
|
||||
1. Execute the [web-clickstreams-parquet.sql](web-clickstreams-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS.
|
||||
|
||||
1. Execute the [product-reviews-hdfs-csv.sql](product-reviews-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
|
||||
## Query data in Oracle from SQL Server master
|
||||
|
||||
In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server.
|
||||
|
||||
**Before you begin**, you need to have an Oracle instance and credentials. Execute the SQL script [inventory-ora.sql](inventory-ora.sql/) in Oracle to create the table and import the "inventory.csv" file created by the bootstrap sample database.
|
||||
**Before you begin**, you need to have an Oracle instance and credentials. Follow the instruction in the [oracle-setup\README.md](oracle-setup\README.md).
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [external-table-oracle.sql](external-table-oracle.sql/).
|
||||
1. Execute the SQL [inventory-oracle.sql](inventory-oracle.sql/).
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create database scoped credential to connect to Oracle server
|
||||
-- Provide appropriate credentials to Oracle server in below statement.
|
||||
-- If you are using SQL Server Management Studio then you can replace the parameters using
|
||||
-- the Query menu, and "Specify Values for Template Parameters" option.
|
||||
CREATE DATABASE SCOPED CREDENTIAL [OracleCredential]
|
||||
WITH IDENTITY = '<oracle_user,nvarchar(100),SYSTEM>', SECRET = '<oracle_user_password,nvarchar(100),manager>';
|
||||
|
||||
-- Create external data source that points to Oracle server
|
||||
--
|
||||
CREATE EXTERNAL DATA SOURCE [OracleSalesSrvr]
|
||||
WITH (LOCATION = 'oracle://<oracle_server,nvarchar(100)>',CREDENTIAL = [OracleCredential]);
|
||||
|
||||
-- Create external table over inventory table on Oracle server
|
||||
-- NOTE: Table names and column names will use ANSI SQL quoted identifier while querying against Oracle.
|
||||
-- As a result, the names are case-sensitive so specify the name in the external table definition
|
||||
-- that matches the exact case of the table and column names in the Oracle metadata.
|
||||
CREATE EXTERNAL TABLE [inventory_ora]
|
||||
([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL,
|
||||
[inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0))
|
||||
WITH (DATA_SOURCE=[OracleSalesSrvr],
|
||||
LOCATION='<oracle_service_name,nvarchar(30),xe>.<oracle_schema,nvarchar(128),HR>.<oracle_table,nvarchar(128),INVENTORY>');
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity
|
||||
FROM [inventory_ora] as i
|
||||
JOIN item as it
|
||||
ON it.i_item_sk = i.inv_item
|
||||
JOIN warehouse as w
|
||||
ON w.w_warehouse_sk = i.inv_warehouse
|
||||
WHERE it.i_category = 'Books' and i.inv_item BETWEEN 1 and 18000 --> get items within specific range
|
||||
GROUP BY w.w_warehouse_name, i.inv_item;
|
||||
GO
|
||||
|
||||
-- Cleanup
|
||||
--
|
||||
DROP EXTERNAL TABLE [inventory_ora];
|
||||
DROP EXTERNAL DATA SOURCE [OracleSalesSrvr] ;
|
||||
DROP DATABASE SCOPED CREDENTIAL [OracleCredential];
|
||||
GO
|
||||
@@ -1,10 +1,44 @@
|
||||
-- Inventory table over which the SQL Server external table will be defined
|
||||
CREATE TABLE "INVENTORY"
|
||||
(
|
||||
"INV_DATE" NUMBER(10,0) NOT NULL,
|
||||
"INV_ITEM" NUMBER(10,0) NOT NULL,
|
||||
"INV_WAREHOUSE" NUMBER(10,0) NOT NULL,
|
||||
"INV_QUANTITY_ON_HAND" NUMBER(10,0)
|
||||
);
|
||||
USE sales
|
||||
GO
|
||||
|
||||
CREATE INDEX INV_ITEM ON HR.INVENTORY(INV_ITEM);
|
||||
-- Create database scoped credential to connect to Oracle server
|
||||
-- Provide appropriate credentials to Oracle server in below statement.
|
||||
-- If you are using SQL Server Management Studio then you can replace the parameters using
|
||||
-- the Query menu, and "Specify Values for Template Parameters" option.
|
||||
CREATE DATABASE SCOPED CREDENTIAL [OracleCredential]
|
||||
WITH IDENTITY = '<oracle_user,nvarchar(100),sales>', SECRET = '<oracle_user_password,nvarchar(100),sql19tw0oracle>';
|
||||
|
||||
-- Create external data source that points to Oracle server
|
||||
--
|
||||
CREATE EXTERNAL DATA SOURCE [OracleSalesSrvr]
|
||||
WITH (LOCATION = 'oracle://<oracle_server,nvarchar(100),oracle-server-name>',CREDENTIAL = [OracleCredential]);
|
||||
|
||||
-- Create external table over inventory table on Oracle server
|
||||
-- NOTE: Table names and column names will use ANSI SQL quoted identifier while querying against Oracle.
|
||||
-- As a result, the names are case-sensitive so specify the name in the external table definition
|
||||
-- that matches the exact case of the table and column names in the Oracle metadata.
|
||||
CREATE EXTERNAL TABLE [inventory_ora]
|
||||
([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL,
|
||||
[inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0))
|
||||
WITH (DATA_SOURCE=[OracleSalesSrvr],
|
||||
LOCATION='<oracle_service_name,nvarchar(30),xe>.SALES.INVENTORY');
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity
|
||||
FROM [inventory_ora] as i
|
||||
JOIN item as it
|
||||
ON it.i_item_sk = i.inv_item
|
||||
JOIN warehouse as w
|
||||
ON w.w_warehouse_sk = i.inv_warehouse
|
||||
WHERE it.i_category = 'Books' and i.inv_item BETWEEN 1 and 18000 --> get items within specific range
|
||||
GROUP BY w.w_warehouse_name, i.inv_item;
|
||||
GO
|
||||
|
||||
-- Cleanup
|
||||
--
|
||||
DROP EXTERNAL TABLE [inventory_ora];
|
||||
DROP EXTERNAL DATA SOURCE [OracleSalesSrvr] ;
|
||||
DROP DATABASE SCOPED CREDENTIAL [OracleCredential];
|
||||
GO
|
||||
@@ -0,0 +1,11 @@
|
||||
# Oracle setup
|
||||
|
||||
This folder contains scripts that can be executed on Oracle server to create the necessary objects for data virtualization in SQL Server 2019 big data cluster.
|
||||
|
||||
# Instructions
|
||||
|
||||
1. Connect to Oracle instance.
|
||||
|
||||
1. Execute the [sales-user.sql](sales-user.sql). This script creates the sample user. If there is name conflict please change the script user/credentials.
|
||||
|
||||
1. Execute the [inventory.sql](inventory.sql). This script creates the inventory table.
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Inventory table over which the SQL Server external table will be defined
|
||||
CREATE TABLE "SALES"."INVENTORY"
|
||||
(
|
||||
"INV_DATE" NUMBER(10,0) NOT NULL,
|
||||
"INV_ITEM" NUMBER(10,0) NOT NULL,
|
||||
"INV_WAREHOUSE" NUMBER(10,0) NOT NULL,
|
||||
"INV_QUANTITY_ON_HAND" NUMBER(10,0)
|
||||
);
|
||||
|
||||
CREATE INDEX INV_ITEM ON "SALES"."INVENTORY"("INV_ITEM");
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE USER "SALES" IDENTIFIED BY "sql19tw0oracle"
|
||||
DEFAULT TABLESPACE "USERS"
|
||||
TEMPORARY TABLESPACE "TEMP"
|
||||
-- QUOTAS
|
||||
QUOTA UNLIMITED ON "USERS";
|
||||
|
||||
-- ROLES
|
||||
GRANT "CONNECT" TO "SALES" ;
|
||||
GRANT "RESOURCE" TO "SALES" ;
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create file format for CSV separated file with appropriate properties.
|
||||
--
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file')
|
||||
CREATE EXTERNAL FILE FORMAT csv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = ',',
|
||||
STRING_DELIMITER = '"',
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
-- is a special data source that is available in any new database in
|
||||
-- SQL Master instance.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [product_reviews_hdfs_csv]
|
||||
("pr_review_sk" BIGINT , "pr_review_content" varchar(8000))
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/product_review_data',
|
||||
FILE_FORMAT = csv_file
|
||||
);
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT
|
||||
p.pr_review_sk, pc.pr_review_content
|
||||
FROM product_reviews as p
|
||||
JOIN (SELECT TOP(10) * FROM product_reviews_hdfs_csv) AS pc
|
||||
ON pc.pr_review_sk = p.pr_review_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[product_reviews_hdfs_csv];
|
||||
GO
|
||||
+9
-9
@@ -3,15 +3,15 @@ GO
|
||||
|
||||
-- Create file format for CSV file with appropriate properties.
|
||||
--
|
||||
CREATE EXTERNAL FILE FORMAT csv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = ',',
|
||||
STRING_DELIMITER = '"',
|
||||
FIRST_ROW = 2,
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'csv_file')
|
||||
CREATE EXTERNAL FILE FORMAT csv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = ',',
|
||||
STRING_DELIMITER = '"',
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
+5
-4
@@ -3,10 +3,11 @@ GO
|
||||
|
||||
-- Create file format for parquet file with appropriate properties.
|
||||
--
|
||||
CREATE EXTERNAL FILE FORMAT parquet_file
|
||||
WITH (
|
||||
FORMAT_TYPE = PARQUET
|
||||
);
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file')
|
||||
CREATE EXTERNAL FILE FORMAT parquet_file
|
||||
WITH (
|
||||
FORMAT_TYPE = PARQUET
|
||||
);
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
@@ -24,7 +24,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Read the CSV into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data/web_clickstreams.csv').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()",
|
||||
"source": "# Read the clickstream CSV file(s) into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
@@ -61,7 +61,46 @@
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 37506| 7933| null| 1384| 2| 39437|\n| 37506| 56044| null| 14689| 2| 26419|\n| 37506| 52706| null| 8541| 2| 44016|\n| 37506| 67325| null| 16129| 2| 83371|\n| 37506| 84857| null| 1869| 2| 13090|\n| 37506| 49599| null| 2994| 2| 8940|\n| 37506| 78150| null| 11392| 2| 65633|\n| 37506| 38720| null| 14366| 2| 22281|\n| 37506| 79915| null| 11102| 2| 81755|\n| 37506| 67253| null| 5380| 2| 46868|\n| 37506| 6507| null| 6813| 2| 49363|\n| 37506| 18280| null| 1458| 2| 49363|\n| 37506| 72258| null| 2869| 2| 67756|\n| 37506| 8045| null| 615| 2| 86035|\n| 37506| 86164| null| 7000| 2| 94821|\n| 37506| 29724| null| 2767| 2| 94821|\n| 37506| 55471| null| 3584| 2| 62792|\n| 37506| 677| null| 1720| 2| 27212|\n| 37506| 66638| null| 9898| 2| 20370|\n| 37506| 48515| null| 9394| 2| 17157|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows",
|
||||
"text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 38250| 4172| 67067| 11504| 40| 16819|\n| 38251| 28919| 67090| 13782| 40| 11283|\n| 38251| 77021| 67096| 4330| 40| 60107|\n| 38251| 29023| 67109| 15796| 40| 31730|\n| 38251| 54047| 67110| 9739| 40| 50449|\n| 38251| 85733| 67117| 6843| 40| 39327|\n| 38252| 53176| 67141| 12525| 40| 37913|\n| 38252| 15873| 67153| 13008| 40| 92546|\n| 38252| 39147| 67167| 5208| 40| 74534|\n| 38252| 79540| 67171| 11552| 40| 94065|\n| 38252| 35200| 67175| 9622| 40| 80502|\n| 38253| 26068| 67191| 8585| 40| 43314|\n| 38253| 63065| 67195| 17486| 40| 63793|\n| 38253| 9687| 67214| 9856| 40| 92780|\n| 38253| 18373| 67219| 406| 40| 38319|\n| 38254| 80201| 67229| 13610| 40| 15342|\n| 38254| 40058| 67239| 13594| 40| 41879|\n| 38254| 79136| 67243| 1933| 40| 42095|\n| 38254| 14684| 67244| 14267| 40| 39119|\n| 38254| 36369| 67248| 641| 40| 82237|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Read the product reviews CSV files into a spark data frame, print schema & top rows\r\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/product_review_data').toDF(\r\n \"pr_review_sk\", \"pr_review_content\"\r\n )\r\nresults.printSchema()\r\nresults.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "root\n |-- pr_review_sk: integer (nullable = true)\n |-- pr_review_content: string (nullable = true)\n\n+------------+--------------------+\n|pr_review_sk| pr_review_content|\n+------------+--------------------+\n| 72621|Works fine. Easy ...|\n| 89334|great product to ...|\n| 89335|Next time will go...|\n| 84259|Great Gift Great ...|\n| 84398|After trip to Par...|\n| 66434|Simply the best t...|\n| 66501|This is the exact...|\n| 66587|Not super magnet;...|\n| 66680|Installed as bath...|\n| 66694|Our home was buil...|\n| 84489|Hi ;We are runnin...|\n| 79052|Terra cotta is th...|\n| 73034|One of my fingern...|\n| 73298|We installed thes...|\n| 66810|needed silicone c...|\n| 66912|Great Gift Great ...|\n| 67028|Laguiole knives a...|\n| 89770|Good sound timers...|\n| 84679|AWESOME FEEDBACK ...|\n| 84953|love the retro gl...|\n+------------+--------------------+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\n",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT pr_review_sk, CHAR_LENGTH(pr_review_content) as len FROM product_reviews LIMIT 100\")\r\nsqlDF.show()",
|
||||
"metadata": {
|
||||
"language": "python"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "+------------+----+\n|pr_review_sk| len|\n+------------+----+\n| 14868| 985|\n| 14869|1601|\n| 14875|1221|\n| 14880| 665|\n| 14886| 91|\n| 14894| 697|\n| 14899| 356|\n| 14903|2361|\n| 14908| 872|\n| 14909| 74|\n| 14917| 908|\n| 14918| 50|\n| 14919| 256|\n| 14921| 723|\n| 14925| 313|\n| 14931|1304|\n| 14939|1023|\n| 14949| 552|\n| 14954|2144|\n| 14955| 123|\n+------------+----+\nonly showing top 20 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user