github/sql-server-samples
1
0
Fork 0
mirror of https://github.com/Microsoft/sql-server-samples.git synced 2025-12-08 14:58:54 +00:00
Code Issues Releases Wiki Activity

PolyBase V1 samples & option to install AW/WWI dbs

Browse Source
This commit is contained in:
Umachandar Jayachandran
2019-03-19 17:26:09 -07:00
parent cf79ffdeb5
commit 6ddfd4d9b1
8 changed files with 337 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd
+50 -12
View File
@@ -7,6 +7,7 @@ set SQL_MASTER_IP=%2
set SQL_MASTER_SA_PASSWORD=%3
set KNOX_IP=%4
set KNOX_PASSWORD=%5
set AW_WWI_SAMPLES=%6
set STARTUP_PATH=%~dp0
set TMP_DIR_NAME=%~nx0
@@ -15,6 +16,7 @@ if NOT DEFINED SQL_MASTER_IP goto :usage
if NOT DEFINED SQL_MASTER_SA_PASSWORD goto :usage
if NOT DEFINED KNOX_IP goto :usage
if NOT DEFINED KNOX_PASSWORD set KNOX_PASSWORD=%SQL_MASTER_SA_PASSWORD%
if NOT DEFINED AW_WWI_SAMPLES set AW_WWI_SAMPLES=no
set SQL_MASTER_INSTANCE=%SQL_MASTER_IP%,31433
set KNOX_ENDPOINT=%KNOX_IP%:30443
@@ -26,38 +28,74 @@ for %%F in (sqlcmd.exe bcp.exe kubectl.exe curl.exe) do (
pushd "%tmp%"
md %TMP_DIR_NAME%
cd %TMP_DIR_NAME%
echo Downloading sample database backup file...
%DEBUG% curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak
if NOT EXIST tpcxbb_1gb.bak (
echo Downloading sample database backup file...
%DEBUG% curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak
)
REM Copy the backup file, restore the database, create necessary objects and data file
echo Copying database backup file...
echo Copying sales database backup file to SQL Master instance...
%DEBUG% kubectl cp tpcxbb_1gb.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
del tpcxbb_1gb.bak >NUL
if /i %AW_WWI_SAMPLES% EQU install_extra_samples (
if NOT EXIST AdventureWorks2016_EXT.bak (
echo Downloading AdventureWorks2016_EXT sample database backup file...
%DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2016_EXT.bak" -o AdventureWorks2016_EXT.bak
)
echo Copying AdventureWorks2016_EXT database backup file to SQL Master instance...
%DEBUG% kubectl cp AdventureWorks2016_EXT.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
echo Configuring sample database...
if NOT EXIST AdventureWorksDW2016_EXT.bak (
echo Downloading AdventureWorksDW2016_EXT sample database backup file...
%DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorksDW2016_EXT.bak" -o AdventureWorksDW2016_EXT.bak
)
echo Copying AdventureWorksDW2016_EXT database backup file to SQL Master instance...
%DEBUG% kubectl cp AdventureWorksDW2016_EXT.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
if NOT EXIST WideWorldImporters-Full.bak (
echo Downloading WideWorldImporters sample database backup file...
%DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImporters-Full.bak" -o WideWorldImporters-Full.bak
)
echo Copying WideWorldImporters-Full database backup file to SQL Master instance...
%DEBUG% kubectl cp WideWorldImporters-Full.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
if NOT EXIST WideWorldImportersDW-Full.bak (
echo Downloading WideWorldImportersDW sample database backup file...
%DEBUG% curl -L -G "https://github.com/Microsoft/sql-server-samples/releases/download/wide-world-importers-v1.0/WideWorldImportersDW-Full.bak" -o WideWorldImportersDW-Full.bak
)
echo Copying WideWorldImportersDW-Full database backup file to SQL Master instance...
%DEBUG% kubectl cp WideWorldImportersDW-Full.bak mssql-master-pool-0:/var/opt/mssql/data -c mssql-server -n %CLUSTER_NAMESPACE% || goto exit
)
echo Configuring sample database(s)...
%DEBUG% sqlcmd -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit
for %%F in (web_clickstreams inventory customer) do (
echo Exporting %%F data...
if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|)
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
if NOT EXIST %%F.csv (
echo Exporting %%F data...
if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|)
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
)
)
echo Exporting product_reviews data...
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
if NOT EXIST product_reviews.csv (
echo Exporting product_reviews data...
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
)
REM Copy the data file to HDFS
echo Uploading web_clickstreams data to HDFS...
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
del /q web_clickstreams.*
:: del /q web_clickstreams.*
echo.
echo Uploading product_reviews data to HDFS...
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
del /q product_reviews.*
:: del /q product_reviews.*
REM %DEBUG% del /q *.out *.err *.csv
echo Bootstrap of the sample database completed successfully.
samples/features/sql-big-data-cluster/bootstrap-sample-db.sql
+97 -19
View File
@@ -18,29 +18,107 @@ BEGIN
END;
GO
IF DB_ID('sales') IS NULL
RESTORE DATABASE sales
FROM DISK=N'/var/opt/mssql/data/tpcxbb_1gb.bak'
WITH
MOVE N'tpcxbb_1gb' TO N'/var/opt/mssql/data/sales.mdf',
MOVE N'tpcxbb_1gb_log' TO N'/var/opt/mssql/data/sales.ldf';
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255))
AS
BEGIN
DECLARE @restore_filelist_tmpl nvarchar(1000) = N'restore filelistonly FROM DISK = N''/var/opt/mssql/data/%F''';
DECLARE @restore_database_tmpl nvarchar(1000) = N'RESTORE DATABASE [%D] FROM DISK = N''/var/opt/mssql/data/%F'' WITH FILE = 1';
DECLARE @move_tmpl nvarchar(1000) = N', MOVE N''%L'' TO N''/var/opt/mssql/data/%F''';
DECLARE @restore_cmd nvarchar(4000), @logical_name nvarchar(128), @filename nvarchar(260), @restore_cur CURSOR;
DECLARE @files TABLE (
[LogicalName] NVARCHAR(128),
[PhysicalName] NVARCHAR(260),
[Type] CHAR(1),
[FileGroupName] NVARCHAR(128),
[Size] NUMERIC(20,0),
[MaxSize] NUMERIC(20,0),
[FileID] BIGINT,
[CreateLSN] NUMERIC(25,0),
[DropLSN] NUMERIC(25,0),
[UniqueID] UNIQUEIDENTIFIER,
[ReadOnlyLSN] NUMERIC(25,0),
[ReadWriteLSN] NUMERIC(25,0),
[BackupSizeInBytes] BIGINT,
[SourceBlockSize] INT,
[FileGroupID] INT,
[LogGroupGUID] UNIQUEIDENTIFIER,
[DifferentialBaseLSN] NUMERIC(25,0),
[DifferentialBaseGUID] UNIQUEIDENTIFIER,
[IsReadOnly] BIT,
[IsPresent] BIT,
[TDEThumbprint] VARBINARY(32),
[SnapshotUrl] NVARCHAR(260)
)
SET @restore_cmd = REPLACE(@restore_filelist_tmpl, '%F', @backup_file);
INSERT INTO @files
EXECUTE(@restore_cmd);
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', LEFT(@backup_file, CHARINDEX('.', @backup_file)-1));
SET @restore_cur = CURSOR FAST_FORWARD FOR SELECT LogicalName, REVERSE(LEFT(REVERSE(PhysicalName), CHARINDEX('\', REVERSE(PhysicalName))-1)) FROM @files;
OPEN @restore_cur;
WHILE(1=1)
BEGIN
FETCH FROM @restore_cur INTO @logical_name, @filename;
IF @@FETCH_STATUS < 0 BREAK;
SET @restore_cmd += REPLACE(REPLACE(@move_tmpl, '%L', @logical_name), '%F', @filename);
END;
EXECUTE(@restore_cmd);
END;
GO
CREATE OR ALTER PROCEDURE #create_data_sources
AS
BEGIN
-- Create database master key (required for database scoped credentials used in the samples)
IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1)
CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!';
-- Create default data sources for SQL Big Data Cluster
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool')
CREATE EXTERNAL DATA SOURCE SqlDataPool
WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default');
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080');
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
CREATE EXTERNAL DATA SOURCE HadoopData
WITH(
TYPE=HADOOP,
LOCATION='hdfs://mssql-master-pool-0.service-master-pool:9000/',
RESOURCE_MANAGER_LOCATION='mssql-master-pool-0.service-master-pool:8032'
);
END;
GO
--- Sample dbs:
DECLARE @sample_dbs CURSOR, @proc nvarchar(255);
SET @sample_dbs = CURSOR FAST_FORWARD FOR
SELECT file_or_directory_name
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak')
WHERE DB_ID(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', '')) IS NULL;
DECLARE @file nvarchar(260);
OPEN @sample_dbs;
WHILE(1=1)
BEGIN
FETCH @sample_dbs INTO @file;
IF @@FETCH_STATUS < 0 BREAK;
EXECUTE #restore_database @file;
SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql');
EXECUTE @proc N'#create_data_sources';
-- Rename TPCx-BB database:
IF DB_ID('tpcxbb_1gb') IS NOT NULL
ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales;
END;
GO
USE sales;
GO
-- Create database master key (required for database scoped credentials used in the samples)
IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = DB_NAME() and is_master_key_encrypted_by_server = 1)
CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'sql19bigdatacluster!';
-- Create default data sources for SQL Big Data Cluster
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool')
CREATE EXTERNAL DATA SOURCE SqlDataPool
WITH (LOCATION = 'sqldatapool://service-mssql-controller:8080/datapools/default');
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')
CREATE EXTERNAL DATA SOURCE SqlStoragePool
WITH (LOCATION = 'sqlhdfs://service-mssql-controller:8080');
GO
-- Create view used for ML services training and scoring stored procedures
CREATE OR ALTER VIEW [dbo].[web_clickstreams_book_clicks]
samples/features/sql-big-data-cluster/data-virtualization/README.md
+2 -1
View File
@@ -6,7 +6,8 @@ In **SQL Server 2019 big data clusters**, the SQL Server engine has gained the a
**Applies to: SQL Server 2019 big data cluster**
In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. The [storage-pool](storage-pool) folder contains SQL scripts that demonstrate how to query data residing in HDFS data inside a big data cluster.
In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. The [storage-pool](storage-pool) folder contains SQL scripts that demonstrate how to query data residing in HDFS data inside a big data cluster. The [hadoop](hadoop) folder contains SQL scripts that demonstrate how to query data residing in HDFS data using the HADOOP data source for
operations that are not yet supported with storage pool (ex: export data to HDFS).
## Query data in Oracle from SQL Server master
samples/features/sql-big-data-cluster/data-virtualization/hadoop/README.md
+23
View File
@@ -0,0 +1,23 @@
# Data virtualization in SQL Server 2019 big data cluster
In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes. Using the PolyBase v1 HADOOP data source, you can manipulate ORC or RCFILE files inside the big data cluster.
## Query data in HDFS from SQL Server master using HADOOP data source
**Applies to:** SQL Server 2019 big data cluster
In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. In this example, you are going to create an external table in the SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster using the HADOOP data source. You will then join the data in the external table with high value data in SQL Master instance. Or export data to HDFS from SQL Master instance.
### Instructions
1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type.
1. Run the [../../spark/spark-sql.ipynb](../../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s).
1. Connect to SQL Server Master instance.
1. Execute the [web-clickstreams-hdfs-orc.sql](web-clickstreams-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS.
1. Execute the [product-reviews-hdfs-orc.sql](product-reviews-hdfs-orc.sql). This script demonstrates how to read ORC file(s) stored in HDFS.
1. Execute the [inventory-hdfs-rcfile.sql](inventory-hdfs-rcfile.sql). This script demonstrates how to export data from SQL Server into HDFS using PolyBase v1 syntax. This script will export data from SQL Server into RCFILE format.
samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql
+51
View File
@@ -0,0 +1,51 @@
USE sales
GO
exec sp_configure 'allow polybase export', 1;
RECONFIGURE WITH OVERRIDE;
GO
-- Create file format for RCFILE with appropriate properties.
--
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'RCFILE')
CREATE EXTERNAL FILE FORMAT rcfile
WITH (
FORMAT_TYPE = RCFILE,
SERDE_METHOD = 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe',
DATA_COMPRESSION = 'org.apache.hadoop.io.compress.DefaultCodec'
);
-- Create external table over HDFS data source using HADOOP type in
-- SQL Server 2019 big data cluster. The HADOOP data source is existing
-- PolyBase v1 syntax available by specifying location to HDFS namenode in
-- SQL Server big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'inventory_hdfs_rcfile')
CREATE EXTERNAL TABLE [inventory_hdfs_rcfile]
("inv_date_sk" BIGINT, "inv_item_sk" BIGINT, "inv_warehouse_sk" BIGINT, "inv_quantity_on_hand" BIGINT)
WITH
(
DATA_SOURCE = HadoopData,
LOCATION = '/inventory_rcfile',
FILE_FORMAT = rcfile
);
GO
-- Export SQL Server table to HDFS
--
INSERT INTO inventory_hdfs_rcfile
SELECT "inv_date_sk", "inv_item_sk", "inv_warehouse_sk", "inv_quantity_on_hand"
FROM inventory;
GO
-- Query the exported data using external table
--
SELECT COUNT(*) FROm inventory_hdfs_rcfile;
GO
-- Cleanup external tables
--
/*
DROP EXTERNAL TABLE inventory_hdfs_rcfile
*/
samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql
+43
View File
@@ -0,0 +1,43 @@
USE sales
GO
-- Create file format for orc file with appropriate properties.
--
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file')
CREATE EXTERNAL FILE FORMAT orc_file
WITH (
FORMAT_TYPE = ORC,
DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec'
);
-- Create external table over HDFS data source using HADOOP type in
-- SQL Server 2019 big data cluster. The HADOOP data source is existing
-- PolyBase v1 syntax available by specifying location to HDFS namenode in
-- SQL Server big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'product_reviews_hdfs_orc')
CREATE EXTERNAL TABLE [product_reviews_hdfs_orc]
("pr_review_sk" BIGINT , "pr_review_content" varchar(8000))
WITH
(
DATA_SOURCE = HadoopData,
LOCATION = '/user/hive/warehouse/product_reviews_orc',
FILE_FORMAT = orc_file
);
GO
-- Join external table with local tables
--
SELECT
p.pr_review_sk, pc.pr_review_content
FROM product_reviews as p
JOIN (SELECT TOP(10) * FROM product_reviews_hdfs_orc) AS pc
ON pc.pr_review_sk = p.pr_review_sk;
GO
-- Cleanup
/*
DROP EXTERNAL TABLE [dbo].[product_reviews_hdfs_orc];
GO
*/
samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql
+54
View File
@@ -0,0 +1,54 @@
USE sales
GO
-- Create file format for orc file with appropriate properties.
--
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'orc_file')
CREATE EXTERNAL FILE FORMAT orc_file
WITH (
FORMAT_TYPE = ORC,
DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec'
);
-- Create external table over HDFS data source using HADOOP type in
-- SQL Server 2019 big data cluster. The HADOOP data source is existing
-- PolyBase v1 syntax available by specifying location to HDFS namenode in
-- SQL Server big data cluster.
--
IF NOT EXISTS(SELECT * FROM sys.external_tables WHERE name = 'web_clickstreams_hdfs_orc')
CREATE EXTERNAL TABLE [web_clickstreams_hdfs_orc]
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
WITH
(
DATA_SOURCE = HadoopData,
LOCATION = '/user/hive/warehouse/web_clickstreams_orc',
FILE_FORMAT = orc_file
);
GO
-- Join external table with local tables
--
SELECT
wcs_user_sk,
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
FROM [dbo].[web_clickstreams_hdfs_orc]
INNER JOIN item it ON (wcs_item_sk = i_item_sk
AND wcs_user_sk IS NOT NULL)
GROUP BY wcs_user_sk;
GO
-- Cleanup
/*
DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_orc];
GO
*/
samples/features/sql-big-data-cluster/spark/spark-sql.ipynb
+17 -29
View File
@@ -25,9 +25,7 @@
{
"cell_type": "code",
"source": "# Read the clickstream CSV file(s) into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()",
"metadata": {
"language": "python"
},
"metadata": {},
"outputs": [
{
"name": "stdout",
@@ -35,14 +33,12 @@
"output_type": "stream"
}
],
"execution_count": 1
"execution_count": 3
},
{
"cell_type": "code",
"source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory where the parquet files will be stored\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\n",
"metadata": {
"language": "python"
},
"source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory where the parquet files will be stored\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet & orc file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\nresults.write.format(\"orc\").mode(\"overwrite\").saveAsTable(\"web_clickstreams_orc\")",
"metadata": {},
"outputs": [
{
"name": "stdout",
@@ -50,29 +46,25 @@
"output_type": "stream"
}
],
"execution_count": 1
"execution_count": 4
},
{
"cell_type": "code",
"source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT * FROM web_clickstreams LIMIT 100\")\r\nsqlDF.show()\r\n\r\nsqlDF = spark.sql(\"SELECT wcs_user_sk, COUNT(*)\\\r\n FROM web_clickstreams\\\r\n WHERE wcs_user_sk IS NOT NULL\\\r\n GROUP BY wcs_user_sk\\\r\n ORDER BY COUNT(*) DESC LIMIT 100\")\r\nsqlDF.show()",
"metadata": {
"language": "python"
},
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 36890| 40052| null| 4379| 34| null|\n| 36890| 41285| null| 6245| 34| null|\n| 36890| 23115| null| 13852| 34| null|\n| 36890| 17702| null| 15975| 34| null|\n| 36890| 62676| null| 2119| 34| null|\n| 36890| 34267| null| 10273| 34| null|\n| 36890| 8502| null| 17790| 34| null|\n| 36890| 54340| null| 3453| 34| null|\n| 36890| 54370| null| 6372| 34| null|\n| 36890| 6578| null| 17203| 34| null|\n| 36890| 75088| null| 4891| 34| null|\n| 36890| 23922| null| 11332| 34| null|\n| 36890| 28761| null| 4484| 34| null|\n| 36890| 21444| null| 5582| 34| null|\n| 36890| 58917| null| 8833| 34| null|\n| 36890| 27578| null| 8599| 34| null|\n| 36890| 8059| null| 6720| 34| null|\n| 36890| 43008| null| 17175| 34| null|\n| 36890| 4378| null| 10644| 34| null|\n| 36890| 55403| null| 8139| 34| null|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows",
"text": "+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 37506| 7933| null| 1384| 2| 39437|\n| 37506| 56044| null| 14689| 2| 26419|\n| 37506| 52706| null| 8541| 2| 44016|\n| 37506| 67325| null| 16129| 2| 83371|\n| 37506| 84857| null| 1869| 2| 13090|\n| 37506| 49599| null| 2994| 2| 8940|\n| 37506| 78150| null| 11392| 2| 65633|\n| 37506| 38720| null| 14366| 2| 22281|\n| 37506| 79915| null| 11102| 2| 81755|\n| 37506| 67253| null| 5380| 2| 46868|\n| 37506| 6507| null| 6813| 2| 49363|\n| 37506| 18280| null| 1458| 2| 49363|\n| 37506| 72258| null| 2869| 2| 67756|\n| 37506| 8045| null| 615| 2| 86035|\n| 37506| 86164| null| 7000| 2| 94821|\n| 37506| 29724| null| 2767| 2| 94821|\n| 37506| 55471| null| 3584| 2| 62792|\n| 37506| 677| null| 1720| 2| 27212|\n| 37506| 66638| null| 9898| 2| 20370|\n| 37506| 48515| null| 9394| 2| 17157|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows\n\n+-----------+--------+\n|wcs_user_sk|count(1)|\n+-----------+--------+\n| 65042| 832|\n| 55928| 821|\n| 15570| 791|\n| 31138| 788|\n| 68188| 784|\n| 88205| 760|\n| 15678| 757|\n| 48063| 741|\n| 77518| 741|\n| 92978| 728|\n| 82129| 727|\n| 21700| 725|\n| 69707| 724|\n| 38895| 719|\n| 97643| 716|\n| 74426| 707|\n| 7813| 704|\n| 49528| 700|\n| 55766| 698|\n| 54355| 697|\n+-----------+--------+\nonly showing top 20 rows",
"output_type": "stream"
}
],
"execution_count": 1
"execution_count": 5
},
{
"cell_type": "code",
"source": "# Read the product reviews CSV files into a spark data frame, print schema & top rows\r\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/product_review_data').toDF(\r\n \"pr_review_sk\", \"pr_review_content\"\r\n )\r\nresults.printSchema()\r\nresults.show()",
"metadata": {
"language": "python"
},
"metadata": {},
"outputs": [
{
"name": "stdout",
@@ -80,31 +72,27 @@
"output_type": "stream"
}
],
"execution_count": 1
"execution_count": 6
},
{
"cell_type": "code",
"source": "# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\n",
"metadata": {
"language": "python"
},
"source": "# Save results as parquet, and orc formats and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\nresults.write.format(\"orc\").mode(\"overwrite\").saveAsTable(\"product_reviews_orc\")\r\n",
"metadata": {},
"outputs": [],
"execution_count": 1
"execution_count": 7
},
{
"cell_type": "code",
"source": "# Execute Spark SQL commands\r\nsqlDF = spark.sql(\"SELECT pr_review_sk, CHAR_LENGTH(pr_review_content) as len FROM product_reviews LIMIT 100\")\r\nsqlDF.show()",
"metadata": {
"language": "python"
},
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": "+------------+----+\n|pr_review_sk| len|\n+------------+----+\n| 14868| 985|\n| 14869|1601|\n| 14875|1221|\n| 14880| 665|\n| 14886| 91|\n| 14894| 697|\n| 14899| 356|\n| 14903|2361|\n| 14908| 872|\n| 14909| 74|\n| 14917| 908|\n| 14918| 50|\n| 14919| 256|\n| 14921| 723|\n| 14925| 313|\n| 14931|1304|\n| 14939|1023|\n| 14949| 552|\n| 14954|2144|\n| 14955| 123|\n+------------+----+\nonly showing top 20 rows",
"text": "+------------+----+\n|pr_review_sk| len|\n+------------+----+\n| 26035| 876|\n| 26037| 109|\n| 26038| 478|\n| 26041| 106|\n| 26043| 332|\n| 26044| 487|\n| 26045| 428|\n| 26048| 87|\n| 26049| 118|\n| 26051|2906|\n| 26053| 464|\n| 26054| 212|\n| 26059| 191|\n| 26060| 207|\n| 26061| 515|\n| 26063| 59|\n| 26069| 487|\n| 26070| 160|\n| 26071| 380|\n| 26072| 234|\n+------------+----+\nonly showing top 20 rows",
"output_type": "stream"
}
],
"execution_count": 1
"execution_count": 8
}
]
}
}}