mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
54 lines
2.1 KiB
Transact-SQL
54 lines
2.1 KiB
Transact-SQL
USE sales
|
|
GO
|
|
|
|
-- Create external table in a data pool in SQL Server 2019 big data cluster.
|
|
-- The SqlDataPool data source is a special data source that is available in
|
|
-- any new database in SQL Master instance. This is used to reference the
|
|
-- data pool in a SQL Server 2019 big data cluster.
|
|
--
|
|
CREATE EXTERNAL TABLE [web_clickstreams_spark_results]
|
|
("wcs_click_date_sk" BIGINT , "wcs_click_time_sk" BIGINT , "wcs_sales_sk" BIGINT , "wcs_item_sk" BIGINT , "wcs_web_page_sk" BIGINT , "wcs_user_sk" BIGINT)
|
|
WITH
|
|
(
|
|
DATA_SOURCE = SqlDataPool,
|
|
DISTRIBUTION = ROUND_ROBIN
|
|
);
|
|
|
|
-- Data can be ingested into the external table from a spark job.
|
|
--
|
|
-- Submit spark job with below parameters. You can use the Spark submit experience from Azure Data Studio.
|
|
-- Right click on server name in a SQL Server big data cluster connection and click "Submit Spark Job".
|
|
--
|
|
-- Specify following values in the Job submission dialog box:
|
|
---- job name: <yourJobName>
|
|
---- switch from "Local" to "HDFS"
|
|
---- Main class: "FileStreaming"
|
|
---- Path to jar: /jar/mssql-spark-lib-assembly-1.0.jar
|
|
---- Arguments:
|
|
---- mssql-master-pool-0.service-master-pool 1433 sa %PASSWORD% sales web_clickstreams_spark_results hdfs:///clickstream_data csv false
|
|
|
|
-- The arguments to jar file are
|
|
-- 1: server name - sql server to connect to read the table schema
|
|
-- 2: port number
|
|
-- 3: username - sql server username for master instance
|
|
-- 4: password - sql server password for master instance
|
|
-- 5: database name
|
|
-- 6: external table name
|
|
-- 7: Source directory for streaming. This must be a full URI - such as "hdfs:///clickstream_data"
|
|
-- 8: Input format. This can be "csv", "parquet", "json".
|
|
-- 9: enable checkpoint: true or false
|
|
--
|
|
|
|
-- After the Spark streaming job has been sucessfully submitted, you can run below query to view the results.
|
|
--
|
|
-- Wait until some rows are available.
|
|
WHILE (1=1)
|
|
IF EXISTS(SELECT * FROM [web_clickstreams_spark_results])
|
|
BREAK;
|
|
|
|
SELECT count(*) FROM [web_clickstreams_spark_results];
|
|
SELECT TOP 10 * FROM [web_clickstreams_spark_results];
|
|
GO
|
|
|
|
DROP EXTERNAL TABLE [dbo].[web_clickstreams_spark_results];
|
|
GO |