mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Split SQL19 & SQL19 big data cluster data-virtualization samples
This commit is contained in:
Umachandar Jayachandran
@@ -1,33 +1,15 @@
|
||||
# Data virtualization in SQL Server 2019 big data cluster
|
||||
# Data virtualization in SQL Server 2019 and SQL Server 2019 big data cluster
|
||||
|
||||
In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes. SQL Server 2019 introduces new ODBC connectors to data sources like SQL Server, Oracle, MongoDB and Teradata.
|
||||
In **SQL Server 2019 big data clusters**, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes. **SQL Server 2019** also introduces **new ODBC connectors** to data sources like SQL Server, Oracle, MongoDB and Teradata.
|
||||
|
||||
## Query data in HDFS from SQL Server master
|
||||
|
||||
In this example, you are going to create an external table in the SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster. Then you will join the data in the external table with high value data in SQL Master instance.
|
||||
**Applies to: SQL Server 2019 big data cluster**
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type.
|
||||
|
||||
1. Run the [../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s).
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
|
||||
1. Execute the [web-clickstreams-parquet.sql](web-clickstreams-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS.
|
||||
|
||||
1. Execute the [product-reviews-hdfs-csv.sql](product-reviews-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. The [storage-pool](storage-pool) folder contains SQL scripts that demonstrate how to query data residing in HDFS data inside a big data cluster.
|
||||
|
||||
## Query data in Oracle from SQL Server master
|
||||
|
||||
In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server.
|
||||
**Applies to: SQL Server 2019 on Windows or Linux, SQL Server 2019 big data cluster**
|
||||
|
||||
**Before you begin**, you need to have an Oracle instance and credentials. Follow the instruction in the [oracle-setup\README.md](oracle-setup\README.md).
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [inventory-oracle.sql](inventory-oracle.sql/).
|
||||
The [oracle](oracle) folder contains SQL scripts that demonstrate how to query data residing in Oracle instance.
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# Data virtualization in SQL Server 2019
|
||||
|
||||
**Applies to: SQL Server 2019 on Windows or Linux, SQL Server 2019 big data cluster**
|
||||
|
||||
SQL Server 2019 introduces new ODBC connectors to data sources like SQL Server, Oracle, MongoDB and Teradata. These connectors can be used from stand-alone SQL Server 2019 on Windows or Linux and SQL Server 2019 big data cluster.
|
||||
|
||||
## Query data in Oracle from SQL Server master
|
||||
|
||||
In this example, you are going to create an external table in SQL Server Master instance over the inventory table that sits on an Oracle server.
|
||||
|
||||
**Before you begin**, you need to have an Oracle instance and credentials. Follow the instruction in the [setup\README.md](setup\README.md).
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the SQL [inventory-oracle.sql](inventory-oracle.sql/).
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
@echo off
|
||||
REM bootstrap sample oracle tables CMD script
|
||||
setlocal enableextensions
|
||||
set ORACLE_SERVER=%1
|
||||
set ORACLE_USER=%2
|
||||
set ORACLE_PASSWORD=%3
|
||||
|
||||
if NOT DEFINED ORACLE_SERVER goto :usage
|
||||
if NOT DEFINED ORACLE_USER goto :usage
|
||||
if NOT DEFINED ORACLE_PASSWORD goto :usage
|
||||
|
||||
echo Verifying sqlplus.exe is in path & CALL WHERE /Q sqlplus.exe || GOTO exit
|
||||
echo Verifying sqlldr.exe is in path & CALL WHERE /Q sqlldr.exe || GOTO exit
|
||||
|
||||
echo Creating user & tables...
|
||||
echo exit | sqlplus -S %ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% @sales-user.sql || GOTO exit
|
||||
echo exit | sqlplus -S %ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% @inventory.sql || GOTO exit
|
||||
echo exit | sqlplus -S %ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% @customer.sql || GOTO exit
|
||||
|
||||
echo Loading tables data...
|
||||
sqlldr CONTROL=inventory.ctl userid=%ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% || GOTO exit
|
||||
sqlldr CONTROL=customer.ctl userid=%ORACLE_USER%/%ORACLE_PASSWORD%@%ORACLE_SERVER% || GOTO exit
|
||||
|
||||
:: del /q *.out *.err *.csv
|
||||
endlocal
|
||||
exit /b 0
|
||||
goto :eof
|
||||
|
||||
:exit
|
||||
echo Bootstrap of the sample tables failed.
|
||||
exit /b 1
|
||||
|
||||
:usage
|
||||
echo USAGE: %0 ^<ORACLE_SERVER^> ^<ORACLE_USER^> ^<ORACLE_PASSWORD^>
|
||||
exit /b 0
|
||||
@@ -0,0 +1,27 @@
|
||||
options (readsize=2048000,bindsize=1600000, rows=100000, silent=(header, feedback) )
|
||||
load data
|
||||
infile '..\..\..\customer.csv' "str '\r\n'"
|
||||
append
|
||||
into table SALES.CUSTOMER
|
||||
fields terminated by '|'
|
||||
OPTIONALLY ENCLOSED BY '"' AND '"'
|
||||
trailing nullcols
|
||||
( C_CUSTOMER_SK,
|
||||
C_CUSTOMER_ID CHAR(16),
|
||||
C_CURRENT_CDEMO_SK,
|
||||
C_CURRENT_HDEMO_SK,
|
||||
C_CURRENT_ADDR_SK,
|
||||
C_FIRST_SHIPTO_DATE_SK,
|
||||
C_FIRST_SALES_DATE_SK,
|
||||
C_SALUTATION CHAR(10),
|
||||
C_FIRST_NAME CHAR(20),
|
||||
C_LAST_NAME CHAR(30),
|
||||
C_PREFERRED_CUST_FLAG CHAR(1),
|
||||
C_BIRTH_DAY,
|
||||
C_BIRTH_MONTH,
|
||||
C_BIRTH_YEAR,
|
||||
C_BIRTH_COUNTRY CHAR(20),
|
||||
C_LOGIN CHAR(13),
|
||||
C_EMAIL_ADDRESS CHAR(50),
|
||||
C_LAST_REVIEW_DATE CHAR(10)
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Customer table over which the SQL Server external table will be defined
|
||||
CREATE TABLE "SALES"."CUSTOMER"
|
||||
(
|
||||
"C_CUSTOMER_SK" NUMBER(10,0),
|
||||
"C_CUSTOMER_ID" VARCHAR2(16 BYTE),
|
||||
"C_CURRENT_CDEMO_SK" NUMBER(10,0),
|
||||
"C_CURRENT_HDEMO_SK" NUMBER(10,0),
|
||||
"C_CURRENT_ADDR_SK" NUMBER(10,0),
|
||||
"C_FIRST_SHIPTO_DATE_SK" NUMBER(10,0),
|
||||
"C_FIRST_SALES_DATE_SK" NUMBER(8,0),
|
||||
"C_SALUTATION" VARCHAR2(10 BYTE),
|
||||
"C_FIRST_NAME" VARCHAR2(20 BYTE),
|
||||
"C_LAST_NAME" VARCHAR2(30 BYTE),
|
||||
"C_PREFERRED_CUST_FLAG" VARCHAR2(1 BYTE),
|
||||
"C_BIRTH_DAY" NUMBER(8,0),
|
||||
"C_BIRTH_MONTH" NUMBER(8,0),
|
||||
"C_BIRTH_YEAR" NUMBER(8,0),
|
||||
"C_BIRTH_COUNTRY" VARCHAR2(20 BYTE),
|
||||
"C_LOGIN" VARCHAR2(13 BYTE),
|
||||
"C_EMAIL_ADDRESS" VARCHAR2(50 BYTE),
|
||||
"C_LAST_REVIEW_DATE" VARCHAR2(20 BYTE)
|
||||
);
|
||||
|
||||
CREATE INDEX "SALES"."CUSTOMER_C_CUSTOMER_SK" ON "SALES"."CUSTOMER"("C_CUSTOMER_SK");
|
||||
@@ -0,0 +1,13 @@
|
||||
options (readsize=2048000,bindsize=1600000, rows=100000, silent=(header, feedback) )
|
||||
load data
|
||||
infile '..\..\..\inventory.csv' "str '\r\n'"
|
||||
append
|
||||
into table SALES.INVENTORY
|
||||
fields terminated by '|'
|
||||
OPTIONALLY ENCLOSED BY '"' AND '"'
|
||||
trailing nullcols
|
||||
( INV_DATE,
|
||||
INV_ITEM,
|
||||
INV_WAREHOUSE,
|
||||
INV_QUANTITY_ON_HAND
|
||||
)
|
||||
+1
-1
@@ -7,4 +7,4 @@ CREATE TABLE "SALES"."INVENTORY"
|
||||
"INV_QUANTITY_ON_HAND" NUMBER(10,0)
|
||||
);
|
||||
|
||||
CREATE INDEX INV_ITEM ON "SALES"."INVENTORY"("INV_ITEM");
|
||||
CREATE INDEX "SALES"."INVENTORY_INV_ITEM" ON "SALES"."INVENTORY"("INV_ITEM");
|
||||
@@ -0,0 +1,23 @@
|
||||
# Data virtualization in SQL Server 2019 big data cluster
|
||||
|
||||
In SQL Server 2019 big data clusters, the SQL Server engine has gained the ability to natively read HDFS files, such as CSV and parquet files, by using SQL Server instances collocated on each of the HDFS data nodes to filter and aggregate data locally in parallel across all of the HDFS data nodes. SQL Server 2019 introduces new ODBC connectors to data sources like SQL Server, Oracle, MongoDB and Teradata.
|
||||
|
||||
## Query data in HDFS from SQL Server master
|
||||
|
||||
**Applies to: SQL Server 2019 big data cluster**
|
||||
|
||||
In SQL Server 2019 big data cluster, the storage pool consists of HDFS data node with SQL Server & Spark endpoints. In this example, you are going to create an external table in the SQL Server Master instance that points to data in HDFS within the SQL Server Big data cluster. You will then join the data in the external table with high value data in SQL Master instance.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Connect to HDFS/Knox gateway from Azure Data Studio using SQL Server big data cluster connection type.
|
||||
|
||||
1. Run the [../../spark/spark-sql.ipynb](../spark/spark-sql.ipynb/) notebook to generate the sample parquet file(s).
|
||||
|
||||
1. Connect to SQL Server Master instance.
|
||||
|
||||
1. Execute the [web-clickstreams-hdfs-csv.sql](web-clickstreams-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
|
||||
1. Execute the [web-clickstreams-parquet.sql](web-clickstreams-hdfs-parquet.sql). This script demonstrates how to read parquet file(s) stored in HDFS.
|
||||
|
||||
1. Execute the [product-reviews-hdfs-csv.sql](product-reviews-hdfs-csv.sql). This script demonstrates how to read CSV file(s) stored in HDFS.
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create file format for parquet file with appropriate properties.
|
||||
--
|
||||
IF NOT EXISTS(SELECT * FROM sys.external_file_formats WHERE name = 'parquet_file')
|
||||
CREATE EXTERNAL FILE FORMAT parquet_file
|
||||
WITH (
|
||||
FORMAT_TYPE = PARQUET
|
||||
);
|
||||
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
-- is a special data source that is available in any new database in
|
||||
-- SQL Master instance.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [product_reviews_hdfs_parquet]
|
||||
("pr_review_sk" BIGINT , "pr_review_content" varchar(8000))
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/user/hive/warehouse/product_review_data',
|
||||
FILE_FORMAT = parquet_file
|
||||
);
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT
|
||||
p.pr_review_sk, pc.pr_review_content
|
||||
FROM product_reviews as p
|
||||
JOIN (SELECT TOP(10) * FROM product_reviews_hdfs_parquet) AS pc
|
||||
ON pc.pr_review_sk = p.pr_review_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[product_reviews_hdfs_parquet];
|
||||
GO
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
USE sales
|
||||
GO
|
||||
|
||||
-- Create file format for tab separated file with appropriate properties.
|
||||
--
|
||||
CREATE EXTERNAL FILE FORMAT tsv_file
|
||||
WITH (
|
||||
FORMAT_TYPE = DELIMITEDTEXT,
|
||||
FORMAT_OPTIONS(
|
||||
FIELD_TERMINATOR = '\t',
|
||||
STRING_DELIMITER = '"',
|
||||
USE_TYPE_DEFAULT = TRUE)
|
||||
);
|
||||
|
||||
-- Create external table over HDFS data source (SqlStoragePool) in
|
||||
-- SQL Server 2019 big data cluster. The SqlStoragePool data source
|
||||
-- is a special data source that is available in any new database in
|
||||
-- SQL Master instance.
|
||||
--
|
||||
CREATE EXTERNAL TABLE [product_reviews_hdfs_tsv]
|
||||
("pr_review_sk" BIGINT , "pr_review_content" varchar(8000))
|
||||
WITH
|
||||
(
|
||||
DATA_SOURCE = SqlStoragePool,
|
||||
LOCATION = '/product_review_data',
|
||||
FILE_FORMAT = tsv_file
|
||||
);
|
||||
GO
|
||||
|
||||
-- Join external table with local tables
|
||||
--
|
||||
SELECT
|
||||
wcs_user_sk,
|
||||
SUM( CASE WHEN i_category = 'Books' THEN 1 ELSE 0 END) AS book_category_clicks,
|
||||
SUM( CASE WHEN i_category_id = 1 THEN 1 ELSE 0 END) AS [Home & Kitchen],
|
||||
SUM( CASE WHEN i_category_id = 2 THEN 1 ELSE 0 END) AS [Music],
|
||||
SUM( CASE WHEN i_category_id = 3 THEN 1 ELSE 0 END) AS [Books],
|
||||
SUM( CASE WHEN i_category_id = 4 THEN 1 ELSE 0 END) AS [Clothing & Accessories],
|
||||
SUM( CASE WHEN i_category_id = 5 THEN 1 ELSE 0 END) AS [Electronics],
|
||||
SUM( CASE WHEN i_category_id = 6 THEN 1 ELSE 0 END) AS [Tools & Home Improvement],
|
||||
SUM( CASE WHEN i_category_id = 7 THEN 1 ELSE 0 END) AS [Toys & Games],
|
||||
SUM( CASE WHEN i_category_id = 8 THEN 1 ELSE 0 END) AS [Movies & TV],
|
||||
SUM( CASE WHEN i_category_id = 9 THEN 1 ELSE 0 END) AS [Sports & Outdoors]
|
||||
FROM [dbo].[web_clickstreams_hdfs_csv]
|
||||
INNER JOIN item it ON (wcs_item_sk = i_item_sk
|
||||
AND wcs_user_sk IS NOT NULL)
|
||||
GROUP BY wcs_user_sk;
|
||||
GO
|
||||
|
||||
DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_csv];
|
||||
GO
|
||||
Reference in New Issue
Block a user