mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Merge pull request #810 from shanyu/fix-bdc-connector
Fix mssql-spark-connector AD cluster sample notebook
This commit is contained in:
shivsood
+88
-125
@@ -29,7 +29,7 @@
|
||||
"- SQL Server 2019 big data cluster is deployed with AD\r\n",
|
||||
"- Have access to AD controller to create keytabs that we will use in this sample. \r\n",
|
||||
"- Download [AdultCensusIncome.csv]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ) to your local machine. Upload this file to hdfs folder named *spark_data*. \r\n",
|
||||
"- The sample uses a SQL database *spark_sql_db* to create/update tables. The database needs to be created before the sample is run.\r\n",
|
||||
"- The sample uses a SQL database *spark_mssql_db* to create/update tables. Refer **data-virtualization/mssql_spark_connector_user_creation.ipynb** on steps to create this database anduser.\r\n",
|
||||
" "
|
||||
],
|
||||
"metadata": {
|
||||
@@ -80,7 +80,7 @@
|
||||
"kinit -kt testusera1.keytab testusera1@AZDATA.LOCAL\r\n",
|
||||
"```\r\n",
|
||||
"\r\n",
|
||||
"Load Keytab to HDFS for use\r\n",
|
||||
"Load Keytab to HDFS for use, you can do it from ADS, or use hadoop commandline inside a BDC container:\r\n",
|
||||
"\r\n",
|
||||
"```sh\r\n",
|
||||
"hadoop fs -mkdir -p /user/testusera1/\r\n",
|
||||
@@ -95,61 +95,6 @@
|
||||
"azdata_cell_guid": "5e733ddd-cbf8-4aa1-9f97-9fff40c28332"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Create a database user\r\n",
|
||||
"\r\n",
|
||||
"``` sql\r\n",
|
||||
"IF NOT EXISTS (select name from sys.server_principals where name='AZDATA.LOCAL\\testusera1')\r\n",
|
||||
"BEGIN\r\n",
|
||||
" CREATE LOGIN [AZDATA.LOCAL\\testusera1] FROM WINDOWS\r\n",
|
||||
"END\r\n",
|
||||
"\r\n",
|
||||
"ALTER SERVER ROLE dbcreator ADD MEMBER [AZDATA.LOCAL\\testusera1]\r\n",
|
||||
"GRANT VIEW SERVER STATE to [AZDATA.LOCAL\\testusera1]\r\n",
|
||||
"\r\n",
|
||||
"# Create a database named \"spark_mssql_db\"\r\n",
|
||||
"IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = N'spark_mssql_db')\r\n",
|
||||
" CREATE DATABASE spark_mssql_db\r\n",
|
||||
"```"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "453e7b2f-e590-4b95-9fbd-5dc8b9d1f02c"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Create Data Pool user\r\n",
|
||||
"\r\n",
|
||||
"```\r\n",
|
||||
"-- To create external tables in data pools\r\n",
|
||||
"grant alter any external data source to [aris\\testuser];\r\n",
|
||||
"\r\n",
|
||||
"-- To create external table\r\n",
|
||||
"grant create table to [aris\\testuser];\r\n",
|
||||
"grant alter any schema to [aris\\testuser];\r\n",
|
||||
"\r\n",
|
||||
"ALTER ROLE [db_datareader] ADD MEMBER [aris\\testuser]\r\n",
|
||||
"ALTER ROLE [db_datawriter] ADD MEMBER [aris\\testuser]\r\n",
|
||||
"```\r\n",
|
||||
"\r\n",
|
||||
"```\r\n",
|
||||
"CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n",
|
||||
"EXECUTE('USE spark_mssql_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n",
|
||||
"\r\n",
|
||||
"-- Create a login in data pools and Provide right permissions to this user\r\n",
|
||||
"EXECUTE( ' Use spark_mssql_db; CREATE LOGIN [aris\\testusera1] FROM WINDOWS ' ) AT DATA_SOURCE connector_ds;\r\n",
|
||||
"\r\n",
|
||||
"EXECUTE( ' Use spark_mssql_db; CREATE USER [aris\\testusera1] ; ALTER ROLE [db_datareader] ADD MEMBER [aris\\testusera1]; ALTER ROLE [db_datawriter] ADD MEMBER [aris\\testusera1] ;') AT DATA_SOURCE connector_ds;\r\n",
|
||||
"\r\n",
|
||||
"```"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "15b7588a-d1ad-4e17-84c6-bf4a862d1905"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
@@ -184,53 +129,23 @@
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "Starting Spark application\n"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "<IPython.core.display.HTML object>",
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>12</td><td>application_1581458669418_0041</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"https://10.193.21.202:30443/gateway/default/yarn/proxy/application_1581458669418_0041/\">Link</a></td><td><a target=\"_blank\" href=\"https://10.193.21.202:30443/gateway/default/yarn/container/container_1581458669418_0041_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
"text/html": "Current session configs: <tt>{'conf': {'spark.files': '/user/testusera1/testusera1.keytab', 'spark.executor.memory': '4g', 'spark.driver.memory': '4g', 'spark.executor.cores': 2, 'spark.driver.cores': 1, 'spark.executor.instances': 4}, 'kind': 'pyspark'}</tt><br>"
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "ae4ec82192fb473e8b77e4065f4c1927"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "SparkSession available as 'spark'.\n"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "<IPython.core.display.HTML object>",
|
||||
"text/html": "Current session configs: <tt>{'conf': {'spark.files': '/user/user1/user1.keytab', 'spark.executor.memory': '4g', 'spark.driver.memory': '4g', 'spark.executor.cores': 2, 'spark.driver.cores': 1, 'spark.executor.instances': 4}, 'kind': 'pyspark'}</tt><br>"
|
||||
"text/html": "No active sessions."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "<IPython.core.display.HTML object>",
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>6</td><td>application_1581458669418_0035</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"https://10.193.21.202:30443/gateway/default/yarn/proxy/application_1581458669418_0041/\">Link</a></td><td><a target=\"_blank\" href=\"https://10.193.21.202:30443/gateway/default/yarn/container/container_1581458669418_0041_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"execution_count": 8
|
||||
"execution_count": 2
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -268,54 +183,54 @@
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "Starting Spark application\n"
|
||||
"text": "Starting Spark application\n",
|
||||
"output_type": "stream"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "<IPython.core.display.HTML object>",
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>7</td><td>application_1581458669418_0036</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"https://10.193.21.202:30443/gateway/default/yarn/proxy/application_1581458669418_0036/\">Link</a></td><td><a target=\"_blank\" href=\"https://10.193.21.202:30443/gateway/default/yarn/container/container_1581458669418_0036_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>4</td><td>application_1597953190209_0006</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"https://knox.azdata.local:30443/gateway/default/yarn/proxy/application_1597953190209_0006/\">Link</a></td><td><a target=\"_blank\" href=\"https://knox.azdata.local:30443/gateway/default/yarn/container/container_1597953190209_0006_01_000001/admin\">Link</a></td><td>✔</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "52afbfffaf3545e28a2d09864d1c4ced"
|
||||
"model_id": "0f33eba1feef441fa9b77e82bdf2c5ee"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "SparkSession available as 'spark'.\n"
|
||||
"text": "SparkSession available as 'spark'.\n",
|
||||
"output_type": "stream"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "7e03eb6a81cd41e4878f12df5ee31dc7"
|
||||
"model_id": "e97636b10a734032a1cdae8c2a16e08a"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education-num| marital-status| occupation| relationship| race| sex|capital-gain|capital-loss|hours-per-week|native-country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows"
|
||||
"text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education-num| marital-status| occupation| relationship| race| sex|capital-gain|capital-loss|hours-per-week|native-country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 5
|
||||
"execution_count": 3
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -336,7 +251,7 @@
|
||||
"#Write from Spark to SQL table using MSSQL Spark Connector\r\n",
|
||||
"print(\"MSSQL Spark Connector write(overwrite) start \")\r\n",
|
||||
"\r\n",
|
||||
"servername = \"jdbc:sqlserver://master-p-svc:1433\"\r\n",
|
||||
"servername = \"jdbc:sqlserver://mssql.azdata.local:31433\"\r\n",
|
||||
"dbname = \"spark_mssql_db\"\r\n",
|
||||
"security_spec = \";integratedSecurity=true;authenticationScheme=JavaKerberos;\"\r\n",
|
||||
"url = servername + \";\" + \"databaseName=\" + dbname + security_spec\r\n",
|
||||
@@ -364,24 +279,24 @@
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "644bfe3ba48142f38f52717cab9eb19d"
|
||||
"model_id": "82fe4fdfbb0c4110871ab3f0e4011600"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "Use MSSQL connector to write to master SQL instance \nMSSQL Connector write(overwrite) succeeded"
|
||||
"text": "MSSQL Spark Connector write(overwrite) start \nMSSQL Connector write(overwrite) done",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 7
|
||||
"execution_count": 11
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -406,12 +321,24 @@
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "3148b605ab214b408412efb9bfe6b14c"
|
||||
}
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "read data from SQL server table \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows"
|
||||
"text": "MSSQL Spark Connector read start \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\nMSSQL Spark Connector read done",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 7
|
||||
"execution_count": 13
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -436,7 +363,7 @@
|
||||
"#Write from Spark to datapools using Spark Connector\r\n",
|
||||
"print(\"MSSQL Spark Connector write(overwrite) start \")\r\n",
|
||||
"\r\n",
|
||||
"servername = \"jdbc:sqlserver://master-p-svc:1433\"\r\n",
|
||||
"servername = \"jdbc:sqlserver://mssql.azdata.local:31433\"\r\n",
|
||||
"dbname = \"spark_mssql_db\"\r\n",
|
||||
"security_spec = \";integratedSecurity=true;authenticationScheme=JavaKerberos;\"\r\n",
|
||||
"url = servername + \";\" + \"databaseName=\" + dbname + security_spec\r\n",
|
||||
@@ -465,8 +392,26 @@
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "9cbe5af3-6ddb-4f19-8423-d10cbb7d48a7"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "17795c5c901e469da197644ce432836b"
|
||||
}
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "MSSQL Spark Connector write(overwrite) start \nMSSQL Connector write(overwrite) done",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 12
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -489,8 +434,26 @@
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "5550bfce-4852-4fb8-9caa-1120c05dafb7"
|
||||
},
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…",
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"version_major": 2,
|
||||
"version_minor": 0,
|
||||
"model_id": "19c6e7b6076b42718229d4ae61ccd61c"
|
||||
}
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"text": "MSSQL Spark Connector read data pool external table start \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\nMSSQL Connector read from data pool external table succeeded",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 14
|
||||
}
|
||||
]
|
||||
}
|
||||
+116
-54
@@ -27,11 +27,79 @@
|
||||
"azdata_cell_guid": "4d6539fc-aa99-4cd8-a9de-357e2d6424e2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## For non-AD cluster, run the following cells to prepare user and table:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "9642f940-770f-4bfd-85ee-e0159ccb5689"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"-- Start with creating a testdatabase. \r\n",
|
||||
"Create DATABASE connector_test_db"
|
||||
"-- Create a database named \"connector_test_db\"\r\n",
|
||||
"IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = N'connector_test_db')\r\n",
|
||||
"BEGIN\r\n",
|
||||
" Create DATABASE connector_test_db\r\n",
|
||||
"END\r\n",
|
||||
"\r\n",
|
||||
"-- Create a login\r\n",
|
||||
"IF NOT EXISTS (select name from sys.server_principals where name='connector_user')\r\n",
|
||||
"BEGIN\r\n",
|
||||
" CREATE LOGIN connector_user WITH PASSWORD ='password123!#' \r\n",
|
||||
"END"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "84932473-88cd-433e-baf7-eeee178021aa"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.335"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 6
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"Use connector_test_db;\r\n",
|
||||
"\r\n",
|
||||
"-- Create a database user\r\n",
|
||||
"CREATE USER connector_user FROM LOGIN connector_user\r\n",
|
||||
"\r\n",
|
||||
"-- To view data pool node configuration\r\n",
|
||||
"grant VIEW DATABASE STATE to connector_user;\r\n",
|
||||
"\r\n",
|
||||
"-- To create external tables in data pools\r\n",
|
||||
"grant alter any external data source to connector_user;\r\n",
|
||||
"\r\n",
|
||||
"-- To create external table\r\n",
|
||||
"grant create table to connector_user;\r\n",
|
||||
"grant alter any schema to connector_user;\r\n",
|
||||
"\r\n",
|
||||
"ALTER ROLE [db_datareader] ADD MEMBER connector_user;\r\n",
|
||||
"ALTER ROLE [db_datawriter] ADD MEMBER connector_user;\r\n",
|
||||
"\r\n",
|
||||
"-- Create external table in data pool\r\n",
|
||||
"CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n",
|
||||
"EXECUTE('USE connector_test_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n",
|
||||
"\r\n",
|
||||
"-- Create a login in data pools and Provide right permissions to this user\r\n",
|
||||
"EXECUTE( ' Use connector_test_db; CREATE LOGIN connector_user WITH PASSWORD = ''password123!#'' ; ' ) AT DATA_SOURCE connector_ds;\r\n",
|
||||
"EXECUTE( ' Use connector_test_db; CREATE USER connector_user ; ALTER ROLE [db_datareader] ADD MEMBER connector_user; ALTER ROLE [db_datawriter] ADD MEMBER connector_user ;') AT DATA_SOURCE connector_ds;"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "8fdf2f56-f89d-4bf9-81be-1dfb92631251"
|
||||
@@ -47,7 +115,7 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:01.345"
|
||||
"text/html": "Total execution time: 00:00:10.334"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
@@ -55,60 +123,31 @@
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Use connector_test_db;\n",
|
||||
"CREATE LOGIN connector_user WITH PASSWORD ='password123!#' \n",
|
||||
"CREATE USER connector_user FROM LOGIN connector_user\n",
|
||||
"\n",
|
||||
"-- to view data pool node configuration\n",
|
||||
"grant VIEW DATABASE STATE to connector_user\n",
|
||||
"\n",
|
||||
"-- To create external tables in data pools\n",
|
||||
"grant alter any external data source to connector_user;\n",
|
||||
"\n",
|
||||
"-- To create external table\n",
|
||||
"grant create table to connector_user;\n",
|
||||
"grant alter any schema to connector_user;\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"ALTER ROLE [db_datareader] ADD MEMBER connector_user\n",
|
||||
"ALTER ROLE [db_datawriter] ADD MEMBER connector_user"
|
||||
"## For AD cluster, run the following cells to prepare user and table:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "4dd976a3-1ed7-43d3-ad29-657fe1046fb3"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.087"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 2
|
||||
"azdata_cell_guid": "11ea89a5-5f60-495d-864e-a80ef7a10510"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"-- Add this user to a data pool user.\r\n",
|
||||
"Use connector_test_db;\r\n",
|
||||
"CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n",
|
||||
"EXECUTE('USE connector_test_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n",
|
||||
"-- Create a database named \"spark_mssql_db\"\r\n",
|
||||
"IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = N'spark_mssql_db')\r\n",
|
||||
"BEGIN\r\n",
|
||||
" CREATE DATABASE spark_mssql_db\r\n",
|
||||
"END\r\n",
|
||||
"\r\n",
|
||||
""
|
||||
"-- Create a login from a domain user\r\n",
|
||||
"IF NOT EXISTS (select name from sys.server_principals where name='AZDATA\\testusera1')\r\n",
|
||||
"BEGIN\r\n",
|
||||
" CREATE LOGIN [AZDATA\\testusera1] FROM WINDOWS\r\n",
|
||||
"END"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "0b06540d-0787-493d-aba6-83420200da14"
|
||||
"azdata_cell_guid": "819012ba-db8f-4c25-9606-1801f46d60e4"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -121,22 +160,45 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:04.496"
|
||||
"text/html": "Total execution time: 00:00:10.921"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
"execution_count": 18
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"Use spark_mssql_db;\r\n",
|
||||
"\r\n",
|
||||
"-- Create a database user\r\n",
|
||||
"CREATE USER [AZDATA\\testusera1];\r\n",
|
||||
"\r\n",
|
||||
"-- To view data pool node configuration\r\n",
|
||||
"GRANT VIEW DATABASE STATE to [AZDATA\\testusera1];\r\n",
|
||||
"\r\n",
|
||||
"-- To create external tables in data pools\r\n",
|
||||
"grant alter any external data source to [AZDATA\\testusera1];\r\n",
|
||||
"\r\n",
|
||||
"-- To create external table\r\n",
|
||||
"grant create table to [AZDATA\\testusera1];\r\n",
|
||||
"grant alter any schema to [AZDATA\\testusera1];\r\n",
|
||||
"\r\n",
|
||||
"ALTER ROLE [db_datareader] ADD MEMBER [AZDATA\\testusera1];\r\n",
|
||||
"ALTER ROLE [db_datawriter] ADD MEMBER [AZDATA\\testusera1];\r\n",
|
||||
"\r\n",
|
||||
"-- Create external table in data pool\r\n",
|
||||
"CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n",
|
||||
"EXECUTE('USE spark_mssql_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n",
|
||||
"\r\n",
|
||||
"-- Create a login in data pools and Provide right permissions to this user\r\n",
|
||||
"EXECUTE( ' Use connector_test_db; CREATE LOGIN connector_user WITH PASSWORD = ''password123!#'' ; ' ) AT DATA_SOURCE connector_ds;\r\n",
|
||||
"EXECUTE( ' Use connector_test_db; CREATE USER connector_user ; ALTER ROLE [db_datareader] ADD MEMBER connector_user; ALTER ROLE [db_datawriter] ADD MEMBER connector_user ;') AT DATA_SOURCE connector_ds;"
|
||||
"EXECUTE( ' USE spark_mssql_db; CREATE LOGIN [AZDATA\\testusera1] FROM WINDOWS ' ) AT DATA_SOURCE connector_ds;\r\n",
|
||||
"\r\n",
|
||||
"EXECUTE( ' USE spark_mssql_db; CREATE USER [AZDATA\\testusera1] ; ALTER ROLE [db_datareader] ADD MEMBER [AZDATA\\testusera1]; ALTER ROLE [db_datawriter] ADD MEMBER [AZDATA\\testusera1] ;') AT DATA_SOURCE connector_ds;"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "5079dfe9-22e3-4a8f-919b-c6219f5a901a"
|
||||
"azdata_cell_guid": "2c853db3-e081-4229-a66f-5c248fb8eb0d"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -149,12 +211,12 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:01.174"
|
||||
"text/html": "Total execution time: 00:00:10.628"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 5
|
||||
"execution_count": 19
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user