diff --git a/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb b/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb index 2af3eb51..def8a13a 100644 --- a/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb +++ b/samples/features/sql-big-data-cluster/spark/spark_to_sql/mssql_spark_connector.ipynb @@ -31,24 +31,7 @@ "cell_type": "code", "source": "%%configure -f\r\n{\"conf\": {\"spark.jars\": \"/jar/spark-mssql-connector-assembly-1.0.0.jar\"}}\r\n\r\n\r\n\r\n", "metadata": {}, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": "", - "text/html": "Current session configs: {'conf': {'spark.jars': '/jar/spark-mssql-connector-assembly-1.0.0.jar'}, 'kind': 'pyspark3'}
" - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/plain": "", - "text/html": "No active sessions." - }, - "metadata": {} - } - ], + "outputs": [], "execution_count": 4 }, { @@ -60,31 +43,14 @@ "cell_type": "code", "source": "#Read a file and then write it to the SQL table\r\ndatafile = \"/spark_data/AdultCensusIncome.csv\"\r\ndf = spark.read.format('csv').options(header='true', inferSchema='true', ignoreLeadingWhiteSpace='true', ignoreTrailingWhiteSpace='true').load(datafile)\r\ndf.show(5)\r\n", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Starting Spark application\n" - }, - { - "output_type": "stream", - "name": "stderr", - "text": "The code failed because of a fatal error:\n\tSession 98 unexpectedly reached final status 'error'. See logs:\nstdout: \n\nstderr: \n19/04/20 02:07:37 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-api-0.5.33476.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-rsc-0.5.33476.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/netty-all-4.1.17.Final.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-api-jdo-3.2.6.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-core-3.2.10.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-rdbms-3.2.9.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/spark/metrics-influxdb-1.1.8.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/spark/spark-influx-sink-0.4.0.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/jar/spark-mssql-connector-assembly-1.0.0.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/repl_2.11-jars/commons-codec-1.9.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/repl_2.11-jars/livy-core_2.11-0.5.33476.jar.\n19/04/20 02:07:38 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/repl_2.11-jars/livy-repl_2.11-0.5.33476.jar.\n19/04/20 02:07:38 INFO client.RMProxy: Connecting to ResourceManager at mssql-master-pool-0.service-master-pool/10.244.1.8:8032\n19/04/20 02:07:38 INFO yarn.Client: Requesting a new application from cluster with 2 NodeManagers\n19/04/20 02:07:38 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (18432 MB per container)\n19/04/20 02:07:38 INFO yarn.Client: Will allocate AM container, with 2432 MB memory including 384 MB overhead\n19/04/20 02:07:38 INFO yarn.Client: Setting up container launch context for our AM\n19/04/20 02:07:38 INFO yarn.Client: Setting up the launch environment for our AM container\n19/04/20 02:07:38 INFO yarn.Client: Preparing resources for our AM container\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs:/spark/spark_libs.zip\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-api-0.5.33476.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-rsc-0.5.33476.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/netty-all-4.1.17.Final.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-api-jdo-3.2.6.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-core-3.2.10.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-rdbms-3.2.9.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/spark/metrics-influxdb-1.1.8.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/spark/spark-influx-sink-0.4.0.jar\n19/04/20 02:07:38 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/jar/spark-mssql-connector-assembly-1.0.0.jar\n19/04/20 02:07:38 INFO yarn.Client: Deleted staging directory hdfs://mssql-master-pool-0.service-master-pool:9000/user/root/.sparkStaging/application_1554316083160_0119\nException in thread \"main\" java.io.FileNotFoundException: File does not exist: hdfs://mssql-master-pool-0.service-master-pool:9000/jar/spark-mssql-connector-assembly-1.0.0.jar\n\tat org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1533)\n\tat org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1526)\n\tat org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)\n\tat org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1541)\n\tat org.apache.spark.deploy.yarn.ClientDistributedCacheManager$$anonfun$1.apply(ClientDistributedCacheManager.scala:71)\n\tat org.apache.spark.deploy.yarn.ClientDistributedCacheManager$$anonfun$1.apply(ClientDistributedCacheManager.scala:71)\n\tat scala.collection.MapLike$class.getOrElse(MapLike.scala:128)\n\tat scala.collection.AbstractMap.getOrElse(Map.scala:59)\n\tat org.apache.spark.deploy.yarn.ClientDistributedCacheManager.addResource(ClientDistributedCacheManager.scala:71)\n\tat org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:478)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10$$anonfun$apply$6.apply(Client.scala:598)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10$$anonfun$apply$6.apply(Client.scala:597)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10.apply(Client.scala:597)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10.apply(Client.scala:596)\n\tat scala.collection.immutable.List.foreach(List.scala:392)\n\tat org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:596)\n\tat org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:864)\n\tat org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:178)\n\tat org.apache.spark.deploy.yarn.Client.run(Client.scala:1134)\n\tat org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1526)\n\tat org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:849)\n\tat org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)\n\tat org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195)\n\tat org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)\n\tat org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)\n\tat org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)\n\tat org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\n19/04/20 02:07:38 INFO util.ShutdownHookManager: Shutdown hook called\n19/04/20 02:07:38 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-57b2c705-b20a-46e7-8fa4-d4b3ef432906\n\nYARN Diagnostics: .\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.\n" - } - ], + "outputs": [], "execution_count": 6 }, { "cell_type": "code", "source": "\r\n#Process this data. Very simple data cleanup steps. Replacing \"-\" with \"_\" in column names\r\ncolumns_new = [col.replace(\"-\", \"_\") for col in df.columns]\r\ndf = df.toDF(*columns_new)\r\ndf.show(5)\r\n\r\n", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": "The code failed because of a fatal error:\n\tSession 96 unexpectedly reached final status 'error'. See logs:\nstdout: \n\nstderr: \n19/04/20 02:02:03 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-api-0.5.33476.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-rsc-0.5.33476.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/netty-all-4.1.17.Final.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-api-jdo-3.2.6.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-core-3.2.10.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-rdbms-3.2.9.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/spark/metrics-influxdb-1.1.8.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/spark/spark-influx-sink-0.4.0.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/jar/spark-mssql-connector-assembly-1.0.0.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/repl_2.11-jars/commons-codec-1.9.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/repl_2.11-jars/livy-core_2.11-0.5.33476.jar.\n19/04/20 02:02:04 WARN deploy.DependencyUtils: Skip remote jar hdfs://mssql-master-pool-0.service-master-pool:9000/livy/repl_2.11-jars/livy-repl_2.11-0.5.33476.jar.\n19/04/20 02:02:04 INFO client.RMProxy: Connecting to ResourceManager at mssql-master-pool-0.service-master-pool/10.244.1.8:8032\n19/04/20 02:02:04 INFO yarn.Client: Requesting a new application from cluster with 2 NodeManagers\n19/04/20 02:02:04 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (18432 MB per container)\n19/04/20 02:02:04 INFO yarn.Client: Will allocate AM container, with 2432 MB memory including 384 MB overhead\n19/04/20 02:02:04 INFO yarn.Client: Setting up container launch context for our AM\n19/04/20 02:02:04 INFO yarn.Client: Setting up the launch environment for our AM container\n19/04/20 02:02:04 INFO yarn.Client: Preparing resources for our AM container\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs:/spark/spark_libs.zip\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-api-0.5.33476.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/livy-rsc-0.5.33476.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/rsc-jars/netty-all-4.1.17.Final.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-api-jdo-3.2.6.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-core-3.2.10.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/livy/spark/datanucleus-rdbms-3.2.9.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/spark/metrics-influxdb-1.1.8.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/spark/spark-influx-sink-0.4.0.jar\n19/04/20 02:02:04 INFO yarn.Client: Source and destination file systems are the same. Not copying hdfs://mssql-master-pool-0.service-master-pool:9000/jar/spark-mssql-connector-assembly-1.0.0.jar\n19/04/20 02:02:04 INFO yarn.Client: Deleted staging directory hdfs://mssql-master-pool-0.service-master-pool:9000/user/root/.sparkStaging/application_1554316083160_0117\nException in thread \"main\" java.io.FileNotFoundException: File does not exist: hdfs://mssql-master-pool-0.service-master-pool:9000/jar/spark-mssql-connector-assembly-1.0.0.jar\n\tat org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1533)\n\tat org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1526)\n\tat org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)\n\tat org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1541)\n\tat org.apache.spark.deploy.yarn.ClientDistributedCacheManager$$anonfun$1.apply(ClientDistributedCacheManager.scala:71)\n\tat org.apache.spark.deploy.yarn.ClientDistributedCacheManager$$anonfun$1.apply(ClientDistributedCacheManager.scala:71)\n\tat scala.collection.MapLike$class.getOrElse(MapLike.scala:128)\n\tat scala.collection.AbstractMap.getOrElse(Map.scala:59)\n\tat org.apache.spark.deploy.yarn.ClientDistributedCacheManager.addResource(ClientDistributedCacheManager.scala:71)\n\tat org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:478)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10$$anonfun$apply$6.apply(Client.scala:598)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10$$anonfun$apply$6.apply(Client.scala:597)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10.apply(Client.scala:597)\n\tat org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$10.apply(Client.scala:596)\n\tat scala.collection.immutable.List.foreach(List.scala:392)\n\tat org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:596)\n\tat org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:864)\n\tat org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:178)\n\tat org.apache.spark.deploy.yarn.Client.run(Client.scala:1134)\n\tat org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1526)\n\tat org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:849)\n\tat org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)\n\tat org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195)\n\tat org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)\n\tat org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)\n\tat org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)\n\tat org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\n19/04/20 02:02:04 INFO util.ShutdownHookManager: Shutdown hook called\n19/04/20 02:02:04 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-fecb2af2-6fa9-4098-aeae-fc5a91bb022e\n\nYARN Diagnostics: .\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.\n" - } - ], + "outputs": [], "execution_count": 8 }, { @@ -94,15 +60,9 @@ }, { "cell_type": "code", - "source": "#Write from Spark to SQL table using MSSQL Spark Connector\r\nprint(\"Use MSSQL connector to write to master SQL instance \")\r\n\r\nservername = \"jdbc:sqlserver://master-0.master-svc\"\r\ndbname = \"MyTestDatabase\"\r\nurl = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n\r\ndbtable = \"dbo.AdultCensus\"\r\nuser = \"sa\"\r\npassword = \"Yukon900\"\r\n\r\n\r\ntry:\r\n df.write \\\r\n .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n .mode(\"overwrite\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password)\\\r\n .save()\r\nexcept ValueError as error :\r\n print(\"MSSQL Connector write failed\", error)\r\n\r\nprint(\"MSSQL Connector write succeeded \")\r\n\r\n\r\n", + "source": "#Write from Spark to SQL table using MSSQL Spark Connector\r\nprint(\"Use MSSQL connector to write to master SQL instance \")\r\n\r\nservername = \"jdbc:sqlserver://master-0.master-svc\"\r\ndbname = \"MyTestDatabase\"\r\nurl = servername + \";\" + \"databaseName=\" + dbname + \";\"\r\n\r\ndbtable = \"dbo.AdultCensus\"\r\nuser = \"sa\"\r\npassword = \"****\" # Please specify password here\r\n\r\n\r\ntry:\r\n df.write \\\r\n .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n .mode(\"overwrite\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password)\\\r\n .save()\r\nexcept ValueError as error :\r\n print(\"MSSQL Connector write failed\", error)\r\n\r\nprint(\"MSSQL Connector write succeeded \")\r\n\r\n\r\n", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Use build in JDBC connector to write to SQLServer master instance in Big data \nMSSQL Connector write succeeded" - } - ], + "outputs": [], "execution_count": 10 }, { @@ -114,13 +74,7 @@ "cell_type": "code", "source": "#Read from SQL table using MSSQ Connector\r\nprint(\"read data from SQL server table \")\r\njdbcDF = spark.read \\\r\n .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n .option(\"url\", url) \\\r\n .option(\"dbtable\", dbtable) \\\r\n .option(\"user\", user) \\\r\n .option(\"password\", password) \\\r\n .load()\r\n\r\njdbcDF.show(5)", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "read data from SQL server table \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows" - } - ], + "outputs": [], "execution_count": 11 } ]