mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
90 lines
7.9 KiB
Plaintext
90 lines
7.9 KiB
Plaintext
{
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"name": "pyspark3kernel",
|
|
"display_name": "PySpark3"
|
|
},
|
|
"language_info": {
|
|
"name": "pyspark3",
|
|
"mimetype": "text/x-python",
|
|
"codemirror_mode": {
|
|
"name": "python",
|
|
"version": 3
|
|
},
|
|
"pygments_lexer": "python3"
|
|
}
|
|
},
|
|
"nbformat_minor": 2,
|
|
"nbformat": 4,
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": "# Spark sample showing read/write methods\nIn this sample notebook, we will read CSV file(s) from HDFS, write it as parquet & orc file(s) and save a Hive table definition.",
|
|
"metadata": {}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": "# Read the clickstream CSV file(s) into a spark data frame, print schema & top rows\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/clickstream_data').toDF(\n \"wcs_click_date_sk\", \"wcs_click_time_sk\", \"wcs_sales_sk\", \"wcs_item_sk\", \"wcs_web_page_sk\", \"wcs_user_sk\"\n )\nresults.printSchema()\nresults.show()",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": "Starting Spark application\n"
|
|
},
|
|
{
|
|
"output_type": "display_data",
|
|
"data": {
|
|
"text/plain": "<IPython.core.display.HTML object>",
|
|
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>0</td><td>application_1555189187089_0001</td><td>pyspark3</td><td>idle</td><td><a target=\"_blank\" href=\"http://master-0.master-svc:8088/proxy/application_1555189187089_0001/\">Link</a></td><td><a target=\"_blank\" href=\"http://storage-0-1.storage-0-svc.demo-ctp25.svc.cluster.local:8042/node/containerlogs/container_1555189187089_0001_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
|
},
|
|
"metadata": {}
|
|
},
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": "SparkSession available as 'spark'.\n"
|
|
},
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": "root\n |-- wcs_click_date_sk: integer (nullable = true)\n |-- wcs_click_time_sk: integer (nullable = true)\n |-- wcs_sales_sk: integer (nullable = true)\n |-- wcs_item_sk: integer (nullable = true)\n |-- wcs_web_page_sk: integer (nullable = true)\n |-- wcs_user_sk: integer (nullable = true)\n\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n|wcs_click_date_sk|wcs_click_time_sk|wcs_sales_sk|wcs_item_sk|wcs_web_page_sk|wcs_user_sk|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\n| 36890| 40052| null| 4379| 34| null|\n| 36890| 41285| null| 6245| 34| null|\n| 36890| 23115| null| 13852| 34| null|\n| 36890| 17702| null| 15975| 34| null|\n| 36890| 62676| null| 2119| 34| null|\n| 36890| 34267| null| 10273| 34| null|\n| 36890| 8502| null| 17790| 34| null|\n| 36890| 54340| null| 3453| 34| null|\n| 36890| 54370| null| 6372| 34| null|\n| 36890| 6578| null| 17203| 34| null|\n| 36890| 75088| null| 4891| 34| null|\n| 36890| 23922| null| 11332| 34| null|\n| 36890| 28761| null| 4484| 34| null|\n| 36890| 21444| null| 5582| 34| null|\n| 36890| 58917| null| 8833| 34| null|\n| 36890| 27578| null| 8599| 34| null|\n| 36890| 8059| null| 6720| 34| null|\n| 36890| 43008| null| 17175| 34| null|\n| 36890| 4378| null| 10644| 34| null|\n| 36890| 55403| null| 8139| 34| null|\n+-----------------+-----------------+------------+-----------+---------------+-----------+\nonly showing top 20 rows"
|
|
}
|
|
],
|
|
"execution_count": 2
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": "# Disable saving SUCCESS file\r\nsc._jsc.hadoopConfiguration().set(\"mapreduce.fileoutputcommitter.marksuccessfuljobs\", \"false\") \r\n\r\n# Print the current warehouse directory where the parquet files will be stored\r\nprint(spark.conf.get(\"spark.sql.warehouse.dir\"))\r\n\r\n# Save results as parquet & orc file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"web_clickstreams\")\r\nresults.write.format(\"orc\").mode(\"overwrite\").saveAsTable(\"web_clickstreams_orc\")",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": "hdfs:///user/hive/warehouse"
|
|
}
|
|
],
|
|
"execution_count": 3
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": "# Read the product reviews CSV files into a spark data frame, print schema & top rows\r\nresults = spark.read.option(\"inferSchema\", \"true\").csv('/product_review_data').toDF(\r\n \"pr_review_sk\", \"pr_review_content\"\r\n )\r\nresults.printSchema()\r\nresults.show()",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": "root\n |-- pr_review_sk: integer (nullable = true)\n |-- pr_review_content: string (nullable = true)\n\n+------------+--------------------+\n|pr_review_sk| pr_review_content|\n+------------+--------------------+\n| 72621|Works fine. Easy ...|\n| 89334|great product to ...|\n| 89335|Next time will go...|\n| 84259|Great Gift Great ...|\n| 84398|After trip to Par...|\n| 66434|Simply the best t...|\n| 66501|This is the exact...|\n| 66587|Not super magnet;...|\n| 66680|Installed as bath...|\n| 66694|Our home was buil...|\n| 84489|Hi ;We are runnin...|\n| 79052|Terra cotta is th...|\n| 73034|One of my fingern...|\n| 73298|We installed thes...|\n| 66810|needed silicone c...|\n| 66912|Great Gift Great ...|\n| 67028|Laguiole knives a...|\n| 89770|Good sound timers...|\n| 84679|AWESOME FEEDBACK ...|\n| 84953|love the retro gl...|\n+------------+--------------------+\nonly showing top 20 rows"
|
|
}
|
|
],
|
|
"execution_count": 5
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": "# Save results as parquet, and orc formats and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"product_reviews\")\r\nresults.write.format(\"orc\").mode(\"overwrite\").saveAsTable(\"product_reviews_orc\")",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"execution_count": 6
|
|
}
|
|
]
|
|
} |