From 5ee198fb5c0da49aabf6318779eba969ce52e34f Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Thu, 18 Apr 2019 14:41:44 -0700 Subject: [PATCH] Updated notebook samples --- .../spark/data-loading/spark-sql.ipynb | 80 +++++++++++++++++-- .../data-loading/transform-csv-files.ipynb | 2 +- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/samples/features/sql-big-data-cluster/spark/data-loading/spark-sql.ipynb b/samples/features/sql-big-data-cluster/spark/data-loading/spark-sql.ipynb index c235aef4..d51dd6f5 100644 --- a/samples/features/sql-big-data-cluster/spark/data-loading/spark-sql.ipynb +++ b/samples/features/sql-big-data-cluster/spark/data-loading/spark-sql.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "source": "%%sql\r\nSELECT * FROM web_clickstreams LIMIT 10", + "source": "%%sql\r\nSHOW DATABASES", "metadata": {}, "outputs": [ { @@ -47,13 +47,77 @@ "output_type": "display_data", "data": { "text/plain": "", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
wcs_click_date_skwcs_click_time_skwcs_item_skwcs_web_page_skwcs_user_sk
03759230656906629994
137592341981294729994
2375925943412816214182
3375936870610695288585
437593139632127241338
537593425575821298662
637593608042259288993
7375933900016154254140
8375937059315148281235
937593206619032281235
\n
" + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n
databaseName
0default
\n
" + }, + "metadata": {} + } + ], + "execution_count": 2 + }, + { + "cell_type": "code", + "source": "%%sql\r\nSHOW TABLES", + "metadata": {}, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "", + "text/vnd.plotly.v1+html": "" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": "\n\n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
databasetableNameisTemporary
0defaultproduct_reviewsFalse
1defaultproduct_reviews_orcFalse
2defaultweb_clickstreamsFalse
3defaultweb_clickstreams_orcFalse
\n
" }, "metadata": {} } ], "execution_count": 3 }, + { + "cell_type": "code", + "source": "%%sql\r\nSELECT * FROM web_clickstreams LIMIT 10", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": "", + "text/vnd.plotly.v1+html": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "", + "text/html": "\n\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
wcs_click_date_skwcs_click_time_skwcs_sales_skwcs_item_skwcs_web_page_skwcs_user_sk
0382005162664573114532850092
138200467246457573562817359
238200145656459026032897378
338200325126459116529285493
4382002472364594106802817741
53820066136460446552898592
638200504986461237672853590
738201505026462666552872118
838201783116462736242844665
93820117468646287442840306
\n
" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "execution_count": 2 + }, { "cell_type": "code", "source": "%%sql\r\nSELECT wcs_user_sk, COUNT(*)\r\n FROM web_clickstreams\r\n WHERE wcs_user_sk IS NOT NULL\r\n GROUP BY wcs_user_sk\r\n ORDER BY COUNT(*) DESC LIMIT 10", @@ -92,28 +156,28 @@ "metadata": {}, "outputs": [ { + "output_type": "display_data", "data": { "text/html": "", "text/vnd.plotly.v1+html": "" }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "display_data", "data": { "text/plain": "", "text/html": "\n\n" }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "display_data", "data": { "text/plain": "", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
pr_review_sklen
026053464
126054212
226059191
326060207
426061515
52606359
626069487
726070160
826071380
926072234
\n
" }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} } ], "execution_count": 2 diff --git a/samples/features/sql-big-data-cluster/spark/data-loading/transform-csv-files.ipynb b/samples/features/sql-big-data-cluster/spark/data-loading/transform-csv-files.ipynb index f4d1f739..286adec0 100644 --- a/samples/features/sql-big-data-cluster/spark/data-loading/transform-csv-files.ipynb +++ b/samples/features/sql-big-data-cluster/spark/data-loading/transform-csv-files.ipynb @@ -19,7 +19,7 @@ "cells": [ { "cell_type": "markdown", - "source": "# Spark sample showing read/write methods\nIn this sample notebook, we will read CSV file from HDFS, write it as parquet file and save a Hive table definition.", + "source": "# Spark sample showing read/write methods\nIn this sample notebook, we will read CSV file(s) from HDFS, write it as parquet & orc file(s) and save a Hive table definition.", "metadata": {} }, {