From 995d4ab8b339169729931128720c5e37a2206834 Mon Sep 17 00:00:00 2001 From: Lixin Gong Date: Wed, 5 Jun 2019 12:54:35 -0700 Subject: [PATCH] fix some typos --- ...in_score_export_ml_models_with_spark.ipynb | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb b/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb index 988a9725..c3219ad8 100644 --- a/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb +++ b/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb @@ -29,34 +29,34 @@ }, { "cell_type": "code", - "source": "![title](datafile = \"/spark_data/AdultCensusIncome.csv\"\r\n\r\n#Read the data to a spark data frame.\r\ndata_all = spark.read.format('csv').options(header='true', inferSchema='true', ignoreLeadingWhiteSpace='true', ignoreTrailingWhiteSpace='true').load(datafile)\r\nprint(\"Number of rows: {}, Number of coulumns : {}\".format(data_all.count(), len(data_all.columns)))\r\ndata_all.printSchema() \r\n\r\n#Replace \"-\" with \"_\" in column names\r\ncolumns_new = [col.replace(\"-\", \"_\") for col in data_all.columns]\r\ndata_all = data_all.toDF(*columns_new)\r\ndata_all.printSchema()\r\n", + "source": "datafile = \"/spark_data/AdultCensusIncome.csv\"\r\n\r\n#Read the data to a spark data frame.\r\ndata_all = spark.read.format('csv').options(header='true', inferSchema='true', ignoreLeadingWhiteSpace='true', ignoreTrailingWhiteSpace='true').load(datafile)\r\nprint(\"Number of rows: {}, Number of coulumns : {}\".format(data_all.count(), len(data_all.columns)))\r\ndata_all.printSchema() \r\n\r\n#Replace \"-\" with \"_\" in column names\r\ncolumns_new = [col.replace(\"-\", \"_\") for col in data_all.columns]\r\ndata_all = data_all.toDF(*columns_new)\r\ndata_all.printSchema()\r\n", "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Starting Spark application\n" + "text": "Starting Spark application\n", + "output_type": "stream" }, { - "output_type": "display_data", "data": { "text/plain": "", - "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
19application_1559313998190_0085pyspark3idleLinkLink✔
" + "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
20application_1559313998190_0086pyspark3idleLinkLink✔
" }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "stream", "name": "stdout", - "text": "SparkSession available as 'spark'.\n" + "text": "SparkSession available as 'spark'.\n", + "output_type": "stream" }, { - "output_type": "stream", "name": "stdout", - "text": "Number of rows: 32561, Number of coulumns : 15\nroot\n |-- age: integer (nullable = true)\n |-- workclass: string (nullable = true)\n |-- fnlwgt: integer (nullable = true)\n |-- education: string (nullable = true)\n |-- education-num: integer (nullable = true)\n |-- marital-status: string (nullable = true)\n |-- occupation: string (nullable = true)\n |-- relationship: string (nullable = true)\n |-- race: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- capital-gain: integer (nullable = true)\n |-- capital-loss: integer (nullable = true)\n |-- hours-per-week: integer (nullable = true)\n |-- native-country: string (nullable = true)\n |-- income: string (nullable = true)\n\nroot\n |-- age: integer (nullable = true)\n |-- workclass: string (nullable = true)\n |-- fnlwgt: integer (nullable = true)\n |-- education: string (nullable = true)\n |-- education_num: integer (nullable = true)\n |-- marital_status: string (nullable = true)\n |-- occupation: string (nullable = true)\n |-- relationship: string (nullable = true)\n |-- race: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- capital_gain: integer (nullable = true)\n |-- capital_loss: integer (nullable = true)\n |-- hours_per_week: integer (nullable = true)\n |-- native_country: string (nullable = true)\n |-- income: string (nullable = true)" + "text": "Number of rows: 32561, Number of coulumns : 15\nroot\n |-- age: integer (nullable = true)\n |-- workclass: string (nullable = true)\n |-- fnlwgt: integer (nullable = true)\n |-- education: string (nullable = true)\n |-- education-num: integer (nullable = true)\n |-- marital-status: string (nullable = true)\n |-- occupation: string (nullable = true)\n |-- relationship: string (nullable = true)\n |-- race: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- capital-gain: integer (nullable = true)\n |-- capital-loss: integer (nullable = true)\n |-- hours-per-week: integer (nullable = true)\n |-- native-country: string (nullable = true)\n |-- income: string (nullable = true)\n\nroot\n |-- age: integer (nullable = true)\n |-- workclass: string (nullable = true)\n |-- fnlwgt: integer (nullable = true)\n |-- education: string (nullable = true)\n |-- education_num: integer (nullable = true)\n |-- marital_status: string (nullable = true)\n |-- occupation: string (nullable = true)\n |-- relationship: string (nullable = true)\n |-- race: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- capital_gain: integer (nullable = true)\n |-- capital_loss: integer (nullable = true)\n |-- hours_per_week: integer (nullable = true)\n |-- native_country: string (nullable = true)\n |-- income: string (nullable = true)", + "output_type": "stream" } ], - "execution_count": 3 + "execution_count": 2 }, { "cell_type": "code", @@ -64,12 +64,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Select few columns to see the data\n+------+---+--------------+---------+\n|income|age|hours_per_week|education|\n+------+---+--------------+---------+\n| <=50K| 39| 40|Bachelors|\n| <=50K| 50| 13|Bachelors|\n| <=50K| 38| 40| HS-grad|\n| <=50K| 53| 40| 11th|\n| <=50K| 28| 40|Bachelors|\n| <=50K| 37| 40| Masters|\n| <=50K| 49| 16| 9th|\n| >50K| 52| 45| HS-grad|\n| >50K| 31| 50| Masters|\n| >50K| 42| 40|Bachelors|\n+------+---+--------------+---------+\nonly showing top 10 rows\n\nNumber of distinct values for income\n+------+\n|income|\n+------+\n| <=50K|\n| >50K|\n+------+\n\nAdded numeric column(income_code) derived from income column\n+------+---+--------------+---------+-----------+\n|income|age|hours_per_week|education|income_code|\n+------+---+--------------+---------+-----------+\n| <=50K| 39| 40|Bachelors| 0|\n| <=50K| 50| 13|Bachelors| 0|\n| <=50K| 38| 40| HS-grad| 0|\n| <=50K| 53| 40| 11th| 0|\n| <=50K| 28| 40|Bachelors| 0|\n| <=50K| 37| 40| Masters| 0|\n| <=50K| 49| 16| 9th| 0|\n| >50K| 52| 45| HS-grad| 1|\n| >50K| 31| 50| Masters| 1|\n| >50K| 42| 40|Bachelors| 1|\n+------+---+--------------+---------+-----------+\nonly showing top 10 rows\n\nPrint a statistical summary of a few columns\n+-------+------+------------------+------------------+------------+-------------------+\n|summary|income| age| hours_per_week| education| income_code|\n+-------+------+------------------+------------------+------------+-------------------+\n| count| 32561| 32561| 32561| 32561| 32561|\n| mean| null| 38.58164675532078|40.437455852092995| null| 0.2408095574460244|\n| stddev| null|13.640432553581356|12.347428681731838| null|0.42758148856469247|\n| min| <=50K| 17| 1| 10th| 0|\n| max| >50K| 90| 99|Some-college| 1|\n+-------+------+------------------+------------------+------------+-------------------+\n\nCalculate Co variance between a few columns to understand features to use\nCovariance between income and hours_per_week is 1.2\nCovariance between income and age is 1.4" + "text": "Select few columns to see the data\n+------+---+--------------+---------+\n|income|age|hours_per_week|education|\n+------+---+--------------+---------+\n| <=50K| 39| 40|Bachelors|\n| <=50K| 50| 13|Bachelors|\n| <=50K| 38| 40| HS-grad|\n| <=50K| 53| 40| 11th|\n| <=50K| 28| 40|Bachelors|\n| <=50K| 37| 40| Masters|\n| <=50K| 49| 16| 9th|\n| >50K| 52| 45| HS-grad|\n| >50K| 31| 50| Masters|\n| >50K| 42| 40|Bachelors|\n+------+---+--------------+---------+\nonly showing top 10 rows\n\nNumber of distinct values for income\n+------+\n|income|\n+------+\n| <=50K|\n| >50K|\n+------+\n\nAdded numeric column(income_code) derived from income column\n+------+---+--------------+---------+-----------+\n|income|age|hours_per_week|education|income_code|\n+------+---+--------------+---------+-----------+\n| <=50K| 39| 40|Bachelors| 0|\n| <=50K| 50| 13|Bachelors| 0|\n| <=50K| 38| 40| HS-grad| 0|\n| <=50K| 53| 40| 11th| 0|\n| <=50K| 28| 40|Bachelors| 0|\n| <=50K| 37| 40| Masters| 0|\n| <=50K| 49| 16| 9th| 0|\n| >50K| 52| 45| HS-grad| 1|\n| >50K| 31| 50| Masters| 1|\n| >50K| 42| 40|Bachelors| 1|\n+------+---+--------------+---------+-----------+\nonly showing top 10 rows\n\nPrint a statistical summary of a few columns\n+-------+------+------------------+------------------+------------+-------------------+\n|summary|income| age| hours_per_week| education| income_code|\n+-------+------+------------------+------------------+------------+-------------------+\n| count| 32561| 32561| 32561| 32561| 32561|\n| mean| null| 38.58164675532078|40.437455852092995| null| 0.2408095574460244|\n| stddev| null|13.640432553581356|12.347428681731838| null|0.42758148856469247|\n| min| <=50K| 17| 1| 10th| 0|\n| max| >50K| 90| 99|Some-college| 1|\n+-------+------+------------------+------------------+------------+-------------------+\n\nCalculate Co variance between a few columns to understand features to use\nCovariance between income and hours_per_week is 1.2\nCovariance between income and age is 1.4", + "output_type": "stream" } ], - "execution_count": 4 + "execution_count": 3 }, { "cell_type": "code", @@ -77,12 +77,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "label = income\nfeatures = ['age', 'hours_per_week', 'education']\nCount of rows that are <=50K 24720\nCount of rows that are >50K 7841" + "text": "label = income\nfeatures = ['age', 'hours_per_week', 'education']\nCount of rows that are <=50K 24720\nCount of rows that are >50K 7841", + "output_type": "stream" } ], - "execution_count": 5 + "execution_count": 4 }, { "cell_type": "markdown", @@ -91,16 +91,16 @@ }, { "cell_type": "code", - "source": "![title](train, test = data.randomSplit([0.75, 0.25], seed=123)\r\n\r\nprint(\"train ({}, {})\".format(train.count(), len(train.columns)))\r\nprint(\"test ({}, {})\".format(test.count(), len(test.columns)))\r\n\r\ntrain_data_path = \"/spark_ml/AdultCensusIncomeTrain\"\r\ntest_data_path = \"/spark_ml/AdultCensusIncomeTest\"\r\n\r\ntrain.write.mode('overwrite').orc(train_data_path)\r\ntest.write.mode('overwrite').orc(test_data_path)\r\nprint(\"train and test datasets saved to {} and {}\".format(train_data_path, test_data_path))", + "source": "train, test = data.randomSplit([0.75, 0.25], seed=123)\r\n\r\nprint(\"train ({}, {})\".format(train.count(), len(train.columns)))\r\nprint(\"test ({}, {})\".format(test.count(), len(test.columns)))\r\n\r\ntrain_data_path = \"/spark_ml/AdultCensusIncomeTrain\"\r\ntest_data_path = \"/spark_ml/AdultCensusIncomeTest\"\r\n\r\ntrain.write.mode('overwrite').orc(train_data_path)\r\ntest.write.mode('overwrite').orc(test_data_path)\r\nprint(\"train and test datasets saved to {} and {}\".format(train_data_path, test_data_path))", "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "train (24469, 4)\ntest (8092, 4)\ntrain and test datasets saved to /spark_ml/AdultCensusIncomeTrain and /spark_ml/AdultCensusIncomeTest" + "text": "train (24469, 4)\ntest (8092, 4)\ntrain and test datasets saved to /spark_ml/AdultCensusIncomeTrain and /spark_ml/AdultCensusIncomeTest", + "output_type": "stream" } ], - "execution_count": 6 + "execution_count": 5 }, { "cell_type": "markdown", @@ -113,12 +113,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Using LogisticRegression model with Regularization Rate of 0.1.\nPipeline Created\nModel Trained\nModel is PipelineModel_8c7a4fdc6110\nModel Stages [StringIndexer_7d244350f55d, OneHotEncoderEstimator_559780c5ce92, StringIndexer_53280e6349e6, VectorAssembler_1051507100cb, LogisticRegressionModel: uid = LogisticRegression_5c0eda4eab78, numClasses = 2, numFeatures = 17]" + "text": "Using LogisticRegression model with Regularization Rate of 0.1.\nPipeline Created\nModel Trained\nModel is PipelineModel_1adfacc01e7a\nModel Stages [StringIndexer_ee8506a28443, OneHotEncoderEstimator_cb5dbefb5cce, StringIndexer_38769cda5ab3, VectorAssembler_a3c2d358bd55, LogisticRegressionModel: uid = LogisticRegression_18837c9488f5, numClasses = 2, numFeatures = 17]", + "output_type": "stream" } ], - "execution_count": 7 + "execution_count": 6 }, { "cell_type": "markdown", @@ -131,12 +131,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Area under ROC: 0.7964496884726682\nArea Under PR: 0.5358180243123482\n+------+-----+----------+\n|income|label|prediction|\n+------+-----+----------+\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n+------+-----+----------+\nonly showing top 20 rows" + "text": "Area under ROC: 0.7964496884726682\nArea Under PR: 0.5358180243123482\n+------+-----+----------+\n|income|label|prediction|\n+------+-----+----------+\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n| <=50K| 0.0| 1.0|\n| >50K| 1.0| 1.0|\n+------+-----+----------+\nonly showing top 20 rows", + "output_type": "stream" } ], - "execution_count": 8 + "execution_count": 7 }, { "cell_type": "markdown", @@ -149,12 +149,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "saved model to /spark_ml/AdultCensus.mml\nSuccessfully loaded from /spark_ml/AdultCensus.mml" + "text": "saved model to /spark_ml/AdultCensus.mml\nSuccessfully loaded from /spark_ml/AdultCensus.mml", + "output_type": "stream" } ], - "execution_count": 9 + "execution_count": 8 }, { "cell_type": "markdown", @@ -167,12 +167,12 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "persist the mleap bundle from local to hdfs" + "text": "persist the mleap bundle from local to hdfs", + "output_type": "stream" } ], - "execution_count": 10 + "execution_count": 9 } ] } \ No newline at end of file