From 541bef23a8020b7aaae6e8d276a2d21c233f8ff0 Mon Sep 17 00:00:00 2001 From: Warren Zhu Date: Thu, 11 Nov 2021 10:27:38 -0800 Subject: [PATCH] Fix train example 1. Use `OneHotEncoder` as `OneHotEncoderEstimator` is removed in 3.0 2. Skip invalid record in StringIndexer to avoid NULL exception --- .../sparkml/train_score_export_ml_models_with_spark.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb b/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb index 4d339123..596016fc 100644 --- a/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb +++ b/samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb @@ -313,7 +313,7 @@ "cell_type": "code", "source": [ "from pyspark.ml import Pipeline, PipelineModel\r\n", - "from pyspark.ml.feature import OneHotEncoderEstimator, StringIndexer, VectorAssembler\r\n", + "from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler\r\n", "from pyspark.ml.classification import LogisticRegression\r\n", "\r\n", "reg = 0.1\r\n", @@ -335,12 +335,12 @@ " \r\n", " tmpCol = \"-\".join([key, \"tmp\"])\r\n", " si_xvars.append(StringIndexer(inputCol=key, outputCol=tmpCol, handleInvalid=\"skip\")) #, handleInvalid=\"keep\"\r\n", - " ohe_xvars.append(OneHotEncoderEstimator(inputCols=[tmpCol], outputCols=[featureCol]))\r\n", + " ohe_xvars.append(OneHotEncoder(inputCols=[tmpCol], outputCols=[featureCol]))\r\n", " else:\r\n", " featureCols.append(key)\r\n", "\r\n", "# string-index the label column into a column named \"label\"\r\n", - "si_label = StringIndexer(inputCol=label, outputCol='label')\r\n", + "si_label = StringIndexer(inputCol=label, outputCol='label').setHandleInvalid("skip")\r\n", "\r\n", "# assemble the encoded feature columns in to a column named \"features\"\r\n", "assembler = VectorAssembler(inputCols=featureCols, outputCol=\"features\")\r\n", @@ -550,4 +550,4 @@ "execution_count": 27 } ] -} \ No newline at end of file +}