github/sql-server-samples
1
0
Fork 0
mirror of https://github.com/Microsoft/sql-server-samples.git synced 2025-12-08 14:58:54 +00:00
Code Issues Releases Wiki Activity

Fix train example

Browse Source 
1. Use `OneHotEncoder` as `OneHotEncoderEstimator` is removed in 3.0
2. Skip invalid record in StringIndexer to avoid NULL exception
This commit is contained in:
Warren Zhu
2021-11-11 10:27:38 -08:00
committed by GitHub
parent 3444e43146
commit 541bef23a8
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/features/sql-big-data-cluster/spark/sparkml/train_score_export_ml_models_with_spark.ipynb
+4 -4
View File
@@ -313,7 +313,7 @@
"cell_type": "code",
"source": [
"from pyspark.ml import Pipeline, PipelineModel\r\n",
"from pyspark.ml.feature import OneHotEncoderEstimator, StringIndexer, VectorAssembler\r\n",
"from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler\r\n",
"from pyspark.ml.classification import LogisticRegression\r\n",
"\r\n",
"reg = 0.1\r\n",
@@ -335,12 +335,12 @@
" \r\n",
" tmpCol = \"-\".join([key, \"tmp\"])\r\n",
" si_xvars.append(StringIndexer(inputCol=key, outputCol=tmpCol, handleInvalid=\"skip\")) #, handleInvalid=\"keep\"\r\n",
" ohe_xvars.append(OneHotEncoderEstimator(inputCols=[tmpCol], outputCols=[featureCol]))\r\n",
" ohe_xvars.append(OneHotEncoder(inputCols=[tmpCol], outputCols=[featureCol]))\r\n",
" else:\r\n",
" featureCols.append(key)\r\n",
"\r\n",
"# string-index the label column into a column named \"label\"\r\n",
"si_label = StringIndexer(inputCol=label, outputCol='label')\r\n",
"si_label = StringIndexer(inputCol=label, outputCol='label').setHandleInvalid("skip")\r\n",
"\r\n",
"# assemble the encoded feature columns in to a column named \"features\"\r\n",
"assembler = VectorAssembler(inputCols=featureCols, outputCol=\"features\")\r\n",
@@ -550,4 +550,4 @@
"execution_count": 27
}
]
}
}