From 2d2104ea097eb8783ebd89b34aa103ac321379c3 Mon Sep 17 00:00:00 2001 From: Umachandar Jayachandran Date: Wed, 7 Nov 2018 10:51:05 -0800 Subject: [PATCH] Added view for ML in the parquet ext table script. --- .../web-clickstreams-hdfs-parquet.sql | 48 ++++++++++++++++++ .../sql/python/book-click-prediction-py.sql | Bin 10802 -> 3240 bytes 2 files changed, 48 insertions(+) diff --git a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql index ec9ee8a6..fd0a0a77 100644 --- a/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql +++ b/samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql @@ -45,6 +45,54 @@ SELECT GROUP BY wcs_user_sk; GO +-- Create view used for ML services training stored procedure +CREATE OR ALTER VIEW [dbo].[web_clickstreams_hdfs_book_clicks] +AS + SELECT + /* There is bug in TPCx-BB data generator which results in data where all users have purchased books. + This will not work for the ML training purposes. So we will treat users with 1-5 clicks in the book category as + not interested in books. */ + CASE WHEN q.clicks_in_category < 6 THEN 0 ELSE q.clicks_in_category END AS clicks_in_category, + CASE WHEN cd.cd_education_status IN ('Advanced Degree', 'College', '4 yr Degree', '2 yr Degree') THEN 1 ELSE 0 END AS college_education, + CASE WHEN cd.cd_gender = 'M' THEN 1 ELSE 0 END AS male, + COALESCE(cd.cd_credit_rating, 'Unknown') as cd_credit_rating, + q.clicks_in_1, + q.clicks_in_2, + q.clicks_in_3, + q.clicks_in_4, + q.clicks_in_5, + q.clicks_in_6, + q.clicks_in_7, + q.clicks_in_8, + q.clicks_in_9, + q.wcs_user_sk + FROM( + SELECT + w.wcs_user_sk, + SUM( CASE WHEN i.i_category = 'Books' THEN 1 ELSE 0 END) AS clicks_in_category, + SUM( CASE WHEN i.i_category_id = 1 THEN 1 ELSE 0 END) AS clicks_in_1, + SUM( CASE WHEN i.i_category_id = 2 THEN 1 ELSE 0 END) AS clicks_in_2, + SUM( CASE WHEN i.i_category_id = 3 THEN 1 ELSE 0 END) AS clicks_in_3, + SUM( CASE WHEN i.i_category_id = 4 THEN 1 ELSE 0 END) AS clicks_in_4, + SUM( CASE WHEN i.i_category_id = 5 THEN 1 ELSE 0 END) AS clicks_in_5, + SUM( CASE WHEN i.i_category_id = 6 THEN 1 ELSE 0 END) AS clicks_in_6, + SUM( CASE WHEN i.i_category_id = 7 THEN 1 ELSE 0 END) AS clicks_in_7, + SUM( CASE WHEN i.i_category_id = 8 THEN 1 ELSE 0 END) AS clicks_in_8, + SUM( CASE WHEN i.i_category_id = 9 THEN 1 ELSE 0 END) AS clicks_in_9 + FROM web_clickstreams_hdfs_parquet as w + INNER JOIN item as i ON (w.wcs_item_sk = i_item_sk + AND w.wcs_user_sk IS NOT NULL) + GROUP BY w.wcs_user_sk + ) AS q + INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk + INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk; +GO + + +-- Inspect top 100 rows +SELECT TOP(100) * FROM web_clickstreams_hdfs_book_clicks; +GO + -- Cleanup /* DROP EXTERNAL TABLE [dbo].[web_clickstreams_hdfs_parquet]; diff --git a/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-py.sql b/samples/features/sql-big-data-cluster/machine-learning/sql/python/book-click-prediction-py.sql index 723820fb993ed27d6d2b2d96b7f21f6ab65e4b35..272d1163b61448a6046b971038bd86d0058fa1bc 100644 GIT binary patch literal 3240 zcmb7GZExE)5dLibibIMu(&`#X`j)QC5Z6&tV~K4@N;@DZ1VyH8B9f?*lpXKGe*2Cu zs;#I)y8=PYY7?-l5nF7TtI6gW8!8a1^BnqyAQ4H~P z=Aqe#U*SVIy@n0FOR}70_fiR(7BX4Q7c#ly{N9=0HGxOCuPCS${)v>8tXzUrT+lgG zf@gGI3ra>|FpLA3hA_N}gAiunbQE0NgaO>n@A#L&tvTxAW4I+3+holo!$3*3RV!Y8 zA>Mhx=QK~sw4ks~MYc+XxwKD)5qTe6jwhsd5sa>e7}xr0w>ws*V(S&@gHL+BzT3el zimgqYjsq*JS|wj=Dz;EIAKIO_ca@o7mFn#4Tt^i1jQKEez{Xi{@hv|C_)KBL@*HYO z;q>Si;0tqLo~m>}dQlL=`Tix&U~;&R)F}cr3&K$_i6I(aPQ0%J3s%C)By;)mVHkl$ z%vsFy(Us>pUD5=hL0`s8XD^U1jf;U^M?3CVPCAws9m~s(x?_3Mv7B}+Z%Oax z2nwl$ujKF@=^-LBMVl38F_*hS{q$L%)1`Iv5r9$>IEVfqm!9e6m9}%_?w}@mgegbzj%jf8t zO34n=cnDXqNLNA9OKD-k?J~Fq1N#&0xo`)3T!CmyWbUQnBK&*o4Ajc zY{Al&a_8k@+RDQV*&jB7y-_PhKFs z%$&y7iG2%|^a5KeB4Omp0Bp-k!sm|xcBtlJ*v$5>RZvu{@gxeu7{-%$YP)!I(%T*C z+W_uA_aGlW4X48~5eZ!_3rGSNg&l S^7|ilKHUeokppDl<@8^#oi&93 literal 10802 zcmcJVZEqXL5y$tf3gkNw4qDM^>O_8an-sQXMR6)if<(nFjDXOy3CR>gNmkq!{pfA} zza5Tu_jpI0Y!(E`eP8M7QY#L_fv%3indWrDtFWd|?vCTUeO;aF&V_#OYTYotkK>$+ zxW?9xG=3JCeITlEgZpyWl6=oXFG{(gUmg8gi@&-4IlR=!YhAt3x;;_&CEU^VmDcZt zzUZBb-+nle3`oEp-s!p@-fI3)ykgavW|XskiTs~QKgh8V^|U5l%i_(QPUZDY_)2$r zqRYGDd{kp8X4p?NM^ry50{ zMlY>ru5;!rPc_F*XB_9h6_@v#(FvodU#RYC#I)}^IG6V7cE`SKKUexT>O)C+A$nI) z+7n&54O~Q7+!m%!-Xs`l9sJ?hk$%r2EhJ1l{LJ8!?}iz0>4YWe@`1kdlXmU69KJhd znGGLDnLlV9I;h*R?+!g<^`?KWuy)hF)3qIZ6~39+3bwH|$RUU>^ta81q(>e;}O> zbq`NtKb%)b{>jd}VNnts$O3mY$Gp5FxejIbX!cO!Uu)i?^mZ2Qi;;BqzQ#UiX8oGG z4WqYpKWT7Dzmf(`pQUbc&UJgciDt~753~oQ{$B4z#m2=ON%T(IA}W`BH>61-(YAE8 zBF*OKu^+X05WSXb+Kw73WI17)!1aV{9HCf);w=%(^%TgW6-BcNo*rdBP)J? zeH552U$X3ZNJPB)T#1qw%|oJlbC4)`>^vkQ;(xA=QgoPyM4*h%l_*7`c}P?ugr@v9 z+p<&Cn}0?ih-^+CKxZzIzG=bm_=gpNqOJ$*n;XSdyyjyljXF^ zYzU?V%dU%WFMjVR1Fi|nq^!6abLEbzt&Zxff9bQCo~=q=^6P|7{r>l3Zl*$eCq01e z96z}pz&-W3*R-%7(KA?!eI-hHu5ZV?uN=o&oJAD9tIE#mMev+UMHaO4$@z{WPNcFy z4yTfzRgNml@xMj=O!o+03Q?X*hC@-rqS)C;-%H_Jy{KS8omOqpRY310dg)2hP2n3` z!HG~)f0})JZN~mm51>i*vcKVV<8?EXRsjQ&Qody)9|V^71o z6D1Ps$NtKB^eFlTeaZ7dT7N4&57h@jzuoXO+6O0isu@Gs1$Ijh2miM&z&O*nd6D-$ zo=dvb==tU)FZ6p^zV>V)cTU}D?nxr}j3rH1+;-LPaNk@I;_cMPM3rLQ36rZ04Y<%r zz=kc3!Lg&e)xFirkZbu|nixxlQMBCaDWDDOjBQJsX6^J}tTSLzQ6~{yO{&xzDf?jb zK3>O_sq10!t`i>1Z~mz9Gj&$*C*$BfAM4bRkyc_Hd>Zv$rx8%_x^Q~4$H57I+Z3&> zSXZt`n%g@2Rjutd)_m38MT(=j2bPv=wq&+oRpND(cXfqV#?|||uX}%EDejy6g6%a| zrEO^4YenmRt62|P&3f2s)}vOl9=Dq1*kYb>*YlC(*2Q3Pcwyp0J^4u48wPWfK8f{X zMcxM_$Bh+P*5n<>vV2C4EH zEwO;!Bz^~7X5!~`wE4uw$!^uIE&fk7wrJ~f$P(lW=t8a8FY^9eSNv}aDY#W8FK>=p zxu1U-IbDkDY0Oo42AF_+aTH}io35wNks1jH+Bf~&aopi3(mxSL>*A+VKogt-XKSZG z&LV2&>p9oHR{P1~w|0+yF>#u;gYa+t>Sr2965eTLK5FQzO1mpX^gt7Y0M&w>6o+ijOJCs@Z%lyWKUYm1`a=38E6MDwf-@7ZL7&+7F}4I?AowT z!WsBV8=LAjSd+_xB|;Tc1%AdFuuyugeO(VVj)lXII0qsyG?Zks(cEk9NCTImITFPt z35<%zQnM(-B&*_)o+S~L((}DCLV+|)R&swrkJad#p0_v(ajz(?k#*Mn=R6$q@YKzo zVtFNJ(yoMmMm)JA&8$QzN7C7$yq_8ZWQR`qZBJl8m7v%>GZ>`>tp-=(!D#+c-*n(j zvtXZmcZP%<5j-vE*@~y)=sBsLALm~pF1O-->^O4X<#}Ia>F~&RIF1>MpFJ zx96K_-|Z1Pbs{IR89$Nj(~){ft14;L@=FmHQ>l*UG*fhN{IM--vFgLE zJDt8&dWkc3ZMnai*WI?O9$`o%_m>h4ll{Od&fB^;R@YhnTu`NGU4HYEgv^p(=a~)} z?Wv_HT?mtp&@{MY8x{ z^atWZ2Vh?yLE8}Z?>K+^r!8y{xsYo>-#ij zmE9OS(zD6rt)?aJ;5VGX_3`VB)TyiYM8C^=CyG? z*E^Pma+4ie96=iEW*|qK`op@xZz}(;;QtWdOtWO8ZGFR>y34|IX)MU_jXleKqyMK^ zC%hB>q&@#5+7fXJ1loocHI37H6|I*#xy{OZ*~;gV!k$LASxG(QJeP$T`5zOVbC9F$ zR9#M`r?pg=cmX<28h;rp1Zaa^ zh^}<>+t8W?=;;kbaQ01}qSg5