Remove extra spacing

This commit is contained in:
Lorin Thwaits
2023-02-20 19:33:29 +00:00
parent 434b798c98
commit b062f2c734
1461 changed files with 13333 additions and 76271 deletions
@@ -1,4 +1,4 @@
**List of data sets**
**List of data sets**
| Data Set Name | Link to the Full Data Set | Full Data Set Size (MB) | Link to Report |
| ---:| ---: | ---: | ---: |
@@ -43,7 +43,7 @@ dim(uid)
# Step 1: RFM analysis
# Call RFM source code
# Call RFM source code
source(wd, "R", "RFM_Analysis_R_Source_Codes_V1.3.R")
@@ -64,7 +64,7 @@ df1 <-getIndependentScore(df)
head(df1)
# Draw the histograms in the R, F, and M dimensions
# Draw the histograms in the R, F, and M dimensions
drawHistograms(df1)
@@ -129,7 +129,7 @@ colnames(RFM_Result) <- c("ID", "R", "F", "M", "R_Score", "F_Score", "M_Score",
head(RFM_Result)
time <- system.time({
sqlSave(channel,
RFM_Result,
rownames=FALSE,
@@ -200,38 +200,38 @@ head(p1)
tail(p1)
# Decision tree
# Grow tree
# Grow tree
fit <- rpart(Cluster~R+F+M,
method="class",
method="class",
data=Train)
# Display the results
# Display the results
printcp(fit)
# Visualize cross-validation results
# Visualize cross-validation results
plotcp(fit)
plotcp(fit)
# Detailed summary of splits
summary(fit)
summary(fit)
# Plot tree
# Plot tree
library(rpart)
plot(fit, uniform=TRUE, main="Classification Tree for CDNOW")
text(fit, use.n=TRUE, all=TRUE, cex=.8)
# Prune the tree
# Prune the tree
pfit <- prune(fit, cp=fit$cptable[which.min(fit$cptable[,"xerror"]), "CP"])
# Plot the pruned tree
# Plot the pruned tree
plot(pfit, uniform=TRUE,
plot(pfit, uniform=TRUE,
main="Pruned Classification Tree for CDNOW")
text(pfit, use.n=TRUE, all=TRUE, cex=.8)
@@ -1,5 +1,5 @@
################################################################
# Title: CRM Demo in-SQL
# Title: CRM Demo in-SQL
# Author: Microsoft
# Date: Dec, 2015
#################################################################
@@ -14,9 +14,9 @@ connectionString <- "Driver=SQL Server;
RFMData <- RxSqlServerData(connectionString=connectionString,
table="RFM_Result")
cc <- RxInSqlServer(connectionString=connectionString,
autoCleanup=FALSE,
cc <- RxInSqlServer(connectionString=connectionString,
autoCleanup=FALSE,
consoleOutput=TRUE)
rxSetComputeContext(cc)
@@ -29,7 +29,7 @@ rxGetInfo(RFMData, getVarInfo=T, numRows=3)
rxHistogram(~R, data=RFMData, xNumTicks=20)
rxHistogram(~F, data=RFMData, rowSelection=F < 30, xNumTicks=20)
rxHistogram(~M, data=RFMData, rowSelection=M < 200, xNumTicks=20)
rxHistogram(~M, data=RFMData, rowSelection=M < 200, xNumTicks=20)
# Count frequency of each RFMscore Level
@@ -38,13 +38,13 @@ results <- rxResultsDF(tmp)
results <- results[results$Counts != 0, ]
results[order(results$Counts, decreasing=TRUE), ]
# Step 2: K-means Clustering
# Step 2: K-means Clustering
KmeansData <- RxSqlServerData(connectionString=connectionString,
table = "Kmeans_Result")
md.km <- rxKmeans(formula=~R+F+M+R_Score+F_Score+M_Score,
data=RFMData,
md.km <- rxKmeans(formula=~R+F+M+R_Score+F_Score+M_Score,
data=RFMData,
outFile=KmeansData,
numClusters=8,
algorithm="lloyd",
@@ -113,7 +113,7 @@ TestData <- RxSqlServerData(connectionString=connectionString,
rxSplit(RFMVIPTrainTestData, outFilesBase=RFMVIPTrainTestData,
splitByFactor='urv', overwrite=T, reportProgress=1)
## Built our Logistic Regression Model with IsVIP as response
r1<- rxLogit(IsVIP~R+F+M,
@@ -124,7 +124,7 @@ r1<- rxLogit(IsVIP~R+F+M,
## r1:stepwise selection shows that Monetary, Recency are significant.
## Build our Logistic Regression Model
r2 <- rxLogit(IsVIP~R+F,
data=RFMVIPData, covCoef=TRUE)
summary(r2)
@@ -132,8 +132,8 @@ summary(r2)
## Predict our Logistic Model on our test Dataset
LogisticPred.xdf <- file.path(output.path,"LogisticPred.xdf")
p2 <- rxPredict(r2, data=test.xdf, outData=LogisticPred.xdf,
writeModelVars=TRUE, extraVarsToWrite="ID",
p2 <- rxPredict(r2, data=test.xdf, outData=LogisticPred.xdf,
writeModelVars=TRUE, extraVarsToWrite="ID",
computeResiduals=TRUE, computeStdErr=TRUE,
interval="confidence", predVarNames='LogitPredict',
overwrite=TRUE)
@@ -182,7 +182,7 @@ title(main="RFM-based Decision Tree on CDNOW Data",line=3)
# Prediction
DTreePred.xdf<-file.path(output.path,"DTreePred.xdf")
rxPredict(d1, data=test.xdf, outData=DTreePred.xdf,
rxPredict(d1, data=test.xdf, outData=DTreePred.xdf,
writeModelVars=TRUE, extraVarsToWrite="ID",
computeResiduals=T, overwrite=TRUE)
@@ -194,7 +194,7 @@ write.table(DTreePredInfoTop10$data, file=DTreePredInfoTop10.txt, sep=" ")
DTreePred <- rxXdfToDataFrame(file=DTreePred.xdf)
sqlSave(channel, DTreePred, rownames=FALSE, append=FALSE,
sqlSave(channel, DTreePred, rownames=FALSE, append=FALSE,
varTypes=list(numeric="float",
integer="int",
Date="date"))
@@ -1,7 +1,7 @@
###############################################################################
#Description: A set of R functions to implement the Independent RFM scoring and the RFM scoring with input breaks.
#Author: Jack Han http://www.DataApple.net email: jackhan2008 # qq.com
#Version: 1.3
#Version: 1.3
#Date: 23 Dec 2013
#Usage: Read the article "RFM Customer Analysis with R Language" http://www.dataapple.net/?p=84
################################################################################
@@ -291,7 +291,7 @@ for (i in 1:f){
c[k]<- dim(tmpdf)[1]
}
if (i==1 & j==1)
if (i==1 & j==1)
barplot(c,col="lightblue",names.arg=names)
else
barplot(c,col="lightblue")
@@ -6,16 +6,16 @@ As a data professional, do you wonder how you can leverage data science for crea
**Example**
In retail, tons of data are generated every data, which imply rich information about the whole market.
In retail, tons of data are generated every data, which imply rich information about the whole market.
To provide market intelligence, the CRM analysis is commonly used to understand customer segmentation, predict customer behavior and better target potential buyers via right recommendation. With growing size of data and higher request on timeliness, it becomes a bit more challenging to do precision marketing on big data in a much more time-efficient way.
To provide market intelligence, the CRM analysis is commonly used to understand customer segmentation, predict customer behavior and better target potential buyers via right recommendation. With growing size of data and higher request on timeliness, it becomes a bit more challenging to do precision marketing on big data in a much more time-efficient way.
In this demo, we address the issue with Microsoft R Server's parallel computing algorithms and build an end-to-end operationalized analytical system using SQL Server R and Power BI.
Using a concrete example of customer relationship management for retail, well share how you can jumpstart by
- Running R scripts using SQL Server as the compute context
- Operationalize your R scripts using stored procedures.
- Operationalize your R scripts using stored procedures.
The insights delivered by these models are visualized using a Power BI dashboard.
@@ -25,7 +25,7 @@ The insights delivered by these models are visualized using a Power BI dashboard
You have to do the following set-up before playing with this demo.
- Install SQL Server 2016 or create a SQL Server 2016 Enterprise VM on Azure with Standalone R Server and R Services installed/configured.
- Install SQL Server 2016 or create a SQL Server 2016 Enterprise VM on Azure with Standalone R Server and R Services installed/configured.
- Install R IDE: R Tools for Visual Studio or R Studio.
- Install PowerBI Desktop.
- Validate the successful installation.
@@ -38,14 +38,14 @@ This sample consists of the following directory structure.
- **Data** - This folder contains the CD sales data CDNOW.
- **R** - This folder contains the R code that you can run in any R IDE.
- **SQL Server** - This folder contains the sql files that you can run to create T-SQL stored procedures (with R code embeded) and try out this precision marketing example.
- **PowerBI** - This folder contains a sample PowerBI report.
- **SQL Server** - This folder contains the sql files that you can run to create T-SQL stored procedures (with R code embeded) and try out this precision marketing example.
- **PowerBI** - This folder contains a sample PowerBI report.
To jumpstart, run the T-SQL files (crm_demo.sql)
**Note**
This is a demo built on SQL 2016 RC1 in Dec 2015. To try out it, please modify it to fit the new version of SQL Server R Services.
This is a demo built on SQL 2016 RC1 in Dec 2015. To try out it, please modify it to fit the new version of SQL Server R Services.
@@ -6,15 +6,15 @@ go
drop procedure if exists get_CDNOW_RFM
go
--create stored procedure to get RFM
--create stored procedure to get RFM
create proc get_CDNOW_RFM (@start datetime = '1900-1-1', @end datetime = '3000-1-1', @now datetime = null)
create proc get_CDNOW_RFM (@start datetime = '1900-1-1', @end datetime = '3000-1-1', @now datetime = null)
as
begin
if @now is null
if @now is null
set @now = getdate()
select
select
ID, DATEDIFF(d,R,@now) as R ,F,M
from
(select
@@ -22,7 +22,7 @@ from
from
[dbo].[CDNOW]
where
[Date] BETWEEN @start AND @end
[Date] BETWEEN @start AND @end
group by ID ) as rfm_tmp
order by cast (ID as int)
end
@@ -38,31 +38,31 @@ go
--create stored procedure to break RFM score
create proc BreakScoreRFM (@start datetime = '1900-1-1', @end datetime = '3000-1-1', @now datetime = null,
@r_cut varchar(254) = null, @f_cut varchar(254) = null, @m_cut varchar(254) = null)
as
create proc BreakScoreRFM (@start datetime = '1900-1-1', @end datetime = '3000-1-1', @now datetime = null,
@r_cut varchar(254) = null, @f_cut varchar(254) = null, @m_cut varchar(254) = null)
as
begin
if @now is null
if @now is null
set @now = getdate()
declare @r_cut1 float = 100, @r_cut2 float = 200, @r_cut3 float = 300, @r_cut4 float = 400
declare @f_cut1 float = 100, @f_cut2 float = 200, @f_cut3 float = 300, @f_cut4 float = 400
declare @m_cut1 float = 100, @m_cut2 float = 200, @m_cut3 float = 300, @m_cut4 float = 400
declare @idx int = 0, @len int = 0
--get cut parameter, should add more code to check cut parameter.
--get cut parameter, should add more code to check cut parameter.
if(@r_cut is not null) -- and @r_cut follow the syntax
begin
set @idx = CHARINDEX('-',@r_cut)
set @idx = CHARINDEX('-',@r_cut)
set @len = len(@r_cut)
set @r_cut1 = substring(@r_cut,1,@idx-1)
set @r_cut=substring(@r_cut,@idx+1,@len-@idx)
set @idx = CHARINDEX('-',@r_cut)
set @idx = CHARINDEX('-',@r_cut)
set @len = len(@r_cut)
set @r_cut2 = substring(@r_cut,1,@idx-1)
set @r_cut=substring(@r_cut,@idx+1,@len-@idx)
set @idx = CHARINDEX('-',@r_cut)
set @idx = CHARINDEX('-',@r_cut)
set @len = len(@r_cut)
set @r_cut3 = substring(@r_cut,1,@idx-1)
set @r_cut4=substring(@r_cut,@idx+1,@len-@idx)
@@ -70,35 +70,35 @@ begin
if(@f_cut is not null) -- and @f_cut follow the syntax
begin
set @idx = CHARINDEX('-',@f_cut)
set @idx = CHARINDEX('-',@f_cut)
set @len = len(@f_cut)
set @f_cut1 = substring(@f_cut,1,@idx-1)
set @f_cut=substring(@f_cut,@idx+1,@len-@idx)
set @idx = CHARINDEX('-',@f_cut)
set @idx = CHARINDEX('-',@f_cut)
set @len = len(@f_cut)
set @f_cut2 = substring(@f_cut,1,@idx-1)
set @f_cut=substring(@f_cut,@idx+1,@len-@idx)
set @idx = CHARINDEX('-',@f_cut)
set @idx = CHARINDEX('-',@f_cut)
set @len = len(@f_cut)
set @f_cut3 = substring(@f_cut,1,@idx-1)
set @f_cut4=substring(@f_cut,@idx+1,@len-@idx)
end
end
if(@m_cut is not null) -- and @m_cut follow the syntax
begin
set @idx = CHARINDEX('-',@m_cut)
set @idx = CHARINDEX('-',@m_cut)
set @len = len(@m_cut)
set @m_cut1 = substring(@m_cut,1,@idx-1)
set @m_cut=substring(@m_cut,@idx+1,@len-@idx)
set @idx = CHARINDEX('-',@m_cut)
set @idx = CHARINDEX('-',@m_cut)
set @len = len(@m_cut)
set @m_cut2 = substring(@m_cut,1,@idx-1)
set @m_cut=substring(@m_cut,@idx+1,@len-@idx)
set @idx = CHARINDEX('-',@m_cut)
set @idx = CHARINDEX('-',@m_cut)
set @len = len(@m_cut)
set @m_cut3 = substring(@m_cut,1,@idx-1)
set @m_cut4=substring(@m_cut,@idx+1,@len-@idx)
@@ -116,21 +116,21 @@ begin
truncate table RFM
drop table RFM
end
-- get RFM table from initial data.
select
-- get RFM table from initial data.
select
ID, DATEDIFF(d,R,@now) as R, F, M into RFM
from
(select
(select
ID, max([Date]) as R, count(Volume) as F, round(avg(Amount),2) as M
from
from
CDNOW
where
[Date] between @start and @end
where
[Date] between @start and @end
group by ID ) as rfm_tmp
order by cast (ID as int)
-- record R_Score at temp table '#R'
select
-- record R_Score at temp table '#R'
select
ID,
case when R <= @r_cut1 then 5
when R > @r_cut1 and R <= @r_cut2 then 4
@@ -140,10 +140,10 @@ begin
else 0
end as R_Score
into #R
from RFM
from RFM
-- score F
select
select
ID,
case when F >= @f_cut4 then 5
when F > @f_cut3 and F <= @f_cut4 then 4
@@ -153,10 +153,10 @@ begin
else 0
end as F_Score
into #F
from RFM
from RFM
-- score M
select
select
ID,
case when M >= @m_cut4 then 5
when M > @m_cut3 and M <= @m_cut4 then 4
@@ -166,12 +166,12 @@ begin
else 0
end as M_Score
into #M
from RFM
from RFM
--union all
select #R.ID, R_Score, F_Score, M_Score, R_Score*100 + F_Score*10 + M_Score as Toltal_Score
into RFM_Score
from #R, #F, #M
--union all
select #R.ID, R_Score, F_Score, M_Score, R_Score*100 + F_Score*10 + M_Score as Toltal_Score
into RFM_Score
from #R, #F, #M
where #R.ID = #F.ID and #R.ID = #M.ID
select * from RFM_Score order by Toltal_Score desc,ID
@@ -189,7 +189,7 @@ go
drop table RFM_Result;
select a.*, b.R_Score, b.F_Score, b.M_Score, b.Toltal_Score
into RFM_Result
from
from
[dbo].[RFM] a left outer join
[dbo].[RFM_Score] b on
a.[ID] = b.[ID];
@@ -217,7 +217,7 @@ begin
tmpdf <-df[df$R_Score==j & df$F_Score==i & df$M_Score==k,]
c[k]<- dim(tmpdf)[1]
}
if (i==1 & j==1)
if (i==1 & j==1)
barplot(c,col="lightblue",names.arg=names)
else
barplot(c,col="lightblue")
@@ -232,7 +232,7 @@ begin
png(filename=ff, width=620, height=240)
print(RFMhist)
dev.off()
OutputDataSet <- data.frame(data=readBin(file(ff, "rb"), what=raw(), n=1e6));
OutputDataSet <- data.frame(data=readBin(file(ff, "rb"), what=raw(), n=1e6));
'
, @input_data_1 = N'select "R", "F", "M" from RFM_Result'
, @input_data_1_name = N'RFM_Result'
@@ -271,8 +271,8 @@ begin
@language = N'R'
, @script = N'
require("RevoScaleR");
CDNOWKmeans <- rxKmeans(formula=~R+F+M+R_Score+F_Score+M_Score,
data=RFM_Result,
CDNOWKmeans <- rxKmeans(formula=~R+F+M+R_Score+F_Score+M_Score,
data=RFM_Result,
#outFile=Kmeans_Result,
numClusters=8,
algorithm="lloyd",
@@ -362,7 +362,7 @@ as
begin
declare @rx_model varbinary(max) = (select model from CDNOW_rx_models where model_name = @model);
-- Predict based on the specified model:
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
@@ -396,20 +396,20 @@ as
begin
declare @rx_model varbinary(max) = (select model from CDNOW_rx_models where model_name = @model);
-- Predict based on the specified model:
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
CDNOWmodel <- unserialize(rx_model);
CDNOWpred <- rxPredict(CDNOWmodel,
data=RFMVIPCluster,
data=RFMVIPCluster,
predVarNames=c("prob1", "prob2", "prob3", "prob4", "prob5", "prob6", "prob7", "prob8"),
writeModelVars=TRUE, extraVarsToWrite="ID",
computeResiduals=T, overwrite=TRUE);
OutputDataSet <- round(cbind(RFMVIPCluster[,1], RFMVIPCluster[,10],
OutputDataSet <- round(cbind(RFMVIPCluster[,1], RFMVIPCluster[,10],
CDNOWpred$prob1,CDNOWpred$prob2,CDNOWpred$prob3,CDNOWpred$prob4,
CDNOWpred$prob5,CDNOWpred$prob6,CDNOWpred$prob7,CDNOWpred$prob8),2);
colnames(OutputDataSet) <- c("ID", "Cluster.Actual", "Cluster1.Prob","Cluster2.Prob","Cluster3.Prob","Cluster4.Prob","Cluster5.Prob","Cluster6.Prob","Cluster7.Prob","Cluster8.Prob");
OutputDataSet<-as.data.frame(OutputDataSet);
'
@@ -12,11 +12,11 @@ input_query <- "
round(CASE WHEN ((orders_items = 0) OR(returns_items IS NULL) OR (orders_items IS NULL) OR ((returns_items / orders_items) IS NULL) ) THEN 0.0 ELSE (cast(returns_items as nchar(10)) / orders_items) END, 7) AS itemsRatio,
round(CASE WHEN ((orders_money = 0) OR (returns_money IS NULL) OR (orders_money IS NULL) OR ((returns_money / orders_money) IS NULL) ) THEN 0.0 ELSE (cast(returns_money as nchar(10)) / orders_money) END, 7) AS monetaryRatio,
round(CASE WHEN ( returns_count IS NULL ) THEN 0.0 ELSE returns_count END, 0) AS frequency
FROM
(
SELECT
ss_customer_sk,
ss_customer_sk,
-- return order ratio
COUNT(distinct(ss_ticket_number)) AS orders_count,
-- return ss_item_sk ratio
@@ -38,7 +38,7 @@ FROM
SUM( sr_return_amt ) AS returns_money
FROM store_returns
GROUP BY sr_customer_sk
) returned ON ss_customer_sk=sr_customer_sk
) returned ON ss_customer_sk=sr_customer_sk
"
# Input customer data that needs to be classified
customer_returns <- RxSqlServerData(sqlQuery = input_query,
@@ -53,9 +53,9 @@ head(customer_data, n = 5);
# Determine number of clusters
#Using a plot of the within groups sum of squares by number of clusters extracted can help determine the appropriate number of clusters.
#We are looking for a bend in the plot. It is at this "elbow" in the plot that we have the appropriate number of clusters
#We are looking for a bend in the plot. It is at this "elbow" in the plot that we have the appropriate number of clusters
wss <- (nrow(customer_data) - 1) * sum(apply(customer_data, 2, var))
for (i in 2:20) {
for (i in 2:20) {
xt = kmeans(customer_data, centers = i)
wss[i] <- sum(kms = kmeans(customer_data, centers = i)$withinss)
}
@@ -4,7 +4,7 @@ DROP PROC IF EXISTS generate_customer_return_clusters;
GO
CREATE procedure [dbo].[generate_customer_return_clusters]
AS
/*
/*
This procedure uses R to classify customers into different groups based on their
purchase & return history.
*/
@@ -21,7 +21,7 @@ SELECT
round(CASE WHEN ((orders_items = 0) OR(returns_items IS NULL) OR (orders_items IS NULL) OR ((returns_items / orders_items) IS NULL) ) THEN 0.0 ELSE (cast(returns_items as nchar(10)) / orders_items) END, 7) AS itemsRatio,
round(CASE WHEN ((orders_money = 0) OR (returns_money IS NULL) OR (orders_money IS NULL) OR ((returns_money / orders_money) IS NULL) ) THEN 0.0 ELSE (cast(returns_money as nchar(10)) / orders_money) END, 7) AS monetaryRatio,
round(CASE WHEN ( returns_count IS NULL ) THEN 0.0 ELSE returns_count END, 0) AS frequency
FROM
(
SELECT
@@ -47,7 +47,7 @@ FROM
SUM( sr_return_amt ) AS returns_money
FROM store_returns
GROUP BY sr_customer_sk
) returned ON ss_customer_sk=sr_customer_sk
) returned ON ss_customer_sk=sr_customer_sk
'
EXEC sp_execute_external_script
@@ -68,7 +68,7 @@ return_cluster = RxSqlServerData(table = "customer_return_clusters", connectionS
# set.seed for random number generator for predictability
set.seed(10);
# generate clusters using rxKmeans and output clusters to a table called "customer_return_clusters".
# generate clusters using rxKmeans and output clusters to a table called "customer_return_clusters".
clust <- rxKmeans( ~ orderRatio + itemsRatio + monetaryRatio + frequency, customer_returns, numClusters = 4
, outFile = return_cluster, outColName = "cluster", writeModelVars = TRUE , extraVarsToWrite = c("customer"), overwrite = TRUE);
'
@@ -95,7 +95,7 @@ SELECT * FROM customer_return_clusters;
--Select email addresses of customers in cluster 1
SELECT customer.[c_email_address], customer.c_customer_sk
FROM dbo.customer
JOIN
JOIN
[dbo].[customer_return_clusters] as r
ON r.customer = customer.c_customer_sk
WHERE r.cluster = 3
@@ -1,6 +1,6 @@
# Perform customer clustering with SQL Server R Services
In this sample, we are going to get ourselves familiar with clustering.
In this sample, we are going to get ourselves familiar with clustering.
Clustering can be explained as organizing data into groups where members of a group are similar in some way.
### Contents
@@ -15,7 +15,7 @@ Clustering can be explained as organizing data into groups where members of a gr
## About this sample
We will be using the Kmeans algorithm to perform the clustering of customers. This can for example be used to target a specific group of customers for marketing efforts.
We will be using the Kmeans algorithm to perform the clustering of customers. This can for example be used to target a specific group of customers for marketing efforts.
Kmeans clustering is an unsupervised learning algorithm that tries to group data based on similarities. Unsupervised learning means that there is no outcome to be predicted, and the algorithm just tries to find patterns in the data.
In this sample, you will learn how to perform Kmeans clustering in R and deploying the solution in SQL Server 2016.
@@ -15,7 +15,7 @@ This sample shows how to create a predictive model in R and operationalize it wi
## About this sample
Predictive modeling is a powerful way to add intelligence to your application. It enables applications to predict outcomes against new data.
The act of incorporating predictive analytics into your applications involves two major phases:
The act of incorporating predictive analytics into your applications involves two major phases:
model training and model operationalization.
In this sample, you will learn how to create a predictive model in R and operationalize it with SQL Server 2016.
@@ -2,7 +2,7 @@
##################### STEP1 - Connect to DB and read data ####################
#Connection string to connect to SQL Server named instance
connStr <- paste("Driver=SQL Server; Server=", "MYSQLSERVER",
connStr <- paste("Driver=SQL Server; Server=", "MYSQLSERVER",
";Database=", "Tutorialdb", ";Trusted_Connection=true;", sep = "");
#Get the data from SQL Server Table
@@ -66,7 +66,7 @@ AS
BEGIN
DECLARE @rx_model varbinary(max) = (select model from rental_rx_models where model_name = @model);
EXEC sp_execute_external_script
EXEC sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
@@ -83,7 +83,7 @@ OutputDataSet <- cbind(rental_predictions[1],rental_predictions[2], rental_predi
, @params = N'@rx_model varbinary(max)'
, @rx_model = @rx_model
with result sets (("RentalCount_Predicted" float, "RentalCount_Actual" float,"Month" float,"Day" float,"WeekDay" float,"Snow" float,"Holiday" float, "Year" float));
END;
GO
@@ -121,14 +121,14 @@ CREATE PROCEDURE predict_rentalcount_new (@model VARCHAR(100),@q NVARCHAR(MAX))
AS
BEGIN
DECLARE @rx_model VARBINARY(MAX) = (SELECT model FROM rental_rx_models WHERE model_name = @model);
EXECUTE sp_execute_external_script
EXECUTE sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
#The InputDataSet contains the new data passed to this stored proc. We will use this data to predict.
rentals = InputDataSet;
#Convert types to factors
rentals$Holiday = factor(rentals$Holiday);
rentals$Snow = factor(rentals$Snow);
@@ -144,7 +144,7 @@ BEGIN
, @params = N'@rx_model varbinary(max)'
, @rx_model = @rx_model
WITH RESULT SETS (("RentalCount_Predicted" FLOAT));
END;
GO
@@ -155,13 +155,13 @@ GO
-------------- STEP 8 - Getting predictions from an Application ----------------------------------
-- Create stored procedure that returns predictions as JSON
-- Create stored procedure that returns predictions as JSON
-- This stored procedure is going to be called from our application
DROP PROCEDURE IF EXISTS get_rental_predictions;
GO
CREATE PROCEDURE get_rental_predictions (@year int)
AS
SELECT
AS
SELECT
"Year",
RentalCount_Predicted ,
RentalCount_Actual ,
@@ -170,10 +170,10 @@ SELECT
"WeekDay" ,
"Snow",
"Holiday"
FROM rental_predictions
FROM rental_predictions
WHERE Year = @year
FOR JSON PATH, root('data')
RETURN
GO
@@ -15,7 +15,7 @@ This sample shows how to create a predictive model in R and operationalize it wi
## About this sample
Predictive modeling is a powerful way to add intelligence to your application. It enables applications to predict outcomes against new data.
The act of incorporating predictive analytics into your applications involves two major phases:
The act of incorporating predictive analytics into your applications involves two major phases:
model training and model operationalization.
In this sample, you will learn how to create a predictive model in R and operationalize it with SQL Server 2016.
@@ -24,7 +24,7 @@ Follow the step by step tutorial [here](http://aka.ms/sqldev/R) to walk through
<!-- Delete the ones that don't apply -->
- **Applies to:** SQL Server 2016 (or higher)
- **Key features:** SQL Server R Services
- **Key features:** SQL Server R Services
- **Workload:** SQL Server R Services
- **Programming Language:** T-SQL, R, JavaScript (NodeJS)
- **Authors:** Nellie Gustafsson
@@ -47,7 +47,7 @@ After that, you can download a DB backup file and restore it using Setup.sql. [D
## Run this sample app
1. From SQL Server Management Studio or SQL Server Data Tools connect to your SQL Server 2016 or vNext SQL database and execute setup.sql to restore the sample DB
2. From SQL Server Management Studio or SQL Server Data Tools, execute Predictive Model.sql script to set up tables, train model, predict using that model etc.
2. From SQL Server Management Studio or SQL Server Data Tools, execute Predictive Model.sql script to set up tables, train model, predict using that model etc.
This is all covered step by step in the [tutorial](http://aka.ms/sqldev/R)
3. Navigate to the folder where you have downloaded sample and run **npm install** in command window, or run setup.bat if you are on Windows operating system. This command will install necessary npm packages defined in project.json.
@@ -84,11 +84,11 @@ The R script that generates a predictive model and uses it to predict rental cou
### Predictive Model.SQL
Takes the R code in PredictiveModel.R and deploys it inside SQL Server. Creating stored procedures and tables for training, storing models and creating stored procedures for prediction.
### app.js
### app.js
File that contains startup code.
### db.js
### db.js
File that contains functions that wrap Tedious library
### predictions.js
### predictions.js
File that contains action that will be called to get the predictions
Service uses Tedious library for data access and built-in JSON functionalities that are available in SQL Server 2016 and Azure SQL Database.
@@ -7,17 +7,17 @@
// If you have a named instance, you need to put the name here:
options: { instanceName: 'MyNamedInstance', database: 'TutorialDB' }
};
var Connection = require('tedious').Connection;
var connection = new Connection(config);
return connection;
}
function createRequest(query, connection) {
var Request = require('tedious').Request;
var req =
new Request(query,
new Request(query,
function (err, rowCount) {
if (err) {
throw err;
@@ -29,33 +29,33 @@ function createRequest(query, connection) {
}
function stream (query, connection, output, defaultContent) {
errorHandler = function (ex) { throw ex; };
var request = query;
if (typeof query == "string") {
request = this.createRequest(query, connection);
}
var empty = true;
request.on('row', function (columns) {
empty = false;
output.write(columns[0].value);
});
request.on('done', function (rowCount, more, rows) {
if (empty) {
output.write(defaultContent);
}
output.end();
});
request.on('doneProc', function (rowCount, more, rows) {
if (empty) {
output.write(defaultContent);
}
output.end();
});
connection.on('connect', function (err) {
if (err) {
throw err;
@@ -8,14 +8,14 @@ var TYPES = require('tedious').TYPES;
/* GET list of all predictions for a given year */
router.get('/', function (req, res) {
router.get('/', function (req, res) {
//Call stored proc and get predictions
var conn = db.createConnection();
var request = db.createRequest("EXEC get_rental_predictions 2015", conn);
db.stream(request, conn, res, '[]');
});
@@ -2,15 +2,15 @@
<html>
<head>
<title>Rental Predictions</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<link rel="shortcut icon" type="image/ico" href="media/images/favicon.ico">
<meta name="viewport" content="initial-scale=1.0, maximum-scale=2.0">
<link href="media/css/Bootstrap.css" rel="stylesheet" />
<link href="media/css/dataTables.bootstrap.css" rel="stylesheet" />
<link href="media/css/demo.css" rel="stylesheet" />
<script src="media/js/lib/jquery.js"></script>
<script src="media/js/lib/Bootstrap.js"></script>
<script src="media/js/lib/jquery.dataTables.js"></script>
@@ -21,11 +21,11 @@
$(document).ready(function() {
var table = $('#example').DataTable(
{
"ajax": "/predictions/",
"columns": [
{ "data": "Year", "width": "10%" },
@@ -37,11 +37,11 @@ $(document).ready(function() {
{ "data": "Snow", "width": "10%" },
{ "data": "Holiday", "width": "10%" }
]
});
} );
@@ -57,7 +57,7 @@ $(document).ready(function() {
<thead><th>Year</th><th>RentalCount_Predicted</th><th>RentalCount_Actual</th><th>Month</th>><th>Day</th><th>WeekDay</th><th>FSnow</th><th>FHoliday</th></thead>
<tbody>
</tbody>
</table>
</div>
</body>
@@ -17,12 +17,12 @@
$(nPrevious).append($('<span>' + (oSettings.oLanguage.oPaginate.sPrevious) + '</span>'));
$(nFirst).append($('<span>1</span>'));
$(nNext).append($('<span>' + (oSettings.oLanguage.oPaginate.sNext) + '</span>'));
nFirst.className = "paginate_button first active";
nPrevious.className = "paginate_button previous";
nNext.className = "paginate_button next";
ul.append(nPrevious);
ul.append(nFirst);
ul.append(nNext);
@@ -80,26 +80,26 @@
for (var i = 0, iLen = an.length ; i < iLen ; i++) {
var buttons = an[i].getElementsByTagName('li');
$(buttons).removeClass("active");
if (oSettings._iDisplayStart === 0) {
buttons[0].className = "paginate_buttons disabled previous";
buttons[buttons.length - 1].className = "paginate_button enabled next";
} else {
buttons[0].className = "paginate_buttons enabled previous";
}
var page = Math.round(oSettings._iDisplayStart / oSettings._iDisplayLength) + 1;
if (page == buttons.length-1 && oSettings.aiDisplay.length > 0) {
$new = $('<li class="dynamic_page_item active"><span>' + page + "</span></li>");
$(buttons[buttons.length - 1]).before($new);
$new.click(function () {
$(oSettings.nTable).DataTable().page(page-1);
fnCallbackDraw(oSettings);
});
} else
$(buttons[page]).addClass("active");
if (oSettings.fnDisplayEnd() == oSettings.fnRecordsDisplay()
||
oSettings.aiDisplay.length < oSettings._iDisplayLength) {
@@ -4,102 +4,102 @@ GO
/****** Object: Database [LendingClub] Script Date: 12/29/2016 9:29:37 PM ******/
CREATE DATABASE [LendingClub]
CONTAINMENT = NONE
ON PRIMARY
( NAME = N'LendingClubData', FILENAME = N'C:\Tiger\DATA\LendingClub.mdf' , SIZE = 19210240KB , MAXSIZE = UNLIMITED, FILEGROWTH = 65536MB ),
ON PRIMARY
( NAME = N'LendingClubData', FILENAME = N'C:\Tiger\DATA\LendingClub.mdf' , SIZE = 19210240KB , MAXSIZE = UNLIMITED, FILEGROWTH = 65536MB ),
FILEGROUP [InMemOLTP] CONTAINS MEMORY_OPTIMIZED_DATA DEFAULT
( NAME = N'InMem', FILENAME = N'C:\Tiger\DATA\InMem' , MAXSIZE = UNLIMITED)
LOG ON
LOG ON
( NAME = N'LendingClubLog', FILENAME = N'C:\Tiger\DATA\LendingClub_log.ldf' , SIZE = 512MB , MAXSIZE = 2048GB , FILEGROWTH = 64MB )
GO
ALTER DATABASE [LendingClub] SET COMPATIBILITY_LEVEL = 130
GO
ALTER DATABASE [LendingClub] SET ANSI_NULL_DEFAULT OFF
ALTER DATABASE [LendingClub] SET ANSI_NULL_DEFAULT OFF
GO
ALTER DATABASE [LendingClub] SET ANSI_NULLS OFF
ALTER DATABASE [LendingClub] SET ANSI_NULLS OFF
GO
ALTER DATABASE [LendingClub] SET ANSI_PADDING OFF
ALTER DATABASE [LendingClub] SET ANSI_PADDING OFF
GO
ALTER DATABASE [LendingClub] SET ANSI_WARNINGS OFF
ALTER DATABASE [LendingClub] SET ANSI_WARNINGS OFF
GO
ALTER DATABASE [LendingClub] SET ARITHABORT OFF
ALTER DATABASE [LendingClub] SET ARITHABORT OFF
GO
ALTER DATABASE [LendingClub] SET AUTO_CLOSE OFF
ALTER DATABASE [LendingClub] SET AUTO_CLOSE OFF
GO
ALTER DATABASE [LendingClub] SET AUTO_SHRINK OFF
ALTER DATABASE [LendingClub] SET AUTO_SHRINK OFF
GO
ALTER DATABASE [LendingClub] SET AUTO_UPDATE_STATISTICS ON
ALTER DATABASE [LendingClub] SET AUTO_UPDATE_STATISTICS ON
GO
ALTER DATABASE [LendingClub] SET CURSOR_CLOSE_ON_COMMIT OFF
ALTER DATABASE [LendingClub] SET CURSOR_CLOSE_ON_COMMIT OFF
GO
ALTER DATABASE [LendingClub] SET CURSOR_DEFAULT GLOBAL
ALTER DATABASE [LendingClub] SET CURSOR_DEFAULT GLOBAL
GO
ALTER DATABASE [LendingClub] SET CONCAT_NULL_YIELDS_NULL OFF
ALTER DATABASE [LendingClub] SET CONCAT_NULL_YIELDS_NULL OFF
GO
ALTER DATABASE [LendingClub] SET NUMERIC_ROUNDABORT OFF
ALTER DATABASE [LendingClub] SET NUMERIC_ROUNDABORT OFF
GO
ALTER DATABASE [LendingClub] SET QUOTED_IDENTIFIER OFF
ALTER DATABASE [LendingClub] SET QUOTED_IDENTIFIER OFF
GO
ALTER DATABASE [LendingClub] SET RECURSIVE_TRIGGERS OFF
ALTER DATABASE [LendingClub] SET RECURSIVE_TRIGGERS OFF
GO
ALTER DATABASE [LendingClub] SET ENABLE_BROKER
ALTER DATABASE [LendingClub] SET ENABLE_BROKER
GO
ALTER DATABASE [LendingClub] SET AUTO_UPDATE_STATISTICS_ASYNC OFF
ALTER DATABASE [LendingClub] SET AUTO_UPDATE_STATISTICS_ASYNC OFF
GO
ALTER DATABASE [LendingClub] SET DATE_CORRELATION_OPTIMIZATION OFF
ALTER DATABASE [LendingClub] SET DATE_CORRELATION_OPTIMIZATION OFF
GO
ALTER DATABASE [LendingClub] SET TRUSTWORTHY OFF
ALTER DATABASE [LendingClub] SET TRUSTWORTHY OFF
GO
ALTER DATABASE [LendingClub] SET ALLOW_SNAPSHOT_ISOLATION OFF
ALTER DATABASE [LendingClub] SET ALLOW_SNAPSHOT_ISOLATION OFF
GO
ALTER DATABASE [LendingClub] SET PARAMETERIZATION SIMPLE
ALTER DATABASE [LendingClub] SET PARAMETERIZATION SIMPLE
GO
ALTER DATABASE [LendingClub] SET READ_COMMITTED_SNAPSHOT OFF
ALTER DATABASE [LendingClub] SET READ_COMMITTED_SNAPSHOT OFF
GO
ALTER DATABASE [LendingClub] SET HONOR_BROKER_PRIORITY OFF
ALTER DATABASE [LendingClub] SET HONOR_BROKER_PRIORITY OFF
GO
ALTER DATABASE [LendingClub] SET RECOVERY SIMPLE
ALTER DATABASE [LendingClub] SET RECOVERY SIMPLE
GO
ALTER DATABASE [LendingClub] SET MULTI_USER
ALTER DATABASE [LendingClub] SET MULTI_USER
GO
ALTER DATABASE [LendingClub] SET PAGE_VERIFY CHECKSUM
ALTER DATABASE [LendingClub] SET PAGE_VERIFY CHECKSUM
GO
ALTER DATABASE [LendingClub] SET DB_CHAINING OFF
ALTER DATABASE [LendingClub] SET DB_CHAINING OFF
GO
ALTER DATABASE [LendingClub] SET FILESTREAM( NON_TRANSACTED_ACCESS = OFF )
ALTER DATABASE [LendingClub] SET FILESTREAM( NON_TRANSACTED_ACCESS = OFF )
GO
ALTER DATABASE [LendingClub] SET TARGET_RECOVERY_TIME = 60 SECONDS
ALTER DATABASE [LendingClub] SET TARGET_RECOVERY_TIME = 60 SECONDS
GO
ALTER DATABASE [LendingClub] SET DELAYED_DURABILITY = DISABLED
ALTER DATABASE [LendingClub] SET DELAYED_DURABILITY = DISABLED
GO
ALTER DATABASE [LendingClub] SET QUERY_STORE = OFF
@@ -132,7 +132,7 @@ GO
ALTER DATABASE SCOPED CONFIGURATION FOR SECONDARY SET QUERY_OPTIMIZER_HOTFIXES = PRIMARY;
GO
ALTER DATABASE [LendingClub] SET READ_WRITE
ALTER DATABASE [LendingClub] SET READ_WRITE
GO
USE [LendingClub]
@@ -259,7 +259,7 @@ CREATE TABLE [dbo].[LoanStatsStaging]
[total_bc_limit] [int] NULL,
[total_il_high_credit_limit] [int] NULL,
INDEX [LoanStats_index] NONCLUSTERED HASH
INDEX [LoanStats_index] NONCLUSTERED HASH
(
[id]
)WITH ( BUCKET_COUNT = 2000000)
@@ -384,7 +384,7 @@ CREATE TABLE [dbo].[LoanStats](
[total_bc_limit] [int] NULL,
[total_il_high_credit_limit] [int] NULL,
[is_bad] int NULL,
CONSTRAINT [PK__LoanStat] PRIMARY KEY CLUSTERED
CONSTRAINT [PK__LoanStat] PRIMARY KEY CLUSTERED
(
[id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
@@ -522,7 +522,7 @@ BEGIN
,[total_bal_ex_mort]
,[total_bc_limit]
,[total_il_high_credit_limit])
SELECT DISTINCT
SELECT DISTINCT
[member_id]
,[loan_amnt]
,[funded_amnt]
@@ -649,10 +649,10 @@ GO
CREATE TABLE [dbo].[LoanStatsPredictions]
(
[is_bad_Pred] [float] NULL,
[is_bad_Pred] [float] NULL,
[id] [int] NULL
INDEX [LoanStats_index] NONCLUSTERED HASH
INDEX [LoanStats_index] NONCLUSTERED HASH
(
[id]
)WITH ( BUCKET_COUNT = 2000000)
@@ -666,10 +666,10 @@ SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[LoanPredictionsWhatIf]
(
[is_bad_Pred] [float] NULL,
[is_bad_Pred] [float] NULL,
[id] [int] NULL
INDEX [LoanStats_index] NONCLUSTERED HASH
INDEX [LoanStats_index] NONCLUSTERED HASH
(
[id]
)WITH ( BUCKET_COUNT = 2000000)
@@ -688,7 +688,7 @@ CREATE TABLE [dbo].[RunTimeStats]
[RunTime] [datetime] NOT NULL,
[Operation] [varchar](255) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL,
INDEX [RunTimeStats_index] NONCLUSTERED HASH
INDEX [RunTimeStats_index] NONCLUSTERED HASH
(
[SessionID]
)WITH ( BUCKET_COUNT = 32)
@@ -719,13 +719,13 @@ SET QUOTED_IDENTIFIER ON
GO
-- Stored procedure for scoring loans for the base predictions
CREATE PROCEDURE [dbo].[ScoreLoans]
CREATE PROCEDURE [dbo].[ScoreLoans]
@start bigint,
@end bigint
AS
BEGIN
AS
BEGIN
-- Declare the variables to get the input data and the scoring model
-- Declare the variables to get the input data and the scoring model
DECLARE @inquery nvarchar(max) = N'SELECT id,revol_util, int_rate, mths_since_last_record, annual_inc_joint, dti_joint, total_rec_prncp, all_util, is_bad FROM [dbo].[LoanStats] where [id] >= ' + CAST(@start as varchar(255)) + 'and [id] <= ' + CAST(@end as varchar(255));
DECLARE @model varbinary(max) = (SELECT TOP 1 [model] FROM [dbo].[models])
@@ -733,11 +733,11 @@ BEGIN
INSERT INTO [dbo].[RunTimeStats] VALUES (@@SPID, GETDATE(),'Start')
-- Score the loans and store them in a table
INSERT INTO [dbo].[LoanStatsPredictions]
EXEC sp_execute_external_script
INSERT INTO [dbo].[LoanStatsPredictions]
EXEC sp_execute_external_script
@language = N'R',
@script = N'
rfModel <- unserialize(as.raw(model));
@script = N'
rfModel <- unserialize(as.raw(model));
OutputDataSet<-rxPredict(rfModel, data = InputDataSet, extraVarsToWrite = c("id"))
',
@input_data_1 = @inquery,
@@ -747,7 +747,7 @@ BEGIN
-- Log end of processing time
INSERT INTO [dbo].[RunTimeStats] VALUES (@@SPID, GETDATE(),'End')
END
END
GO
@@ -757,25 +757,25 @@ SET QUOTED_IDENTIFIER ON
GO
-- Stored procedure to in
CREATE PROCEDURE [dbo].[ScoreLoansWhatIf]
CREATE PROCEDURE [dbo].[ScoreLoansWhatIf]
@start bigint,
@end bigint,
@incr float
AS
BEGIN
AS
BEGIN
-- Declare the variables to get the input data and the scoring model
-- Declare the variables to get the input data and the scoring model
DECLARE @inquery nvarchar(max) = N'SELECT id,revol_util, (int_rate+ ' + CAST (@Incr as varchar(5)) + ') as int_rate, mths_since_last_record, annual_inc_joint, dti_joint, total_rec_prncp, all_util,is_bad FROM [dbo].[LoanStats] where [id] >= ' + CAST(@start as varchar(255)) + 'and [id] <= ' + CAST(@end as varchar(255));
DECLARE @model varbinary(max) = (SELECT TOP 1 [model] FROM [dbo].[models])
-- Log beginning of processing time
INSERT INTO [dbo].[RunTimeStats] VALUES (@@SPID, GETDATE(),'Start')
INSERT INTO [dbo].[LoanPredictionsWhatIf]
EXEC sp_execute_external_script
INSERT INTO [dbo].[LoanPredictionsWhatIf]
EXEC sp_execute_external_script
@language = N'R',
@script = N'
rfModel <- unserialize(as.raw(model));
@script = N'
rfModel <- unserialize(as.raw(model));
OutputDataSet<-rxPredict(rfModel, data = InputDataSet, extraVarsToWrite = c("id"))
',
@input_data_1 = @inquery,
@@ -786,7 +786,7 @@ BEGIN
INSERT INTO [dbo].[RunTimeStats] VALUES (@@SPID, GETDATE(),'End')
END
END
GO
@@ -2,7 +2,7 @@
Author: Amit Banerjee
Contact: @mssqltiger | @banerjeeamit | http://aka.ms/sqlserverteam
Description:
Description:
1. First step is to use Lending Club CSV files and import them into an in-memory staging table
2. Second step is to transform the staging data and move them into the table which will be used for the predictions
@@ -19,7 +19,7 @@ function ImportData ($csvFile)
$SqlServer = "." # TODO: Change the name of SQL Server instance name
$dbName = "LendingClub" # TODO: Change the name of the database
$csvData = Get-Content -Path $csvFile | Select-Object -Skip 1 | Where-Object {$_.id -notcontains "*Total amount funded in policy code*"} | ConvertFrom-Csv
$csvData = Get-Content -Path $csvFile | Select-Object -Skip 1 | Where-Object {$_.id -notcontains "*Total amount funded in policy code*"} | ConvertFrom-Csv
foreach ($line in $csvData)
{
@@ -31,9 +31,9 @@ function ImportData ($csvFile)
#Write-Host "Removing NULLs"
$Query = $Query.Replace(",,",",NULL,")
}
#Write-Host $query
Invoke-Sqlcmd -ServerInstance $SqlServer -Database $dbName -Query $Query -Verbose
Invoke-Sqlcmd -ServerInstance $SqlServer -Database $dbName -Query $Query -Verbose
}
}
@@ -1,7 +1,7 @@
USE [LendingClub]
GO
UPDATE [dbo].[LoanStats]
UPDATE [dbo].[LoanStats]
SET [is_bad] = (CASE WHEN loan_status IN ('Late (16-30 days)', 'Late (31-120 days)', 'Default', 'Charged Off') THEN 1 ELSE 0 END);
CREATE NONCLUSTERED COLUMNSTORE INDEX [ncci_LoanStats] ON [dbo].[LoanStats]
@@ -14,11 +14,11 @@ CREATE TABLE [dbo].[models](
GO
INSERT INTO [dbo].[models]
EXEC sp_execute_external_script
@language = N'R',
@script = N'
EXEC sp_execute_external_script
@language = N'R',
@script = N'
randomForestObj <- rxDForest(is_bad ~ revol_util + int_rate + mths_since_last_record + annual_inc_joint + dti_joint + total_rec_prncp + all_util, InputDataSet)
model <- data.frame(payload = as.raw(serialize(randomForestObj, connection=NULL)))
',
@input_data_1 = N'SELECT revol_util, int_rate, mths_since_last_record, annual_inc_joint, dti_joint, total_rec_prncp, all_util,is_bad FROM [dbo].[LoanStats] WHERE (ABS(CAST((BINARY_CHECKSUM(id, NEWID())) as int)) % 100) < 75',
@input_data_1 = N'SELECT revol_util, int_rate, mths_since_last_record, annual_inc_joint, dti_joint, total_rec_prncp, all_util,is_bad FROM [dbo].[LoanStats] WHERE (ABS(CAST((BINARY_CHECKSUM(id, NEWID())) as int)) % 100) < 75',
@output_data_1_name = N'model';
@@ -2,19 +2,19 @@ create external resource pool "lcerp1" with (affinity numanode = (0));
create external resource pool "lcerp2" with (affinity numanode = (1));
create resource pool "lcrp1" with (affinity numanode = (0));
create resource pool "lcrp2" with (affinity numanode = (1));
create workload group "rg0" using "lcrp1", external "lcerp1";
create workload group "rg1" using "lcrp2", external "lcerp2";
USE [master]
@@ -3,7 +3,7 @@
$StartCtr = 1
$Increment = 250000
$EndCtr = $Increment
$EndCtr = $Increment
$FinalCount = 1195907
$vServerName = $env:computername
$vDatabaseName = "LendingClub"
@@ -27,7 +27,7 @@ while ($EndCtr -le $FinalCount)
# Wait till jobs complete
while (Get-Job -State Running)
{
Start-Sleep 1
}
@@ -4,7 +4,7 @@
# Counter values
$StartCtr = 1
$Increment = 250000
$EndCtr = $Increment
$EndCtr = $Increment
$FinalCount = 1195907
# Server name
@@ -35,7 +35,7 @@ Write-Host "Starting parallel jobs to score " $count "loans" -ForegroundColor Ye
while ($EndCtr -le $FinalCount)
{
$SqlScript = [ScriptBlock]::Create("Invoke-Sqlcmd -ServerInstance `"" + $vServerName + "`" -Query `"EXEC [dbo].[ScoreLoansWhatIf] " + $StartCtr + "," + $EndCtr + "," + $IntRate + "`" -Database `"$vDatabaseName`"")
Start-Job -ScriptBlock $SqlScript
Start-Job -ScriptBlock $SqlScript
$StartCtr += $Increment
$EndCtr += $Increment
}
@@ -43,7 +43,7 @@ while ($EndCtr -le $FinalCount)
# Wait till jobs complete
while (Get-Job -State Running)
{
Start-Sleep 1
}
@@ -1,5 +1,5 @@
# Loan Classification using SQL Server 2016 R Services #
During an Ignite keynote session, we had shown how customers are able to achieve a scale up of 1 million predictions/sec using SQL Server 2016 R Services. This sample contains all the scripts required to emulate a similar setup using Lending Club data with SQL Server 2016 R Services and an Azure Data Science VM.
**Scripts**
@@ -1,15 +1,15 @@
Implementing Predictive Analytics in Your Applications
============================================================
Predictive analytics are a powerful way to add intelligence to your application, it enables you to predict outcomes against data that is new to your application. The Microsoft data platform provides numerous ways you can add predictive analytics to your applications
Predictive analytics are a powerful way to add intelligence to your application, it enables you to predict outcomes against data that is new to your application. The Microsoft data platform provides numerous ways you can add predictive analytics to your applications
What is predictive analytics?
What is predictive analytics?
-------------------------------
Before we get into the implementation, lets address a fundamental question—**what is predictive analytics?** At their core, predictive tasks are those that predict one value given a set of other values as input. In other words, predictive tasks learn (or are taught) how to make predictions. This learning is captured in a model by an algorithm. Think of the model as the way the learnings are compactly summarized. When you want to make a prediction, you invoke a prediction operation and provide the model as one of the inputs, along with the input values against which you want to form a prediction. Predictive analytics is the act of applying prediction (and your model) to your data to gain new insights.
Before we get into the implementation, lets address a fundamental question—**what is predictive analytics?** At their core, predictive tasks are those that predict one value given a set of other values as input. In other words, predictive tasks learn (or are taught) how to make predictions. This learning is captured in a model by an algorithm. Think of the model as the way the learnings are compactly summarized. When you want to make a prediction, you invoke a prediction operation and provide the model as one of the inputs, along with the input values against which you want to form a prediction. Predictive analytics is the act of applying prediction (and your model) to your data to gain new insights.
So, **what are some examples of predictive analytics?** These fall out into two basic categories. You have _prediction that aims to predict the class (or category) of something_. For example, you can have single class classification that tries to predict if an email is spam or not spam—so the class is either “spam” or “not spam”. You can also have multi-class classification, that predicts an outcome from a set of possible outcomes. For example, you can have a multi-class classification that predicts if a consumer is at “high risk”, “moderate risk”, or “low risk” of default on a loan.
So, **what are some examples of predictive analytics?** These fall out into two basic categories. You have _prediction that aims to predict the class (or category) of something_. For example, you can have single class classification that tries to predict if an email is spam or not spam—so the class is either “spam” or “not spam”. You can also have multi-class classification, that predicts an outcome from a set of possible outcomes. For example, you can have a multi-class classification that predicts if a consumer is at “high risk”, “moderate risk”, or “low risk” of default on a loan.
You also have _numeric prediction_. Instead of trying to predict a class from a fixed set of options, numeric prediction tries to predict a numeric value from a continuous range of numbers. For example, you might try to predict how long of a delay in minutes a flight will experience or the currency value of a particular stock in the stock market.
You also have _numeric prediction_. Instead of trying to predict a class from a fixed set of options, numeric prediction tries to predict a numeric value from a continuous range of numbers. For example, you might try to predict how long of a delay in minutes a flight will experience or the currency value of a particular stock in the stock market.
Prediction on the Microsoft Data Platform
----------------------------------------------
@@ -17,7 +17,7 @@ The Microsoft Data Platform provide numerous ways you can build predictive model
![Alternatives to train and use a model](imgs/UseModelForPrediction.png "Model Train and use")
As you can infer from the diagram, the act of incorporating predictive analytics into your applications involves two major phases: model creation and model operationalization. Conceptually, these are very simple to understand.
As you can infer from the diagram, the act of incorporating predictive analytics into your applications involves two major phases: model creation and model operationalization. Conceptually, these are very simple to understand.
**Model Creation:** During model creation, you train your predictive model (by showing it sample data along with the outcomes) and test that it works (at least that it predicts results better than random chance would). You save this model so you can use it later when you want to make predictions against new data.
@@ -25,7 +25,7 @@ As you can infer from the diagram, the act of incorporating predictive analytics
Options for Model Creation
-----------------------------
Lets begin by understanding the various ways you can train and test your model. When creating your model, you can train your model locally. This is amounts to authoring and running R or Python scripts on your development workstation. For example, you might use the integrated development environment R GUI (a component in Microsoft R Open) or R Tools for Visual Studio to author your R scripts that train your model, help you test it and visualize the results.
Lets begin by understanding the various ways you can train and test your model. When creating your model, you can train your model locally. This is amounts to authoring and running R or Python scripts on your development workstation. For example, you might use the integrated development environment R GUI (a component in Microsoft R Open) or R Tools for Visual Studio to author your R scripts that train your model, help you test it and visualize the results.
Alternately, the training can be done using resources that are remote to your development workstation. The Microsoft Data Platform offers the following options for this:
@@ -33,19 +33,19 @@ Alternately, the training can be done using resources that are remote to your de
* **HDInsight:** HDInsight provides numerous ways you can train predictive models using a cluster of servers running in Azure. With R Server on Spark and R Server on Hadoop, you author R scripts whose execution runs across the cluster to train (and test) your model. If you deploy an HDInsight with Spark, you can use Spark ML to program the training and testing models using Scala, Java, Python or R. Generally, the data used to train models in HDInsight comes from a form of highly scalable block storage such as HDFS, Azure Data Lake Store or Azure Storage Blobs.
* **SQL R Services:** SQL in R Services enable you to train and test predictive models in the context of SQL Server 2016. You author T-SQL procedures that contain embedded R scripts, and the SQL Server database engine takes care of the execution. Because it executes in the context of SQL Server, your models can be easily trained against data stored within tables within your database.
* **SQL R Services:** SQL in R Services enable you to train and test predictive models in the context of SQL Server 2016. You author T-SQL procedures that contain embedded R scripts, and the SQL Server database engine takes care of the execution. Because it executes in the context of SQL Server, your models can be easily trained against data stored within tables within your database.
Option for Model Operationalization
---------------------------------------
The Microsoft Data Platform also provide multiple ways of adding predictive analytics to an application. As you can see in the diagram, while there are many ways to train a model there tend to be only a few practical ways to use the model from an application.
The Microsoft Data Platform also provide multiple ways of adding predictive analytics to an application. As you can see in the diagram, while there are many ways to train a model there tend to be only a few practical ways to use the model from an application.
* **Invoke Predict within a Script:** When running within a local environment, you can easily use the trained model as input into your predictive script.
* **Invoke Predict within a Script:** When running within a local environment, you can easily use the trained model as input into your predictive script.
* **Invoke Predictive Web Service:** When running in a remote environment, a common approach is to encapsulate the call to prediction in a web service/Rest API operation that is readily invoked from an application. For Azure ML, this is as easy as a few clicks to deploy a predictive experiment as a web service. For HDInsight, this amounts to exporting your trained model to a file and importing the model into a compatible host such a Microsofts DeployR (for models created with R Server on Spark or Hadoop), which wraps a web services layer around a prediction script written in R.
* **Invoke Predictive Web Service:** When running in a remote environment, a common approach is to encapsulate the call to prediction in a web service/Rest API operation that is readily invoked from an application. For Azure ML, this is as easy as a few clicks to deploy a predictive experiment as a web service. For HDInsight, this amounts to exporting your trained model to a file and importing the model into a compatible host such a Microsofts DeployR (for models created with R Server on Spark or Hadoop), which wraps a web services layer around a prediction script written in R.
* **Invoke a Predictive Store Procedure:** When using SQL R Services, you can package the code that invokes prediction using your model within a stored procedure. Therefore, integrating a prediction into your application becomes a matter of executing a stored a procedure in SQL Server—something that most applications can easily accomplish regardless of whether they are written in .NET, node.js, Java...
While each approach has its merits, in the accompanying lab, well examine how to augment a node.js application with predictive analytics using this last approach that invokes a predictive stored procedure running in SQL Server 2016.
While each approach has its merits, in the accompanying lab, well examine how to augment a node.js application with predictive analytics using this last approach that invokes a predictive stored procedure running in SQL Server 2016.
**Done with the intro?**
[Start the lab](scripts/Lab.md)
@@ -1,33 +1,33 @@
use taxidata
go
CREATE PROCEDURE [dbo].[TrainTipPredictionModel]
AS
BEGIN
DECLARE @inquery nvarchar(max) = N'
select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance
from nyctaxi_features
'
CREATE PROCEDURE [dbo].[TrainTipPredictionModel]
AS
BEGIN
DECLARE @inquery nvarchar(max) = N'
select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance
from nyctaxi_features
'
--delete previous stored models
truncate table dbo.nyc_taxi_models
-- Insert the trained model into a database table
INSERT INTO nyc_taxi_models
-- Insert the trained model into a database table
INSERT INTO nyc_taxi_models
EXEC sp_execute_external_script
@language = N'R',
@script = N'
@language = N'R',
@script = N'
## Create model
logitObj <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs + direct_distance, data = InputDataSet)
## Create model
logitObj <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs + direct_distance, data = InputDataSet)
## Serialize model and put it in data frame
trained_model <- data.frame(model=as.raw(serialize(logitObj, NULL)));
',
@input_data_1 = @inquery,
@output_data_1_name = N'trained_model'
;
## Serialize model and put it in data frame
trained_model <- data.frame(model=as.raw(serialize(logitObj, NULL)));
',
@input_data_1 = @inquery,
@output_data_1_name = N'trained_model'
;
END
END
GO
@@ -11,8 +11,8 @@ dbhandle <- odbcDriverConnect('driver={SQL Server};server=<yourservername>;datab
##Run the query to brin the data we'll use to create the model
res <- sqlQuery(dbhandle, 'select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance from nyctaxi_features')
##Create the model...
model <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs + direct_distance, res)
##Create the model...
model <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs + direct_distance, res)
summary(model)
##Now, let's create the frame with the parameters for the prediction
@@ -8,84 +8,84 @@ Requirements
-----------------
* [SQL Server 2016 Developer Edition](https://www.microsoft.com/en-us/sql-server/sql-server-editions-developers) or higher
* [Visual Studio Code](http://code.visualstudio.com)
* Node.js
* Node.js
A tool to run SQL scripts against your SQL Server database, such as [SQL Server Management Studio (SSMS)](https://msdn.microsoft.com/library/mt238290.aspx)
A tool to run SQL scripts against your SQL Server database, such as [SQL Server Management Studio (SSMS)](https://msdn.microsoft.com/library/mt238290.aspx)
This lab assumes you have setup SQL Server 2016 Developer Edition locally on your workstation or a remote instance.
Required SQL Server Configuration
* Make sure that your installation of SQL Server includes R Services.
* Using SQL Server Configuration Manager, make sure that TCP/IP connections are enabled to your instance of SQL Server.
* Using SQL Server Configuration Manager, make sure that TCP/IP connections are enabled to your instance of SQL Server.
* Be sure that the SQL Server, SQL Server Launchpad and SQL Server Browser services are all running.
Download the Project
----------------------
Clone this repo to have the sample application and setup scripts.
### Setup the sample database
The following steps will get your taxidata database setup and loaded with data.
1. Using the SQL tool of your choice (SQL Server Management Studio, Visual Studio Code with the MSSQL extension), connect to your database and execute the following scripts (provided with the project files) to create the database, a table to hold the taxi data, and load the 1.7M records into the taxi trip data table.
- *CreateDatabase.sql*
- *Create nyctaxi_features Table.sql*
1. Using the SQL tool of your choice (SQL Server Management Studio, Visual Studio Code with the MSSQL extension), connect to your database and execute the following scripts (provided with the project files) to create the database, a table to hold the taxi data, and load the 1.7M records into the taxi trip data table.
- *CreateDatabase.sql*
- *Create nyctaxi_features Table.sql*
- *Load nyctaxi_features using BCP.sql* **Important**, make sure that you edit the script to point to the folder where you've cloned the .bcp file
2. Next, create a table valued function that will package inputs received by the stored procedure into a tabular format by executing the following script. You will use this function later within the stored procedure that makes predictions.
- *Create Function fnEngineerFeatures.sql*
- *Create Function fnEngineerFeatures.sql*
3. Execute the following script to create a table that will persist the predictive model you will generate. Observe that this table has a schema that consists of one column of type varbinary(max). This column will hold the serialized representation of your model.
- *Create nyc_taxi_models Table.sql*
- *Create nyc_taxi_models Table.sql*
### Train the Model
To train your model, you will create a stored procedure that you can run at any time to train your model and store its serialized form in the nyctaxi_features table.
1. Execute the following script to define the stored procedure:
- *Create Procedure TrainTipPredictionModel.sql*
2. Execute the following script to train the model and store it.
1. Execute the following script to define the stored procedure:
- *Create Procedure TrainTipPredictionModel.sql*
2. Execute the following script to train the model and store it.
- *Exec TrainTipPredictionModel.sql*. You may need to [configure external scripts](https://msdn.microsoft.com/en-us/library/mt590884.aspx) and restart SQL.
Lets take a closer look at the contents of the stored procedure.
```
CREATE PROCEDURE [dbo].[TrainTipPredictionModel]
AS
BEGIN
DECLARE @inquery nvarchar(max) = N'
select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance
from nyctaxi_features
'
-- Before insterting a new model, we delete the previous one
CREATE PROCEDURE [dbo].[TrainTipPredictionModel]
AS
BEGIN
DECLARE @inquery nvarchar(max) = N'
select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance
from nyctaxi_features
'
-- Before insterting a new model, we delete the previous one
truncate table dbo.nyc_taxi_models
-- Insert the trained model into a database table
INSERT INTO nyc_taxi_models
-- Insert the trained model into a database table
INSERT INTO nyc_taxi_models
EXEC sp_execute_external_script
@language = N'R',
@script = N'
@language = N'R',
@script = N'
##Create model
##Create model
logitObj <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs +
direct_distance, data = InputDataSet)
direct_distance, data = InputDataSet)
##Serialize model and put it in data frame
trained_model <- data.frame(model=as.raw(serialize(logitObj, NULL)));
',
@input_data_1 = @inquery,
@output_data_1_name = N'trained_model'
;
##Serialize model and put it in data frame
trained_model <- data.frame(model=as.raw(serialize(logitObj, NULL)));
',
@input_data_1 = @inquery,
@output_data_1_name = N'trained_model'
;
END
GO
END
GO
```
The procedure begins by defining a query that retrieves the sample data contained in the nyctaxi_features table.
The procedure begins by defining a query that retrieves the sample data contained in the nyctaxi_features table.
```
DECLARE @inquery nvarchar(max) = N'
select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance
from nyctaxi_features
'
```
This query is passed as one of the parameters to sp_execute_external_script, via the @input_data_1 parameter.
DECLARE @inquery nvarchar(max) = N'
select tipped, passenger_count, trip_time_in_secs, trip_distance, direct_distance
from nyctaxi_features
'
```
This query is passed as one of the parameters to sp_execute_external_script, via the @input_data_1 parameter.
Next, a call to sp_execute_external_script is constructed. The return value of this stored procedure call is the serialized model, which is saved into the nyc_taxi_models table. The inputs to sp_execute_external_script are:
* __@language__: needs to indicate that the script is written in the R language.
@@ -96,11 +96,11 @@ Next, a call to sp_execute_external_script is constructed. The return value of t
Looking at the R script specifically, we have:
```
##Create model
logitObj <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs +
direct_distance, data = InputDataSet)
##Serialize model and put it in data frame
trained_model <- data.frame(model=as.raw(serialize(logitObj, NULL)));
##Create model
logitObj <- rxLogit(tipped ~ passenger_count + trip_distance + trip_time_in_secs +
direct_distance, data = InputDataSet)
##Serialize model and put it in data frame
trained_model <- data.frame(model=as.raw(serialize(logitObj, NULL)));
```
The second line trains the model using the **rxLogit** method, which performs a **logistic regression**. Observe that the inputs are expressed in a formula syntax that describes what feature to predict and what features to use in its prediction:
@@ -111,17 +111,17 @@ tipped ~ passenger_count + trip_distance + trip_time_in_secs + direct_distance
The formula reads as such: predict **tipped** given the *passenger_count*, *trip_distance*, *trip_time_in_secs* and *direct_distance*. The source of these features comes from the data set made available via the InputDataSet variable.
After that, we serialize the model into a **data.frame** and store it in the *trained_model* variable, which is returned as an output result set consisting of one cell with the column name *trained_model* and value of the serialized model.
After that, we serialize the model into a **data.frame** and store it in the *trained_model* variable, which is returned as an output result set consisting of one cell with the column name *trained_model* and value of the serialized model.
### Operationalize the Model
With trained model in hand, you are ready operationalize the model and make it available to your application.
### Operationalize the Model
With trained model in hand, you are ready operationalize the model and make it available to your application.
1. Execute the following script to operationalize the model in a stored procedure.
- *Create Procedure PredictTip.sql*
1. Execute the following script to operationalize the model in a stored procedure.
- *Create Procedure PredictTip.sql*
Lets take a closer look at this stored procedure.
```
CREATE PROCEDURE [dbo].[PredictTip]
CREATE PROCEDURE [dbo].[PredictTip]
@passenger_count int = 0,
@trip_distance float = 0,
@trip_time_in_secs int = 0,
@@ -131,7 +131,7 @@ BEGIN
-- Package the inputs as a table
DECLARE @inquery nvarchar(max) = N'
SELECT * FROM [dbo].[fnEngineerFeatures](
SELECT * FROM [dbo].[fnEngineerFeatures](
@passenger_count,
@trip_distance,
@trip_time_in_secs,
@@ -142,17 +142,17 @@ BEGIN
DECLARE @lmodel2 varbinary(max) = (SELECT TOP 1 model FROM nyc_taxi_models);
-- Invoke the prediction
EXEC sp_execute_external_script
EXEC sp_execute_external_script
@language = N'R',
@script = N'
mod <- unserialize(as.raw(model));
OutputDataSet<-rxPredict(modelObject = mod, data = InputDataSet,
outData = NULL,
predVarNames = "Score", type = "response",
OutputDataSet<-rxPredict(modelObject = mod, data = InputDataSet,
outData = NULL,
predVarNames = "Score", type = "response",
writeModelVars = FALSE, overwrite = TRUE);
',
@input_data_1 = @inquery,
@params = N'@model varbinary(max),
@params = N'@model varbinary(max),
@passenger_count int,
@trip_distance float,
@trip_time_in_secs int ,
@@ -171,7 +171,7 @@ The PredictTip procedure takes as input the passenger count, trip distance (odom
The first query uses the fnEngineerFeatures table valued function to package the values of the input parameters as a table:
```
DECLARE @inquery nvarchar(max) = N'
SELECT * FROM [dbo].[fnEngineerFeatures](
SELECT * FROM [dbo].[fnEngineerFeatures](
@passenger_count,
@trip_distance,
@trip_time_in_secs,
@@ -189,18 +189,18 @@ Following that we invoke the prediction using a call to sp_execute_external_scri
```
--Invoke the prediction
EXEC sp_execute_external_script
EXEC sp_execute_external_script
@language = N'R',
@script = N'
mod <- unserialize(as.raw(model));
OutputDataSet<-rxPredict(modelObject = mod, data = InputDataSet,
outData = NULL,
predVarNames = "Score", type = "response",
outData = NULL,
predVarNames = "Score", type = "response",
writeModelVars = FALSE, overwrite = TRUE);
',
@input_data_1 = @inquery,
@params = N'@model varbinary(max),
@params = N'@model varbinary(max),
@passenger_count int,
@trip_distance float,
@trip_time_in_secs int ,
@@ -226,12 +226,12 @@ The R script used for prediction has only the following two lines:
mod <- unserialize(as.raw(model));
OutputDataSet<-rxPredict(modelObject = mod, data = InputDataSet,
outData = NULL, predVarNames = "Score", type = "response",
outData = NULL, predVarNames = "Score", type = "response",
writeModelVars = FALSE, overwrite = TRUE);
```
The first line deserializes the model so it is in a form useable by the **rxPredict** method. The second line invokes rxPredict which uses the model against the supplied row of data (within InputDataSet, which is sourced from the query in @inquery).
The first line deserializes the model so it is in a form useable by the **rxPredict** method. The second line invokes rxPredict which uses the model against the supplied row of data (within InputDataSet, which is sourced from the query in @inquery).
Finally, the script ends using the following line:
@@ -239,24 +239,24 @@ Finally, the script ends using the following line:
WITH RESULT SETS ((Score float));
```
This schematizes the result set returned in OutputDataSet within the R script. This data set has one column, labeled Score with a type of float. The label of “Score” was configured in the call to rxPredict via the predVarNames parameter.
This schematizes the result set returned in OutputDataSet within the R script. This data set has one column, labeled Score with a type of float. The label of “Score” was configured in the call to rxPredict via the predVarNames parameter.
### Execute a prediction in T-SQL
Now you are ready to give your predictive stored procedure a test run.
1. Run the following script to predict the probability of a tip using the PredictTip stored procedure.
- *Exec PredictTip.sql*
- *Exec PredictTip.sql*
2. You should get a result set consisting of one row and one column (labeled Score), for example:
### Execute the sample in Node.js
Now lets integrate a call to this stored procedure from our node.js sample application.
1. Within the root of the project directory, at the command line execute:
1. Within the root of the project directory, at the command line execute:
```
npm install tedious
npm install tedious
```
2. This will install the tedious package which we use to connect to SQL Server.
@@ -291,7 +291,7 @@ connection.on('connect', function(err) {
console.log("Unable to Connect: " + err);
return;
}
// If no error, then good to go...
console.log("Connected.");
@@ -300,7 +300,7 @@ connection.on('connect', function(err) {
});
```
7. Observe that this method connects to SQL Server using tedious. If it connects successfully, it executes the method executeStatement().
7. Observe that this method connects to SQL Server using tedious. If it connects successfully, it executes the method executeStatement().
8. Look at the implementation for executeStatement().
```
@@ -340,7 +340,7 @@ function executeStatement() {
11. Run the sample application by typing:
```
node TipPredictor.js
node TipPredictor.js
```
12. You should see output like the following (which in this case means there is a 53% chance of a tip):
@@ -22,7 +22,7 @@ connection.on('connect', function(err) {
console.log("Unable to Connect: " + err);
return;
}
// If no error, then good to go...
console.log("Connected.");
@@ -18,7 +18,7 @@
<DataSet Name="Feature_Settings">
<Query>
<DataSourceName>SqlConnection</DataSourceName>
<CommandText>/*
<CommandText>/*
Retrieve the R Services installation setting &amp; configuration options:
Implied Authentication Configuration is checked by verifying if login exists for SQLRUserGroup
*/
@@ -70,12 +70,12 @@ begin
, @script = N'
# Retrieve properties like R.home, libPath &amp; default packages
OutputDataSet &lt;- data.frame(
property_name = c("R.home", "libPaths", "defaultPackages"),
property_name = c("R.home", "libPaths", "defaultPackages"),
property_value = c(R.home(), .libPaths(), paste(getOption("defaultPackages"), collapse=", "))
)
# Transform R version properties to data.frame
OutputDataSet &lt;- rbind(OutputDataSet, data.frame(
property_name = names(R.version),
property_name = names(R.version),
property_value = matrix(unlist(R.version), nrow = length(R.version), byrow = TRUE),
stringsAsFactors = FALSE)
)
@@ -19,11 +19,11 @@
<CommandText>/*
Get list of extended events from SQLSatellite package.
*/
select o.name as event_name, o.description
from sys.dm_xe_objects o
join sys.dm_xe_packages p
on o.package_guid = p.guid
where o.object_type = 'event'
select o.name as event_name, o.description
from sys.dm_xe_objects o
join sys.dm_xe_packages p
on o.package_guid = p.guid
where o.object_type = 'event'
and p.name = 'SQLSatellite';</CommandText>
<rd:UseGenericDesigner>true</rd:UseGenericDesigner>
</Query>
@@ -61,23 +61,23 @@ Custom reports for R Services allow you to perform the following tasks from Obje
### R Services - Configuration.rdl
This report can be used to view the installation settings of R Services and properties of the R runtime. You can also use this report to configure R Services after installation.
This report can be used to view the installation settings of R Services and properties of the R runtime. You can also use this report to configure R Services after installation.
### R Services - Packages.rdl
This report lists the R packages installed on the SQL Server instance and properties like version, name.
This report lists the R packages installed on the SQL Server instance and properties like version, name.
### R Services - Resource Usage.rdl
This report can be used to view the CPU, Memory, IO consumption of SQL Server & R scripts execution. You can also view the memory setting of external resource pools.
This report can be used to view the CPU, Memory, IO consumption of SQL Server & R scripts execution. You can also view the memory setting of external resource pools.
### R Services - Extended Events.rdl
This report can be used to view the extended events that are available to get more insights into R scripts execution.
This report can be used to view the extended events that are available to get more insights into R scripts execution.
### R Services - Execution Statistics.rdl
This report can be used to view the execution statistics of R services. For example, you can get the total number of R scripts executions, number of parallel executions and frequently used RevoScaleR functions.
This report can be used to view the execution statistics of R services. For example, you can get the total number of R scripts executions, number of parallel executions and frequently used RevoScaleR functions.
<a name=related-links></a>
@@ -10,11 +10,11 @@
**Description**
- **telcoChurn-setUp.R** - Setting up relevant R packages
- **telcoChurn-evaluate.R** - Defining pre-functions for model evaluation
- **telcoChurn-evaluate.R** - Defining pre-functions for model evaluation
- **telcoChurn-dataExploration.R** - Creating a Shiny application to explore and visualize the data
- **telcoChurn-dataPreparation.R** - Defining functions to do data pre-processing and spliting in order to generate suitable training and testing data sets
- **telcoChurn-trainModel.R** - Defining a function to train the telco churn model with rxDForest algorithm
- **telcoChurn-main.R** - Main R file driving the demo execution
- **telcoChurn-modelComparison.R** - R file to build and compare different tree-based classification models, including CRAN R algorithms - randomForest and xgboost, RevoScaleR algorithms rxDForest and rxBTrees
- **telcoChurn-modelComparison.R** - R file to build and compare different tree-based classification models, including CRAN R algorithms - randomForest and xgboost, RevoScaleR algorithms rxDForest and rxBTrees
----------
@@ -34,9 +34,9 @@ data <- cdrDF %>%
ui <- fluidPage(
tags$style(HTML("
@import url('https://fonts.googleapis.com/css?family=Poppins');
body {
font-family: 'Poppins', 'Lucida Grande', Verdana, Lucida, Helvetica, Arial, Calibri, sans-serif;
color: rgb(0,0,0);
background-color: #d2d2d2;
@@ -45,7 +45,7 @@ ui <- fluidPage(
titlePanel("Telco Customer Churn"),
# Sidebar with a slider input for number of bins
# Sidebar with a slider input for number of bins
sidebarLayout(
sidebarPanel(
sliderInput("sc", "Scale size of circles (also redraws map to show only the last added state)",
@@ -65,9 +65,9 @@ ui <- fluidPage(
p("Impact of call failure rate on churn"),
plotOutput("MyPlot3", height = "200px"),
h2("About"),
HTML("<p>Created by Fang Zhou with R and Shiny leaflet. R users can download the
HTML("<p>Created by Fang Zhou with R and Shiny leaflet. R users can download the
cleaned and tidy call detail record data from <a href = 'https://github.com/Microsoft/sql-server-samples/tree/master/samples/features/r-services/Telco%20Customer%20Churn'>
https://github.com/Microsoft/sql-server-samples/tree/master/samples/features/r-services/Telco%20Customer%20Churn</a>.
https://github.com/Microsoft/sql-server-samples/tree/master/samples/features/r-services/Telco%20Customer%20Churn</a>.
The latitute and longitute for each USA state can be found from <a href = 'http://dev.maxmind.com/geoip/legacy/codes/state_latlon/'>
http://dev.maxmind.com/geoip/legacy/codes/state_latlon/</a>.")
),
@@ -1,6 +1,6 @@
####################################################################################################
## Title: Telco Customer Churn
## Description: Data Preparation
## Description: Data Preparation
## Author: Microsoft
## Note: Prepare the training and testing data sets by pre-processing and spliting on raw data
####################################################################################################
@@ -13,10 +13,10 @@ dataPreparation <- function(sqlSettings, trainTable, testTable) {
dataVars <- dataVars[!dataVars %in% c("year", "month")]
dataVars <- paste(dataVars, collapse = ", ")
dataQuery <- paste("select", dataVars, "from", inputTable)
## Create sql server data sources
inputDataSQL = RxSqlServerData(sqlQuery = dataQuery,
connectionString = sqlConnString,
inputDataSQL = RxSqlServerData(sqlQuery = dataQuery,
connectionString = sqlConnString,
colInfo = cdrColInfo)
trainDataSQL <- RxSqlServerData(connectionString = sqlConnString,
table = trainTable,
@@ -30,7 +30,7 @@ dataPreparation <- function(sqlSettings, trainTable, testTable) {
}
preProcess <- function(inData, outData1, outData2) {
## Clean missing data
## Clean missing data
## Remove duplicate rows
cdrDF <- rxDataStep(inData = inData,
removeMissings = TRUE,
@@ -48,8 +48,8 @@ preProcess <- function(inData, outData1, outData2) {
overwrite = TRUE)
trainFile <- splitFile[[2]]
testFile <- splitFile[[1]]
## SMOTE on training data
## SMOTE on training data
trainDF <- rxDataStep(inData = trainFile, varsToDrop = c("ind"))
testDF <- rxDataStep(inData = testFile, varsToDrop = c("ind"))
@@ -62,7 +62,7 @@ preProcess <- function(inData, outData1, outData2) {
smotetrainDF <- cbind(smotetrain$X, smotetrain$Y)
names(smotetrainDF)[names(smotetrainDF) == "smotetrain$Y"] <- "churn"
trainDF <- smotetrainDF
## Load final training data and testing data into SQL
rxDataStep(inData = trainDF, outFile = outData1, overwrite = TRUE)
rxDataStep(inData = testDF, outFile = outData2, overwrite = TRUE)
@@ -8,7 +8,7 @@
## Define functions for model evaluation
####################################################################################################
## Define evaluation metrics
evaluateModel <- function(data, observed, predicted)
evaluateModel <- function(data, observed, predicted)
{
confusion <- table(data[[observed]], data[[predicted]])
print(confusion)
@@ -27,8 +27,8 @@ evaluateModel <- function(data, observed, predicted)
return(metrics)
}
## Define ROC curve
rxrocCurve <- function(data, observed, predicted)
## Define ROC curve
rxrocCurve <- function(data, observed, predicted)
{
data <- data[, c(observed, predicted)]
data[[observed]] <- as.numeric(as.character(data[[observed]]))
@@ -7,7 +7,7 @@
####################################################################################################
## Settings
# In order to run this script, you need to set the values of the parameters in this section to your
# own values.
# own values.
####################################################################################################
## SQL database and login credentials. Please change this part to your own values.
@@ -107,8 +107,8 @@ trainTable <- "edw_cdr_train"
testTable <- "edw_cdr_test"
predTable <- "edw_cdr_pred"
## Data preparation.
# We now delete unnecessary columns, clean missing values, remove duplicate rows,
## Data preparation.
# We now delete unnecessary columns, clean missing values, remove duplicate rows,
# but more importantly, split the raw data into training and testing data sets followed by SMOTE.
system.time({
dataPreparation(sqlSettings, trainTable, testTable)
@@ -129,7 +129,7 @@ rxSummary( ~ churn, data = testDataSQL)
####################################################################################################
## Train model
####################################################################################################
## Switch to sql compute context.
## Switch to sql compute context.
# From now on, all the executions will be done in the SQL server
rxSetComputeContext(sqlCompute)
@@ -21,12 +21,12 @@ train_df <- rxDataStep(inData = train_table)
test_df <- rxDataStep(inData = test_table)
####################################################################################################
## Random forest modeling with randomForest on the data frame
## Random forest modeling with randomForest on the data frame
####################################################################################################
library(randomForest)
## Train model
system.time({
system.time({
forest_model <- randomForest(churn ~ .,
data = train_df,
ntree = 8,
@@ -66,7 +66,7 @@ rxrocCurve(data = pred_df,
predicted = "randomForest_Probability")
####################################################################################################
## Extreme gradient boost modeling with xgboost on the data frame
## Extreme gradient boost modeling with xgboost on the data frame
####################################################################################################
library(Matrix)
library(xgboost)
@@ -118,7 +118,7 @@ rxrocCurve(data = pred_df,
predicted = "xgboost_Probability")
####################################################################################################
## Decision forest modeling with rxDForest on SQL data source
## Decision forest modeling with rxDForest on SQL data source
####################################################################################################
## Train model
@@ -1,25 +1,25 @@
Data Science for Database Professionals
As a data professional, do you wonder how you can leverage data science for creating new value in your organization? In this sample, learn how you can leverage your familiar knowledge on working with databases, and learn how you can get started with doing data science with databases.
As a data professional, do you wonder how you can leverage data science for creating new value in your organization? In this sample, learn how you can leverage your familiar knowledge on working with databases, and learn how you can get started with doing data science with databases.
----------
**Example**
Businesses need an effective strategy for managing customer churn. Customer churn includes customers stopping the use of a service, switching to a competitor service, switching to a lower-tier experience in the service or reducing engagement with the service.
Businesses need an effective strategy for managing customer churn. Customer churn includes customers stopping the use of a service, switching to a competitor service, switching to a lower-tier experience in the service or reducing engagement with the service.
In this use case, we look at how a mobile phone carrier company can proactively identify customers more likely to churn in the near term in order to improve the service and create custom outreach campaigns that help retain the customers.
In this use case, we look at how a mobile phone carrier company can proactively identify customers more likely to churn in the near term in order to improve the service and create custom outreach campaigns that help retain the customers.
Mobile phone carriers face an extremely competitive market. Many mobile carriers lose revenue from postpaid customers due to churn. Hence the ability to proactively and accurately identify customer churn at scale can be a huge competitive advantage. Some of the factors contributing to mobile phone customer churn includes: Perceived frequent service disruptions, poor customer service experiences in online/retail stores, offers from other competing carriers (better family plan, data plan, etc.).
Mobile phone carriers face an extremely competitive market. Many mobile carriers lose revenue from postpaid customers due to churn. Hence the ability to proactively and accurately identify customer churn at scale can be a huge competitive advantage. Some of the factors contributing to mobile phone customer churn includes: Perceived frequent service disruptions, poor customer service experiences in online/retail stores, offers from other competing carriers (better family plan, data plan, etc.).
Using a concrete example of building a predictive customer churn model for mobile service provider, well share how you can jumpstart by
- Running R scripts using SQL Server as the compute context
- Operationalize your R scripts using stored procedures.
- Operationalize your R scripts using stored procedures.
The insights delivered by these models are visualized using a Power BI dashboard
(e.g.[ https://powerbi.microsoft.com/en-us/industries/telco]( https://powerbi.microsoft.com/en-us/industries/telco)) or a SQL Report (telcoChurn-reportBuilder.rdl).
(e.g.[ https://powerbi.microsoft.com/en-us/industries/telco]( https://powerbi.microsoft.com/en-us/industries/telco)) or a SQL Report (telcoChurn-reportBuilder.rdl).
----------
@@ -27,7 +27,7 @@ The insights delivered by these models are visualized using a Power BI dashboard
You have to do the following set-up before playing with this demo.
- Install SQL Server 2016 or create a SQL Server 2016 Enterprise VM on Azure with Standalone R Server and R Services installed/configured.
- Install SQL Server 2016 or create a SQL Server 2016 Enterprise VM on Azure with Standalone R Server and R Services installed/configured.
- Install R IDE: R Tools for Visual Studio or R Studio.
- Install ReportBuilder for SQL Server 2016 Enterprise.
- Validate the successful installation.
@@ -40,8 +40,8 @@ This sample consists of the following directory structure.
- **Data** - Download the data files [edw_cdr.csv](https://sqlchoice.blob.core.windows.net/sqlchoice/samples/telco-customer-churn-v1/edw_cdr.csv) and [state_latlon.csv](https://sqlchoice.blob.core.windows.net/sqlchoice/samples/telco-customer-churn-v1/state_latlon.csv).
- **R** - This folder contains the R code that you can run in any R IDE.
- **SQL Server** - This folder contains the sql files that you can run to create T-SQL stored procedures (with R code embeded) and try out this telco churn example.
- **ReportBuilder** - This folder contains a sample SQL report created by ReportBuilder.
- **SQL Server** - This folder contains the sql files that you can run to create T-SQL stored procedures (with R code embeded) and try out this telco churn example.
- **ReportBuilder** - This folder contains a sample SQL report created by ReportBuilder.
To jumpstart, run the T-SQL files (telcoChurn-operationalize.sql and telcoChurn-main.sql)
@@ -20,7 +20,7 @@ The database consists of the following tables
- **edw\_cdr\_train**- Training data
- **edw\_cdr\_test** - Testing data
- **edw\_cdr\_pred** - Predicted results
and the following stored procedures
- **generate_cdr_rx_forest** - Train decision forest model with the rxDForest algorithm in RevoScaleR library
@@ -80,7 +80,7 @@ as
begin
declare @rx_model varbinary(max) = (select model from cdr_models where model_name = @model);
-- Predict based on the specified model:
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
@@ -137,7 +137,7 @@ begin
execute sp_execute_external_script
@language = N'R'
, @script = N'
evaluateModel <- function(data, observed, predicted)
evaluateModel <- function(data, observed, predicted)
{
confusion <- table(data[[observed]], data[[predicted]])
print(confusion)
@@ -176,7 +176,7 @@ go
exec model_evaluate
go
--Create a stored procedure to generate roc curve
--Create a stored procedure to generate roc curve
drop procedure if exists model_roccurve;
go
create procedure model_roccurve
@@ -186,7 +186,7 @@ begin
@language = N'R'
, @script = N'
require("RevoScaleR");
rxrocCurve <- function(data, observed, predicted)
rxrocCurve <- function(data, observed, predicted)
{
data <- data[, c(observed, predicted)]
data[[observed]] <- as.numeric(as.character(data[[observed]]))
@@ -205,7 +205,7 @@ begin
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'
select * from edw_cdr_pred'
, @input_data_1_name = N'edw_cdr_pred'
@@ -226,7 +226,7 @@ create procedure pie
as
begin
exec sp_execute_external_script
@language = N'R',
@language = N'R',
@script = N'
# Set output directory for files
# Prior to plotting ensure there are no files with same file names as the out files below in the above directory.
@@ -248,7 +248,7 @@ exec sp_execute_external_script
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'select * from edw_cdr_pred'
, @input_data_1_name = N'edw_cdr_pred'
with result sets ((plot varbinary(max)));
@@ -266,7 +266,7 @@ create procedure stackedbar
as
begin
exec sp_execute_external_script
@language = N'R',
@language = N'R',
@script = N'
# Set output directory for files
# Prior to plotting ensure there are no files with same file names as the out files below in the above directory.
@@ -290,7 +290,7 @@ exec sp_execute_external_script
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'select * from edw_cdr_pred'
, @input_data_1_name = N'edw_cdr_pred'
with result sets ((plot varbinary(max)));
@@ -7,12 +7,12 @@ Use the R files provided in this folder, and is the “driver” program
1. **TelcoChurn-Setup.R -** Setup/Configuration (e.g. install the relevant R packages, setup relevant R functions, setup your working directory/etc)
2. **TelcoChurn-DataExploration.R** - Data Exploration, Visualization (e.g. on data distributions, etc).
2. **TelcoChurn-DataExploration.R** - Data Exploration, Visualization (e.g. on data distributions, etc).
3. **TelcoChurn-DataPreparation.R** - Prepare training and testing data sets in two ways.
Upload and prepare the data stored in the SQL Server database. Performs feature engineering, split on raw data in R and load them into SQL. This includes persisting relevant train/test data back into the database.
4. **TelcoChurn-ModelBuilding.R** - Train the model, and Evaluate the model
4. **TelcoChurn-ModelBuilding.R** - Train the model, and Evaluate the model
----------
@@ -46,7 +46,7 @@ ggplot(data = myData,
bw = "SJ",
colour = NA) +
geom_rug(colour = "salmon") +
theme_minimal()
theme_minimal()
#boxplot of age by churn
ggplot(data = myData,
@@ -66,8 +66,8 @@ ggplot(data = myData,
bw = "SJ",
colour = NA) +
geom_rug(colour = "salmon") +
theme_minimal()
theme_minimal()
#boxplot of annualincome by churn
ggplot(data = myData,
aes(x = reorder(churn, - annualincome),
@@ -13,7 +13,7 @@ local <- RxLocalParallel()
rxOptions(reportProgress = 0)
####################################################################################################
##Method 1: Prepare the raw, training and testing data sets all in SQL database
##Method 1: Prepare the raw, training and testing data sets all in SQL database
####################################################################################################
col_info <- list(age = list(type = "integer"),
annualincome = list(type = "integer"),
@@ -162,7 +162,7 @@ table(testData$churn)
library(unbalanced)
myvars <- names(trainData) %in% c("churn")
SMOTEData <- ubSMOTE(X = trainData[!myvars], Y = trainData$churn, perc.over = 200, k = 3, perc.under = 500, verbose = TRUE)
SMOTEData <- ubSMOTE(X = trainData[!myvars], Y = trainData$churn, perc.over = 200, k = 3, perc.under = 500, verbose = TRUE)
newSMOTEData <- cbind(SMOTEData$X, SMOTEData$Y)
colnames(newSMOTEData)
names(newSMOTEData)[names(newSMOTEData) == "SMOTEData$Y"] <- "churn"
@@ -40,7 +40,7 @@ str(trainData)
str(testData)
####################################################################################################
## Random forest modeling with randomForest on the data frame
## Random forest modeling with randomForest on the data frame
####################################################################################################
library(randomForest)
@@ -88,7 +88,7 @@ plot(M.ROC.randomForest[1,], M.ROC.randomForest[2,], main = "ROC Curves for Rand
text(0.2, 0, paste("AUC=", round(randomForest.auc, 2)))
####################################################################################################
## Extreme gradient boost modeling with xgboost on the data frame
## Extreme gradient boost modeling with xgboost on the data frame
####################################################################################################
library(Matrix)
library(xgboost)
@@ -140,7 +140,7 @@ plot(M.ROC.xgboost[1,], M.ROC.xgboost[2,], main = "ROC Curves for Xgboost", col
text(0.5, 0, paste("AUC=", round(xgboost.auc, 2)))
####################################################################################################
## Decision forest modeling with rxDForest on SQL data source
## Decision forest modeling with rxDForest on SQL data source
####################################################################################################
##Train Model
@@ -1,21 +1,21 @@
Data Science for Database Professionals
As a data professional, do you wonder how you can leverage data science for creating new value in your organization? In this sample, learn how you can leverage your familiar knowledge on working with databases, and learn how you can get started with doing data science with databases.
As a data professional, do you wonder how you can leverage data science for creating new value in your organization? In this sample, learn how you can leverage your familiar knowledge on working with databases, and learn how you can get started with doing data science with databases.
----------
**Example**
Businesses need an effective strategy for managing customer churn. Customer churn includes customers stopping the use of a service, switching to a competitor service, switching to a lower-tier experience in the service or reducing engagement with the service.
Businesses need an effective strategy for managing customer churn. Customer churn includes customers stopping the use of a service, switching to a competitor service, switching to a lower-tier experience in the service or reducing engagement with the service.
In this use case, we look at how a mobile phone carrier company can proactively identify customers more likely to churn in the near term in order to improve the service and create custom outreach campaigns that help retain the customers.
In this use case, we look at how a mobile phone carrier company can proactively identify customers more likely to churn in the near term in order to improve the service and create custom outreach campaigns that help retain the customers.
Mobile phone carriers face an extremely competitive market. Many mobile carriers lose revenue from postpaid customers due to churn. Hence the ability to proactively and accurately identify customer churn at scale can be a huge competitive advantage. Some of the factors contributing to mobile phone customer churn includes: Perceived frequent service disruptions, poor customer service experiences in online/retail stores, offers from other competing carriers (better family plan, data plan, etc.).
Mobile phone carriers face an extremely competitive market. Many mobile carriers lose revenue from postpaid customers due to churn. Hence the ability to proactively and accurately identify customer churn at scale can be a huge competitive advantage. Some of the factors contributing to mobile phone customer churn includes: Perceived frequent service disruptions, poor customer service experiences in online/retail stores, offers from other competing carriers (better family plan, data plan, etc.).
Using a concrete example of building a predictive customer churn model for mobile service provider, well share how you can jumpstart by
- Running R scripts using SQL Server as the compute context
- Operationalize your R scripts using stored procedures.
- Operationalize your R scripts using stored procedures.
The insights delivered by these models are visualized using a Power BI dashboard
@@ -29,7 +29,7 @@ This sample consists of the following directory structure.
- **R** - This folder contains the R code that you can run in any R IDE.
- **SQL Server** - This folder contains the SQL Server backup file (telcoedw2.bak) that you can restore to a SQL Server 2016 instance. The database telcoedw2.bak contains the data (train, test and new data) that you can use for exploring the example. The stored procedures (with R code) are included in the database.
- **SQL Server** - This folder contains the SQL Server backup file (telcoedw2.bak) that you can restore to a SQL Server 2016 instance. The database telcoedw2.bak contains the data (train, test and new data) that you can use for exploring the example. The stored procedures (with R code) are included in the database.
To jumpstart, run the T-SQL file (TelcoChurn-Main.sql)
@@ -14,27 +14,27 @@ RECONFIGURE;
GO
--InstallPackage using sp_execute_external_script
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'install.packages("ggplot")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
-- using Download.file command
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'download.file("https://cran.r-project.org/bin/windows/contrib/3.4/ggplot2_2.1.0.zip","ggplot")
install.packages("ggplot", repos = NULL, type = "source")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
--InstallPackage using sp_execute_external_script
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'install.packages("gplots")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
-- using Download.file command
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'download.file("https://cran.r-project.org/bin/windows/contrib/3.4/gplots_3.0.1.zip","gplots")
install.packages("gplots", repos = NULL, type = "source")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
@@ -42,28 +42,28 @@ WITH RESULT SETS (( ResultSet VARCHAR(50)));
--InstallPackage using sp_execute_external_script
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'install.packages("xgboost")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
-- using Download.file command
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'download.file("https://cran.r-project.org/bin/windows/contrib/3.4/xgboost_0.4-4.zip","xgboost")
install.packages("xgboost", repos = NULL, type = "source")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
--InstallPackage using sp_execute_external_script
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'install.packages("qcc")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
-- using Download.file command
EXECUTE sp_execute_external_script
@language = N'R'
EXECUTE sp_execute_external_script
@language = N'R'
,@script=N'download.file("https://cran.r-project.org/bin/windows/contrib/3.4/qcc_2.6.zip","qcc")
install.packages("qcc", repos = NULL, type = "source")'
WITH RESULT SETS (( ResultSet VARCHAR(50)));
@@ -19,6 +19,6 @@ The database consists of the following tables
- **edw\_cdr\_train**- Training data
- **edw\_cdr\_test** - Test data
- **edw\_cdr\_test\_pred** - Predicted results
----------
@@ -28,7 +28,7 @@ create table cdr_rx_models(
);
go
--Create a stored procedure to train Xgboost model
--Create a stored procedure to train Xgboost model
drop procedure if exists generate_cdr_xgboost;
go
create procedure generate_cdr_xgboost
@@ -45,12 +45,12 @@ begin
dtrainData$data <- Matrix(ntrainData, sparse = TRUE)
dtrainData$label <- as.numeric(trainData$churn) - 1
str(dtrainData)
xgboost_model <- xgboost(data = dtrainData$data,
label = dtrainData$label,
max.depth = 32,
eta = 1,
nthread = 2,
nround = 2,
xgboost_model <- xgboost(data = dtrainData$data,
label = dtrainData$label,
max.depth = 32,
eta = 1,
nthread = 2,
nround = 2,
objective = "binary:logistic")
xgboost_model <- data.frame(payload = as.raw(serialize(xgboost_model, connection=NULL)));
'
@@ -157,7 +157,7 @@ create procedure importance (@model varchar(100))
as
begin
declare @rx_model varbinary(max) = (select model from cdr_rx_models where model_name = @model);
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
@@ -190,7 +190,7 @@ as
begin
declare @rx_model varbinary(max) = (select model from cdr_rx_models where model_name = @model);
-- Predict based on the specified model:
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
library(Matrix)
@@ -244,7 +244,7 @@ as
begin
declare @rx_model varbinary(max) = (select model from cdr_rx_models where model_name = @model);
-- Predict based on the specified model:
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
@@ -297,7 +297,7 @@ as
begin
declare @rx_model varbinary(max) = (select model from cdr_rx_models where model_name = @model);
-- Predict based on the specified model:
exec sp_execute_external_script
exec sp_execute_external_script
@language = N'R'
, @script = N'
require("RevoScaleR");
@@ -419,7 +419,7 @@ roc_curve(data = edw_cdr_test_pred,
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'
select * from edw_cdr_test_pred'
, @input_data_1_name = N'edw_cdr_test_pred'
@@ -441,17 +441,17 @@ create procedure pareto
as
begin
exec sp_execute_external_script
@language = N'R',
@language = N'R',
@script = N'
# Set output directory for files
# Prior to plotting ensure there are no files with same file names as the out files below in the above directory.
# Calculate counts(percentages) of customers churned or non-churned
# Calculate counts(percentages) of customers churned or non-churned
require("RevoScaleR");
tmp<- rxCube(~ F(churn),edw_cdr,means=FALSE)
Results_df<- rxResultsDF(tmp)
library(qcc)
CountOfChurn<-
CountOfChurn<-
setNames(as.numeric(Results_df[,2]), Results_df[,1])
# Open a jpeg file and output plot in that file.
@@ -465,7 +465,7 @@ pareto<-pareto.chart(CountOfChurn,
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'select * from edw_cdr'
, @input_data_1_name = N'edw_cdr'
with result sets ((plot varbinary(max)));
@@ -483,7 +483,7 @@ create procedure histogram
as
begin
exec sp_execute_external_script
@language = N'R',
@language = N'R',
@script = N'
require("RevoScaleR");
# Set output directory for files
@@ -497,7 +497,7 @@ rxHistogram(~age|F(churn),edw_cdr,reportProgress=0)
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'select * from edw_cdr'
, @input_data_1_name = N'edw_cdr'
with result sets ((plot varbinary(max)));
@@ -515,7 +515,7 @@ create procedure heatmap
as
begin
exec sp_execute_external_script
@language = N'R',
@language = N'R',
@script = N'
# Set output directory for files
@@ -530,14 +530,14 @@ library(gplots)
image_file = tempfile();
jpeg(filename=image_file, width=800, height = 550);
print(
heatmap.2(data.matrix(Results_df[,-1]),
heatmap.2(data.matrix(Results_df[,-1]),
labRow=Results_df[,1], col=cm.colors(255),
trace="none",dendrogram ="none",
na.color=par("bg"))
);
dev.off();
OutputDataSet <- data.frame(data=readBin(file(image_file, "rb"), what=raw(), n=1e6));
'
'
, @input_data_1 = N'select * from edw_cdr'
, @input_data_1_name = N'edw_cdr'
with result sets ((plot varbinary(max)));