I am trying to calculate the gini index for a classification model done using GBTClassifier from the pyspark ml models. I cant seem to find a metrics which gives the roc_auc_score like the one in python sklearn.
Below is the code that I have used so far on databricks. I am currently using a dataset from the databricks
%fs ls databricks-datasets/adult/adult.data
from pyspark.sql.functions import *
from pyspark.ml.classification import RandomForestClassifier, GBTClassifier
from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, VectorSlicer
from pyspark.ml import Pipeline
from pyspark.ml.evaluation import BinaryClassificationEvaluator,MulticlassClassificationEvaluator
from pyspark.mllib.evaluation import BinaryClassificationMetrics
from pyspark.ml.linalg import Vectors
from pyspark.ml.tuning import ParamGridBuilder, TrainValidationSplit
dataset = spark.table("adult")
# spliting the train and test data frames
splits = dataset.randomSplit([0.7, 0.3])
train_df = splits[0]
test_df = splits[1]
def churn_predictions(train_df,
target_col,
# algorithm,
# model_parameters = conf['model_parameters']
):
"""
#Function attributes
dataframe - training df
target - target varibale in the model
Algorithm - Algorithm used
model_parameters - model parameters used to fine tune the model
"""
# one hot encoding and assembling
encoding_var = [i[0] for i in train_df.dtypes if (i[1]=='string') & (i[0]!=target_col)]
num_var = [i[0] for i in train_df.dtypes if ((i[1]=='int') | (i[1]=='double')) & (i[0]!=target_col)]
string_indexes = [StringIndexer(inputCol = c, outputCol = 'IDX_' + c, handleInvalid = 'keep') for c in encoding_var]
onehot_indexes = [OneHotEncoderEstimator(inputCols = ['IDX_' + c], outputCols = ['OHE_' + c]) for c in encoding_var]
label_indexes = StringIndexer(inputCol = target_col, outputCol = 'label', handleInvalid = 'keep')
assembler = VectorAssembler(inputCols = num_var + ['OHE_' + c for c in encoding_var], outputCol = "features")
gbt = GBTClassifier(featuresCol = 'features', labelCol = 'label',
maxDepth = 5,
maxBins = 45,
maxIter = 20)
pipe = Pipeline(stages = string_indexes + onehot_indexes + [assembler, label_indexes, gbt])
model = pipe.fit(train_df)
return model
gbt_model = churn_predictions(train_df = train_df,
target_col = 'income')
#### prediction in test sample ####
gbt_predictions = gbt_model.transform(test_df)
# display(gbt_predictions)
gbt_evaluator = MulticlassClassificationEvaluator(
labelCol="label", predictionCol="prediction", metricName="accuracy")
accuracy = gbt_evaluator.evaluate(gbt_predictions) * 100
print("Accuracy on test data = %g" % accuracy)
gini_train = 2 * metrics.roc_auc_score(Y, pred_prob) - 1
as you can see in the last line of code there is clearly no metric called roc_auc_score to calculate the gini.
Really appreciate any help on this.
Normally Gini is used to evaluate a binary classification model.
You can calculate it in pyspark in the next way:
from pyspark.ml.evaluation import BinaryClassificationEvaluator
evaluator = BinaryClassificationEvaluator()
auc = evaluator.evaluate(gbt_predictions, {evaluator.metricName: "areaUnderROC"})
gini = 2 * auc - 1.0
Related
I am interested to have statistics summary as obtained in statsmodel, while using pyspark. I am new to pyspark. As a first step, I tried to run logistic regression using statstmodel and pyspark, to match the intercept and coefficients. However I get them different.
Common part for both:
from pyspark.sql import SparkSession
file_location = "/FileStore/tables/bank.csv"
spark = SparkSession.builder.appName('ml-bank').getOrCreate()
df = spark.read.csv(file_location, header = True, inferSchema = True)
temp_table_name = "bank_csv"
df.createOrReplaceTempView(temp_table_name)
numeric_features = [t[0] for t in df.dtypes if t[1] == 'int']
from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler
categoricalColumns = ['marital' ]
stages = []
for categoricalCol in categoricalColumns:
stringIndexer = StringIndexer(inputCol = categoricalCol, outputCol = categoricalCol + 'Index')
encoder = OneHotEncoder(inputCols=[stringIndexer.getOutputCol()], outputCols=[categoricalCol +
"classVec"])
stages += [stringIndexer, encoder]
label_stringIdx = StringIndexer(inputCol = 'deposit', outputCol = 'label')
stages += [label_stringIdx]
numericCols = ['age', 'balance']
assemblerInputs = [c + "classVec" for c in categoricalColumns] + numericCols
assembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features")
stages += [assembler]
from pyspark.ml import Pipeline
pipeline = Pipeline(stages = stages)
pipelineModel = pipeline.fit(df)
df = pipelineModel.transform(df)
selectedCols = ['label', 'features'] + cols
df = df.select(selectedCols)
Separate content starts: For Mllib:
#Test train split
train, test = df.randomSplit([0.7, 0.3], seed = 2018)
from pyspark.ml.classification import LogisticRegression
lr = LogisticRegression(featuresCol = 'features', labelCol = 'label',
maxIter=10,standardization=False,regParam=0.0,elasticNetParam=1)
lrModel = lr.fit(train)
predict_train=lrModel.transform(train)
predict_test=lrModel.transform(test)
print("Coefficients: " + str(lrModel.coefficients))
print("Intercept: " + str(lrModel.intercept))
It gives an output of coefficient and intercept as:
Coefficients: [-0.15224852297849636,0.4353532847935295,0.01418389527062097,4.6707000626430065e-05]
Intercept: -0.8298919632117885
Now statsmodel is used to find the coeffcient and intercept as follows:
import statsmodels.formula.api as smf
df2=df_stats.toPandas()
m1 = smf.logit(
formula='label ~ C(maritalIndex) + age+ balance',
data=df2) \
.fit()
m1.summary()
The intercept is obtained as seen in the image in this link:
The intercept is -1.0338. Similarly other coefficient are different from that from mllib (pyspark).
What did I try
Setting regularization to L1 (elasticNetParam=1) with regParam=0. This is to check if its something
related to regularization. The intention was to turn of regularization as discussed in
Replicate logistic regression model from pyspark in scikit-learn
scikit-learn
Could someone help me to find why I get different intercept and coefficients with pyspark.ml.classification and statstmodel.
Thanks
I am down loading stock data from you finance.
I am trying to generate stock trading signal using ANN.
I am getting Coefficient of determination is close to -1 and prediction is seem to be mirror image.
Can someone suggest.
I am getting similar Coefficient of determination with others ML.
It seems to be strange.
import numpy as np
import yfinance as yf
import talib as ta
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score, accuracy_score
from sklearn.metrics import classification_report
from sklearn.preprocessing import StandardScaler
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
# Ignore warnings
import warnings
warnings.filterwarnings('ignore')
import random
random.seed(42)
# YYYY-MM-DD
start_date = '2010-01-01'
end_date = '2020-08-03'
#df = yf.download(tickers="^NSEI", start=start_date, end=end_date, interval="1d", progress=False)
df = yf.download("SBIN.NS", start=start_date, end=end_date, interval="1d", progress=False)
def create_trading_condition(df):
df['RSI'] = ta.RSI(df['Adj Close'].values, timeperiod = 9)
df['MACD'] = ta.MACD(df['Adj Close'].values, fastperiod=12, slowperiod=26, signalperiod=9)[0]
df['Williams %R'] = ta.WILLR(df['High'].values, df['Low'].values, df['Adj Close'].values, 7)
df['C-O'] = df['Adj Close'] - df['Open']
df['H-L'] = df['High'] - df['Low']
df['STDEV']= df['Adj Close'].rolling(5).std()
df['TARGET'] = np.where(df['Adj Close'].shift(-1) > df['Adj Close'], 1, 0)
df = df.dropna()
X = df[['RSI', 'MACD', 'Williams %R', 'C-O', 'H-L', 'STDEV']]
Y = df['TARGET']
#print(df)
return (df, X, Y)
df_new, X, Y = create_trading_condition(df)
X_train,X_test,Y_train,Y_test = train_test_split(X, Y, shuffle=False, train_size=0.8)
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
clf = Sequential()
clf.add(Dense(units = 128, kernel_initializer = 'uniform', activation= 'relu', input_dim = X.shape[1]))
clf.add(Dense(units = 128, kernel_initializer = 'uniform', activation= 'relu'))
clf.add(Dense(units = 1, kernel_initializer = 'uniform', activation= 'sigmoid'))
clf.compile(optimizer = 'adam', loss = 'mean_squared_error', metrics= ['accuracy'])
clf.fit(X_train, Y_train, batch_size = 10, epochs = 100)
Y_pred = clf.predict(X_test)
#print(Y_pred)
Y_pred = (Y_pred > 0.5)
df_new['Y_pred'] = np.NaN
df_new.iloc[(len(df_new) - len(Y_pred)):,-1:] = Y_pred
trade_df = df_new.dropna()
trade_df['Tomorrows Returns'] = 0.
trade_df['Tomorrows Returns'] = np.log(trade_df['Adj Close']/trade_df['Adj Close'].shift(1))
trade_df['Tomorrows Returns'] = trade_df['Tomorrows Returns'].shift(-1)
# Y_pred = true for long position otherwise short position
trade_df['Strategy Returns'] = 0.
trade_df['Strategy Returns'] = np.where(trade_df['Y_pred'] == True, trade_df['Tomorrows Returns'], -trade_df['Tomorrows Returns'])
trade_df['Cumulative Market Returns'] = np.cumsum(trade_df['Tomorrows Returns'])
trade_df['Cumulative Strategy Returns'] = np.cumsum(trade_df['Strategy Returns'])
#accuracy_test = accuracy_score(Y_test, clf.predict(X_test))
# mean squared error
mse_test = mean_squared_error(Y_test, clf.predict(X_test))
# Coefficient of determination
#r2_test = r2_score(Y_test, clf.predict(X_test))
r2_test = r2_score(Y_test, Y_pred)
#report_test = classification_report(Y_test, clf.predict(X_test))
#print("Test accuracy score: %.2f" % accuracy_test)
print("Test mean squared error: %.2f" % mse_test)
print('Test R-square: %.2f\n' % r2_test)
#print(report_test)
import matplotlib.pyplot as plt
plt.figure(figsize=(10,5))
plt.plot(trade_df['Cumulative Market Returns'], color='r', label='Market Returns')
plt.plot(trade_df['Cumulative Strategy Returns'], color='g', label='Strategy Returns')
plt.legend()
plt.show()
I'm trying to add an L1 penalty to a specific layer of a neural network, and I have the code below (in which I attempt to add l1 penalty to the first layer). If I run it for lambda = 0 (i.e. no penalty), the output gets very close to the expected weights those being [10, 12, 2, 11, -0.25]) and if I run for enough epochs or reduce batch size it will get it exactly, as in the output below:
mlp.0.weight
Parameter containing:
tensor([[ 9.8657, -11.8305, 2.0242, 10.8913, -0.1978]],
requires_grad=True)
Then, when I run it for a large lambda, say 1000, I would expect these weights to shrink towards zero as there is a large penalty being added to the loss that we are trying to minimise. However, the opposite happens and the weights explode, as in the output below (for lam = 1000)
mlp.0.weight
Parameter containing:
tensor([[-13.9368, 9.9072, 2.2447, -11.6870, 26.7293]],
requires_grad=True)
If anyone could help me, that'd be great. I'm new to pytorch (but not the idea of regularisation), so I'm guessing it's something in my code that is the problem.
Thanks
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import numpy as np
from sklearn.linear_model import LinearRegression
class TrainDataset(Dataset):
def __init__(self, data):
self.data = data
def __len__(self):
return self.data.shape[0]
def __getitem__(self, ind):
x = self.data[ind][1:]
y = self.data[ind][0]
return x, y
class TestDataset(TrainDataset):
def __getitem__(self, ind):
x = self.data[ind]
return x
torch.manual_seed(94)
x_train = np.random.rand(1000, 5)
y_train = x_train[:, 0] * 10 - x_train[:, 1] * 12 + x_train[:, 2] * 2 + x_train[:, 3] * 11 - x_train[:, 4] * 0.25
y_train = y_train.reshape(1000, 1)
x_train.shape
y_train.shape
train_data = np.concatenate((y_train, x_train), axis=1)
train_set = TrainDataset(train_data)
batch_size = 100
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
class MLP(nn.Module):
def __init__(self):
super(MLP, self).__init__()
self.mlp = nn.Sequential(nn.Linear(5, 1, bias=False))
def forward(self, x_mlp):
out = self.mlp(x_mlp)
return out
device = 'cpu'
model = MLP()
optimizer = torch.optim.SGD(model.parameters(), lr=0.02, momentum=0.82)
criterion = nn.MSELoss()
epochs = 5
lam = 0
model.train()
for epoch in range(epochs):
losses = []
for batch_num, input_data in enumerate(train_loader):
optimizer.zero_grad()
x, y = input_data
x = x.to(device).float()
y = y.reshape(batch_size, 1)
y = y.to(device)
output = model(x)
for name, param in model.named_parameters():
if name == 'mlp.0.weight':
l1_norm = torch.norm(param, 1)
loss = criterion(output, y) + lam * l1_norm
loss.backward()
optimizer.step()
print('\tEpoch %d | Batch %d | Loss %6.2f' % (epoch, batch_num, loss.item()))
for name, param in model.named_parameters():
if param.requires_grad:
print(name)
print(param)
I found that if I use Adagrad as the optimiser instead of SGD, it acts as expected. Will need to look into the difference of those now, but this can be considered answered.
I am trying to plot the ROC curve for a gradient boosting model. I have come across this post but it doesn't seem to work for the GBTclassifier model. pyspark extract ROC curve?
I am using a dataset in databricks and below is my code. It gives the following error
AttributeError: 'PipelineModel' object has no attribute 'summary'
%fs ls databricks-datasets/adult/adult.data
from pyspark.sql.functions import *
from pyspark.ml.classification import RandomForestClassifier, GBTClassifier
from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, VectorSlicer
from pyspark.ml import Pipeline
from pyspark.ml.evaluation import BinaryClassificationEvaluator,MulticlassClassificationEvaluator
from pyspark.ml.linalg import Vectors
from pyspark.ml.tuning import ParamGridBuilder, TrainValidationSplit
import pandas as pd
dataset = spark.table("adult")
# spliting the train and test data frames
splits = dataset.randomSplit([0.7, 0.3])
train_df = splits[0]
test_df = splits[1]
def predictions(train_df,
target_col,
):
"""
#Function attributes
dataframe - training df
target - target varibale in the model
"""
# one hot encoding and assembling
encoding_var = [i[0] for i in train_df.dtypes if (i[1]=='string') & (i[0]!=target_col)]
num_var = [i[0] for i in train_df.dtypes if ((i[1]=='int') | (i[1]=='double')) & (i[0]!=target_col)]
string_indexes = [StringIndexer(inputCol = c, outputCol = 'IDX_' + c, handleInvalid = 'keep') for c in encoding_var]
onehot_indexes = [OneHotEncoderEstimator(inputCols = ['IDX_' + c], outputCols = ['OHE_' + c]) for c in encoding_var]
label_indexes = StringIndexer(inputCol = target_col, outputCol = 'label', handleInvalid = 'keep')
assembler = VectorAssembler(inputCols = num_var + ['OHE_' + c for c in encoding_var], outputCol = "features")
gbt = GBTClassifier(featuresCol = 'features', labelCol = 'label',
maxDepth = 5,
maxBins = 45,
maxIter = 20)
pipe = Pipeline(stages = string_indexes + onehot_indexes + [assembler, label_indexes, gbt])
model = pipe.fit(train_df)
return model
gbt_model = predictions(train_df = train_df,
target_col = 'income')
import matplotlib.pyplot as plt
plt.figure(figsize=(5,5))
plt.plot([0, 1], [0, 1], 'r--')
plt.plot(gbt_model.summary.roc.select('FPR').collect(),
gbt_model.summary.roc.select('TPR').collect())
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.show()
Based on your error, have a look at PipelineModel in this doc:
https://spark.apache.org/docs/2.4.3/api/python/pyspark.ml.html#pyspark.ml.PipelineModel
There is not attribute summary in on an object of this class. Instead, I believe you need to access the stages of the PipelineModel individually, such as gbt_model.stages[-1] (which should give access to your last stage - the GBTClassifier. Then try and play around with the attributes there, such as:
gbt_model.stages[-1].summary
And if your GBTClassifier has a summary, you'll find it there. Hope this helps.
I follow the tutorial to write some linear regression code about boston price, it worked very well and the loss became smaller, when I wanted to paint the graph in matplotlib, I found the graph not showed as what in my mind.
I searched,but could not solve my question.
import pandas as pd
from sklearn.datasets import load_boston
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import torch
from torch.autograd import Variable
import matplotlib.pyplot as plt
if __name__ == '__main__':
boston = load_boston()
col_names = ['feature_{}'.format(i) for i in range(boston['data'].shape[1])]
df_full = pd.DataFrame(boston['data'], columns=col_names)
scalers_dict = {}
for col in col_names:
scaler = StandardScaler()
df_full[col] = scaler.fit_transform(df_full[col].values.reshape(-1, 1))
scalers_dict[col] = scaler
x_train, x_test, y_train, y_test = train_test_split(df_full.values, boston['target'], test_size=0.2, random_state=2)
model = torch.nn.Sequential(torch.nn.Linear(x_train.shape[1], 1), torch.nn.ReLU())
criterion = torch.nn.MSELoss(reduction='mean')
optimizer = torch.optim.Adam(model.parameters(), lr=0.1)
n_epochs = 2000
train_loss = []
test_loss = []
x_train = Variable(torch.from_numpy(x_train).float(), requires_grad=True)
y_train = Variable(torch.from_numpy(y_train).float())
for epoch in range(n_epochs):
y_hat = model(x_train)
loss = criterion(y_hat, y_train)
optimizer.zero_grad()
loss.backward()
optimizer.step()
epoch_loss = loss.data ** (1/2)
train_loss.append(epoch_loss)
if (epoch + 1) % 250 == 0:
print("{}:loss = {}".format(epoch + 1, epoch_loss))
order = y_train.argsort()
y_train = y_train[order]
x_train = x_train[order, :]
model.eval()
predicted = model(x_train).detach().numpy()
actual = y_train.numpy()
print('predicted:", predicted[:5].flatten(), actual[:5])
plt.plot(predicted.flatten(), 'r-', label='predicted')
plt.plot(actual, 'g-', label='actual')
plt.show()
why the predict were the same result like [22.4413, 22.4413, ...],
in the picture, it's a horizontal line.
I'm a very beginner to deeplearning, thank you very much for your help!