kaggle-titanic 数据分析过程

5.完整的数据处理过程

train=pd.read_csv('base_data/train.csv')
test=pd.read_csv('base_data/test.csv')
full=train.append(test,ignore_index=True)#保证train和test的数据格式一样
titanic=full[:891]
titanic_pre=full[891:]
deltrain,test
print('DataSets:','full:',full.shape,'titanic:',titanic.shape)

full=full.drop(['Cabin','PassengerId','Ticket'],axis=1)

full['Sex']=full['Sex'].map({'female':1,'male':4}).astype(int)

full['Pclass*Sex']=full.Pclass*full.Sex

full['Pclass*Sex']=pd.factorize(full['Pclass*Sex'])[0]

full['Title']=full.Name.str.extract('([A-Za-z]+)\.',expand=False)

full['Title']=full['Title'].replace(['Capt','Col','Don','Dr','Jonkheer','Major','Rev','Sir'],'Male_Rare')

full['Title']=full['Title'].replace(['Countess','Lady','Mlle','Mme','Ms'],'Female_Rare')

title_mapping={"Mr":1,"Miss":2,"Mrs":3,"Master":4,"Female_Rare":5,'Male_Rale':6}
full['Title']=full['Title'].map(title_mapping)
full['Title']=full['Title'].fillna(0)

full.head()

full['FamilySize']=full['Parch']+full['SibSp']+1

full.loc[full['FamilySize']==1,'Family']=0
full.loc[(full['FamilySize']>1)&(full['FamilySize']<5),'Family']=1
full.loc[(full['FamilySize']>=5),'Family']=2

dataset=full[['Age','Sex','Pclass']]

guess_ages=np.zeros((2,3))

l=[1,4]
foriinrange(len(l)):
forjinrange(0,3):
guess_df=dataset[(dataset['Sex']==l[i])&(dataset['Pclass']==j+1)]['Age'].dropna()

#age_mean=guess_df.mean()
#age_std=guess_df.std()
#age_guess=rnd.uniform(age_mean-age_std,age_mean+age_std)

age_guess=guess_df.median()

#Convertrandomagefloattonearest.5age
guess_ages[i,j]=int(age_guess/0.5+0.5)*0.5

print(guess_ages)

foriinrange(len(l)):
forjinrange(0,3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex==l[i])&(dataset.Pclass==j+1),'Age']=guess_ages[i,j]

full['Age']=dataset['Age'].astype(int)

full.loc[full['Age']<=20,'Age']=4
full.loc[(full['Age']>20)&(full['Age']<=40),'Age']=5
full.loc[(full['Age']>40)&(full['Age']<=60),'Age']=6
full.loc[full['Age']>60,'Age']=7

full.head()

full['Pclass*Age']=full.Pclass*full.Age

freq_port=full.Embarked.dropna().mode()[0]
full['Embarked']=full['Embarked'].fillna(freq_port)

full[['Embarked','Survived']].groupby(['Embarked'],as_index=False).mean().sort_values(by='Survived',ascending=False)

full['Embarked']=full['Embarked'].map({'S':0,'C':1,'Q':2}).astype(int)
full.head()

#Fare预测数据特征里面也有缺失值，需要填充
#用中位数进行填充，然后划分Fare段

full['Fare'].fillna(full['Fare'].dropna().median(),inplace=True)

full.loc[full['Fare']<=128.082,'Fare']=0
full.loc[(full['Fare']>128.082)&(full['Fare']<=256.165),'Fare']=1
full.loc[(full['Fare']>256.165)&(full['Fare']<=384.247),'Fare']=2
full.loc[full['Fare']>384.247,'Fare']=3
full['Fare']=full['Fare'].astype(int)

full=full.drop(['Name'],axis=1)

full=full.drop(['Parch','SibSp','FamilySize'],axis=1)

full['Sex']=full['Sex'].astype('int').astype('str')
full=pd.get_dummies(full,prefix='Sex')
full.head()

full['Embarked']=full['Embarked'].astype('str')
full=pd.get_dummies(full,prefix='Embarked')

full.head()

full['Title']=full['Title'].astype('str')

full=pd.get_dummies(full,prefix='Title')

full=full.drop(['Survived'],axis=1)

6.选择模型

defCOMPARE_MODEL(train_valid_X,train_valid_y):

defcross_model(model,train_X,train_y):
cvscores=cross_val_score(model,train_X,train_y,cv=3,n_jobs=-1)
returnround(cvscores.mean()*100,2)

train_X,valid_X,train_y,valid_y=train_test_split(train_valid_X,train_valid_y,train_size=.78,random_state=0)

defonce_model(clf):
clf.fit(train_X,train_y)
y_pred=clf.predict(valid_X)
returnround(accuracy_score(y_pred,valid_y)*100,2)

logreg=LogisticRegression()
acc_log=cross_model(logreg,train_valid_X,train_valid_y)
acc_log_once=once_model(logreg)

xgbc=XGBClassifier()
acc_xgbc=cross_model(xgbc,train_valid_X,train_valid_y)
acc_xgbc_once=once_model(xgbc)

svc=SVC()
acc_svc=cross_model(svc,train_valid_X,train_valid_y)
acc_svc_once=once_model(svc)

knn=KNeighborsClassifier(n_neighbors=3)
acc_knn=cross_model(knn,train_valid_X,train_valid_y)
acc_knn_once=once_model(knn)

gaussian=GaussianNB()
acc_gaussian=cross_model(gaussian,train_valid_X,train_valid_y)
acc_gaussian_once=once_model(gaussian)

perceptron=Perceptron()
acc_perceptron=cross_model(perceptron,train_valid_X,train_valid_y)
acc_perceptron_once=once_model(perceptron)

linear_svc=LinearSVC()
acc_linear_svc=cross_model(linear_svc,train_valid_X,train_valid_y)
acc_linear_svc_once=once_model(linear_svc)

sgd=SGDClassifier()
acc_sgd=cross_model(sgd,train_valid_X,train_valid_y)
acc_sgd_once=once_model(sgd)

decision_tree=DecisionTreeClassifier()
acc_decision_tree=cross_model(decision_tree,train_valid_X,train_valid_y)
acc_decision_tree_once=once_model(decision_tree)

random_forest=RandomForestClassifier(n_estimators=100)
acc_random_forest=cross_model(random_forest,train_valid_X,train_valid_y)
acc_random_forest_once=once_model(random_forest)

gbc=GradientBoostingClassifier()
acc_gbc=cross_model(gbc,train_valid_X,train_valid_y)
acc_gbc_once=once_model(gbc)

models=pd.DataFrame({
'Model':['XGBC','SupportVectorMachines','KNN','LogisticRegression',
'RandomForest','NaiveBayes','Perceptron',
'StochasticGradientDecent','LinearSVC',
'DecisionTree','GradientBoostingClassifier'],
'Score':[acc_xgbc,acc_svc,acc_knn,acc_log,
acc_random_forest,acc_gaussian,acc_perceptron,
acc_sgd,acc_linear_svc,acc_decision_tree,acc_gbc]})

models_once=pd.DataFrame({
'Model':['XGBC','SupportVectorMachines','KNN','LogisticRegression',
'RandomForest','NaiveBayes','Perceptron',
'StochasticGradientDecent','LinearSVC',
'DecisionTree','GradientBoostingClassifier'],
'Score':[acc_xgbc_once,acc_svc_once,acc_knn_once,acc_log_once,
acc_random_forest_once,acc_gaussian_once,acc_perceptron_once,
acc_sgd_once,acc_linear_svc_once,acc_decision_tree_once,acc_gbc_once]})

models=models.sort_values(by='Score',ascending=False)
models_once=models_once.sort_values(by='Score',ascending=False)

returnmodels,models_once

train_valid_X=full[0:891]
train_valid_y=titanic.Survived
test_X=full[891:]
train_X,valid_X,train_y,valid_y=train_test_split(train_valid_X,train_valid_y,train_size=.78,random_state=0)

print(full.shape,train_X.shape,valid_X.shape,train_y.shape,valid_y.shape,test_X.shape)

models_cross,models_once=COMPARE_MODEL(train_valid_X,train_valid_y)

xgbc=XGBClassifier()
xgbc.fit(train_valid_X,train_valid_y)
test_Y_xgbc=xgbc.predict(test_X)

passenger_id=titanic_pre.PassengerId

test=pd.DataFrame({'PassengerId':passenger_id,'Survived':np.round(test_Y_xgbc).astype('int32')})

printtest.head()

test.to_csv('titanic_pred_gbdc_sc.csv',index=False)