Building a student intervention system: MCA for dimensionality reduction

# Import libraries
import numpy as np
import pandas as pd
from time import time
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
%pylab inline
# Read student data
from udacityP2_funs import * 
from mca import *
from mpl_toolkits.mplot3d import Axes3D

from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import LinearRegression
from sklearn.metrics import log_loss, accuracy_score, f1_score
from sklearn.feature_selection import SelectKBest, f_regression,chi2
%load_ext autoreload
%autoreload 2

student_data = pd.read_csv("student-data.csv")
print "Student data read successfully!"

student_data['absentee_class'] = student_data.apply(lambda row: absentee_class(row['absences']), axis=1)
student_data['age_class'] = student_data.apply(lambda row: age_class(row['age']), axis=1)
#student_data = student_data.drop('age',axis=1)

student_data[['absentee_class','absences']].head(5)

student_data[['age_class','age']].head(5)

student_data = student_data.drop('absences',axis=1)
student_data = student_data.drop('age',axis=1)

target = student_data['passed']
features = student_data.drop('passed',axis=1)
features.columns

var_index = []
val_index = []
col_names_DF = []


unique_names = {'school' : ['GP', 'MS'] ,
        'sex' : ['F','M'] ,
        'address' : ['U' ,'R'] ,
        'famsize' : ['GT3', 'LE3'] ,
        'Pstatus' : ['A' ,'T'] ,
        'Medu' : [0 ,1 ,2, 3 ,4] ,
        'Fedu' : [0, 1, 2 ,3, 4] ,
        'Mjob' : ['at_home', 'health', 
                  'other', 'services', 'teacher'] ,
        'Fjob' : ['teacher' ,'other', 
                  'services', 'health','at_home'] ,
        'reason' : ['course', 'other' ,
                    'home' ,'reputation'] ,
        'guardian' : ['mother', 'father' ,
                      'other'] ,
        'traveltime' : [1,2,3,4] ,
        'studytime' : [1,2,3,4] ,
        'failures' : [0,1,2,3] ,
        'schoolsup' : ['yes', 'no'] ,
        'famsup' : ['yes', 'no'] ,
        'paid' : ['yes', 'no'] ,
        'activities' : ['yes', 'no'] ,
        'nursery' : ['yes', 'no'] ,
        'higher' : ['yes', 'no'] ,
        'internet' : ['yes', 'no'] ,
        'romantic' : ['yes', 'no'] ,
        'famrel' : [1,2,3,4,5] ,
        'freetime' : [1,2,3,4,5] ,
        'goout' : [1,2,3,4,5] ,
        'Dalc' : [1,2,3,4,5] ,
        'Walc' : [1,2,3,4,5] ,
        'health' : [1,2,3,4,5] ,
        'absentee_class' : ['<=3','>3 & <=7', 
                            '>7 & <=14', 
                            '>14 & <=21','>21'] ,
        'age_class' : ['<=16','>16 & <=17',
                       '>17 & <=18','>18'] ,}

col_names = features.columns


print 'Number of features is %d'%len(col_names) 
i = 0
str_row = []
for c_name in col_names:
    print 'Feature value :' ,features[c_name][i] 
    print 'Possible values:' ,unique_names[c_name]
    var_index += [c_name]*len(unique_names[c_name])
    val_index += unique_names[c_name]
    print 'Ind-matrix values:',[1 if (features[c_name][i] == c_val) else 0\
     for c_val in unique_names[c_name]]
    
    col_names_DF+=[c_name+'-'+str(c_val) for c_val in unique_names[c_name]]
    
    str_row+=[1 if (features[c_name][i] == c_val) else 0\
     for c_val in unique_names[c_name]]
    
print 'Number of variables :', len(str_row)

col_index = pd.MultiIndex.from_arrays([var_index, val_index], 
                                      names=['variable', 'value'])
ind_data = []
for i in np.arange(0,len(features)):
    ind_data.append(make_row_indicator(features,i,0))
    
table1 = pd.DataFrame(data=ind_data, columns=col_index)
table1.head()

ind_data_DF = pd.DataFrame(ind_data,columns = col_names_DF)
ind_data_DF.head()

ncols = len(col_names)
mca_ben = MCA(ind_data_DF, ncols=ncols)

N_eig = 3
mca_ben = MCA(ind_data_DF, ncols=ncols)
mca_ind = MCA(ind_data_DF, ncols=ncols, benzecri=False)
data = {'Iλ': pd.Series(mca_ind.L[:N_eig]),
        'τI': mca_ind.expl_var(greenacre=False, N=N_eig),
        'Zλ': pd.Series(mca_ben.L[:N_eig]),
        'τZ': mca_ben.expl_var(greenacre=False, N=N_eig),
        'cλ': pd.Series(mca_ben.L[:N_eig]),
        'τc': mca_ben.expl_var(greenacre=True, N=N_eig)}

# 'Indicator Matrix', 'Benzecri Correction', 'Greenacre Correction'
columns = ['Iλ', 'τI', 'Zλ', 'τZ', 'cλ', 'τc']
table2 = pd.DataFrame(data=data, columns=columns).fillna(0)
table2.index += 1
table2.loc['Σ'] = table2.sum()
table2.index.name = 'Factor'

table2

N_eig_all = np.linspace(1,100,100)
Expl_var_nc = []
Expl_var_bn = []
Expl_var_bnga = []
for N_eig in N_eig_all:
    Expl_var_nc.append(np.sum(mca_ind.expl_var(greenacre=False, 
                                        N=N_eig)))
    Expl_var_bn.append(np.sum(mca_ben.expl_var(greenacre=False,
                                               N=N_eig)))
    Expl_var_bnga.append(np.sum(mca_ben.expl_var(greenacre=True, 
                                      N=N_eig)))

plt.figure(figsize=(8,5))
plt.plot(N_eig_all,Expl_var_nc, label='No-correction')
plt.plot(N_eig_all,Expl_var_bn, label='Benzecri correction')
plt.plot(N_eig_all,Expl_var_bnga,label='Benzecri & Greenacre correction')
plt.legend(loc='lower right')
plt.ylim(0,1.1)
plt.xlim(5,100)

expl_var = mca_ben.expl_var(greenacre=True, N=N_eig)
target_01 = [1.0 if (tar_i=='yes') else 0.0 for tar_i in target]
target_01 = np.asarray(target_01)
color_target = [[0,0,1,.5] if (tar_i=='yes') else [1,0,0,.5] for tar_i in target]

plt.figure(figsize(10,10))
plt.subplot(2,2,1)
plot_low_Dims(plt,features_LD,target_01,0,0,1)

plt.subplot(2,2,2)
plot_low_Dims(plt,features_LD,target_01,0,1,2)

plt.subplot(2,2,3)
plot_low_Dims(plt,features_LD,target_01,0,0,2)

x = np.asarray(features_LD[target_01==0,10])
y = np.asarray(features_LD[target_01==0,11])
z = np.asarray(features_LD[target_01==0,12])

fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(x, y, z,  s=30,color = 'r',alpha = 0.5)
x = np.asarray(features_LD[target_01==1,10])
y = np.asarray(features_LD[target_01==1,11])
z = np.asarray(features_LD[target_01==1,12])
ax.scatter(x, y, z,  s=30,color = 'b',alpha = 0.5)

from sklearn.manifold import TSNE

model = TSNE(n_components=2, init='pca', random_state=0)
tSNE_data = model.fit_transform(ind_data_DF) 
plt.figure()
plt.plot(tSNE_data[target_01==1,0],tSNE_data[target_01==1,1],'rs')
plt.plot(tSNE_data[target_01==0,0],tSNE_data[target_01==0,1],'bo')