Improved Austin Animal Shelter Analysis with Data Visualizations

from sqlalchemy import create_engine
import psycopg2
import pandas as pd
from datetime import datetime
import numpy as np
import re

import category_encoders as ce
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import normalize
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import log_loss
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import GridSearchCV


import matplotlib.pyplot as plt
# import plotly.plotly
# plotly.offline.init_notebook_mode()
import plotly.plotly as py
import plotly.graph_objs as go

import seaborn as sns

%matplotlib inline


pd.set_option('display.max_columns', 100)

import plotly 
plotly.tools.set_credentials_file(username='****', api_key='*****')

intake = pd.read_csv('intakes.csv')
outcome = pd.read_csv('outcomes.csv')

intake.columns

outcome.columns

intake = intake[intake['Animal Type'].isin(['Dog', 'Cat'])]
outcome = outcome[outcome['Animal Type'].isin(['Dog', 'Cat'])]

intake = intake.rename(columns = {'Animal ID': 'ID', 'Found Location': 'Found_Location', 'Intake Type': 'Intake_Type',
                                 'Intake Condition': 'Intake_Condition', 'Animal Type': 'Animal_Type',
                                 'Sex upon Intake':'Sex_Intake','Age upon Intake': 'Age_Intake'})

outcome = outcome.rename(columns = {'Animal ID': 'ID', 'Date of Birth': 'Birth_Date', 'Outcome Type': 'Outcome',
                                   'Outcome Subtype': 'Outcome_Subtype','Animal Type': 'Animal_Type',
                                    'Sex upon Outcome': 'Sex_Outcome', 'Age upon Outcome': 'Outcome_Age'})

intake.head()

outcome.head()

df = pd.merge(intake, outcome, on = 'ID', how = 'inner', suffixes = ('_intake', '_outcome'))

df[df.duplicated()].head()

df = df.drop_duplicates('ID')
df.shape

df.head()

df.columns

df = df.drop(['Name_intake', 'Animal_Type_intake', 'Breed_intake', 'Color_intake'], axis = 1).reset_index(drop = True)

df.head(10)

intake_dates = [df.DateTime_intake[i].split()[0] for i in range(len(df.DateTime_intake))]
intake_date = [datetime.strptime(d,'%m/%d/%Y') for d in intake_dates]

outcome_dates = [df.DateTime_outcome[i].split()[0] for i in range(len(df.DateTime_outcome))]
outcome_date = [datetime.strptime(d,'%m/%d/%Y') for d in outcome_dates]

days_in = [abs(intake_date[i] - outcome_date[i]).days for i in range(len(intake_date))]

df['Days_In'] = days_in

def convert_age(ages):
    Ages = ages.astype(str)
    y = np.asarray([float(age.split()[0]) if 'year' in age
     else float(age.split()[0])/52. if 'week' in age
     else float(age.split()[0])/12. if 'month' in age
     else float(age.split()[0])/352. if 'week' in age
     else -99.0
     for age in Ages])
    y[y == -99] = np.mean(y[y != -99])
    return y

df['Age_Intake'] = convert_age(df.Age_Intake)

df['Outcome_Age'] = convert_age(df.Outcome_Age)

df.head()

df['Sex_Intake'].unique()

df['Sex_Intake'] = df['Sex_Intake'].replace(np.nan,'Unknown')
df['Sex_Intake'].unique()

def convert_sex(sex):
    n = len(sex)
    gender = ['male' if 'Male' in str(sex[i]) 
           else 'female' if 'Female' in str(sex[i] )
           else 'unknown'
           for i in range(n)]

    spayed_neutered = ['fixed' if any(x in str(sex[i]) for x in ['Neutered','Spayed'])
                        else 'not_fixed' if 'Intact' in str(sex[i])
                        else 'unknown_fixed'
                        for i in range(n)]
    return gender, spayed_neutered

df['Sex'], df['Fixed'] = convert_sex(df['Sex_Outcome'])

df.head()

len(df[df.Animal_Type_outcome == 'Dog' ].Breed_outcome.unique())

len(df[df.Animal_Type_outcome == 'Cat' ].Breed_outcome.unique())

breeds = list(df[df['Animal_Type_outcome'] == 'Dog']['Breed_outcome'])
dog_breeds = [re.sub(' Mix', '', dog) if 'Mix' in dog else dog.split("/")[0] for dog in breeds]

df.ix[df.Animal_Type_outcome == 'Dog', 'Breed_outcome'] = dog_breeds

df.head()

len(df.Breed_outcome.unique())

# number of distinct colors for dogs
len(df[df.Animal_Type_outcome == 'Dog' ].Color_outcome.unique())

# number of distinct colors for dogs
len(df[df.Animal_Type_outcome == 'Cat' ].Color_outcome.unique())

# Take the first color
def get_color(animal_color):
    color = [c.split("/")[0] for c in animal_color]
    return color

df['Color_outcome'] = get_color(df['Color_outcome'])

len(df.Color_outcome.unique())

df.head()

dates = [df['DateTime_intake'][i].split()[0] for i in range(len(df['DateTime_intake']))]
date = [datetime.strptime(d,'%m/%d/%Y') for d in dates]

month = ['Janurary' if d.month  == 1
        else 'February' if d.month == 2
        else 'March' if d.month == 3
        else 'April' if d.month == 4
        else 'May' if d.month == 5
        else 'June' if d.month == 6
        else 'July' if d.month == 7
        else 'August' if d.month == 8
        else 'September' if d.month == 9
        else 'October' if d.month == 10
        else 'November' if d.month == 11
        else 'December'
        for d in date]

df['Month_Intake'] = month

df.head()

df = df[df.Outcome.notnull()]

df['Given_Name'] = df.Name_outcome.notnull() * 1.0

df.head()

f = {'Breed_outcome':'count', 'Days_In': 'mean'}

days_avg = df.groupby('Breed_outcome').agg(f)

df2 = days_avg.sort_values(['Breed_outcome','Days_In'])[-10:]

df2 = df2.rename(columns = {'Breed_outcome': 'Count'})
df2

ann_text = ['Count = ' + list(df2['Count'].astype(str))[i] for i in range(len(df2))]

trace1 = go.Bar(
    y = df2['Days_In'], 
    x = df2.index,
    name = 'Average Days',
    text = ann_text
)

layout = go.Layout(
    title = 'Average Days in Shelter by Breed (2013 - 2017)', 
    yaxis = dict(title = 'Days')

)
data = [trace1]
fig = go.Figure(data = data, layout = layout)
# plotly.offline.iplot(fig)
py.iplot(fig)

summary_outcome = pd.crosstab(df.Breed_outcome, df.Outcome).apply(lambda r: r/r.sum() * 100., axis = 1)

summary_outcome.head()

cols = summary_outcome.columns
summary = summary_outcome[summary_outcome.index.isin(list(df2.index))][-10:]
data = []
for col in cols:
    trace = go.Bar(
        y = summary[col],
        x = summary.index,
        name = col
    )
    data.append(trace)
    
layout = go.Layout(
    title = 'Outcome by Breed (2013 - 2017)',
    yaxis = dict(title = 'Percentage'),
#     xaxis = dict(title = 'Breed')
)

fig = go.Figure(data = data, layout = layout)
# plotly.offline.iplot(fig)
py.iplot(fig)

year = [date[i].year for i in range(len(date))]

df3 = pd.concat([df, pd.DataFrame({'year': year})], axis = 1)

df3.head()

summary_by_year = pd.crosstab([df3.Breed_outcome, df3.Outcome], df3.year)

summary_by_year.head()

df4 = summary_by_year[summary_by_year.index.get_level_values(0).isin(df2.index)]

breeds = df4.index.get_level_values(0).unique().tolist()
show = np.array(range(1, 65))
# s = (show < 1e6).tolist()
# button.append(dict(args = ['visible', s], label = 'All', method = 'restyle'))
button = []
data = []
count = 0
for breed in breeds:
    # Determine visiblility for drop down menu
    outcomes = df4.loc[breed].index.get_level_values(0).unique().tolist()
    s = ((show <= (len(outcomes) + count)) & (show > count)).tolist()
    count += len(outcomes)
    button.append(dict(args = ['visible', s], label = breed, method = 'restyle'))
    for outcome in outcomes:
        xlab = df4.loc[breed].columns.tolist()
        totals = df4.groupby(level = 0).sum().loc[breed]
        ylab = df4.loc[breed].loc[outcome]/totals * 100
        #Use to define first breed visibility in plot
        if breed == breeds[0]:
            v = True
        else:
            v = False
        trace = go.Scatter(
            x = xlab,
            y = ylab,
            name = outcome,
            visible = v
        )
        data.append(trace)
        

layout = go.Layout(
title = 'Outcome Percentages by Breed',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Percentage %'),
updatemenus = list([
    dict(
    x = -0.05, 
    y = 1,
    yanchor = 'top',
    buttons = button
    )
])
)

fig = go.Figure(data = data, layout = layout)

# plotly.offline.iplot(fig)
py.iplot(fig)

features = ['Intake_Type', 'Intake_Condition', 'Age_Intake', 'Given_Name', 'Animal_Type_outcome', 'Breed_outcome',
           'Color_outcome', 'Sex', 'Fixed', 'Month_Intake','Days_In', 'Outcome']
df1 = df[features]

df1.head()

categorical_vars = ['Intake_Type', 'Intake_Condition', 'Animal_Type_outcome', 'Breed_outcome', 'Color_outcome', 
                   'Sex','Fixed', 'Month_Intake']

binary = ce.binary.BinaryEncoder(cols = categorical_vars)
binary.fit(df1)
dat = binary.transform(df1)

labels = dat['Outcome']
labels.value_counts()

le = LabelEncoder()
lab = le.fit_transform(labels)

le.classes_

dat = dat.drop(['Outcome', 'Days_In'], axis = 1)

dat.head()

lab[10:25]

X_train, X_test, Y_train, Y_test = train_test_split(dat, lab, test_size = 0.30)

np.bincount(Y_train)/float(len(Y_train))

# Normalize training and testing set
X_train = normalize(X_train, axis = 0)
X_test = normalize(X_test, axis = 0)

params = {'n_estimators':[250, 300, 350], 'max_depth': [3, 4, 5], 'max_features':['sqrt', 'log2']}
gb_gs = GridSearchCV(GradientBoostingClassifier(), params, verbose = 1)

gb_gs.fit(X_train, Y_train)

gb_gs.best_params_

# Get best parameters
depth = gb_gs.best_params_['max_depth']
feat = gb_gs.best_params_['max_features']
estimators = gb_gs.best_params_['n_estimators']

gb_model = GradientBoostingClassifier(n_estimators = estimators, max_depth = depth, max_features = feat)

gb_model.fit(X_train, Y_train)

print('Test Accuracy :', gb_model.score(X_test, Y_test) * 100)

log_loss(Y_test, gb_model.predict_proba(X_test))

cm = confusion_matrix(Y_test, gb_model.predict(X_test))
cm

list(le.classes_)

fig = plt.figure(figsize = (10,8))
sns.heatmap(cm, annot = True, fmt = 'd', annot_kws = {'size': 20}, cmap = 'coolwarm', 
           cbar = 0, xticklabels = list(le.classes_), yticklabels = list(le.classes_) )
plt.xticks(rotation = 35, fontsize = 12)
plt.yticks(rotation = 25, fontsize = 12)
plt.xlabel('Predicted Labels')
plt.ylabel('True Labels')
plt.title('Confusion Matrix for Test Set Predictions', fontsize = 16)
plt.tight_layout(h_pad = 5.0, w_pad = 3.0)

feat_imp = pd.DataFrame({'Features':dat.columns.values,'Values':gb_model.feature_importances_})
top_feat = feat_imp.sort_values(by = 'Values').iloc[-10:]
top_feat

# labels.value_counts()[0]/float(len(labels))