
import numpy as np                                          # Algebra.
import pandas as pd                                         # Data processing.


import matplotlib.pyplot as plt                             # Plot the data and display graphics inside jupyter.
%matplotlib inline                            
from matplotlib.colors import ListedColormap                # Create color map.
import seaborn as sns                                       # Seaborn, style plug-in for matplotlib.


from sklearn.impute import KNNImputer                       # Allows us to use k nearest neighbors.
from sklearn.model_selection import train_test_split        # It allows us to separate our set in two to work with the predictive model.
from sklearn.linear_model import LogisticRegression         # Allows us to use logistic regression on the model.
from sklearn.preprocessing import StandardScaler            # Standardize the characteristics by removing the mean and scaling to the unit variance.


url = 'https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv'        # We store the csv in variable.
rms_titanic = pd.read_csv(url, index_col='PassengerId')                                      # we access the dataset by storing it in the var rms_titanic and using the PassengerID column as index.


rms_titanic.info()                                                                           # General dataset information.

<class 'pandas.core.frame.DataFrame'>
Index: 891 entries, 1 to 891
Data columns (total 11 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   Survived  891 non-null    int64  
 1   Pclass    891 non-null    int64  
 2   Name      891 non-null    object 
 3   Sex       891 non-null    object 
 4   Age       714 non-null    float64
 5   SibSp     891 non-null    int64  
 6   Parch     891 non-null    int64  
 7   Ticket    891 non-null    object 
 8   Fare      891 non-null    float64
 9   Cabin     204 non-null    object 
 10  Embarked  889 non-null    object 
dtypes: float64(2), int64(4), object(5)
memory usage: 83.5+ KB


rms_titanic                                  # Dataset overview. 

# I prefer to use this display shape instead of .head or .tail because calling the set in a single view displays the same data along with a .shape


titanic = rms_titanic.copy()
titanic = pd.concat([rms_titanic, rms_titanic])     # we will use the concat() method of pandas to place the dataset behind itself.

print('rms_titanic: ', rms_titanic.shape)           # we use shape to show the size of the Datasets.
print('titanic: ', titanic.shape)

rms_titanic:  (891, 11)
titanic:  (1782, 11)


titanic = titanic.drop_duplicates()                 # using the drop_duplicates() method we remove possible duplicate data.
print('titanic: ', titanic.shape)                   # we check the size of the dataset.

titanic:  (891, 11)


titanic.isnull().sum()

Survived      0
Pclass        0
Name          0
Sex           0
Age         177
SibSp         0
Parch         0
Ticket        0
Fare          0
Cabin       687
Embarked      2
dtype: int64


del titanic['Cabin']                                # deletes the cabin column. 
titanic.isnull().sum()

Survived      0
Pclass        0
Name          0
Sex           0
Age         177
SibSp         0
Parch         0
Ticket        0
Fare          0
Embarked      2
dtype: int64


most_comun_port = titanic['Embarked'].mode()[0]                        # we use the mode() method to find the mode of the embarked column .
print('Most comun port: ', most_comun_port)                            # print it on the screen.

print('\n','-'*80, '\n')                                               # separator.

titanic['Embarked'] = titanic['Embarked'].fillna(most_comun_port)      # with the fillna() method we search and fill the null data of the embarked column with the new variable.
 
titanic.isnull().sum()

Most comun port:  S

 --------------------------------------------------------------------------------

Survived      0
Pclass        0
Name          0
Sex           0
Age         177
SibSp         0
Parch         0
Ticket        0
Fare          0
Embarked      0
dtype: int64


titanic.head()                                                      # Checking.


change_sex = {'female': 0, 'male': 1}                               # We create a dictionary with the data to be transformed.
titanic['Sex'] = titanic['Sex'].map(change_sex)                     # we map the series sex transform the binary data.

titanic.head()                                                      # Checking.


embarked_port = {'C': 1, 'Q': 2, 'S': 3}                            
titanic['Embarked'] = titanic['Embarked'].map(embarked_port)       

titanic.head()


x = titanic[['Survived', 'Pclass', 'Sex', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Age']]   # Select the columns to be imputed (all but name and ticket).
imputer = KNNImputer(n_neighbors=5)                                                       # we create an object for the call to imputation with 5 nearest objects.
x_imputed = imputer.fit_transform(x)                                                      # invoke the fit_transform() method to work on the data and store the result in another variable .
titanic.loc[titanic['Age'].isnull(), 'Age'] = x_imputed[:, -1][titanic['Age'].isnull()]   # we use .loc to select the rows with nulls from the Age column and replace the nulls with the imputed data.

titanic.isnull().sum()

Survived    0
Pclass      0
Name        0
Sex         0
Age         0
SibSp       0
Parch       0
Ticket      0
Fare        0
Embarked    0
dtype: int64


titanic.head()                                                      # Checking.


titanic.describe()                       # Summary statistics .


high_contrast = 'Set1'
class_color = ['skyblue', 'lightgreen', 'salmon']
embarked_color = ['skyblue', 'lightgreen', 'salmon']
survival_color = ['salmon', 'skyblue']
survival_dictionary = {0: 'salmon', 1: 'skyblue'}


# Statements
pclass_count = titanic['Pclass'].value_counts().sort_index()                                                 # We create a variable and store in it a series with the categories and the sum of all of them.
print(pclass_count)

# Graph
plt.figure(figsize=(11, 6))                                                                                  # we create a figure and give it dimension.

plt.bar(pclass_count.index, pclass_count, color = class_color)                                               # we create a bar chart indicating the axes and color it.
plt.xlabel('')                                                                                               # we empty the x-axis
plt.xticks([1,2,3], ['1st', '2nd', '3rd'], fontsize=14)                                                      # we rename the x-axis ticks by selecting and replacing them.
plt.ylabel('nº of passengers',fontsize=14)                                                                   # we rename the y-axis and give it size.
plt.title('No. of passengers per class', fontdict={'fontsize': 11, 'fontweight': 'bold'})                    # retitle and align in the center, give size and weight (This reminds me a lot of CSS :D).

plt.show()                                                                                                   # we plot.

Pclass
1    216
2    184
3    491
Name: count, dtype: int64


print(titanic['Fare'].describe())

plt.figure(figsize=(11, 6))                                                                                # We create a figure and give it dimension.

sns.boxplot(x=titanic['Fare'], color='skyblue')                                                            # we use seaborn to create a boxplot and use the Fare series on the x-axis, we color the box.   
plt.xlabel('')                                                                                             # we empty the x-axis label.
plt.title('ticket prices', fontdict={'fontsize': 11, 'fontweight': 'bold'})                                # we retitle and align in the center, give size and weight.                                              

plt.show()                                                                                                 # we plot.

count    891.000000
mean      32.204208
std       49.693429
min        0.000000
25%        7.910400
50%       14.454200
75%       31.000000
max      512.329200
Name: Fare, dtype: float64


print(titanic['Age'].describe())

plt.figure(figsize=(11, 6))                                                                              # We create a figure and give it dimension.

sns.histplot(data=titanic, x='Age', bins=80, color='skyblue')                                            # we create a histiogram with the age series data and spread it over 80 cells (which is the maximum age) and color it.
plt.xlabel('Age of passengers', fontsize=14)                                                             # we customize the x-axis label and size it.
plt.ylabel('no. of passengers', fontsize=14)                                                             # we customize the y-axis label and size it.
plt.title('No. of passengers per age', fontdict={'fontsize': 11, 'fontweight': 'bold'})                  # we retitle and align in the center, give size and weight.   

plt.show()                                                                                               # we plot.

count    891.000000
mean      30.061452
std       13.644962
min        0.420000
25%       21.000000
50%       29.000000
75%       38.000000
max       80.000000
Name: Age, dtype: float64


# Statements
sibsp_count = titanic['SibSp'].value_counts().sort_index()                                          # Creates a series within the variable and calculates the number of passengers in each group SibSp (number of siblings/spouses on board).
parch_count = titanic['Parch'].value_counts().sort_index()                                          # we do the same with Parch (parents/children abroad).
width = 0.40                                                                                        # we select a width for the histiogram bar, this will allow to place the sibsp bar next to the parch bar.
index = np.arange(len(sibsp_count))                                                                 # we create a list with the range of sibsp, which we know has the most values.

print(sibsp_count,'\n', '-'*20, '\n', parch_count, '\n', '-'*20, '\n')

# Graph
fig, x = plt.subplots(figsize=(11, 6))                                                              # We create a figure and give it dimension.

bar_sibsp = x.bar(index, sibsp_count, width, label='Siblings / spouses', color='Salmon')            # we create the bars for sibsp. we tell it where they have to go, their content, the width of the bar, how we want it to be identified and we color it.
bar_parch = x.bar(index + width, parch_count, width, label='Parents / children', color='Skyblue')   # we do the same for the parch bars, but this time we tell it that the position is the same but to add the width of a bar, that way it doesn't stack with the previous one. 
x.set_xlabel('No. of family members on board', fontsize=14)                                         # we customize the x-axis label and size it.                             
x.set_ylabel('No. of passengers', fontsize=14)                                                      # we customize the y-axis label and size it.
x.set_title('Passengers by number of family members on board', fontdict={'fontsize': 11, 'fontweight': 'bold'})    # we retitle and align in the center, give size and weight.  
x.set_xticks(index + width / 2)                                                                     # the positions on the x-axis where the bar labels will be placed. They are placed in the middle between the bars for SibSp and Parch.
x.set_xticklabels(sibsp_count.index)                                                                # we define the labels for the bars on the x-axis as the unique values of SibSp.
x.legend(title='Family on board')                                                                   # we create a title for the legend and the tags we have created.

plt.show()                                                                                          # we plot.

SibSp
0    608
1    209
2     28
3     16
4     18
5      5
8      7
Name: count, dtype: int64 
 -------------------- 
 Parch
0    678
1    118
2     80
3      5
4      4
5      5
6      1
Name: count, dtype: int64 
 --------------------


# Statements
gender = titanic['Sex'].value_counts()                                                   # we create a series to count the number of passengers of each genre.             
print(gender)

# Graph
fig, plot = plt.subplots(figsize=(11, 6))                                                # We create a figure and give it dimension.

plt.bar(gender.index, gender, color=survival_color)                                      # we create a bar chart with the index of the series we have generated and tell it to use the stored values, then we colorize.
plt.xlabel('')                                                                           # we empty the x-axis label.
plt.xticks([0, 1], ['Women', 'Men'], fontsize=14)                                        # we change the X-axis ticks, select the desired ones and then choose the custom ones.
plt.ylabel('No. of passengers', fontsize=14)                                             # we customize the y-axis label and size it.
plt.title('No. of passengers per gender', fontdict={'fontsize': 11, 'fontweight': 'bold'})    # we retitle it and align it in the center, give it size and weight.  

plt.show()                                                                               # we plot.

Sex
1    577
0    314
Name: count, dtype: int64


# Statements
embarked_count =  titanic['Embarked'].value_counts().sort_index()                                    # We create a series using the boarding ports as an index and adding each passenger. 
print(embarked_count)

# Graph
fig, plot = plt.subplots(figsize=(11, 6))                                                            # We create a figure and give it dimension.

plt.bar(embarked_count.index, embarked_count, color=embarked_color)                                  # we create a bar chart using as an index the one of the created series, load its values and color it.
plt.xlabel('')                                                                                       # we empty the x-axis label .
plt.xticks([1, 2, 3], ['Cherbourg', 'Queenstown', 'Southampton'], fontsize=14)                       # we change the X-axis ticks, select the desired ones and then choose the custom ones.os.
plt.ylabel('Nº of Passengers', fontsize=14)                                                          # we customize the y-axis label and size it.
plt.title('No. of passengers per port', fontdict={'fontsize': 11, 'fontweight': 'bold'})             # we retitle it and align it in the center, give it size and weight .

plt.show()                                                                                           # we plot .

Embarked
1    168
2     77
3    646
Name: count, dtype: int64


# Statements
survival_count = titanic['Survived'].value_counts()                                                         # We create a new series with the data from the suvived table.
print(survival_count)

# Graph
fig, plot = plt.subplots(figsize=(11, 6))                                                                   # We create a figure and give it dimension.

# We create a pie chart with labels for each part, percentage format with a single decimal, angle to start from, colors, font size, spacing between pie parts and shading. 
plt.pie(survival_count, labels=['Deceased', 'Survivors'], autopct='%1.1f%%', startangle=90, colors= survival_color , textprops={'fontsize': 14}, explode=[0.1, 0], shadow=True)  
plt.title('survival rate', fontdict={'fontsize': 11, 'fontweight': 'bold'})                                 # we retitle it and align it in the center, give it size and weight .

plt.show()

Survived
0    549
1    342
Name: count, dtype: int64


# Statements
corr = titanic.corr(numeric_only=True)                                       # We call the corr() method and tell it to use only the columns with numeric data.
print(corr)

# Graph
fig, ax = plt.subplots(figsize=(17, 8))                                      
sns.heatmap(corr, annot=True)                                                # we create a heat map with the above variable and tell it to use annotations.
plt.title('Correlation matrix of the DataSet', fontdict={'fontsize': 11, 'fontweight': 'bold'}) 

plt.show()

          Survived    Pclass       Sex       Age     SibSp     Parch  \
Survived  1.000000 -0.338481 -0.543351 -0.110313 -0.035322  0.081629   
Pclass   -0.338481  1.000000  0.131900 -0.361498  0.083081  0.018443   
Sex      -0.543351  0.131900  1.000000  0.139182 -0.114631 -0.245489   
Age      -0.110313 -0.361498  0.139182  1.000000 -0.215897 -0.183820   
SibSp    -0.035322  0.083081 -0.114631 -0.215897  1.000000  0.414838   
Parch     0.081629  0.018443 -0.245489 -0.183820  0.414838  1.000000   
Fare      0.257307 -0.549500 -0.182333  0.092513  0.159651  0.216225   
Embarked -0.167675  0.162098  0.108262 -0.008357  0.068230  0.039798   

              Fare  Embarked  
Survived  0.257307 -0.167675  
Pclass   -0.549500  0.162098  
Sex      -0.182333  0.108262  
Age       0.092513 -0.008357  
SibSp     0.159651  0.068230  
Parch     0.216225  0.039798  
Fare      1.000000 -0.224719  
Embarked -0.224719  1.000000


embarked_count

Embarked
1    168
2     77
3    646
Name: count, dtype: int64


plt.figure(figsize=(16, 6))                                                                                                                                # We create a figure and give it a size.

# Graph 1
plt.subplot(1, 2, 1)                                                                                                                                       # we place the first graph in the first row of a two-row grid.                                                                                                                                               
sns.countplot(x='Embarked', data=titanic, palette=embarked_color)                                                                                          # We use the seaborn countplot function to autogenerate a plot with the 'Embarked' data as input, from the titanic dataset, and paint it with the variable we have stored.
plt.xlabel('')                                                                                                                                             # we empty the X-axis label.
plt.xticks([0, 1, 2], ['Cherbourg', 'Queenstown', 'Southampton'], fontsize=14)                                                                             # we replace the ticks on the x-axis with the name of the ports and size them .
plt.ylabel('no. of passengers', fontsize=14)                                                                                                               # we customize the y-axis label and shape it.
plt.title('Relationship between the number of passengers and the port of embarkation', fontdict={'fontsize': 11, 'fontweight': 'bold'})                    # We customize the title of the chart, center it and give it size and weight. 

# Graph 2
plt.subplot(1, 2, 2)                                                                                                                                       # we place the second graph in the first row of a two-row grid.
sns.countplot(x='Embarked', hue='Survived', data=titanic, palette=survival_color)                                                                          # we use the seaborn countplot() function to autogenerate a plot with the 'Embarked' data as input by the 'Survived' data, from the titanic dataset, and paint it with the variable we have stored.
plt.xlabel('')                                                                                                                                             # we empty the X-axis label.
plt.xticks([0, 1, 2], ['Cherbourg', 'Queenstown', 'Southampton'], fontsize=14)                                                                             # we replace the ticks on the x-axis with the name of the ports and size them .
plt.ylabel('no. of passengers', fontsize=14)                                                                                                               # we customize the y-axis label and shape it.
plt.title('Relationship between the number of survivors and the port of embarkation', fontdict={'fontsize': 11, 'fontweight': 'bold'})                     # we customize the title of the chart, center it and give it size and weigh.
plt.legend(title='Status', labels=['Deceased', 'Survivors'])                                                                                               # we create a legend, generate a title for the legend and the labels for the values of the "Survivors" series.

plt.tight_layout()                                                                                                                                         # we use the .tight_layout() function of matplotlib to avoid overlaps.
plt.show()                                                                                                                                                 # we paint the grid.

	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked
PassengerId
1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	NaN	S
2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	C85	C
3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	NaN	S
4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	C123	S
5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	NaN	S
...	...	...	...	...	...	...	...	...	...	...	...
887	0	2	Montvila, Rev. Juozas	male	27.0	0	0	211536	13.0000	NaN	S
888	1	1	Graham, Miss. Margaret Edith	female	19.0	0	0	112053	30.0000	B42	S
889	0	3	Johnston, Miss. Catherine Helen "Carrie"	female	NaN	1	2	W./C. 6607	23.4500	NaN	S
890	1	1	Behr, Mr. Karl Howell	male	26.0	0	0	111369	30.0000	C148	C
891	0	3	Dooley, Mr. Patrick	male	32.0	0	0	370376	7.7500	NaN	Q

	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Embarked
PassengerId
1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	S
2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	C
3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	S
4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	S
5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	S

	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Embarked
PassengerId
1	0	3	Braund, Mr. Owen Harris	1	22.0	1	0	A/5 21171	7.2500	S
2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	0	38.0	1	0	PC 17599	71.2833	C
3	1	3	Heikkinen, Miss. Laina	0	26.0	0	0	STON/O2. 3101282	7.9250	S
4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	0	35.0	1	0	113803	53.1000	S
5	0	3	Allen, Mr. William Henry	1	35.0	0	0	373450	8.0500	S

	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Embarked
PassengerId
1	0	3	Braund, Mr. Owen Harris	1	22.0	1	0	A/5 21171	7.2500	3
2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	0	38.0	1	0	PC 17599	71.2833	1
3	1	3	Heikkinen, Miss. Laina	0	26.0	0	0	STON/O2. 3101282	7.9250	3
4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	0	35.0	1	0	113803	53.1000	3
5	0	3	Allen, Mr. William Henry	1	35.0	0	0	373450	8.0500	3

	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Embarked
PassengerId
1	0	3	Braund, Mr. Owen Harris	1	22.0	1	0	A/5 21171	7.2500	3
2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	0	38.0	1	0	PC 17599	71.2833	1
3	1	3	Heikkinen, Miss. Laina	0	26.0	0	0	STON/O2. 3101282	7.9250	3
4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	0	35.0	1	0	113803	53.1000	3
5	0	3	Allen, Mr. William Henry	1	35.0	0	0	373450	8.0500	3

Python final practice - EDA¶

1. Question or problem definition.¶

2. Titanic DataSet¶

libraries and imports¶

3. Types of data available and exploratory analysis¶

Data legend¶

4. Prepare and cleanse the data.¶

Checking for duplicate data¶

Null data management¶

Normalization or standardization of data¶

5-7 Descriptive statistics, data visualization and analysis¶

(let's create our color range for plotting.)¶

Next, we continue our analysis by exploring the correlation between the data.¶

As we can see this heat map is much nicer than the raw data we have stored.¶

Embarked¶

	Survived	Pclass	Sex	Age	SibSp	Parch	Fare	Embarked
count	891.000000	891.000000	891.000000	891.000000	891.000000	891.000000	891.000000	891.000000
mean	0.383838	2.308642	0.647587	30.061452	0.523008	0.381594	32.204208	2.536476
std	0.486592	0.836071	0.477990	13.644962	1.102743	0.806057	49.693429	0.791503
min	0.000000	1.000000	0.000000	0.420000	0.000000	0.000000	0.000000	1.000000
25%	0.000000	2.000000	0.000000	21.000000	0.000000	0.000000	7.910400	2.000000
50%	0.000000	3.000000	1.000000	29.000000	0.000000	0.000000	14.454200	3.000000
75%	1.000000	3.000000	1.000000	38.000000	1.000000	0.000000	31.000000	3.000000
max	1.000000	3.000000	1.000000	80.000000	8.000000	6.000000	512.329200	3.000000