Author: Agnieszka

Pandas

Agnieszka

Pandas is an open source data analysis and manipulation tool, built on top of the Python programming language. Offers data structures and operations for manipulating numerical tables and time series.

Installation:

#checking version
import pandas as pd
print(pd.__version__)

Pandas Series

#series
vec = [0,12,99,54,89,10,28,30]
res = pd.Series(vec)
#print(res)
print(res[0])
print(res[2:])
print(res[:-4])

vec=[1,2,3,4]
own_index = pd.Series(vec,["I","II","III","IV"])
print(own_index)

#series with key values
random_values = {"I": 10, "II": 200, "III": 3000, "IV":40000}
res = pd.Series(random_values)
print(res)

Data Frames

#DataFrames
data = {

  "cardio": [20, 26, 39, 41, 29],
  "steps": [2700, 3400, 4520, 4700, 3700],
  "weight": [65.0,64.5,63.0,62.0, 61.5]

}
fit_table = pd.DataFrame(data, index = ["weekI", "weekII","weekIII","weekIV","weekV"])
print(fit_table)

#refers to one column only:
print(fit_table.loc["weekI"])

Stats

To see some results for various statistic tests we need te copy some data into PyCharm. I’ve created simple data file .csv :

man_and_womanDownload
import scipy.stats as sps
import pandas
FileData = pandas.read_csv('man_and_woman.csv', sep=',', na_values='.')

#T-test
T_test_result = sps.ttest_rel(FileData['Age'],FileData['IQ'])
T_test_result1 = sps.ttest_1samp(FileData['Age'] - FileData['IQ'],0)
print(T_test_result)
print(T_test_result1)

#Normal test
normaltest_results = sps.normaltest(FileData['Age'])
print(normaltest_results)

# Mann-Whitney test
mw_results = sps.mannwhitneyu(FileData['Age'], FileData['IQ'])
print(mw_results)

# Wilcoxona test
wilcoxon_results = sps.wilcoxon(FileData['Age'], FileData['IQ'])
print(wilcoxon_results)

version with file and data processing in columns

example with data downloaded from Kaggle platform: Divorce/marriage dataset with birth dates

import scipy.stats as sps
import pandas
FileData = pandas.read_csv('divorces_2000-2015_translated.csv', sep=',', na_values='0')

# print(FileData.head())
# print(FileData.dtypes)

#T-test
T_test_result = sps.ttest_rel(FileData['Monthly_income_partner_woman_peso'],
                              FileData['Monthly_income_partner_man_peso'])
T_test_result1 = sps.ttest_1samp(FileData['Monthly_income_partner_woman_peso'],
                              FileData['Monthly_income_partner_man_peso'],0)
# print(T_test_result)
# print(T_test_result1)

print(FileData['Monthly_income_partner_woman_peso'].isna().sum())
print(FileData['Monthly_income_partner_man_peso'].isna().sum())

# select columns from file
FileData_cols = ['Monthly_income_partner_woman_peso',
        'Monthly_income_partner_man_peso']

FileData_clean = FileData[FileData_cols].dropna()

#delete rows with gaps in both columns at once
sps.ttest_rel(FileData_clean.iloc[:, 0], FileData_clean.iloc[:, 1])

#T-test
result = sps.ttest_rel(
    FileData_clean['Monthly_income_partner_woman_peso'],
    FileData_clean['Monthly_income_partner_man_peso']
)

print(result)

# Normal test
normaltest_results = sps.normaltest(FileData_clean['Monthly_income_partner_woman_peso'])
print(normaltest_results)

# Mann-Whitney test
mw_results = sps.mannwhitneyu(FileData_clean['Monthly_income_partner_woman_peso'],
                              FileData_clean['Monthly_income_partner_man_peso'])
print(mw_results)

# Wilcoxona test
wilcoxon_results = sps.wilcoxon(FileData_clean['Monthly_income_partner_woman_peso'],
                                FileData_clean['Monthly_income_partner_man_peso'])
print(wilcoxon_results)

SciPy

Agnieszka

SciPy is an open-source, scientific computation library for mathematics, science and engineering. Uses NumPy underneath and rovides more utility functions for optimization, stats and signal processing.

Similar to Numpy, in order to start programming with this library we need to “switch it on”, which means ‘pip install’ on terminal:

and check if it works:
Lets try some simple options offered in SciPy, like constant values:
from scipy import constants as const

print(const.pi)
print(const.day)
print(const.kilo)
print(const.light_year)
print(const.micro)
print(const.tera)
print(const.e)
print(const.eV)

#Spars Data
import numpy as np
from scipy.sparse import csr_matrix

arr1 = np.array([0,1,2,3,4,5,6,7,8,9])
print(arr1)
print(csr_matrix(arr1))

arr2 = np.array([0,0,0,0,0,1,1,2,2,3])
print(arr2)
print(csr_matrix(arr2))

arr3 = np.array([[0, 0, 0], [0, 0, 1], [1, 2, 2]])
print(arr3)
print(csr_matrix(arr3).data)
print(csr_matrix(arr3).count_nonzero())

arr4 = np.array([[0, 0, 0], [0, 0, 1], [1, 0, 2]])
arr4 = csr_matrix(arr4).tocsc()
print(arr4)

Let’s try with some data. Write some numbers, f.e:

156,158,167,188,197,167,197,178,186,157,198,199,178,168,156,151,167,167

in one line in notepad, save it and copy into PyCharm:

#SciPy stats
import numpy as np
import scipy.stats as scs

#T-test
data = np.loadtxt("Numbers.csv", delimiter=',', skiprows=0, unpack=True)
print(data)
results = scs.ttest_1samp(data,160)
print(results)

p = 0.3
data = scs.bernoulli.rvs(p, size=100)
mean, var, skew, kurt = scs.bernoulli.stats(p, moments = 'mvsk')
print(data)

#normality test shapiro-wilk 
import scipy.stats as sps
random_data = sps.norm.rvs(size=1000)
shapiro_test_results = sps.shapiro(random_data)
print(shapiro_test_results)

#kolmogorov smirnov
random_data = sps.norm.rvs(size=1000)
KS_test_results = sps.kstest(random_data,'norm')
print(KS_test_results)

#Levene test
normal_data = sps.norm.rvs(size=1000)
normal_data1 = sps.norm.rvs(size=1000)
normal_data2 = sps.norm.rvs(loc=1, scale=20, size=1000)
levene_test_results = sps.levene(normal_data, normal_data1, normal_data2)
print(levene_test_results)

#Bartlett test
normal_data = sps.norm.rvs(size=1000)
normal_data1 = sps.norm.rvs(size=1000)
normal_data2 = sps.norm.rvs(loc=1, scale=20, size=1000)
bartlett_test_results = sps.bartlett(normal_data, normal_data1, normal_data2)
print(bartlett_test_results)