Python3 fstrings

vars = 'abc'

print(f 'Variable  is : {vars}')

This is much better than 'Variable is : {}'.format(vars)

Pandas Tutorial 3

1) Drop Nulls of one column in Pandas

Find isnull in a column:

isnull(master["playerID"]).value_counts()

Output:

False    7520

True      241

Name: playerID, dtype: int64

Drop nulls of specific columns (dropna)

master_orig = master.copy()

master = master.dropna(subset=["playerID"])

master.shape

2) Drop Nulls of multiple column in Pandas

how = 'all' # if all subset cols are nulls

how = 'any' # if any of the subset cols are nulls

df.dropna(subset=[col_list], how='all')

master = master.dropna(subset=["firstNHL", "lastNHL"], how="all")

3)

master1 = master[master["lastNHL"] >= 1980]

master1.shape # (4627, 31)

Vs

master1 = master.loc[master["lastNHL"] >= 1980]

master1.shape # (4627, 31)

But later is good, more performance with huge data

4) filter columns

master.filter(columns_to_keep).head()

(or)

master = master.filter(regex="(playerID|pos|^birth)|(Name$)")

5) Find DF memory usage

df.memory_usage()

def mem_mib(df):

    mem = df.memory_usage().sum() / (1024 * 1024)

    print(f'{mem}.2f Mib')

    

mem_mib(master) # 0.39 MiB

mem_mib(master_orig) # 1.84 MiB

6) Categorical

# A string variable consisting of only a few different values. 

# Converting such a string variable to a categorical variable will save some memory.

def make_categorical(df, col_name):

    df.loc[:, col_name] = pd.Categorical(df[col_name]) 

# to save memory

make_categorical(master, "pos")

make_categorical(master, "birthCountry")

make_categorical(master, "birthState")

7)

pd.read_pickle()

8) Joins

Default is inner join

pd.merge(df1, df2, how='left')

# We joining based on player id of both dfs

# If left df has PlayerId & right df has plrId

pd.merge(df1, df2, left_on='PlayerId', right_on='plrId')

# We joining based on player id of both DFs

# Say if left DF has player id as index

# Here resultant merge DF has index from right DF 

# left DF index (left_index) is not considered in merge dF

pd.merge(df1, df2, left_index=True, right_on='plrId')

# We joining based on player id of both dfs

# Say if right df has player id as index

# Here resultant merge DF has index from left DF 

# right DF index (right_index) is not considered in merge dF

pd.merge(df1, df2, left_on='PlayerId', right_index=True)

# We can even set DF index (set_index) and use

# left_index and right_index 

pd.merge(df1, df2.set_index("playerID", drop=True),

                            left_index=True, right_index=True).head()

# Indicator

# It creates additional column _merge

# It indicates both, left_only, right_only

merged = pd.merge(master2, scoring, left_index=True,

                  right_on="playerID", how="right", indicator=True)

merged["_merge"].value_counts()

both          28579

right_only       37

left_only         0

Name: _merge, dtype: int64

# Filter only right_only

merged[merged["_merge"] == "right_only"].head()

# Filter only right_only or left_only

merged[(merged["_merge"] == "right_only") | (merged["_merge"] == "left_only")].sample(3)

or

# 

merged[merged["_merge"].str.endswith("only")].sample(5)

# Filter out 1:m (one to many)

try:

pd.merge(df1, df2, left_index=True, right_on='plrId', validate="1:m").head()

except Exception as e:

pass

8) Drop random records

df.drop(drop.sample(5).index)

-------

9) Longer to Wider format (pivot)

df.show()

playerID year Goals

10320 hlavaja01 2001 7.0

10322 hlavaja01 2002 1.0

10324 hlavaja01 2003 5.0

15873 markoan01 2001 5.0

15874 markoan01 2002 13.0

15875 markoan01 2003 6.0

18899 nylanmi01 2001 15.0

18900 nylanmi01 2002 0.0

18902 nylanmi01 2003 0.0

# Longer to Wider format conversion

pivot = df.pivot(index="playerID", columns="year", values="Goals")

year 2001 2002 2003

playerID

hlavaja01 7.0 1.0 5.0

markoan01 5.0 13.0 6.0

nylanmi01 15.0 0.0 0.0

pivot = pivot.reset_index()

pivot.columns.name = None

pivot

playerID 2001 2002 2003

0 hlavaja01 7.0 1.0 5.0

1 markoan01 5.0 13.0 6.0

2 nylanmi01 15.0 0.0 0.0

10) Wide to Long format (melt)

# melt()

# Pandas melt() function is used to change the DataFrame format from wide to long.

pivot.melt(id_vars="playerID", var_name="year", value_name="goals")

playerID year goals

0 hlavaja01 2001 7.0

1 markoan01 2001 5.0

2 nylanmi01 2001 15.0

3 hlavaja01 2002 1.0

4 markoan01 2002 13.0

5 nylanmi01 2002 0.0

6 hlavaja01 2003 5.0

7 markoan01 2003 6.0

8 nylanmi01 2003 0.0

-------

Pandas Multi-level Index

1) How to set multi-index

mi = df.set_index(['playerID', 'year'])

mi.head()

2) List multi-index values 

mi.index

MultiIndex([('aaltoan01', 1997),

            ('aaltoan01', 1998),

            ('zyuzian01', 2005),

            ('zyuzian01', 2006),

            ('zyuzian01', 2007)],

           names=['playerID', 'year'], length=28616)

3) len(mi.index.levels) # 2

4) mi.index.levels[0]

Index(['aaltoan01', 'abdelju01', 'abidra01', 'abrahth01', 'actonke01',

       'adamlu01', 'adamru01'], dtype='object', name='playerID', length=4627)

5) mi.index.levels[1]

Int64Index([1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990],

           dtype='int64', name='year')

6) mi.groupby(level="year")['G'].max().head()

year

1980    68.0

1981    92.0

1982    71.0

1983    87.0

1984    73.0

Name: G, dtype: float64

7) idmax (gives index)

mi.groupby(level="year")['G'].idmax().head()

year

1980    (bossymi01, 1980)

1981    (gretzwa01, 1981)

1982    (gretzwa01, 1982)

1983    (gretzwa01, 1983)

1984    (gretzwa01, 1984)

Name: G, dtype: object

8) Filter based on above

mi.loc[mi.groupby(level="year")['G'].idxmax()].head()

firstName lastName pos Year Mon Day Country State City tmID GP G A Pts SOG

playerID year

bossymi01 1980 Mike Bossy R 1957.0 1.0 22.0 Canada QC Montreal NYI 79.0 68.0 51.0 119.0 315.0

gretzwa01 1981 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 80.0 92.0 120.0 212.0 369.0

1982 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 80.0 71.0 125.0 196.0 348.0

1983 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 74.0 87.0 118.0 205.0 324.0

1984 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 80.0 73.0 135.0 208.0 358.0

Python 2 Vs 3

Python 2	Python 3
input() may store as int, string raw_input() stores str always	input() function was fixed in Python 3 so that it always stores the user inputs as str
print "Hi" print("Hi")	print("Hi")
3/2 ==> floor(1.5) => 1 (defaults to floor), return int	3/2 ==> 1.5
Strings default stores as Ascii	Strings default stores as unicode Unicode is a superset of ASCII and hence, can encode more characters including foreign ones.
sorted(employees.items(), key=lambda(x,y): y['age'])	sorted(employees.items(), key=lambda x: x[1]['age'])
	AsyncIO
	Fstrings
	It is recommended to use __future__ imports it if you are planning Python 3.x support for your code
xrange() - Lazy evaluation	range() - Lazy evaluation
except NameError, err:	except NameError as err:
my_generator = (letter for letter in 'abcdefg') next(my_generator) my_generator.next()	my_generator = (letter for letter in 'abcdefg') next(my_generator)
print 'Python', python_version() i = 1 print 'before: i =', i print 'comprehension: ', [i for i in range(5)] print 'after: i =', i Python 2.7.6 before: i = 1 comprehension: [0, 1, 2, 3, 4] after: i = 4	Python 3.x for-loop variables don’t leak into the global namespace anymore! print ('Python', python_version()) i = 1 print 'before: i =', i print 'comprehension: ', [i for i in range(5)] print 'after: i =', i Python 3.4.1 before: i = 1 comprehension: [0, 1, 2, 3, 4] after: i = 1
print range(3) print type(range(3)) [0, 1, 2] <type 'list'>	print range(3) print type(range(3)) print(list(range(3))) range(0, 3) <class 'range'> [0, 1, 2]
round(15.5) # 16.0 round(16.5) # 17.0	Bankers rounding round(15.5) # 16 round(16.5) # 16

Python itertools

# Itertools

# Iterate over data structures that can be stepped over using a for-loop.

# Such data structures are also known as iterables.

# Itertools are more readable, fast, memory-efficient

# provides various functions that work on iterators to produce complex iterators.

# Infinite iterators: count, cycle, repeat

# Finite iterators: chain, compress, dropwhile

import itertools

# 1) Count - Infinite iterator

# print the first four even numbers

result = itertools.count(start = 0, step = 2)

for number in result:

if number < 8:

print (number)

else:

break

# Output:

# 0

# 2

# 4

# 6

# 2) Cycle - Infinite iterator

result = itertools.cycle('test')

counter = 0

for each in result:

if counter > 10:

break

counter += 1

print(each)

# Output:

# t

# e

# s

# t

# t

# e

# s

# t

# t

# e

# s

# 3) Repeat - Infinite iterator

result = itertools.repeat('test', 2)

for each in result:

print(each)

# Output:

# test

# test

# 1) Chain - Finite iterator

l1 = ['aaa', 'bbb', 'ccc']

l2 = ['ddd', 'eee', 'fff']

result = itertools.chain(l1, l2)

for each in result:

print(each)

# Output:

# aaa

# bbb

# ccc

# ddd

# eee

# fff

# 2) Compress - Finite iterator

l1 = ['aaa', 'bbb', 'ccc']

l2 = [True, False, False]

result = itertools.compress(l1, l2)

for each in result:

print(each)

# Output:

# aaa

# 3) Dropwhile - Finite iterator

# keeps on dropping values from the iterable until it encounters the first element

def is_positive(n):

return n > 0

value_list =[5, 6, -8, -4, 2]

result = list(itertools.dropwhile(is_positive, value_list))

print(result)

# Output:

# [-8, -4, 2]

# 4) groupby

ll = [("aaa", 1), ("aaa", 2), ("bbb", 3), ("bbb", 4)]

# Key function

key_func = lambda y: y[0]

for key, group in itertools.groupby(ll, key_func):

print(key + " :", list(group))

# Output:

# aaa : [('aaa', 1), ('aaa', 2)]

# bbb : [('bbb', 3), ('bbb', 4)]

Pandas Tutorial 2 (data formats, db, files, pickle)

import numpy as np

import pandas as pd

# nrows - just read first N rows

# usecols - read only those columns

df = pd.read_csv("input.csv", index_col="Id")

print(df.head(10))

# Name Age City

# Id

# 1 John 30 Bangalore

# 2 Doe 25 Chennai

# 3 Mary 22 Hyderabad

# 4 Tom 35 Mumbai

df = pd.read_csv("input.csv", index_col="Id", nrows=2, usecols=['Id', 'Name'])

print(df.head(10))

# Name

# Id

# 1 John

# 2 Doe

# Serialize & save to disk

df = pd.read_csv("input.csv", index_col="Id")

df.to_pickle('data_frame.pickle')

df1 = pd.read_pickle('data_frame.pickle')

print(df1.head(10))

# Name Age City

# Id

# 1 John 30 Bangalore

# 2 Doe 25 Chennai

# 3 Mary 22 Hyderabad

# 4 Tom 35 Mumbai

# Dataframe To Table

table_dtype = {

'name': VARCHAR(),

'city': VARCHAR(),

'state': VARCHAR(),

'pincode': INTEGER()

}

df1.to_sql(con=db_conn, name=table_name, dtype=table_dtype,

if_exists='replace', index=False)

# From Table to Dataframe

raw_data_sql = f""" select id, name, city, state

from event_raw_data

"""

df1 = pd.read_sql(raw_data_sql, con=db_conn, index_col="id")

Pandas Tutorial 1

import numpy as np

import pandas as pd

la = np.random.rand(3)

dd = pd.Series(la)

print(dd.head(10))

# 0 0.145517

# 1 0.718904

# 2 0.448772

# dtype: float64

ll = range(3)

dd = pd.Series(ll)

print(dd.head(10))

# 0 0

# 1 1

# 2 2

# dtype: int64

dd = pd.Series(ll, index=['First', 'Second', 'Third'])

print(dd.head(10))

# First 0

# Second 1

# Third 2

# dtype: int64

print(dd["First"]) # 0

print(dd[0]) # 0

nd = np.random.rand(3,2)

df = pd.DataFrame(nd)

df.columns = ["First", "Second"]

print(df.head(10))

# First Second

# 0 0.567850 0.275874

# 1 0.839382 0.792727

# 2 0.445246 0.537417

print(df.loc[0])

# First 0.567850

# Second 0.275874

# Name: 0, dtype: float64

print(df.iloc[0,1]) #0.275874

print(df.loc[0, "Second"]) #0.275874

Pyspark Tutorial 1

#............................................................................
##   Loading and Storing Data
#............................................................................

#Create a Spark Session
SpSession = SparkSession \
    .builder \
    .master("local[2]") \ #[2] is no of partitions
    .appName("learnSpark") \
    .config("spark.executor.memory", "1g") \
    .config("spark.cores.max","2") \
    .getOrCreate()

#Get the Spark Context from Spark Session
SpContext = SpSession.sparkContext

#Get the Spark Context from Spark Session
SpContext = SpSession.sparkContext
print SpContext

#Create an RDD by loading from a file
tweetsRDD = SpContext.textFile("movietweets.csv")
print tweetsRDD

#Action - Count the number of tweets
print tweetsRDD.count()

#show top 5 records
print tweetsRDD.take(5)

#Transform Data - change to upper Case
ucRDD = tweetsRDD.map( lambda x : x.upper() )
print ucRDD.take(5)

#Load from a collection
collData = SpContext.parallelize([4,3,8,5,8])
collData.collect()  # bring the entire RDD to the driver node, could be expensive

#Load the file. Lazy initialization
autoData = SpContext.textFile("auto-data.csv")
autoData.cache()
#Loads only now.
print autoData.count() #198
print autoData.first() #prints 1st line
print autoData.take(5) #gives you a list of 5 elements

#Save to a local file. First collect the RDD to the master
#and then save as local file.
autoDataFile = open("auto-data-saved.csv","w")
autoDataFile.write("\n".join(autoData.collect()))
autoDataFile.close()

#............................................................................
##   Transformations
#............................................................................

#Map and create a new RDD
tsvData=autoData.map(lambda x : x.replace(",","\t"))
print tsvData.take(5)

#Filter and create a new RDD
toyotaData=autoData.filter(lambda x: "toyota" in x)
print toyotaData #RDD object it prints
print toyotaData.count() #32

#FlatMap
words=toyotaData.flatMap(lambda line: line.split(","))
print words.count() #384 words in 32 lines of toyota
print words.take(20) #it takes 20 words from 384 and print
print '----'    

#Distinct
for numbData in collData.distinct().collect():
    print(numbData)
print '----'    

#Set operations
words1 = SpContext.parallelize(["hello","war","peace","world"])
words2 = SpContext.parallelize(["war","peace","universe"])

for unions in words1.union(words2).distinct().collect():
    print(unions)

print '----'    
for intersects in words1.intersection(words2).collect():
    print(intersects)

print '----'    
#Using functions for transformation
#cleanse and transform an RDD
def cleanseRDD(autoStr) :
    if isinstance(autoStr, int) :
        return autoStr
    attList=autoStr.split(",")
    #convert doors to a number str
    if attList[3] == "two" :
         attList[3]="2"
    else :
         attList[3]="4"
    #Convert Drive to uppercase
    attList[5] = attList[5].upper()
    return ",".join(attList)
    
cleanedData=autoData.map(cleanseRDD)
print cleanedData.collect() #it returns a List

#............................................................................
##   Actions
#............................................................................

#reduce - compute the sum
collData.collect()  #[4,3,8,5,8]
print collData.reduce(lambda x,y: x+y) #28

print '----'    
#find the shortest line - reduce() RDD function 
print autoData.reduce(lambda x,y: x if len(x) < len(y) else y)

print '----'    

#Use a function to perform reduce 
def getMPG( autoStr) :
    if isinstance(autoStr, int) :
        return autoStr
    attList=autoStr.split(",")
    if attList[9].isdigit() :
        return int(attList[9])
    else:
        return 0

#find average MPG-City for all cars    
print autoData.reduce(lambda x,y : getMPG(x) + getMPG(y)) \
    / (autoData.count()-1.0)  # account for header line

#............................................................................
##   Working with Key/Value RDDs
#............................................................................

#create a Key Value RDD of auto Brand and Horsepower
cylData = autoData.map( lambda x: ( x.split(",")[0], \
    x.split(",")[7]))
print cylData.count() #198
print '----'    
print cylData.take(5) #take first 5 key-value pairs
print '----'    
print cylData.keys().collect() #get all keys from key-value pair RDD cylData
print '----'    

#Remove header row
header = cylData.first()
print header
cylHPData= cylData.filter(lambda line: line != header) #get data without header
print cylHPData.count() #197

print '----'    

#Find average HP by Brand
#Add a count 1 to each record and then reduce to find totals of HP and counts
addOne = cylHPData.mapValues(lambda x: (x, 1))
print addOne.collect() #(u'bmw', (u'182', 1)), (u'mercedes-benz', (u'123', 1))......, (u'bmw', (u'182', 1)), (u'mercedes-benz', (u'184', 1)) ]

print '----'    
brandValues= addOne \
    .reduceByKey(lambda x, y: (int(x[0]) + int(y[0]), x[1] + y[1])) 
print brandValues.collect() #(u'mercedes-benz', (1170, 8)), (u'mitsubishi', (1353, 13)), (u'saab', (760, 6)), (u'volkswagen', (973, 12))
print '----'    

#find average by dividing HP total by count total
print brandValues.mapValues(lambda x: int(x[0])/int(x[1])). \
    collect()
#[(u'dodge', 84), (u'mercury', 175), (u'jaguar', 204), (u'alfa-romero', 125), (u'nissan', 102), (u'toyota', 92), (u'plymouth', 86), (u'mazda', 86), (u'subaru', 86), (u'peugot', 99), (u'porsche', 191), (u'isuzu', 84), (u'chevrolet', 62), (u'honda', 80), (u'volvo', 128), (u'bmw', 138), (u'mercedes-benz', 146), (u'mitsubishi', 104), (u'saab', 126), (u'volkswagen', 81), (u'audi', 114)]

print '----'    

#............................................................................
##   Advanced Spark : Accumulators & Broadcast Variables
#............................................................................

#function that splits the line as well as counts sedans and hatchbacks
#Speed optimization

    
#Initialize accumulator
sedanCount = SpContext.accumulator(0)
hatchbackCount = SpContext.accumulator(0)

#Set Broadcast variable
sedanText=SpContext.broadcast("sedan")
hatchbackText=SpContext.broadcast("hatchback")

def splitLines(line) :
    global sedanCount
    global hatchbackCount

    #Use broadcast variable to do comparison and set accumulator
    if sedanText.value in line:
       sedanCount += 1
    if hatchbackText.value in line:
       hatchbackCount += 1
        
    return line.split(",")


#do the map
splitData=autoData.map(splitLines)

#Make it execute the map (lazy execution)
print splitData.count()
print '----'    
print(sedanCount, hatchbackCount)
print '----'    

#............................................................................
##   Advanced Spark : Partitions
#............................................................................
print collData.getNumPartitions()

#Specify no. of partitions.
collData = SpContext.parallelize([3,5,4,7,4], 4)
print collData.cache()
print collData.count()

print '----'    
print collData.getNumPartitions()

print '----'    
#localhost:4040 shows the current spark instance

RDD 

• Transformation - map
• Action - reduce, take
• lazy loads

Persist()

• percentofDelayedFlights = flightsParsed.filter(lambda x:x.delay>0).count() / flaot(flightsParsed.count()) 
• Here we are using flightsParsed twice, instead we can force RDD to be materialized once using persist()
• flightsParsed.persist()  # Cached, huge performance saving
• Any RDD you use over and over again, you need to persist() to improve performance
• flightsParsed.unpersist() # UnCache, free-up memory

aggregate()

• totalDistance = flightsParsed.filter(lambda x:x.distance).reduce(lambda x,y:x+y)
• avgDistance = totalDistance / flightsParsed.count()
• Instead of two actions reduce & count, we can go with one using aggregate
• 
  

Freq Dist by Hours

•  flightsParsed.filter(lambda x: int(x.dep_delay/60)).countByValue()

Python Virtual Enviroment

Install python3 on top of python2 (using virtual environment)

python3 -m venv < python3env>

or

pip3 install virtualenv

virtualenv -p /usr/local/bin/python3 python3env

Activate:

    source python3env/bin/activate

    

    #Install modules within the virtual ENV

    pip install requests

    #pip Install requirements.txt, python3

    pip3 install -r requirements.txt

Deactivate:

python3env>> deactivate

Python Local Vs Global

#########################
#   Global Scope Vs Enclosing Scope Vs Local Scope 
#   LEGB rule
#   Local(L): Defined inside function/class
#   Enclosed(E): Defined inside enclosing functions(Nested function concept)
#   Global(G): Defined at the uppermost level
#   Built-in(B): Reserved names in Python builtin modules
#########################

message = 'global'

def enclosing():
    message = 'enclosing'
    def local():
        message = 'local'
    print('enclosing message: ', message)   # enclosing
    local()
    print('enclosing message: ', message)   # enclosing


def enclosing_nonlocal():
    message = 'enclosing'
    def local():
        nonlocal message    # This refers to the above message in enclosing scope, not in global scope
        message = 'local'
    print('enclosing message: ', message)   # enclosing
    local()
    print('enclosing message: ', message)   # local


def enclosing_global():
    message = 'enclosing'
    def local():
        global message
        message = 'local'  # Here you are updating message in global scope, not in enclosing scope
    print('enclosing message: ', message)   # enclosing
    local()
    print('enclosing message: ', message)    # enclosing 


if __name__ == '__main__':
    print('------------------------------------')
    print('global message: ', message)
    enclosing()
    print('global message: ', message)
    print('----------------NONLOCAL------------------')
    print('global message: ', message)
    enclosing_nonlocal()
    print('global message: ', message)
    print('----------------GLOBAL--------------------')
    print('global message: ', message)
    enclosing_global()
    print('global message: ', message)
    print('------------------------------------')

"""
Output:

------------------------------------
global message:  global
enclosing message:  enclosing
enclosing message:  enclosing
global message:  global
----------------NONLOCAL------------------
global message:  global
enclosing message:  enclosing
enclosing message:  local
global message:  global
----------------GLOBAL--------------------
global message:  global
enclosing message:  enclosing
enclosing message:  enclosing
global message:  local
------------------------------------

"""