Pandas Tutorial 3

1) Drop Nulls of one column in Pandas

Find isnull in a column:

isnull(master["playerID"]).value_counts()

Output:

False    7520

True      241

Name: playerID, dtype: int64

Drop nulls of specific columns (dropna)

master_orig = master.copy()

master = master.dropna(subset=["playerID"])

master.shape

2) Drop Nulls of multiple column in Pandas

how = 'all' # if all subset cols are nulls

how = 'any' # if any of the subset cols are nulls

df.dropna(subset=[col_list], how='all')

master = master.dropna(subset=["firstNHL", "lastNHL"], how="all")

3)

master1 = master[master["lastNHL"] >= 1980]

master1.shape # (4627, 31)

Vs

master1 = master.loc[master["lastNHL"] >= 1980]

master1.shape # (4627, 31)

But later is good, more performance with huge data

4) filter columns

master.filter(columns_to_keep).head()

(or)

master = master.filter(regex="(playerID|pos|^birth)|(Name$)")

5) Find DF memory usage

df.memory_usage()

def mem_mib(df):

    mem = df.memory_usage().sum() / (1024 * 1024)

    print(f'{mem}.2f Mib')

    

mem_mib(master) # 0.39 MiB

mem_mib(master_orig) # 1.84 MiB

6) Categorical

# A string variable consisting of only a few different values. 

# Converting such a string variable to a categorical variable will save some memory.

def make_categorical(df, col_name):

    df.loc[:, col_name] = pd.Categorical(df[col_name]) 

# to save memory

make_categorical(master, "pos")

make_categorical(master, "birthCountry")

make_categorical(master, "birthState")

7)

pd.read_pickle()

8) Joins

Default is inner join

pd.merge(df1, df2, how='left')

# We joining based on player id of both dfs

# If left df has PlayerId & right df has plrId

pd.merge(df1, df2, left_on='PlayerId', right_on='plrId')

# We joining based on player id of both DFs

# Say if left DF has player id as index

# Here resultant merge DF has index from right DF 

# left DF index (left_index) is not considered in merge dF

pd.merge(df1, df2, left_index=True, right_on='plrId')

# We joining based on player id of both dfs

# Say if right df has player id as index

# Here resultant merge DF has index from left DF 

# right DF index (right_index) is not considered in merge dF

pd.merge(df1, df2, left_on='PlayerId', right_index=True)

# We can even set DF index (set_index) and use

# left_index and right_index 

pd.merge(df1, df2.set_index("playerID", drop=True),

                            left_index=True, right_index=True).head()

# Indicator

# It creates additional column _merge

# It indicates both, left_only, right_only

merged = pd.merge(master2, scoring, left_index=True,

                  right_on="playerID", how="right", indicator=True)

merged["_merge"].value_counts()

both          28579

right_only       37

left_only         0

Name: _merge, dtype: int64

# Filter only right_only

merged[merged["_merge"] == "right_only"].head()

# Filter only right_only or left_only

merged[(merged["_merge"] == "right_only") | (merged["_merge"] == "left_only")].sample(3)

or

# 

merged[merged["_merge"].str.endswith("only")].sample(5)

# Filter out 1:m (one to many)

try:

pd.merge(df1, df2, left_index=True, right_on='plrId', validate="1:m").head()

except Exception as e:

pass

8) Drop random records

df.drop(drop.sample(5).index)

-------

9) Longer to Wider format (pivot)

df.show()

playerID year Goals

10320 hlavaja01 2001 7.0

10322 hlavaja01 2002 1.0

10324 hlavaja01 2003 5.0

15873 markoan01 2001 5.0

15874 markoan01 2002 13.0

15875 markoan01 2003 6.0

18899 nylanmi01 2001 15.0

18900 nylanmi01 2002 0.0

18902 nylanmi01 2003 0.0

# Longer to Wider format conversion

pivot = df.pivot(index="playerID", columns="year", values="Goals")

year 2001 2002 2003

playerID

hlavaja01 7.0 1.0 5.0

markoan01 5.0 13.0 6.0

nylanmi01 15.0 0.0 0.0

pivot = pivot.reset_index()

pivot.columns.name = None

pivot

playerID 2001 2002 2003

0 hlavaja01 7.0 1.0 5.0

1 markoan01 5.0 13.0 6.0

2 nylanmi01 15.0 0.0 0.0

10) Wide to Long format (melt)

# melt()

# Pandas melt() function is used to change the DataFrame format from wide to long.

pivot.melt(id_vars="playerID", var_name="year", value_name="goals")

playerID year goals

0 hlavaja01 2001 7.0

1 markoan01 2001 5.0

2 nylanmi01 2001 15.0

3 hlavaja01 2002 1.0

4 markoan01 2002 13.0

5 nylanmi01 2002 0.0

6 hlavaja01 2003 5.0

7 markoan01 2003 6.0

8 nylanmi01 2003 0.0

-------

Pandas Multi-level Index

1) How to set multi-index

mi = df.set_index(['playerID', 'year'])

mi.head()

2) List multi-index values 

mi.index

MultiIndex([('aaltoan01', 1997),

            ('aaltoan01', 1998),

            ('zyuzian01', 2005),

            ('zyuzian01', 2006),

            ('zyuzian01', 2007)],

           names=['playerID', 'year'], length=28616)

3) len(mi.index.levels) # 2

4) mi.index.levels[0]

Index(['aaltoan01', 'abdelju01', 'abidra01', 'abrahth01', 'actonke01',

       'adamlu01', 'adamru01'], dtype='object', name='playerID', length=4627)

5) mi.index.levels[1]

Int64Index([1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990],

           dtype='int64', name='year')

6) mi.groupby(level="year")['G'].max().head()

year

1980    68.0

1981    92.0

1982    71.0

1983    87.0

1984    73.0

Name: G, dtype: float64

7) idmax (gives index)

mi.groupby(level="year")['G'].idmax().head()

year

1980    (bossymi01, 1980)

1981    (gretzwa01, 1981)

1982    (gretzwa01, 1982)

1983    (gretzwa01, 1983)

1984    (gretzwa01, 1984)

Name: G, dtype: object

8) Filter based on above

mi.loc[mi.groupby(level="year")['G'].idxmax()].head()

firstName lastName pos Year Mon Day Country State City tmID GP G A Pts SOG

playerID year

bossymi01 1980 Mike Bossy R 1957.0 1.0 22.0 Canada QC Montreal NYI 79.0 68.0 51.0 119.0 315.0

gretzwa01 1981 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 80.0 92.0 120.0 212.0 369.0

1982 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 80.0 71.0 125.0 196.0 348.0

1983 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 74.0 87.0 118.0 205.0 324.0

1984 Wayne Gretzky C 1961.0 1.0 26.0 Canada ON Brantford EDM 80.0 73.0 135.0 208.0 358.0

Python 2 Vs 3

Python 2	Python 3
input() may store as int, string raw_input() stores str always	input() function was fixed in Python 3 so that it always stores the user inputs as str
print "Hi" print("Hi")	print("Hi")
3/2 ==> floor(1.5) => 1 (defaults to floor), return int	3/2 ==> 1.5
Strings default stores as Ascii	Strings default stores as unicode Unicode is a superset of ASCII and hence, can encode more characters including foreign ones.
sorted(employees.items(), key=lambda(x,y): y['age'])	sorted(employees.items(), key=lambda x: x[1]['age'])
	AsyncIO
	Fstrings
	It is recommended to use __future__ imports it if you are planning Python 3.x support for your code
xrange() - Lazy evaluation	range() - Lazy evaluation
except NameError, err:	except NameError as err:
my_generator = (letter for letter in 'abcdefg') next(my_generator) my_generator.next()	my_generator = (letter for letter in 'abcdefg') next(my_generator)
print 'Python', python_version() i = 1 print 'before: i =', i print 'comprehension: ', [i for i in range(5)] print 'after: i =', i Python 2.7.6 before: i = 1 comprehension: [0, 1, 2, 3, 4] after: i = 4	Python 3.x for-loop variables don’t leak into the global namespace anymore! print ('Python', python_version()) i = 1 print 'before: i =', i print 'comprehension: ', [i for i in range(5)] print 'after: i =', i Python 3.4.1 before: i = 1 comprehension: [0, 1, 2, 3, 4] after: i = 1
print range(3) print type(range(3)) [0, 1, 2] <type 'list'>	print range(3) print type(range(3)) print(list(range(3))) range(0, 3) <class 'range'> [0, 1, 2]
round(15.5) # 16.0 round(16.5) # 17.0	Bankers rounding round(15.5) # 16 round(16.5) # 16

Python List Vs Array

# Arrays Vs Lists

• Arrays need to be declared. Lists don’t
• Arrays can store data very compactly
• Arrays are great for numerical operations

import array

# Array (stores single data type)

array.array('i', [1, 22, 30, 44, 51]) # integer

array.array('d', [2.5, 3.2, 3.3]) # float

array.array('u', ['a', 'b', 'c']) # unicode

#List

ll = ['abc', 10, ['a', 'b', 'c'], (1,2,3)] # List can store anything 

import numpy as np

# Numpy Array (it can store various data types)

array_2 = np.array(["numbers", 3, 6, 9, 12])

print (array_2)

print(type(array_2))

 

Python random

import random

random.random() ---> 0 to 1 float

random.randint() * 100 ---> 0 to 100 integer

random.randint() * 100 - 50       ---> -50 to 50 integer

random.randint(1, 40) ---> b/w 1 to 40 integer

random.uniform(1,50) ---> b/w 1 to 50 float

Python itertools

# Itertools

# Iterate over data structures that can be stepped over using a for-loop.

# Such data structures are also known as iterables.

# Itertools are more readable, fast, memory-efficient

# provides various functions that work on iterators to produce complex iterators.

# Infinite iterators: count, cycle, repeat

# Finite iterators: chain, compress, dropwhile

import itertools

# 1) Count - Infinite iterator

# print the first four even numbers

result = itertools.count(start = 0, step = 2)

for number in result:

if number < 8:

print (number)

else:

break

# Output:

# 0

# 2

# 4

# 6

# 2) Cycle - Infinite iterator

result = itertools.cycle('test')

counter = 0

for each in result:

if counter > 10:

break

counter += 1

print(each)

# Output:

# t

# e

# s

# t

# t

# e

# s

# t

# t

# e

# s

# 3) Repeat - Infinite iterator

result = itertools.repeat('test', 2)

for each in result:

print(each)

# Output:

# test

# test

# 1) Chain - Finite iterator

l1 = ['aaa', 'bbb', 'ccc']

l2 = ['ddd', 'eee', 'fff']

result = itertools.chain(l1, l2)

for each in result:

print(each)

# Output:

# aaa

# bbb

# ccc

# ddd

# eee

# fff

# 2) Compress - Finite iterator

l1 = ['aaa', 'bbb', 'ccc']

l2 = [True, False, False]

result = itertools.compress(l1, l2)

for each in result:

print(each)

# Output:

# aaa

# 3) Dropwhile - Finite iterator

# keeps on dropping values from the iterable until it encounters the first element

def is_positive(n):

return n > 0

value_list =[5, 6, -8, -4, 2]

result = list(itertools.dropwhile(is_positive, value_list))

print(result)

# Output:

# [-8, -4, 2]

# 4) groupby

ll = [("aaa", 1), ("aaa", 2), ("bbb", 3), ("bbb", 4)]

# Key function

key_func = lambda y: y[0]

for key, group in itertools.groupby(ll, key_func):

print(key + " :", list(group))

# Output:

# aaa : [('aaa', 1), ('aaa', 2)]

# bbb : [('bbb', 3), ('bbb', 4)]

Pandas Tutorial 1

import numpy as np

import pandas as pd

la = np.random.rand(3)

dd = pd.Series(la)

print(dd.head(10))

# 0 0.145517

# 1 0.718904

# 2 0.448772

# dtype: float64

ll = range(3)

dd = pd.Series(ll)

print(dd.head(10))

# 0 0

# 1 1

# 2 2

# dtype: int64

dd = pd.Series(ll, index=['First', 'Second', 'Third'])

print(dd.head(10))

# First 0

# Second 1

# Third 2

# dtype: int64

print(dd["First"]) # 0

print(dd[0]) # 0

nd = np.random.rand(3,2)

df = pd.DataFrame(nd)

df.columns = ["First", "Second"]

print(df.head(10))

# First Second

# 0 0.567850 0.275874

# 1 0.839382 0.792727

# 2 0.445246 0.537417

print(df.loc[0])

# First 0.567850

# Second 0.275874

# Name: 0, dtype: float64

print(df.iloc[0,1]) #0.275874

print(df.loc[0, "Second"]) #0.275874

Python Super Tutorial 2

super() 

• will allow us not to call explicitly
• Enable multiple inheritance

class Computer():

    def __init__(self, computer, ram, storage):

        self.computer = computer

        self.ram = ram

        self.storage = storage

# Class Mobile inherits Computer

class Mobile(Computer):

    def __init__(self, computer, ram, storage, model):

        super().__init__(computer, ram, storage)

        self.model = model

Apple = Mobile('Apple', 2, 64, 'iPhone X')

print('The mobile is:', Apple.computer)

print('The RAM is:', Apple.ram)

print('The storage is:', Apple.storage)

print('The model is:', Apple.model)

Python Unittest

import unittest

class TestStringMethods(unittest.TestCase):

    def test_upper(self):

         self.assertEqual('foo'.upper(), 'FOO')

    def test_isupper(self):

         self.assertTrue('FOO'.isupper())

         self.assertFalse('Foo'.isupper())

    def test_split(self):

         s = 'hello world'

         self.assertEqual(s.split(), ['hello', 'world'])

         # check that s.split fails when the separator is not a string

         with self.assertRaises(TypeError):

              s.split(2)

if __name__ == '__main__':

    unittest.main()

python collections Counter

from collections import Counter

listA = [40, 10, 20, 30, 10]
c = Counter(listA)

print(c)  #Counter({10: 2, 40: 1, 20: 1, 30: 1})

python collections OrderedDict

from collections import namedtuple
from collections import OrderedDict, defaultdict

# count no elements
# print dict in same order
d = OrderedDict() 
n = 5
ll = ['aaa', 'bbb', 'ccc', 'aaa']
for i in ll:
    d.setdefault(i, 0)  #it initialises to, else index error may come first time 
    d[i]+= 1

print(len(d))            # 3 
print(*d.values())   # 2 1 1
print(d)              #OrderedDict([('aaa', 2), ('bbb', 1), ('ccc', 1)])

Python Collections Counter, Defaultdict

1) Counter

from collections import Counter
myList = [10,10,20,30,4,5,3,2,3,4,2,1,2,30]
print(Counter(myList))
# Counter({2: 3, 10: 2, 30: 2, 4: 2, 3: 2, 20: 1, 5: 1, 1: 1})

print(Counter(myList).items())
# dict_items([(10, 2), (20, 1), (30, 2), (4, 2), (5, 1), (3, 2), (2, 3), (1, 1)])

print(Counter(myList).keys())
# dict_keys([10, 20, 30, 4, 5, 3, 2, 1])

print(Counter(myList).values())
# dict_values([2, 1, 2, 2, 1, 2, 3, 1])

2) defaultdict

from collections import defaultdict

a)

d = defaultdict(list)

# Even if key not exists it defaults to list/int
d['python'].append("awesome")
d['others'].append("not relevant")
d['python'].append("language")
d['test']

for i in d.items():
    print(i)

# O/P:
# ('python', ['awesome', 'language'])
# ('others', ['not relevant'])
# ('test', [])


b)

d = defaultdict(int)
d['without_val']
d['with_val'] = 100
for i in d.items():
    print(i)

# O/P:
# ('without_val', 0)
# ('with_val', 100)

c)

demo = defaultdict(int)

print(demo[300]) # 0

Python collections namedtuple

import collections

fields = ['OBJECTID', 'Identifier', 'Occurrence_Date', 'Day_of_Week', 'Occurrence_Month', 'Occurrence_Day', 'Occurrence_Year', 'Occurrence_Hour', 'CompStat_Month', 'CompStat_Day', 'CompStat_Year', 'Offense', 'Offense_Classification', 'Sector', 'Precinct', 'Borough', 'Jurisdiction', 'XCoordinate', 'YCoordinate', 'Location_1']

Crime = collections.namedtuple('Crime', fields)

row1_value = ['1', 'f070032d', '09/06/1940 07:30:00 PM', 'Friday', 'Sep', '6', '1940', '19', '9', '7', '2010', 'BURGLARY', 'FELONY', 'D', '66', 'BROOKLYN', 'N.Y. POLICE DEPT', '987478', '166141', '(40.6227027620001, -73.9883732929999)']

row1_obj = Crime(*row1_value)

print(row1_obj)


Output:
Crime(OBJECTID='1', Identifier='f070032d', Occurrence_Date='09/06/1940 07:30:0
0 PM', Day_of_Week='Friday', Occurrence_Month='Sep', Occurrence_Day='6', Occur
rence_Year='1940', Occurrence_Hour='19', CompStat_Month='9', CompStat_Day='7',
 CompStat_Year='2010', Offense='BURGLARY', Offense_Classification='FELONY', Se
ctor='D', Precinct='66', Borough='BROOKLYN', Jurisdiction='N.Y. POLICE DEPT', 
XCoordinate='987478', YCoordinate='166141', Location_1='(40.6227027620001, -73
.9883732929999)')

Python Puzzle Remove even numbers

Python Puzzle Remove even numbers

# Wrong approach (incorrect - using For loop)

def removeEven(List):

    print(id(List)) # 139909029905664

    for each in List:

       i f each % 2 == 0:

          List.remove(each)

myList = [152, 168, 154, 32, -55, 81, 146, -34, -124, -9, 4, -31, -131, -86, -190, -38]

print(id(myList)) # 139909029905664

print(myList) # [1, 2, 4, 5, 10, 6, 3]

removeEven(myList)

print(myList) # [168, 32, -55, 81, -34, -9, -31, -131, -190]

# Wrong as when element gets deleted, index goes down

print('-' * 60)

# Correct approach (using While loop)

def removeEvenNew(List):

    print(id(List))

    i = 0

    while i < len(List):

      if List[i] % 2 == 0:

          List.remove(List[i])

      else:

          i += 1 

myList = [152, 168, 154, 32, -55, 81, 146, -34, -124, -9, 4, -31, -131, -86, -190, -38]

print(id(myList))

print(myList) # 

removeEvenNew(myList)

print(myList) # [-55, 81, -9, -31, -131]

Output:

140407115793920

[152, 168, 154, 32, -55, 81, 146, -34, -124, -9, 4, -31, -131, -86, -190, -38]

140407115793920

[168, 32, -55, 81, -34, -9, -31, -131, -190]

------------------------------------------------------------

140407114861888

[152, 168, 154, 32, -55, 81, 146, -34, -124, -9, 4, -31, -131, -86, -190, -38]

140407114861888

[-55, 81, -9, -31, -131]

Python Lists Advanced

ll = [10, 20, 30, 40, 50]

# insert, remove, pop
ll.remove(20) #[10, 30, 40, 50]
ll.pop() # #[10, 30, 40]

ll = [1, 3, 5, 'seven']
ll.insert(0, 2) 
print(ll) # [2, 1, 3, 5, 'seven']

ll.pop(2) # pops 2nd index element
print(ll) # [2, 1, 5, 'seven']

ll.pop() # takes out last item
print(ll) # [2, 1, 5]

# Slice
gg = [1, 3, 5, 'seven', 'eight', 'nine', [10, 20,]]
print(gg[1:4])  # [3, 5, 'seven']
print(gg[3:])  # ['seven', 'eight', 'nine', [10, 20]]
print(gg[:3])  # [1, 3, 5]
print(gg[:])  # [1, 3, 5, 'seven', 'eight', 'nine', [10, 20]]

print(gg[-1:]) # [[10, 20]]
print(gg[:-1]) # [1, 3, 5, 'seven', 'eight', 'nine']

print(gg[-3:-1]) # ['eight', 'nine']

# list[start:stop:step]
print(gg[0:7:2]) # [1, 5, 'eight', [10, 20]]

ff = [1, 3, 5, 'seven', 'eight', 'nine']
print(ff) #[1, 3, 5, 'seven', 'eight', 'nine']
ff[2:2] = ['test']
print(ff) # [1, 3, 'test', 5, 'seven', 'eight', 'nine']
ff[1:3] = []
print(ff) # [1, 5, 'seven', 'eight', 'nine']

del ff[::2] # Delete even numbred indeces
print(ff)  # [5, 'eight']

# Concatenate
kk = [1, 2, 3, 4] # [1, 2, 3, 4]
kk += 'ab' # since string, it takes as two elements
print(kk) # [1, 2, 3, 4, 'a', 'b']

kk += ['c', 'd']
print(kk) # [1, 2, 3, 4, 'a', 'b', 'c', 'd']

kk.extend(['e', 'f'])
print(kk) # [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f']

# List Vs Array
# Array has homogenous elements
# Python arrays are just wrappers for C language
import array
# type: 'd' (float), initializer list: [1, 2, 3]
newArray = array.array('i', [1, 2, 3])
print(newArray) # array('i', [1, 2, 3])

Python AsyncIO Example

import asyncio
import aiohttp
import time

async def crawl_one_url(url, session):
    get_request = session.get(url)
    print(url)
    res = await get_request
    txt = await res.text()
    get_request.close()
    return txt


async def crawl_urls(urls_to_crawl):
    session = aiohttp.ClientSession()

    work_to_do = list()
    for url in urls_to_crawl:
        work_to_do.append(crawl_one_url(url, session))
    print(*work_to_do)
    res = await asyncio.gather(*work_to_do)
    # print(res)
    await session.close()
    return res


def main():
    t0 = time.time()
    urls_to_crawl = list()
    urls_to_crawl.append('http://blog.prabhathkota.com/search/label/python')
    urls_to_crawl.append('http://blog.prabhathkota.com/search/label/perl')
    urls_to_crawl.append('http://blog.prabhathkota.com/search/label/unix')
    urls_to_crawl.append('http://blog.prabhathkota.com/search/label/aws')
    urls_to_crawl.append('http://blog.prabhathkota.com/search/label/java')
    asyncio.run(crawl_urls(urls_to_crawl))
    elapsed = time.time() - t0
    print(f"{len(urls_to_crawl)} URLS downloaded in {elapsed:.2f}")


if __name__ == '__main__':
    main()


Output:
<coroutine object crawl_one_url at 0x7f5fe36181c0> <coroutine object crawl_on
e_url at 0x7f5fe3618240> <coroutine object crawl_one_url at 0x7f5fe36182c0> <
coroutine object crawl_one_url at 0x7f5fe3618340> <coroutine object crawl_one
_url at 0x7f5fe36183c0>
http://blog.prabhathkota.com/search/label/python
http://blog.prabhathkota.com/search/label/perl
http://blog.prabhathkota.com/search/label/unix
http://blog.prabhathkota.com/search/label/aws
http://blog.prabhathkota.com/search/label/java
5 URLS downloaded in 0.48

Python Generator Send & Recieve

#############################################
# Yield returns a generator object
# use next() to access the return value
# If you attempt to invoke next() on a generator object that had already produced (yielded) all its values, it will throw StopIteration exception
# We can pass data to a generator function using the send() method defined
# Use generators to generate values and coroutines to consume values. Generator functions receive values are called coroutines. 

#############################################


############# Test1 (send & receive) ############ 
def test_yield_send():
    while True:
        item = yield
        print(f'Received item: {item}')

if __name__ == '__main__':
    gen = test_yield_send()
    print(gen)  #<generator object test_yield at 0x7ff2f0510c80>
    next(gen)
    gen.send(100)

Output:

<generator object test_yield_send at 0x7ff700b45c80>

Received item: 100

############ Test2 (send & receive) ############ 
def generate_num():
    i = 0
    while True:
        i += 1
        t = (yield i)
        print(t)


if __name__ == "__main__":
    gen = generate_num()

    item = gen.send(None)
    print("First received " + str(item))

    for i in range(0, 5):
        item = gen.send(100 + i)
        print("Other received " + str(item))


Output:
First received 1
100
Other received 2
101
Other received 3
102
Other received 4
103
Other received 5
104
Other received 6

Python multiprocessing spawn

Fork Vs Spawn
Fork child inherit all resources from the parent process

Fork is the default method for multi-processing
Spawn child doesn't inherit any resources from the parent process other than those required to execute the specified callable target.
Spawning a process is slower (because it re-imports) than forking a process

Spawn
Spawn is essentially a combination of fork followed by an exec system call
When a child process is spawned, anything imported at the module level in the (above )__main__ module of the parent process gets reimported in the child
Anything below __main__ will not be copied

test1.py
#########
Inside test1

test1.py
#########
Inside test2

spawn_test.py
############
from multiprocessing import Process
import test1
import test2

def process_task():
    print("I am inside child process")


if __name__ == '__main__':

    # Change the method to 'spawn' and verify
    # that the modules are reimported in the child process

    ### Spawn
    multiprocessing.set_start_method('spawn') 
    process = Process(target=process_task)
    process.start()
    process.join()
    print("I am inside parent process")


# Child reimports the test1, test2 module again...

Output:
Inside test1
Inside test2
Inside test1
Inside test2
I am child process
I am parent process

Python multiprocessing fork

Fork
# When we fork, the entire Python process is duplicated in memory including the Python interpreter, code, libraries, current stack, etc.
# This creates a new copy of the python interpreter.
# Fork creates two python interpreters each with its own GIL.
# Fork is faster than Spawn (Fork child inherit all resources from the parent process, Spawn re-imports all above main() method)

# Fork is the default method for multi-processing

 
#Disadvantages of Fork
# It won't work on windows
# When child shares parent libraries, values, data-structures, if a lock acquired by parent, child ends up waiting for that lock ever
# Very hard to debug when you import a third-party module/library that uses threads behind the scenes
# Fork and Multi-threading won't go well

from multiprocessing import Process
import multiprocessing
import os

file_desc = None

def process_task1():
    # write to the file in child process
    file_desc.write(f"\nWritten by child process with id {os.getpid()}")
    file_desc.flush()


if __name__ == '__main__':
    # create a file in the parent process
    file_desc = open("sample.txt", "w")
    file_desc.write(f"\nWritten by parent process with id {os.getpid()}")
    file_desc.flush()

    # Fork is default method to create a process
    # multiprocessing.set_start_method('fork')

    p = Process(target=process_task1)
    p.start()
    p.join()
    file_desc.close()

    file_des = open("sample.txt", "r")
    print(file_des.read())

    os.remove("sample.txt")


Output:
Written by parent process with id 288
Written by child process with id 294

python how to overcome GIL

Ref: https://realpython.com/python-gil/

# single_threaded.py
import time
from threading import Thread

COUNT = 50000000

def countdown(n):
    while n>0:
        n -= 1

start = time.time()
countdown(COUNT)
end = time.time()

print('Time taken in seconds -', end - start)
#Time taken in seconds - 6.20024037361145


# multi_threaded.py
import time
from threading import Thread

COUNT = 50000000

def countdown(n):
    while n>0:
        n -= 1

t1 = Thread(target=countdown, args=(COUNT//2,))
t2 = Thread(target=countdown, args=(COUNT//2,))

start = time.time()
t1.start()
t2.start()
t1.join()
t2.join()
end = time.time()

print('Time taken in seconds -', end - start)
# Time taken in seconds - 6.924342632293701
# No improvement here due to GIL & due to sharing of lock & releases


# multi_processing.py
# Uses cores instead of threads
from multiprocessing import Pool
import time

COUNT = 50000000
def countdown(n):
    while n>0:
        n -= 1

if __name__ == '__main__':
    pool = Pool(processes=2)
    start = time.time()
    r1 = pool.apply_async(countdown, [COUNT//2])
    r2 = pool.apply_async(countdown, [COUNT//2])
    pool.close()
    pool.join()
    end = time.time()
    print('Time taken in seconds -', end - start)
    #Time taken in seconds - 4.060242414474487

Python sum using multiprocessing pool

#####
# Read all text files & calculate Sum of all numbers in txt files

# Threading/input1.txt
# 100
# 200
# 300
# 400
# 500

# Threading/input2.txt
# 600
# 700
# 800
# 900
# 1000
#####

import glob
import multiprocessing

def get_txt_files():
    my_files = []
    for each in glob.glob('Threading/*.txt'):
        my_files.append(each)
    return my_files

def calculate_sum(input_file):
    with open(input_file) as fh:
      return sum(map(int, fh.readlines()))


if __name__ == '__main__':
    my_files = get_txt_files()
    #['Threading/input1.txt', 'Threading/input2.txt']
    print(my_files)

    # sys.exit()
    pool = multiprocessing.Pool(processes=2)
    results = pool.map(calculate_sum, my_files)
    print(f'Sum of individual files: {results}')
    total_sum = sum(results)
    print(f'Total Sum: {total_sum}')


Output:
######
['Threading/input1.txt', 'Threading/input2.txt']
Sum of individual files: [1500, 4000]
Total Sum: 5500