Python Generator Send & Recieve

#############################################
# Yield returns a generator object
# use next() to access the return value
# If you attempt to invoke next() on a generator object that had already produced (yielded) all its values, it will throw StopIteration exception
# We can pass data to a generator function using the send() method defined
# Use generators to generate values and coroutines to consume values. Generator functions receive values are called coroutines. 

#############################################


############# Test1 (send & receive) ############ 
def test_yield_send():
    while True:
        item = yield
        print(f'Received item: {item}')

if __name__ == '__main__':
    gen = test_yield_send()
    print(gen)  #<generator object test_yield at 0x7ff2f0510c80>
    next(gen)
    gen.send(100)

Output:

<generator object test_yield_send at 0x7ff700b45c80>

Received item: 100

############ Test2 (send & receive) ############ 
def generate_num():
    i = 0
    while True:
        i += 1
        t = (yield i)
        print(t)


if __name__ == "__main__":
    gen = generate_num()

    item = gen.send(None)
    print("First received " + str(item))

    for i in range(0, 5):
        item = gen.send(100 + i)
        print("Other received " + str(item))


Output:
First received 1
100
Other received 2
101
Other received 3
102
Other received 4
103
Other received 5
104
Other received 6

Python Generator yield

"""
Yield returns a generator object
use next() to access the return value
If you attempt to invoke next() on a generator object that had already produced (yielded) all its values, it will throw StopIteration exception
It can yield multiple items as well (as shown in second example)

"""

################# Test1 #################
def test_yield():
    yield 'hi'

if __name__ == '__main__':
  a = test_yield()
  print(a)  #<generator object test_yield at 0x7ff2f0510c80>
  val = next(a)
  print(val) # hi

Output:

hi

################# Test2 #################
# You can yield multiple values
def test_yield(name):
    yield 'hi'
    yield name

if __name__ == '__main__':
  a = test_yield('John')
  print(a)  #<generator object test_yield at 0x7ff2f0510c85>
  for item in a:
    print(item)

Output:

<generator object test_yield at 0x7f93f8066c80>

hi

John

Python AsyncIO

Ref: https://realpython.com/async-io-python/

• Threading Vs Multi-Processing
• Threading is better for I/O based tasks
• Multi-Processing is better for CPU based tasks
• What’s important to know about threading is that it’s better for IO-bound tasks.
• Concurrency Vs Parallelism
• Concurrency is when two tasks can start, run, and complete in overlapping time periods. e.g., Threading, AsyncIO
• Parallelism is when tasks literally run at the same time, eg. multi-processing. 
• While a CPU-bound task is characterised by the computer’s cores continually working hard from start to finish, an IO-bound job is dominated by a lot of waiting on input/output to complete.
• Preemptive multitasking Vs Cooperative multitasking
• OS preempts a thread forcing it to give up the use of CPU (E.g., Threading)
• Cooperative multitasking on the other hand, the running process voluntarily gives up the CPU to other processes E.g., (AsyncIO)
• Coroutine Vs Method/Function/Subroutine
• Method or Function returns a value and don't remember the state between invocations
• A coroutine is a special function that can give up control to its caller without losing its state
• Coroutine Vs Generator
• Generator yield back value to invoker
• Coroutine yields control to another coroutine and can resume execution from point it gave the control
• A generator can't accept arguments once it is started where as a coroutine can accept arguments once it started
• AsyncIO is a single-threaded, single-process design: it uses cooperative multitasking
• AsyncIO gives a feeling of concurrency despite using a single thread in a single process
• Coroutines (a central feature of async IO) can be scheduled concurrently, but they are not inherently concurrent.
• Asynchronous routines are able to “pause” while waiting on their ultimate result and let other routines run in the meantime.

import asyncio

import time

async def count_func():

    print("Line One")

    await asyncio.sleep(1) # await non-blocking call

    print("Line Two")

async def main():

    await asyncio.gather(count_func(), count_func(), count_func())

if __name__ == "__main__":

    t1 = time.time()

    asyncio.run(main())

    elapsed = time.time() - t1

    #This is supposed to take more than 3 secs

    print(f"{__file__} executed in {elapsed:0.2f} seconds.")

Output:

#####

Line One

Line One

Line One

Line Two

Line Two

Line Two

main.py executed in 1.10 seconds.

python context managers using contextlib

"""
# Using contextlib you don't have to explicitly write __enter__, __exit__
# yield instead of return
"""
import contextlib
import sys
import time

@contextlib.contextmanager
def context_manager_def_test():
    print('context_manager_def_test: ENTER')
    try:
        yield 'You are in with-block'
        print('context_manager_def_test: NORMAL EXIT')
    except Exception:
        print('context_manager_def_test: EXCEPTION EXIT', sys.exc_info())
        raise

print('*'*75)

with context_manager_def_test() as cm:
    print('Inside ContextManagerTest')
    print(cm)

print('*'*75)
time.sleep(1)

with context_manager_def_test() as cm:
    print('Inside ContextManagerTest')
    print(cm)
    raise ValueError('something is wrong')

print('*'*75)


"""
***************************************************************************
context_manager_def_test: ENTER
Inside ContextManagerTest
You are in with-block
context_manager_def_test: NORMAL EXIT
***************************************************************************
context_manager_def_test: ENTER
Inside ContextManagerTest
You are in with-block
context_manager_def_test: EXCEPTION EXIT (<class 'ValueError'>, ValueError('something is wrong'), <traceback object at 0x1023ed0c8>)
Traceback (most recent call last):
  File "/Users/prabhathkota/Workspace/prabhath/personal/Python_Scripts/context_managers/contextlib_example.py", line 31, in <module>
    raise ValueError('something is wrong')
ValueError: something is wrong
***************************************************************************
"""

Python context manager with exceptions

###################
# __enter__
# __enter__ is called before executing with-statement body
# __exit__
# __exit__ called after with-statement body
# File opening is context managers
###################


class ContextManagerTest:
    def __init__(self):
        print('Inside __init__')

    def __enter__(self):
        print('Inside __enter__')
        return 'returning ... Inside with block'
        # return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_type is None:
            print('Inside __exit__ without exception')
        else:
            print('Inside __exit__ with exception ({} - {} - {})' .format(exc_type, exc_val, exc_tb))


with ContextManagerTest() as cm:
    print('Inside ContextManagerTest')
    print(cm)
    raise ValueError('something is wrong')


"""
Traceback (most recent call last):
Inside __enter__
  File "...../Python_Scripts/context_managers/context_manager_with_exception.py", line 30, in <module>
Inside ContextManagerTest
    raise ValueError('something is wrong')
returning ... Inside with block
ValueError: something is wrong
Inside __exit__ with exception (<class 'ValueError'> - something is wrong - <traceback object at 0x1034ba608>)

"""

Python context manager

###################
# __enter__
# __enter__ is called before executing with-statement body
# __exit__
# __exit__ called after with-statement body
# File opening is context managers
###################


class ContextManagerTest:
    def __init__(self):
        print('Inside __init__')

    def __enter__(self):
        print('Inside __enter__')
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_type is None:
            print('Inside __exit__ without exception')
        else:
            print('Inside __exit__ with exception ({} - {} - {})'.format(exc_type, exc_val, exc_tb))
        return


with ContextManagerTest() as cm:
    print('Inside ContextManagerTest')
    print(cm)


"""
Inside __init__
Inside __enter__
Inside ContextManagerTest
<__main__.ContextManagerTest object at 0x10a920160>
Inside __exit__ without exception
"""

python open very big file, python memory

Python open very big file: 

#Say you are looping through a big 2 TB file

logfile = open("huge_log_file.txt","r")
info_lines = [(line,len(line)) for line in logfile if line.startswith("INFO")]
#Here it will get a huge list - costs RAM, this list could contain 2 TB of content

logfile = open("huge_log_file.txt","r")
info_lines = ((line,len(line)) for line in logfile if line.startswith("INFO"))
#Here it will get generator object - memory efficient

Opening a file (read mode) does NOT implicitly read nor load its contents into memory.  
Even when you do so using Python's context management protocol (the with keyword).

e.g.,

with open('huge_log_file.txt', 'r') as f:
   for each_line in f:
     do_something_each_line(each_line)

Then your peak memory utilization shouldn't be much larger than the longest line of the file

If you really are reading the full content of the file into a data structure like list, then it's no wonder that your RAM usage peaks like that. 
It's not that python puts the full contents of the file in RAM, but that you do.
e.g.,

#Here it will load all into memory as you are storing/dumping into a list called info_lines
info_lines = [(line,len(line)) for line in logfile if line.startswith("INFO")] 


#Memory efficinet - since it return generator
info_lines = ((line,len(line)) for line in logfile if line.startswith("INFO"))



file = '/tmp/huge_log_file.txt'

#This is fine
with open(file, 'r') as fh:
   for each in fh:
       print each

#this is a blunder, this will crash, it loads every thing into memory
#with open(file, 'r') as fh:
#    lines = fh.readlines()
#    print len(lines)

Python Generator

#################################################
## Uses of Generators:
##   1) It will automatically takes care of __iter__() and next()/__next__() 
##   2) More easy to use
##   3) It won't load everything in memory, so it consumes less memory (memory efficient)
#################################################

def generatorFunction(listA):
for each in listA:
yield each

print '------'
ic = generatorFunction(['A','B','C'])
for each in ic:
print(each)

print '------'
ic = generatorFunction(['A','B','C'])
print (ic)
print(next(ic))
print(next(ic))
print(next(ic))
#print(next(ic)) #This raises StopIteration

print (ic)

print '$$$$$$$$'
# This will not print anything, since generator got exhausted as we earlier called next() many times already
# You have to re-initialize generator object again
for each in ic: 
print(each)

ic = generatorFunction(['A','B','C'])
print '#########'

for each in ic:

print(each)


Output:

------

A

B

C

------

<generator object generatorFunction at 0x7f82ddcaca50>

A

B

C

<generator object generatorFunction at 0x7f82ddcaca50>

--$$$$$----

--#####----

A

B

C

Python Iterator Iterbale

################################################
## Writing own iterators
## Two ways:
##    1) using __iter__ and __next__ for Python 3.0
##       using __iter__ and next() for Python 2.7      
##    2) using generator functions
##    3) They can go only forward, no backwards
################################################

class IterClass:
def __init__(self, listA):
self.index = 0
self.elments = listA
def __iter__(self): #To make an object sequence
return self
def next(self): #in 2.7 use next(), in 3.0 use __next__()
if self.index >= len(self.elments):
raise StopIteration
index = self.index
self.index += 1
return self.elments[index]

ic = IterClass(['A','B','C'])
for each in ic:
print each
print '------'
ic = IterClass(['A','B','C'])
print(next(ic))
print(next(ic))
print(next(ic))
#print(next(ic)) #This raoses StopIteration
print'-------'

ll = range(0,5)
print ll

#List object is iterable but not iterator
print dir(ll) #it has __iter__ only, but no next/__next__ method
#next(ll) will fail

ll_iter = ll.__iter__()
print dir(ll_iter) #it has next/__next__ method


print ll_iter
print dir(ll_iter)
print next(ll_iter)
print next(ll_iter)
print next(ll_iter)
print next(ll_iter)
print next(ll_iter)
#print next(ll_iter) #It will throw StopIteration

print '###########'
ll_iter = ll.__iter__()

while True:
try:
item = next(ll_iter)
print item
#except StopIteration:
# print e
# break
except Exception,e:
break

print '$$$$$$$$$$$$'
ll_iter = ll.__iter__()
for each in ll_iter:
print each

Output:
A
B
C
------
A
B
C
-------
[0, 1, 2, 3, 4]
['__add__', '__class__', '__contains__', '__delattr__', '__delitem__', '__delslice__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__getslice__', '__gt__', '__hash__', '__iadd__', '__imul__', '__init__', '__iter__', '__le__', '__len__', '__lt__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__reversed__', '__rmul__', '__setattr__', '__setitem__', '__setslice__', '__sizeof__', '__str__', '__subclasshook__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort']
['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__iter__', '__length_hint__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'next']
<listiterator object at 0x03978190>
['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__iter__', '__length_hint__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'next']
0
1
2
3
4
###########
0
1
2
3
4
$$$$$$$$$$$$
0
1
2
3

4