Python concurrent.futures ProcessPoolExecutor

"""
The ProcessPoolExecutor class is an Executor subclass that uses a pool of processes to execute calls asynchronously. 

ProcessPoolExecutor uses the multiprocessing module
"""

from concurrent.futures import ProcessPoolExecutor
import math
import multiprocessing
import os
import sys
import time

PRIMES = [
    112272535095293,
    112582705942171,
    112272535095293,
    115280095190773,
    115797848077099,
    109972689928541]

def is_prime(n):
    if n == 2:

        return True

    if n % 2 == 0:
        return False

    sqrt_n = int(math.floor(math.sqrt(n)))
    for i in range(3, sqrt_n + 1, 2):
        if n % i == 0:
            return False
    return True

def main():
    print('No of CPUs/Processors: {}' . format(multiprocessing.cpu_count()))
    a = time.time()
    #default max_workers is number of processors on the machine
    with ProcessPoolExecutor() as executor:
        for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)):
            print('%d is prime: %s' % (number, prime))
    b = time.time()
    print('Time taken: {:.2f} secs'.format(b-a))

if __name__ == '__main__':
    main()


Output:
No of CPUs/Processors: 4
112272535095293 is prime: True
112582705942171 is prime: True
112272535095293 is prime: True
115280095190773 is prime: True
115797848077099 is prime: True
109972689928541 is prime: False
Time taken: 12.67 secs

Python concurrent.futures ThreadPoolExecutor as_completed

import urllib.request 
from concurrent.futures import ThreadPoolExecutor, as_completed

URLS = ['https://www.google.com', 
               'http://www.cnn.com/',
               'http://europe.wsj.com/', 
               'http://www.bbc.co.uk/', 
               'http://abc.abc.com'   #invalid
             ]

def load_url(url, timeout):
  with urllib.request.urlopen(url, timeout=timeout) as conn:
    txt = conn.read()
    return txt

with ThreadPoolExecutor(max_workers = 5) as executor:
  #Forming Key-Value pairs
  future_to_url = {executor.submit(load_url, url, 50): url for url in URLS}
  print(future_to_url)
  print('----')
  for future in as_completed(future_to_url):
    url = future_to_url[future]
    try:
      data = future.result()
      print('%s length is %d' % (url, len(data)))
    except Exception as e:
      print('Error in URL: %s is %s' % (url, e))


Output:
{<Future at 0x7f40d262d2d0 state=running>: 'https://www.google.com', <Future at 0x7f40cadc4ad0 state=running>: 'http://www.cnn.com/', <Future at 0x7f40cadcd710 state=running>: 'http://europe.wsj.com/', <Future at 0x7f40cadcd410 state=running>: 'http://www.bbc.co.uk/', <Future at 0x7f40cade0a10 state=running>: 'http://abc.abc.com'}
----
Error in URL: http://abc.abc.com is <urlopen error [Errno -2] Name or servicenot known>
https://www.google.com length is 12571
http://www.cnn.com/ length is 1134562
http://europe.wsj.com/ length is 1006417
http://www.bbc.co.uk/ length is 311008

Python ThreadPoolExecutor submit

from concurrent.futures import ThreadPoolExecutor import threading def task(n): print("Processing {} - {}".format(n, threading.current_thread())) def main(): print("Starting ThreadPoolExecutor") with ThreadPoolExecutor(max_workers=3) as executor: future = executor.submit(task, (2)) future = executor.submit(task, (3)) future = executor.submit(task, (4)) print("All tasks complete") if __name__ == '__main__': main()

Output:

Starting ThreadPoolExecutor Processing 2 - <Thread(ThreadPoolExecutor-0_0, started daemon 140052642395904)> Processing 3 - <Thread(ThreadPoolExecutor-0_1, started daemon 140052634003200)> Processing 4 - <Thread(ThreadPoolExecutor-0_2, started daemon 140052625610496)> All tasks complete

Python ThreadPoolExecutor, map

import urllib.request 
from concurrent.futures import ThreadPoolExecutor
import threading

urls = [
  'http://www.python.org', 
  'http://www.python.org/about/',
  'http://www.onlamp.com/pub/a/python/2003/04/17/metaclasses.html',
  'http://www.python.org/doc/',
  'http://www.python.org/download/',
  'http://www.python.org/getit/',
  'http://www.python.org/community/',
  'https://wiki.python.org/moin/',
]

def fun(url):
  print(url, threading.current_thread())
  r = urllib.request.urlopen(url)
  return r

# make the Pool of workers
pool = ThreadPoolExecutor(4) 

results = pool.map(fun, urls)
real_results = list(results)
print('----')
print(real_results)


Output:
http://www.python.org <Thread(ThreadPoolExecutor-0_0, started daemon 139989036611328)>
http://www.python.org/about/ <Thread(ThreadPoolExecutor-0_1, started daemon 139988956083968)>
http://www.onlamp.com/pub/a/python/2003/04/17/metaclasses.html <Thread(ThreadPoolExecutor-0_2, started daemon 139988947691264)>
http://www.python.org/doc/ <Thread(ThreadPoolExecutor-0_3, started daemon 139988939298560)>
http://www.python.org/download/ <Thread(ThreadPoolExecutor-0_1, started daemon 139988956083968)>
http://www.python.org/getit/ <Thread(ThreadPoolExecutor-0_3, started daemon 139988939298560)>
http://www.python.org/community/ <Thread(ThreadPoolExecutor-0_0, started daemon 139989036611328)>
https://wiki.python.org/moin/ <Thread(ThreadPoolExecutor-0_3, started daemon 139988939298560)>
----
[<http.client.HTTPResponse object at 0x7f51be3d7910>, <http.client.HTTPResponse object at 0x7f51be3c5f90>, <http.client.HTTPResponse object at 0x7f51be3d73d0>, <http.client.HTTPResponse object at 0x7f51be3d77d0>, <http.client.HTTPResponse object at 0x7f51be3b3b90>, <http.client.HTTPResponse object at 0x7f51be3c5610>, <http.client.HTTPResponse object at 0x7f51be3e1c90>, <http.client.HTTPResponse object at 0x7f51be3d7250>]