Mar 25, 2020

Python sum using multiprocessing pool

#####
# Read all text files & calculate Sum of all numbers in txt files

# Threading/input1.txt
# 100
# 200
# 300
# 400
# 500

# Threading/input2.txt
# 600
# 700
# 800
# 900
# 1000
#####

import glob
import multiprocessing

def get_txt_files():
    my_files = []
    for each in glob.glob('Threading/*.txt'):
        my_files.append(each)
    return my_files

def calculate_sum(input_file):
    with open(input_file) as fh:
      return sum(map(int, fh.readlines()))


if __name__ == '__main__':
    my_files = get_txt_files()
    #['Threading/input1.txt', 'Threading/input2.txt']
    print(my_files)

    # sys.exit()
    pool = multiprocessing.Pool(processes=2)
    results = pool.map(calculate_sum, my_files)
    print(f'Sum of individual files: {results}')
    total_sum = sum(results)
    print(f'Total Sum: {total_sum}')


Output:
######
['Threading/input1.txt', 'Threading/input2.txt']
Sum of individual files: [1500, 4000]
Total Sum: 5500

No comments:

Post a Comment