python open very big file, python memory

Python open very big file: 

#Say you are looping through a big 2 TB file

logfile = open("huge_log_file.txt","r")
info_lines = [(line,len(line)) for line in logfile if line.startswith("INFO")]
#Here it will get a huge list - costs RAM, this list could contain 2 TB of content

logfile = open("huge_log_file.txt","r")
info_lines = ((line,len(line)) for line in logfile if line.startswith("INFO"))
#Here it will get generator object - memory efficient

Opening a file (read mode) does NOT implicitly read nor load its contents into memory.  
Even when you do so using Python's context management protocol (the with keyword).

e.g.,

with open('huge_log_file.txt', 'r') as f:
   for each_line in f:
     do_something_each_line(each_line)

Then your peak memory utilization shouldn't be much larger than the longest line of the file

If you really are reading the full content of the file into a data structure like list, then it's no wonder that your RAM usage peaks like that. 
It's not that python puts the full contents of the file in RAM, but that you do.
e.g.,

#Here it will load all into memory as you are storing/dumping into a list called info_lines
info_lines = [(line,len(line)) for line in logfile if line.startswith("INFO")] 


#Memory efficinet - since it return generator
info_lines = ((line,len(line)) for line in logfile if line.startswith("INFO"))



file = '/tmp/huge_log_file.txt'

#This is fine
with open(file, 'r') as fh:
   for each in fh:
       print each

#this is a blunder, this will crash, it loads every thing into memory
#with open(file, 'r') as fh:
#    lines = fh.readlines()
#    print len(lines)

Python Zip

"""
input_data.txt
#########
First row contains column name
Second row contains data type
From third row, it contains student records.
How to map all the reocords using zip function?

name, age, city
varchar, int, varchar
prabhath, 32, bangalore
vamsi, 30, hyderabad
lakshmi, 30, vizag
ramesh, 50, chennai

"""

with open('input_data.txt') as input_file:
    rows = input_file.readlines()

rows = [row.strip() for row in rows]
rows = [row.split(',') for row in rows]
print rows

columns = rows[0]
data_types = rows[1]
rows = rows[2:]

for row in rows:
print '----'
for column, data_type, val in zip(columns, data_types, row):
print column.strip(), data_type.strip(), val.strip()

"""
Output:

[['name', ' age', ' city'], ['varchar', ' int', ' varchar'], ['prabhath', ' 32', ' bangalore'], ['vamsi', ' 30', ' hyderabad'], ['lakshmi', ' 30', ' vizag'], ['ramesh', ' 50', ' chennai']]
----
name varchar prabhath
age int 32
city varchar bangalore
----
name varchar vamsi
age int 30
city varchar hyderabad
----
name varchar lakshmi
age int 30
city varchar vizag
----
name varchar ramesh
age int 50
city varchar chennai
"""

Python read characters vertically in a file

with open('input_vertical.txt') as input_file:
    rows = input_file.readlines()
print rows
print '---'
rows = [row.strip() for row in rows]
print rows
##The single star * unpacks the sequence/collection into positional arguments
rows = zip(*rows)
print rows
rows = [''.join(row) for row in rows]
print rows



"""
Output:

['prabhath\n', 'test\n', 'vertical\n', 'lines']
---
['prabhath', 'test', 'vertical', 'lines']
[('p', 't', 'v', 'l'), ('r', 'e', 'e', 'i'), ('a', 's', 'r', 'n'), ('b', 't', 't', 'e')]
['ptvl', 'reei', 'asrn', 'btte']
"""

Python csv content to dictionary

import csv

file_name = 'student.csv'
input_file_handle = csv.DictReader(open(file_name, 'rb'))
for row in input_file_handle:
     print row  #This will print row as dictionary
     print row['id']
     print row['name']
     print row['email']

Python copy file

#copy txt file
rf = open('input.txt', 'r')
wf = open('output.txt', 'w')

for i in rf:
    wf.write(i)


#copy image - binary
rf = open('relax.jpg', 'rb')
wf = open('relax1.jpg', 'wb')

for i in rf:
    wf.write(i)

How to remove duplicate lines from a file in Python

Today, we will discuss removing duplicate lines from a file in Python

Lets discuss in two ways as mentioned below :

1) Removing duplicate lines and print the lines in order (When Order is important)

        - Using normal way

2) Removing duplicate lines and print the lines in any order (When Order is NOT important) 

        - Using SET concept in Python. SET concept in python does not consider Order.

Note:
Both the scripts read duplicated content from file_with_duplicates.txt, 
Read the above file and remove duplicate lines and finally
Print to file_without_duplicates.txt 

Input: file_with_duplicates.txt

Mother Teresa
Winston Churchill
Abraham Lincoln
Mahatma Gandhi
Winston Churchill
Mother Teresa
Abraham Lincoln  

1)

infile = open('file_with_duplicates.txt', 'r')

outfile = open('file_without_duplicates.txt', 'w')



lines_seen = set()

for line in infile:

    if line not in lines_seen:

        outfile.write(line)

        lines_seen.add(line)

outfile.close()

1) remove_duplicate_lines_from_file_with_order.py
- Using Normal Way - it take cares of order of the lines

#!/usr/bin/python

try:
    input_file = open("file_with_duplicates.txt", "r")
    output_file = open("file_without_duplicates.txt", "w")

    unique = []
    for line in input_file:
        line = line.strip()
        if line not in unique:
            unique.append(line)
    input_file.close()
    
    for i in range(0, len(unique)-1):
        unique[i] += "\n"

    output_file.writelines(unique)
    output_file.close
    
except FileNotFoundError:
    print('\n File NOT Found Error')
    sys.exit
except IOError:
    print('\n IO Error')
    sys.exit  

Output: file_without_duplicates.txt

Mother Teresa
Winston Churchill
Abraham Lincoln
Mahatma Gandhi  

2) remove_duplicate_lines_from_file_without_order.py
- Using SET concept - it does not consider the order of the lines

#!/usr/bin/python

try:
    input_file = open("file_with_duplicates.txt", "r")
    output_file = open("file_without_duplicates.txt","w")

    #The main drawback of using sets is, the order of the lines may not be same as in input file
    uniquelines = set(input_file.read().split("\n"))
    output_file.write("".join([line + "\n" for line in uniquelines]))
    
    input_file.close()
    output_file.close()
    
except FileNotFoundError:
    print('\n File NOT Found Error')
    sys.exit
except IOError:
    print('\n IO Error')
    sys.exit      

Output: file_without_duplicates.txt

Abraham Lincoln
Winston Churchill
Mother Teresa
Mahatma Gandhi  

You may also wish to read Object Oriented Concepts in Python as mentioned :
Python Class and Object Example
Inheritance in Python
Packages in Python
Exceptions in Python
How to remove duplicate lines from a file in perl

Reading a File with line numbers

file = open("read.txt","r")
text = file.readlines()
file.close()

counter = 1
for line in text:
    print counter, line,
    counter = counter +1
print

File Operations in Python

file = open("read.txt","r")
text = file.readlines()    #Read lines from read.txt
file.close()

file2 = open ("write.txt", "w")
#Write all lines from read.txt into write.txt
#It overrides if any content already present in write.txt
file2.writelines(text)
file2.close

file2 = open ("append.txt", "a")
#It appends content from read.txt into append.txt
file2.writelines(text)
file2.close

Print File Content in Python

file = open("read.txt","r")
text = file.readlines()   #Reads the file content
file.close()

for line in text:
    print line        #Python 2.X Versions
    #print(line)      #Python 3.X Versions

Print File in Reverse Order in Python

file = open("read.txt","r")
text = file.readlines()   #Reads the file content
file.close()

text.reverse()    #It reverses the content

for line in text:
    print line        #Python 2.X Versions
    #print(line)    #Python 3.X Versions