Boto S3 How to access buckets other than us-east-1

Boto S3 - How to access buckets other than us-east-1?

Trying to connect mumbai region (ap-south-1) bucket, but not able to connect.
Esp in Boto 2 versions, there is some issue, getting below errors:

• boto.exception.S3ResponseError: S3ResponseError: 301 Moved Permanently
• boto.exception.S3ResponseError: S3ResponseError: 400 Bad Request

Solution:

You need to mark S3_USE_SIGV4 flag to True, else it will throw you error.

from boto.s3.connection import S3Connection, Location
import os

os.environ['S3_USE_SIGV4'] = 'True'
conn = S3Connection(S3_KEY, S3_SECRET, host='s3.ap-south-1.amazonaws.com', calling_format=boto.s3.connection.OrdinaryCallingFormat())
print conn
client_buk_obj = conn.get_bucket('test-mumbai-region-test')
print client_buk_obj
os.environ['S3_USE_SIGV4'] = 'False'

Python compare lists

#Comapre lists

student1_scores = [61, 73, 84, 90, 85, 45]
student2_scores = [40, 37, 45, 87, 99, 54]

#set - will change the order
print list(set(student1_scores)) #[73, 45, 84, 85, 90, 61]
print list(set(student2_scores)) #[99, 37, 40, 45, 54, 87]

#Commom elements in two lists
print list(set(student1_scores) & set(student2_scores)) #[45]

#Commom elements in two lists
print list(set(student1_scores).intersection(set(student2_scores))) #[45]

#Union of two lists
print list(set(student1_scores).union(set(student2_scores))) 
#[99, 37, 40, 73, 45, 84, 85, 54, 87, 90, 61]

#Commom elements in two lists - using list comprehension
print [i for i in student1_scores if i in student2_scores] #[45]

#compare common elements with for the same subject (same index)
student1_scores = [61, 73, 84, 90, 85, 45]
student2_scores = [40, 37, 45, 90, 99, 54]

print [i for i, j in zip(student1_scores, student2_scores) if i == j] #[90]

Python MySQL

import mysql.connector

mydb = mysql.connector.connect(
  host="localhost",
  user="root",
  passwd="password",
  database="myproject"
)

my_cursor = mydb.cursor()

sql = "SELECT user.name, product.name FROM user JOIN product ON user.product_id = product.id"

my_cursor.execute(sql)

results = my_cursor.fetchall()

for each in results:
  print each

MySQL Joins

  #INNER JOIN - this gives common elements from user & product
  SELECT user.name, product.name 
  FROM user INNER JOIN product ON user.product_id = product.id
  
  #LEFT JOIN - this gives all user & matching product
  SELECT user.name, product.name 
  FROM user LEFT JOIN product ON user.product_id = product.id
  
  #RIGHT JOIN - this gives all product with user
  SELECT   user.name, product.name 
  FROM user RIGHT JOIN product ON user.product_id = product.id

Python reverse a string

Python reverse a string

ss = 'Hello World'
print ss[::-1] #dlroW olleH
print reversed(ss)
print ''.join(reversed(ss)) #dlroW olleH

O/P:
dlroW olleH 

<reversed object at 0x7f5813ce73d0> 

dlroW olleH

Python datetime

from datetime import datetime

datetime_obj = datetime.now() 
print datetime_obj #2019-01-25 13:45:28.087000
print datetime_obj.date() #2019-01-25
print datetime_obj.year  #2019
print datetime_obj.month #1
print datetime_obj.day   #25
print datetime_obj.hour  #13
print datetime_obj.minute #45
print datetime_obj.second #28

datetime_object = datetime(2011, 07, 01, 11, 50, 55)
print datetime_object
datetime_object_str =  datetime_object.strftime("%Y-%m-%d %H:%M:%S")
print datetime_object_str #2011-07-01 11:50:55
print type(datetime_object_str) #<type 'str'>

datetime_object = datetime.strptime('2018-07-11 10:55:55', '%Y-%m-%d %H:%M:%S')
print datetime_object #2011-07-01 11:50:55
print type(datetime_object) #<type 'datetime'>

Python re subn

import re

text = 'python is good, python is better, python is best';

#subn - returns a tuple with no of substitutions made

print re.subn('python', 'PYTHON', text, count=0, flags=0) #replaces all
#('PYTHON is good, PYTHON is better, PYTHON is best', 3)

print re.subn('python', 'PYTHON', text, count=1, flags=0) #replaces first
#('PYTHON is good, python is better, python is best', 1)

print re.subn('python', 'PYTHON', text, count=2, flags=0) #replaces first & second
#('PYTHON is good, PYTHON is better, python is best', 2)

print re.subn('python', 'PYTHON', text, count=3, flags=0) #replaces first, second, third
#('PYTHON is good, PYTHON is better, PYTHON is best', 3)

Python substitute nth occurance

import re

##Substitue 3rd occurrence of 'python' with 'PYTHON'

nth_occurance = 3
text = 'python is good, python is better, python is best';

count = text.count('python')
if count <= 1:
print re.sub('python', r'PYTHON', text)
else:
print re.sub('^((.*?python.*?){' + str(nth_occurance-1) + '})python', r'\1PYTHON', text)


Output:
python is good, python is better, PYTHON is best

Python Shallow Vs Deep Copy

import copy

#Shallow Copy
#   copy.copy(x)
#   A shallow copy creates a new object which stores the reference of the original elements.
#   A shallow copy doesn't create a copy of nested objects, instead it just copies the reference of nested objects. 

#Deep Copy
#   copy.deepcopy(x)
#   A deep copy creates a new object and recursively adds the copies of nested objects present in the original elements.


print ' --------- Copy by Ref ------ '
dic1 = {'StudnetId': 'test1', 'Scores': [60,70,80,90,85,45]}
dic2 = dic1

print dic1
print dic2

print 'Updating/Appending score 50'
dic1['Scores'].append(50) #update existing

print dic1 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}

#Both dic1 & dic2 point to same memory locations
print id(dic1) #46138688
print id(dic2) #46138688

print 'Adding new field Age'
dic1['Age'] = 25

print dic1 #{'Age': 25, 'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}
print dic2 #{'Age': 25, 'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}

print ' --------- Shallow Copy ------ '
dic1 = {'StudnetId': 'test1', 'Scores': [60,70,80,90,85,45]}
dic2 = copy.copy(dic1)

print dic1 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}

print "Memory locations of dic1 & dic2 respectively"
#Both dic1 & dic2 point to different memory location
print id(dic1) #57641696
print id(dic2) #57641840

print 'Updating/Appending score 50'
#Updating the existing will 
dic1['Scores'].append(50) #update existing

#Both dic1 & dic2 point to different memory locations
print dic1 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}

#Shallow Copy -> Internal data points to the same memory location
print "Memory locations of dic1['Scores'] & dic2['Scores'] respectively"
print id(dic1['Scores']) #52223472
print id(dic2['Scores']) #52223472

print 'Adding new field Age'
dic1['Age'] = 25

print dic1 #{'Age': 25, 'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}

print ' --------- Deep Copy ------ '
dic1 = {'StudnetId': 'test1', 'Scores': [60,70,80,90,85,45]}
dic2 = copy.deepcopy(dic1)

#Both dic1 & dic2 point to different memory locations
print dic1 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}

print id(dic1) #54035776
print id(dic2) #54037504

print 'Updating/Appending score 50'
dic1['Scores'].append(50) #update existing

print dic1 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}

print 'Adding new field Age'
dic1['Age'] = 25

print dic1 #{'Age': 25, 'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45, 50]}
print dic2 #{'StudnetId': 'test1', 'Scores': [60, 70, 80, 90, 85, 45]}

Python Local Vs Global

#########################
#   Global Scope Vs Enclosing Scope Vs Local Scope 
#   LEGB rule
#   Local(L): Defined inside function/class
#   Enclosed(E): Defined inside enclosing functions(Nested function concept)
#   Global(G): Defined at the uppermost level
#   Built-in(B): Reserved names in Python builtin modules
#########################

message = 'global'

def enclosing():
    message = 'enclosing'
    def local():
        message = 'local'
    print('enclosing message: ', message)   # enclosing
    local()
    print('enclosing message: ', message)   # enclosing


def enclosing_nonlocal():
    message = 'enclosing'
    def local():
        nonlocal message    # This refers to the above message in enclosing scope, not in global scope
        message = 'local'
    print('enclosing message: ', message)   # enclosing
    local()
    print('enclosing message: ', message)   # local


def enclosing_global():
    message = 'enclosing'
    def local():
        global message
        message = 'local'  # Here you are updating message in global scope, not in enclosing scope
    print('enclosing message: ', message)   # enclosing
    local()
    print('enclosing message: ', message)    # enclosing 


if __name__ == '__main__':
    print('------------------------------------')
    print('global message: ', message)
    enclosing()
    print('global message: ', message)
    print('----------------NONLOCAL------------------')
    print('global message: ', message)
    enclosing_nonlocal()
    print('global message: ', message)
    print('----------------GLOBAL--------------------')
    print('global message: ', message)
    enclosing_global()
    print('global message: ', message)
    print('------------------------------------')

"""
Output:

------------------------------------
global message:  global
enclosing message:  enclosing
enclosing message:  enclosing
global message:  global
----------------NONLOCAL------------------
global message:  global
enclosing message:  enclosing
enclosing message:  local
global message:  global
----------------GLOBAL--------------------
global message:  global
enclosing message:  enclosing
enclosing message:  enclosing
global message:  local
------------------------------------

"""

Python Closure

#########################
# Closure in Python
# A nested function references a value in its enclosing scope.
# We should have a nested function (function within a function).
# The nested function should refer to a value defined in the enclosing function.
# The enclosing function must return the nested function.
# Closures are used as callback functions, this helps in data hiding. This helps to reduce the use of global variables.
# When we have few functions in our code, closures are helpful. But if we have many functions, then we may go for a class
#########################

# Nested Function
# inner_function() can easily be accessed inside the outer_function body but not outside of it’s body.
# Hence, inner_function() is treated as nested Function which uses text as non-local variable.
def outer_function(text):
    text = text
    def inner_function():
        print(text)
    inner_function()


# A closure — unlike a plain function as above — allows the function to access those enclosed captured variables through
# the closure’s copies of their values or references, even when the function is invoked outside their scope.
def closure_outer_function(text):
    text = text
    def closure_inner_function():
        print(text)
    return closure_inner_function  # without parentheses() / callback function


def enclosed_function(x):
    print('In enclosed_function: ' + str(x))

    def nested_function(y):
        print('###')
        print('In nested_function x : ' + str(x))
        print('In nested_function y : ' + str(y))

    return nested_function    # without parentheses() / callback function


if __name__ == '__main__':
    print('------------------------------------------------------')
    outer_function('Hi Nested Function')

    print('------------------------------------------------------')
    func_obj = closure_outer_function('Hi Closure')
    func_obj()
    print('------------------------------------------------------')
    func_obj = enclosed_function(100)
    print('After calling enclosed_function')
    func_obj(111)
    print('------------------------------------------------------')


"""
------------------------------------------------------
Hi Nested Function
------------------------------------------------------

Hi Closure
------------------------------------------------------
In enclosed_function: 100
After calling enclosed_function
###
In nested_function x : 100
In nested_function y : 111
------------------------------------------------------
"""

Mysql dump commands

Mysql dump


Copy/Import SQL to DB:

• mysql -uroot -pekdo123 test_db < test_db.sql
• mysql --max_allowed_packet=1000M -uroot -pekdo123 test_db < /tmp/test_db.sql

MySQL Dump:

• mysqldump -uroot -pekdo123 test_db > test_db.sql
• mysqldump -t -u MyUserName -pMyPassword MyDatabase MyTable --where="ID = 10"

MySQL Dump - Avoid a TABLE

• mysqldump -uroot -pekdo123 <database> --ignore-table=<database>.table1 > database.sql
• mysqldump -uroot -pekdo123 test_db --ignore-table=test_db.table1 > database.sql

Ignore multiple/heavy/not-so-important tables while dumping/backup DB:
e.g. session tables like django_session

• mysqldump --host=<> --port=3306 -u<root> -p<password> test_db --ignore-table=test_db.table1 --ignore-table=test_db.table2 > /tmp/test_db.sql

MySQL Dump - only Insert queries

• --no-create-info - This will not write create table (drop & create)
• mysqldump -u root -pekdo123 test_db table1 --no-create-info --where="g_id = 43818 and id < 691672 and is_cancelled = 0" > /tmp/test_db_table1.sql
• --lock-tables=false
• mysqldump -u root -pekdo123 test_db table1 --lock-tables=false --no-create-info --where="id1 in (select id from table2 where gid = 43818 and id < 691672 and is_cancelled = 0)" > /tmp/test_db_table2.sql

For more information:

• mysqldump --help

Python Generator

#################################################
## Uses of Generators:
##   1) It will automatically takes care of __iter__() and next()/__next__() 
##   2) More easy to use
##   3) It won't load everything in memory, so it consumes less memory (memory efficient)
#################################################

def generatorFunction(listA):
for each in listA:
yield each

print '------'
ic = generatorFunction(['A','B','C'])
for each in ic:
print(each)

print '------'
ic = generatorFunction(['A','B','C'])
print (ic)
print(next(ic))
print(next(ic))
print(next(ic))
#print(next(ic)) #This raises StopIteration

print (ic)

print '$$$$$$$$'
# This will not print anything, since generator got exhausted as we earlier called next() many times already
# You have to re-initialize generator object again
for each in ic: 
print(each)

ic = generatorFunction(['A','B','C'])
print '#########'

for each in ic:

print(each)


Output:

------

A

B

C

------

<generator object generatorFunction at 0x7f82ddcaca50>

A

B

C

<generator object generatorFunction at 0x7f82ddcaca50>

--$$$$$----

--#####----

A

B

C

Python Iterator Iterbale

################################################
## Writing own iterators
## Two ways:
##    1) using __iter__ and __next__ for Python 3.0
##       using __iter__ and next() for Python 2.7      
##    2) using generator functions
##    3) They can go only forward, no backwards
################################################

class IterClass:
def __init__(self, listA):
self.index = 0
self.elments = listA
def __iter__(self): #To make an object sequence
return self
def next(self): #in 2.7 use next(), in 3.0 use __next__()
if self.index >= len(self.elments):
raise StopIteration
index = self.index
self.index += 1
return self.elments[index]

ic = IterClass(['A','B','C'])
for each in ic:
print each
print '------'
ic = IterClass(['A','B','C'])
print(next(ic))
print(next(ic))
print(next(ic))
#print(next(ic)) #This raoses StopIteration
print'-------'

ll = range(0,5)
print ll

#List object is iterable but not iterator
print dir(ll) #it has __iter__ only, but no next/__next__ method
#next(ll) will fail

ll_iter = ll.__iter__()
print dir(ll_iter) #it has next/__next__ method


print ll_iter
print dir(ll_iter)
print next(ll_iter)
print next(ll_iter)
print next(ll_iter)
print next(ll_iter)
print next(ll_iter)
#print next(ll_iter) #It will throw StopIteration

print '###########'
ll_iter = ll.__iter__()

while True:
try:
item = next(ll_iter)
print item
#except StopIteration:
# print e
# break
except Exception,e:
break

print '$$$$$$$$$$$$'
ll_iter = ll.__iter__()
for each in ll_iter:
print each

Output:
A
B
C
------
A
B
C
-------
[0, 1, 2, 3, 4]
['__add__', '__class__', '__contains__', '__delattr__', '__delitem__', '__delslice__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__getslice__', '__gt__', '__hash__', '__iadd__', '__imul__', '__init__', '__iter__', '__le__', '__len__', '__lt__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__reversed__', '__rmul__', '__setattr__', '__setitem__', '__setslice__', '__sizeof__', '__str__', '__subclasshook__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort']
['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__iter__', '__length_hint__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'next']
<listiterator object at 0x03978190>
['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__iter__', '__length_hint__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'next']
0
1
2
3
4
###########
0
1
2
3
4
$$$$$$$$$$$$
0
1
2
3

4

Python csv content to dictionary

import csv

file_name = 'student.csv'
input_file_handle = csv.DictReader(open(file_name, 'rb'))
for row in input_file_handle:
     print row  #This will print row as dictionary
     print row['id']
     print row['name']
     print row['email']

Python Operator Overloading

#Operator overloading
class operatorOverloadObject:
    def __init__(self, x = 0, y = 0):
        self.x = x
        self.y = y
    
    def __str__(self):
        return "({0},{1})".format(self.x,self.y)
    
    def __add__(self,other): #Fucntion to overload + operator
        x = self.x + other.x
        y = self.y + other.y
        return operatorOverloadObject(x,y)

obj1 = operatorOverloadObject(1, 2)
obj2 = operatorOverloadObject(3, 4)
print(obj1)  #(1,2)
print(obj2)  #(3,4)
print(obj1 + obj2)  #(4,6) 

#Other operations to overload...
#Addition    p1 + p2, p1.__add__(p2)
#Subtraction p1 - p2, p1.__sub__(p2)
#Multiplication  p1 * p2, p1.__mul__(p2)
#Power   p1 ** p2,    p1.__pow__(p2)
#Division    p1 / p2, p1.__truediv__(p2)
#Floor Division  p1 // p2,    p1.__floordiv__(p2)
#Remainder (modulo)  p1 % p2, p1.__mod__(p2)
#Bitwise Left Shift  p1 << p2,    p1.__lshift__(p2)
#Bitwise Right Shift p1 >> p2,    p1.__rshift__(p2)
#Bitwise AND p1 & p2, p1.__and__(p2)
#Bitwise OR  p1 | p2, p1.__or__(p2)
#Bitwise XOR p1 ^ p2, p1.__xor__(p2)
#Bitwise NOT ~p1, p1.__invert__()

Python Class methods static vs class

#class method demo
class Pets:
    name = "pet animals"

    @classmethod
    def about(cls):
        print("This class is about {}!".format(cls.name))
    
class Dogs(Pets):
    name = "'man's best friends'"

class Cats(Pets):
    name = "cats"

p = Pets() #parent class
p.about() #This class is about pet animals!

d = Dogs() #inherited class
d.about() #This class is about 'man's best friends'!

c = Cats() #inherited class
c.about() #This class is about cats!


#static method demo
class Pets:
    name = "pet animals"

    @staticmethod
    def about():
        print("This class is about {}!".format(Pets.name))   
    
class Dogs(Pets):
    name = "'man's best friends'"

class Cats(Pets):
    name = "cats"

p = Pets()
p.about() #This class is about pet animals!
d = Dogs() 
d.about() #This class is about pet animals!
c = Cats()
c.about() #This class is about pet animals!

Python encapsulation getter setter

#Python OOPS Getter / Setter
class Person(object):
    def __init__(self, p_name=None):
        self._name = p_name

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, new_name):
        if type(new_name) == str: #type checking for name property
            self._name = new_name
        else:
        print 'Error: Invalid type to set'

    @name.deleter
    def name(self):
        del self._name

print '############# Getter/Setter'
p = Person('Mike')
print(p.name)  #Mike
p.name = 'George'  #Grorge
print(p.name)
p.name = 2.3 # Causes an exception, Error: Invalid type to set
print(p.__dict__)  #{'_name': 'George'}
del p.name
print(p.__dict__) #{}

Python Inheritance Detail

#Multiple inheritance
class Base1:
    @classmethod
    def f1(self):
    print  'Base1 f1'
    def f2(self):
    print  'Base1 f2'   

class Base2:
    def f1(self):
    print  'Base2 f1'
    def f2(self):
    print  'Base2 f2'
    def f3(self):
    print  'Base2 f3'   

class MultiDerived(Base1, Base2):
    def f1(self):
    print  'MultiDerived f1'

print '###########Multiple inheritance'
md = MultiDerived()
md.f1()  #MultiDerived f1
md.f2()  #Base1 f2
md.f3()  #Base2 f3
Base1.f1()  #Base1 f1  #classmethod
#Base2.f1()  #fail
#MultiDerived.f1()  #fail


#Multi-level inheritance
class Base:
    def f1(self):
    print  'Base f1'

class Derived1(Base):
    def f1(self):
    print  'Derived1 f1'
    def f2(self):
    print  'Derived1 f2'

class Derived2(Derived1):
    pass

print '###########Multi-level inheritance'
d2 = Derived2()
d2.f1() #Derived1 f1
d2.f2() #Derived1 f2      

Python csv

import io
import csv

output = io.BytesIO()
writer = csv.writer(output)

row = ['Name', 'City', 'Phone']
writer.writerow(row)
row1 = ['test1', 'city1', '123456789']
writer.writerow(row1)
row2 = ['test2', 'city2', '234567890']
writer.writerow(row2)
data =  output.getvalue()
fname = "output2.csv"
f = open(fname, 'wb')
f.write(data)
f.close()

Python Thread Functions

import time
import threading
from threading import Thread

def sleepFunc(i):
    print("Thread %s going to sleep for 5 seconds..." % threading.current_thread())
    time.sleep(5)
    print("Thread %i is awake now..." % i)


for i in range(10):
    th = Thread(target=sleepFunc, args=(i, ))
    th.start()
    print("Current Threads count: %i." % threading.active_count())


for thread in threading.enumerate():
    print("Thread name is %s ..." % thread.getName())

Python Thread concepts

from threading import Thread
import time

class abc(Thread):
def run(self):
for i in range(5):
print 'abc'
time.sleep(1)

class xyz(Thread):
def run(self):
for i in range(5):
print 'xyz'
time.sleep(1)

a = abc()
b = xyz()
a.start()
time.sleep(0.2)
b.start()

a.join()
b.join()

print 'bye'

Python copy file

#copy txt file
rf = open('input.txt', 'r')
wf = open('output.txt', 'w')

for i in rf:
    wf.write(i)


#copy image - binary
rf = open('relax.jpg', 'rb')
wf = open('relax1.jpg', 'wb')

for i in rf:
    wf.write(i)

Python anagram puzzle

An anagram is a word or phrase formed by rearranging the letters of a different word or phrase, typically using all the original letters exactly once.

E.g., Fried, Fired

def is_anagram(s1, s2):
print (f'{s1}, {s2}')
s1 = s1.replace(' ', '')
s2 = s2.replace(' ', '')
return sorted(s1) == sorted(s2)


print(is_anagram('silent', 'listen'))                                 # True
print(is_anagram('public relations', 'crap built on lies'))   # True