java.lang.ClassNotFoundException: com.mysql.jdbc.Driver

Error:

java.lang.ClassNotFoundException: com.mysql.jdbc.Driver

Solution:
Both spark driver and executor need mysql driver on class path so specify

• spark.driver.extraClassPath = /usr/share/java/mysql-connector-java.jar 
• spark.executor.extraClassPath = /usr/share/java/mysql-connector-java.jar

#Initialize SparkSession and SparkContext

from pyspark.sql import SparkSession

from pyspark import SparkContext

#Create a Spark Session

SpSession = SparkSession \

    .builder \

    .master("local[2]") \

    .appName("prabhath") \

    .config("spark.executor.memory", "1g") \

    .config("spark.cores.max","2") \

    .config("spark.driver.extraClassPath", "/usr/share/java/mysql-connector-java.jar") \

    .config("spark.executor.extraClassPath", "/usr/share/java/mysql-connector-java.jar") \

    .config("spark.sql.warehouse.dir", "/Users/jlyang/Spark/spark-warehouse")\

    .getOrCreate()

#Get the Spark Context from Spark Session

SpContext = SpSession.sparkContext

demoDf = SpSession.read.format("jdbc").options(

    url="jdbc:mysql://localhost:3306/testpraba1",

    driver = "com.mysql.jdbc.Driver",

    dbtable = "users_userprofile",

    user="root",

    password="XXXXX").load()

demoDf.show()

ERROR MasterWebUI: Failed to bind MasterWebUI

Problem: 
While starting start-master.sh, getting ERROR MasterWebUI: Failed to bind MasterWebUI

Solution:
check SPARK_LOCAL_IP is set to correct IP Address
export SPARK_LOCAL_IP=192.168.254.122 

Error:
Spark Command: /opt/java/jdk1.8.0_201/bin/java -cp /opt/spark/conf/:/opt/spark/jars/* -Xmx1g org.apache.spark.deploy.master.Master --host 127.0.0.1 --port 7077 --webui-port 8080
========================================
19/03/12 14:03:50 ERROR MasterWebUI: Failed to bind MasterWebUI
java.net.BindException: Cannot assign requested address: Service 'MasterUI' failed after 16 retries (starting from 8080)! Consider explicitly setting the appropriate port for the service 'MasterUI' (for example spark.ui.port for SparkUI) to an available port or increasing spark.port.maxRetries.
    at sun.nio.ch.Net.bind0(Native Method)
    at sun.nio.ch.Net.bind(Net.java:433)
    at sun.nio.ch.Net.bind(Net.java:425)
    at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
    at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
    at org.spark_project.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:351)
    at org.spark_project.jetty.server.ServerConnector.open(ServerConnector.java:319)
    at org.spark_project.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
    at org.spark_project.jetty.server.ServerConnector.doStart(ServerConnector.java:235)
    at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
    at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$newConnector$1(JettyUtils.scala:353)
    at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$httpConnect$1(JettyUtils.scala:380)
    at org.apache.spark.ui.JettyUtils$$anonfun$7.apply(JettyUtils.scala:383)
    at org.apache.spark.ui.JettyUtils$$anonfun$7.apply(JettyUtils.scala:383)
    at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:2269)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:2261)
    at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:383)
    at org.apache.spark.ui.WebUI.bind(WebUI.scala:132)
    at org.apache.spark.deploy.master.Master.onStart(Master.scala:140)
    at org.apache.spark.rpc.netty.Inbox$$anonfun$process$1.apply$mcV$sp(Inbox.scala:122)
    at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:205)
    at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101)
    at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:221)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)

python reduce using lambda

#Using reduce get average MGP-CITY (miles per Gallon) from sample car data
from functools import reduce

car_data = ['MAKE,FUELTYPE,ASPIRE,4,BODY,DRIVE,CYLINDERS,HP,RPM,MPG-CITY,MPG-HWY,PRICE',\
 'mercedes-benz,gas,std,2,convertible,RWD,eight,155,4750,16,18,35056', \
 'jaguar,gas,std,4,sedan,RWD,six,176,4750,15,19,35550', \
 'jaguar,gas,std,2,sedan,RWD,twelve,262,5000,13,17,36000', \
 'bmw,gas,std,4,sedan,RWD,six,182,5400,15,20,36880', \
 'porsche,gas,std,2,convertible,RWD,six,207,5900,17,25,37028', \
 'mercedes-benz,gas,std,4,sedan,RWD,eight,184,4500,14,16,40960', \
 'bmw,gas,std,2,sedan,RWD,six,182,5400,16,22,41315', \
 'mercedes-benz,gas,std,2,hardtop,RWD,eight,184,4500,14,16,45400']

#Use a function to perform reduce
def getMPGCity( autoStr) :
    if isinstance(autoStr, int) :
        return autoStr
    attList=autoStr.split(",")
    if attList[9].isdigit() : #this will ignore header
        return int(attList[9])
    else:
        return 0

total_mgp = reduce(lambda x,y : getMPGCity(x) + getMPGCity(y), car_data)
print 'Total MPG: {}'.format(total_mgp)

data_length_without_header = len(car_data)-1.0  # account for header line
print 'Total lines without header: {}'.format(data_length_without_header)

print 'Average MPG: {} '.format(total_mgp/data_length_without_header)

Output:
Total MPG: 120
Total lines without header: 8.0
Average MPG: 15.0

python reduce

from functools import reduce

input_list = [10, 20, 30, 5, 7, 1, 34, 45, 566]

#Find smallest no
print reduce(lambda x,y: x if x < y else y, input_list)  #1

#Find largest no
print reduce(lambda x,y: x if x > y else y, input_list)  #566

python change conf variables

conf.py
x=10

a.py
import conf
print 'Inside A: %d ' % (conf.x)
conf.x=20  #Here overwriting conf.x
print 'Inside A after changing value to 20 : %d ' % (conf.x)

b.py
import a
import conf
print 'Inside B: %d' % (conf.x)

Output:
python a.py
Inside A: 10
Inside A after changing value to 20 : 20

python b.py
Inside A: 10


Inside A after changing value to 20 : 20
Inside B: 20
#Here you get 20 as output, since you imported a 


When you import using 2 ways:
1) from conf import x 
#this will create local variable scope

2) import conf
conf.x  #refers to original location


a.py
from conf import x   #This creates variable in local scope
print 'Inside A: %d ' % (x)
x=20
print 'Inside A after changing value to 20 : %d ' % (x)

Output: python a.py
Inside A: 10
Inside A after changing value to 20 : 20

python b.py
Inside A: 10
Inside A after changing value to 20 : 20
Inside B: 10

Conclusion:
Always safer to use 
from conf import x
instead of 
import x

python ConfigParser

#file.ini
#[HEADER]
#name = Matt

import ConfigParser
config = ConfigParser.ConfigParser()
config.read('file.ini')
name = config.get('HEADER', 'NAME') #Matt
print name
name = 'Will'
config.set('HEADER', 'NAME', name)
print config.get('HEADER', 'NAME') #Will

# Writing our configuration file to 'example.cfg'
with open('file.ini', 'w') as configfile:
    config.write(configfile)

python open very big file, python memory

Python open very big file: 

#Say you are looping through a big 2 TB file

logfile = open("huge_log_file.txt","r")
info_lines = [(line,len(line)) for line in logfile if line.startswith("INFO")]
#Here it will get a huge list - costs RAM, this list could contain 2 TB of content

logfile = open("huge_log_file.txt","r")
info_lines = ((line,len(line)) for line in logfile if line.startswith("INFO"))
#Here it will get generator object - memory efficient

Opening a file (read mode) does NOT implicitly read nor load its contents into memory.  
Even when you do so using Python's context management protocol (the with keyword).

e.g.,

with open('huge_log_file.txt', 'r') as f:
   for each_line in f:
     do_something_each_line(each_line)

Then your peak memory utilization shouldn't be much larger than the longest line of the file

If you really are reading the full content of the file into a data structure like list, then it's no wonder that your RAM usage peaks like that. 
It's not that python puts the full contents of the file in RAM, but that you do.
e.g.,

#Here it will load all into memory as you are storing/dumping into a list called info_lines
info_lines = [(line,len(line)) for line in logfile if line.startswith("INFO")] 


#Memory efficinet - since it return generator
info_lines = ((line,len(line)) for line in logfile if line.startswith("INFO"))



file = '/tmp/huge_log_file.txt'

#This is fine
with open(file, 'r') as fh:
   for each in fh:
       print each

#this is a blunder, this will crash, it loads every thing into memory
#with open(file, 'r') as fh:
#    lines = fh.readlines()
#    print len(lines)

What is the difference between __init__ and __call__?

# __init__ method is used when the class is called to initialize the instance
# while the __call__ method is called when the instance is called

class Foo:
    def __init__(self, a, b, c):
        print 'inside Foo __init__'
    def __call__(self, a, b, c):
        print 'inside Foo __call__'

class Bar:
    def __init__(self):
        print 'inside Bar __init__'
    def __call__(self, a, b, c):
        print 'inside Bar __call__'


f = Foo(1, 2, 3) # __init__
b = Bar()
b(1, 2, 3) # __call__

Output:

inside Foo __init__

inside Bar __init__

inside Bar __call__

__all__ in Python

__all__ in a module

e.g. module.py:

__all__ = ['foo', 'Bar']
means that when you import * from the module, only those names in the __all__ are imported:

from module import *   
# imports only foo and Bar (through wild card)

test1.py

__all__ = ['var1', 'func1']

var1 = 100

def func1():
    return 'func1'

def func2():
    return 'func2'


test2.py
from test import var1, func1, func2
print var1
print func1()
print func2()

Output:
100
func1
func2


test3.py
from test import *

print var1
print func1()
print func2() #NameError: name 'func2' is not defined

Output:
100
func1
NameError: name 'func2' is not defined

Python __name__, __main__

test1.py
if __name__ == '__main__':
   print 'inside test1 main...'
else:
   print 'test1 imported: ' + __name__


test2.py
import test1

if __name__ == '__main__':
   print 'inside test2 main...'
else:
   print 'test2 imported: ' + __name__

test3.py
import test2

if __name__ == '__main__':
   print 'inside test3 main...'
else:
   print 'test3 imported: ' + __name__

python test1.py

Output:

inside test1 main...

python test2.py

Output:

test1 imported: test1

inside test2 main...

python test3.py    ##(from test1 -> test2 -> test3)

Output:

test1 imported: test1

test2 imported: test2

inside test3 main...

Python multi threading Vs multi processing

Python multi threading Vs multi processing

Ref:
https://medium.com/@nbosco/multithreading-vs-multiprocessing-in-python-c7dc88b50b5b

#Threading
#The Python threading module uses threads instead of processes. Threads run in the same unique memory heap.
#Whereas Processes run in separate memory heaps. This, makes sharing information harder with processes and object instances.
#One problem arises because threads use the same memory heap, multiple threads can write to the same location in the memory heap which is why the global interpreter lock(GIL) in CPython was created as a mutex to prevent it from happening.

#Multiprocessing

#The multiprocessing library uses separate memory space, multiple CPU cores, bypasses GIL limitations in CPython, child processes are killable(ex. function calls in program) and is much easier to use.

#Some caveats of the module are a larger memory footprint and IPC’s a little more complicated with more overhead.

import threading

#Threading
#The Python threading module uses threads instead of processes. Threads run in the same unique memory heap.
#Whereas Processes run in separate memory heaps. This, makes sharing information harder with processes and object instances.
#One problem arises because threads use the same memory heap, multiple threads can write to the same location in the memory heap which is why the global interpreter lock(GIL) in CPython was created as a mutex to prevent it from happening.

def calc_square(number):
    print('Square:' , number * number)

def calc_quad(number):
    print('Quad:' , number * number * number * number)

if __name__ == "__main__":
    number = 7
    thread1 = threading.Thread(target=calc_square, args=(number,))
    thread2 = threading.Thread(target=calc_quad, args=(number,))
    # Will execute both in parallel
    thread1.start()
    thread2.start()
    # Joins threads back to the parent process, which is this
    # program
    thread1.join()
    thread2.join()
    # This program reduces the time of execution by running tasks in parallel

import multiprocessing

#The multiprocessing library uses separate memory space, multiple CPU cores, bypasses GIL limitations in CPython, child processes are killable(ex. function calls in program) and is much easier to use.

#Some caveats of the module are a larger memory footprint and IPC’s a little more complicated with more overhead.

def calc_square(number):

    print('Square:' , number * number)

    result = number * number

    print(result)

def calc_quad(number):

    print('Quad:' , number * number * number * number)

if __name__ == "__main__":

    number = 7

    result = None

    p1 = multiprocessing.Process(target=calc_square, args=(number,))

    p2 = multiprocessing.Process(target=calc_quad, args=(number,))

    p1.start()

    p2.start()

    p1.join()

    p2.join()

    # Wont print because processes run using their own memory location

    print(result)

Python read from XLS/XLSX

import xlrd

loc = ("/tmp/Students.xlsx")

wb = xlrd.open_workbook(loc)

sheet1 = wb.sheet_by_index(0)
sheet2 = wb.sheet_by_index(1)

#Sheet1
for i in range(1, sheet1.nrows):
each_list = sheet1.row_values(i)
print each_list

#Sheet2
for i in range(1, sheet1.nrows):
each_list = sheet2.row_values(i)
print each_list

Python Find second smallest number

Python Find second smallest number

def get_second_smallest(numbers):
    num1, num2 = float('inf'), float('inf')
    print num1, num2
    print '#####'
    for x in numbers:
        if x <= num1:
            num1, num2 = x, num1
            print num1, '-------', num2
        elif x < num2:
            num2 = x
            print num1, '-------', num2
        else:
            print 'pass... ' + str(x)
    return num2

out = get_second_smallest([2, 3, 4, 5, 1, 1.3, 1.5])
print '\nOutput: ' + str(out)

Output:
2 ------- inf
2 ------- 3
pass... 4
pass... 5
1 ------- 2
1 ------- 1.3
pass... 1.5

Output: 1.3

AWS Create/change key pair for EC2 instance

AWS Create/change key pair for EC2 instance

Under EC2 -> Network & Security -> Key Pairs -> Create with name: Mirror
It will give you Mirror.pem

PuyttyGen:
PuttyGen -> Load this PEM -> Generate
Download public key as Mirror.pub & private key as Mirror.ppk

Login in instance:
vim ~/.ssh/authorized_keys
Open Mirror.pub, copy whole key as single line, without new line  characters (after Comment: and before END SSH2 PUBLIC KEY)

ssh-rsa <copied text from above> <Mirror(without pem extension)>

e.g.,
ssh-rsa  AAAAB3NzaC1yc2EAAA...urR2A5IUkqscHRU1Nc7TFz363UFJW6XMYae1116PO4  Mirror

Setup Mirror under Putty like below:
Host Name -> EC2 Host Name
Connection -> SSH -> Auth -> Add Mirror.pek path & save

Now login into EC2 as root

(it should not ask any password)

How to install pyspark in centos

Install spark ref:
http://devopspy.com/python/apache-spark-pyspark-centos-rhel/

cd /opt
wget http://www-eu.apache.org/dist/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz
ln -s spark-2.4.0-bin-hadoop2.7 spark
check /etc/hosts

How to set path?
export SPARK_HOME = /opt/spark
export PATH = $PATH:/opt/spark
export export PYTHONPATH=$SPARK_HOME/python/lib/py4j-0.10.4-src.zip:$SPARK_HOME/python/lib/pyspark.zip:$PYTHONPATH
export PATH = $SPARK_HOME/python:$PATH

How to start master?
./sbin/start-master.sh

    1) If you get error like blow:
"hostname: Unknown host" start-master.sh
set the hostname properly
hostname test.com
hostname -f #should give you some output

    2) If you get error like below:
Getting "Unsupported major.minor version 52.0" exception while using  
        Spark Web Application framework
check Java version of jar files (/opt/spark/jars) and your installed java

How to start spark master?
cd /opt/spark
./sbin/start-master.sh
This internally runs command like below:
Spark Command: /opt/java/jdk1.8.0_201/bin/java -cp /opt/spark/conf/:/opt/spark/jars/* -Xmx1g org.apache.spark.deploy.master.Master --host test.com --port 7077 --webui-port 8080

How to access from web?
test.com:8080 (port: 8080)

How to start spark shell?
cd /opt/spark
./bin/pyspark
FYI, This internally runs command like below:
/opt/java/jdk1.8.0_201/bin/java -cp /opt/spark/conf/:/opt/spark/jars/* -Xmx1g org.apache.spark.deploy.SparkSubmit --name PySparkShell pyspark-shell

How to access the spark process in ps commands?
ps -ef | grep spark
      e.g.,root     13770     1  0 14:18 pts/0    00:00:10 /opt/java/jdk1.8.0_201/bin/java -cp /opt/spark/conf/:/opt/spark/jars/* -Xmx1g org.apache.spark.deploy.master.Master --host test.com --port 7077 --webui-port 8080

PIP modules to install
pip install py4j

How to access in web?

http://localhost:8080

How to install java on centos

How to install java on centos?

Ref:
https://tecadmin.net/install-java-8-on-centos-rhel-and-fedora/

How to find java version of jar file

unzip test.jar
check for the file: 
META-INF/MANIFEST.MF (to get more details)

You see details like:
Created-By: 1.8.0_144 (Oracle Corporation)
Bundle-Version: 1.3.2.2

Backup Apache log files using logrotate

/etc/logrotate.d/httpd

If you want to backup apache logs for all instances (you can copy this logrotate script/snippet in /etc/logrotate.d/httpd)

Prerequisite: setup s3cmd

/var/log/httpd/*log {
    daily
    dateext
    #dateext dateformat -%Y-%m-%d-%s
    missingok
    notifempty
    #size 3M
    sharedscripts
    delaycompress
    rotate 4
    create
    postrotate
        /sbin/service httpd reload > /dev/null 2>/dev/null || true

        BUCKET=logs-backup
        INSTANCE_ID=`curl --silent http://169.254.169.254/latest/meta-data/instance-id`
        /usr/local/bin/s3cmd -c /root/.s3cfg -m text/plain sync /var/log/httpd/access_log* s3://${BUCKET}/system_logs/httpd/${INSTANCE_ID}/
        /usr/local/bin/s3cmd -c /root/.s3cfg -m text/plain sync /var/log/httpd/error_log* s3://${BUCKET}/system_logs/httpd/${INSTANCE_ID}/

    endscript
}

Python csv write

from users.models import *
import sys

up = UserProfile.objects.all()
print up.count()

header = "Name, User Type, Email, Phone, City, Country, TimeZone, Created Date\n"

fname = "/tmp/user_data.csv"
with open(fname, 'w') as FW:
   FW.write(header)
   for each in up:
     try:
        name = each.name
        name = name.encode('utf-8').strip()
        dts = each.created_date
        dts = dts.strftime("%d %b %Y") #dts.strftime("%b %d %Y %I:%M %p")
        str1 = "%s,%s,%s,%s,%s,%s,%s,%s\n" % (str(name), str(each.usertype), str(each.email_id), str(each.phone), str(each.city), str(each.country), str(each.timezone), str(dts))
        #print str1
        FW.write(str1)
     except Exception, e:
        print '---- Error: ' + str(e)

print fname


Output:
Name, User Type, Email, Phone, City, Country, TimeZone, Created Date
User2,Student,user2@abc.com,None,Bangalore,IN,Asia/Kolkata,16 Mar 2018
User3,Student,user3@gmail.com,None,Pune,IN,Asia/Kolkata,28 Mar 2018
....

Python Zip

"""
input_data.txt
#########
First row contains column name
Second row contains data type
From third row, it contains student records.
How to map all the reocords using zip function?

name, age, city
varchar, int, varchar
prabhath, 32, bangalore
vamsi, 30, hyderabad
lakshmi, 30, vizag
ramesh, 50, chennai

"""

with open('input_data.txt') as input_file:
    rows = input_file.readlines()

rows = [row.strip() for row in rows]
rows = [row.split(',') for row in rows]
print rows

columns = rows[0]
data_types = rows[1]
rows = rows[2:]

for row in rows:
print '----'
for column, data_type, val in zip(columns, data_types, row):
print column.strip(), data_type.strip(), val.strip()

"""
Output:

[['name', ' age', ' city'], ['varchar', ' int', ' varchar'], ['prabhath', ' 32', ' bangalore'], ['vamsi', ' 30', ' hyderabad'], ['lakshmi', ' 30', ' vizag'], ['ramesh', ' 50', ' chennai']]
----
name varchar prabhath
age int 32
city varchar bangalore
----
name varchar vamsi
age int 30
city varchar hyderabad
----
name varchar lakshmi
age int 30
city varchar vizag
----
name varchar ramesh
age int 50
city varchar chennai
"""

Python read characters vertically in a file

with open('input_vertical.txt') as input_file:
    rows = input_file.readlines()
print rows
print '---'
rows = [row.strip() for row in rows]
print rows
##The single star * unpacks the sequence/collection into positional arguments
rows = zip(*rows)
print rows
rows = [''.join(row) for row in rows]
print rows



"""
Output:

['prabhath\n', 'test\n', 'vertical\n', 'lines']
---
['prabhath', 'test', 'vertical', 'lines']
[('p', 't', 'v', 'l'), ('r', 'e', 'e', 'i'), ('a', 's', 'r', 'n'), ('b', 't', 't', 'e')]
['ptvl', 'reei', 'asrn', 'btte']
"""

Python week of the month

from math import ceil
from datetime import datetime

def week_of_month(dt):
#This will add the first day of the month weekday & present day of month
    first_day = dt.replace(day=1)
    day_of_month = dt.day
    #print day_of_month
    adj_day_of_month = day_of_month + first_day.weekday()
    return int(ceil(adj_day_of_month/7.0))

if __name__ == '__main__':
    #dt = datetime.now()
    #dt = datetime.strptime('Feb 21 2019  1:00PM', '%b %d %Y %I:%M%p')
    dt = datetime.strptime('Feb 19 2019', '%b %d %Y')
    print dt
    print week_of_month(dt)

"""
Output:
2019-02-19 00:00:00
3
"""

Django update UTC time to MySQL

from datetime import datetime
from django.utils.timezone import utc

sch = Schedule.objects.get(id=25)
sch.start_time = datetime.utcnow().replace(tzinfo=utc)
sch.save()

Compare List Vs List Comprehension, timeit

import timeit

# timeit.timeit(stmt, setup, timer, number)
# stmt   - which is the statement you want to measure it defaults to 'pass'.
# setup  - which is the code that you run before running the stmt; it defaults to 'pass'. We generally use this to import the required modules for our code.
# number - number of executions you like to run the stmt.

# List Comprehension

lst = [3, 2, 41, 3, 34, 99]

print(lst)

# If Condition alone - keep after the For loop

print([number for number in lst if number % 2 != 0])

# If-Else Condition - keep before For loop

print([number if number % 2 != 0 else -1 for number in lst])

#List Vs List Comprehension
#Ref: https://stackoverflow.com/questions/16341775/what-is-the-advantage-of-a-list-comprehension-over-a-for-loop
#List comprehensions are more compact and faster than an explicit for loop building a list:
#This is because calling .append() on a list causes the list object to grow (in chunks) to make space for new elements individually, while the list comprehension gathers all elements first before creating the list to fit the elements in one go:

def for_loop_test():
    result = []
    for elem in iter_var:
        result.append(elem)
    return result

def list_comprehension_test():
    return [elem for elem in iter_var]

if __name__ == '__main__':
iter_var = range(1000)
print timeit.timeit('f()', 'from __main__ import for_loop_test as f', number=10000)
print timeit.timeit('f()', 'from __main__ import list_comprehension_test as f', number=10000)


"""
Output:
1.41242463295
0.474811508482
"""'

Boto S3 How to access buckets other than us-east-1

Boto S3 - How to access buckets other than us-east-1?

Trying to connect mumbai region (ap-south-1) bucket, but not able to connect.
Esp in Boto 2 versions, there is some issue, getting below errors:

• boto.exception.S3ResponseError: S3ResponseError: 301 Moved Permanently
• boto.exception.S3ResponseError: S3ResponseError: 400 Bad Request

Solution:

You need to mark S3_USE_SIGV4 flag to True, else it will throw you error.

from boto.s3.connection import S3Connection, Location
import os

os.environ['S3_USE_SIGV4'] = 'True'
conn = S3Connection(S3_KEY, S3_SECRET, host='s3.ap-south-1.amazonaws.com', calling_format=boto.s3.connection.OrdinaryCallingFormat())
print conn
client_buk_obj = conn.get_bucket('test-mumbai-region-test')
print client_buk_obj
os.environ['S3_USE_SIGV4'] = 'False'

Python compare lists

#Comapre lists

student1_scores = [61, 73, 84, 90, 85, 45]
student2_scores = [40, 37, 45, 87, 99, 54]

#set - will change the order
print list(set(student1_scores)) #[73, 45, 84, 85, 90, 61]
print list(set(student2_scores)) #[99, 37, 40, 45, 54, 87]

#Commom elements in two lists
print list(set(student1_scores) & set(student2_scores)) #[45]

#Commom elements in two lists
print list(set(student1_scores).intersection(set(student2_scores))) #[45]

#Union of two lists
print list(set(student1_scores).union(set(student2_scores))) 
#[99, 37, 40, 73, 45, 84, 85, 54, 87, 90, 61]

#Commom elements in two lists - using list comprehension
print [i for i in student1_scores if i in student2_scores] #[45]

#compare common elements with for the same subject (same index)
student1_scores = [61, 73, 84, 90, 85, 45]
student2_scores = [40, 37, 45, 90, 99, 54]

print [i for i, j in zip(student1_scores, student2_scores) if i == j] #[90]

Python MySQL

import mysql.connector

mydb = mysql.connector.connect(
  host="localhost",
  user="root",
  passwd="password",
  database="myproject"
)

my_cursor = mydb.cursor()

sql = "SELECT user.name, product.name FROM user JOIN product ON user.product_id = product.id"

my_cursor.execute(sql)

results = my_cursor.fetchall()

for each in results:
  print each

MySQL Joins

  #INNER JOIN - this gives common elements from user & product
  SELECT user.name, product.name 
  FROM user INNER JOIN product ON user.product_id = product.id
  
  #LEFT JOIN - this gives all user & matching product
  SELECT user.name, product.name 
  FROM user LEFT JOIN product ON user.product_id = product.id
  
  #RIGHT JOIN - this gives all product with user
  SELECT   user.name, product.name 
  FROM user RIGHT JOIN product ON user.product_id = product.id

Python reverse a string

Python reverse a string

ss = 'Hello World'
print ss[::-1] #dlroW olleH
print reversed(ss)
print ''.join(reversed(ss)) #dlroW olleH

O/P:
dlroW olleH 

<reversed object at 0x7f5813ce73d0> 

dlroW olleH

Python datetime

from datetime import datetime

datetime_obj = datetime.now() 
print datetime_obj #2019-01-25 13:45:28.087000
print datetime_obj.date() #2019-01-25
print datetime_obj.year  #2019
print datetime_obj.month #1
print datetime_obj.day   #25
print datetime_obj.hour  #13
print datetime_obj.minute #45
print datetime_obj.second #28

datetime_object = datetime(2011, 07, 01, 11, 50, 55)
print datetime_object
datetime_object_str =  datetime_object.strftime("%Y-%m-%d %H:%M:%S")
print datetime_object_str #2011-07-01 11:50:55
print type(datetime_object_str) #<type 'str'>

datetime_object = datetime.strptime('2018-07-11 10:55:55', '%Y-%m-%d %H:%M:%S')
print datetime_object #2011-07-01 11:50:55
print type(datetime_object) #<type 'datetime'>