Setup python app in centos from scratch (centos 6.9+uwsgi+nginx+flask+mysql)

Initial setup

$ sudo yum update
$ sudo yum install epel-release
$ sudo yum groupinstall "Development tools"
$ sudo yum install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel telnet htop
$ sudo yum install python-devel python-virtualenv
$ sudo yum install mysql-connector-python mysql-devel mysql-server

Install Python

Download and install Python : https://www.python.org/

./configure && make && make altinstall

Install uWSGI

$ wget https://bootstrap.pypa.io/get-pip.py
$ which python2.7
$ sudo /usr/local/bin/python2.7 get-pip.py
$ which pip2.7
$ sudo /usr/local/bin/pip2.7 install uWSGI
$ which uwsgi
$ uwsgi --version

Setup vassels

$ sudo mkdir -p /etc/uwsgi/vassels

Setup Emperor service

$ sudo vim /etc/init.d/emperor
#!/bin/sh
# chkconfig: 2345 99 10
# Description: Starts and stops the emperor-uwsgi
# See how we were called.

RUNEMPEROR="/usr/local/bin/uwsgi --emperor=/etc/uwsgi/vassels"

PIDFILE=/var/run/emperor-uwsgi.pid
LOGFILE=/var/log/uwsgi/emperor.log

start() {
  if [ -f /var/run/$PIDNAME ] && kill -0 $(cat /var/run/$PIDNAME); then
    echo 'Service emperor-uwsgi already running' >&2
    return 1
  fi
  echo 'Starting Emperor...' >&2
  local CMD="$RUNEMPEROR &> \"$LOGFILE\" & echo \$!"
  su -c "$CMD" > "$PIDFILE"
  echo 'Service started' >&2
}

stop() {
  if [ ! -f "$PIDFILE" ] || ! kill -0 $(cat "$PIDFILE"); then
    echo 'Service emperor-uwsgi not running' >&2
    return 1
  fi
  echo 'Stopping emperor-uwsgi' >&2
  kill -7 $(cat "$PIDFILE") && rm -f "$PIDFILE"
  echo 'Service stopped' >&2
}

status() {
    if [ ! -f "$PIDFILE" ]; then
	echo "Emperor is not running." >&2
	return 1
    else
    	echo "Emperor (pid  `cat ${PIDFILE}`) is running..."
    	ps -ef |grep `cat $PIDFILE`| grep -v grep
    fi
}

case "$1" in
start)
      start
      ;;
stop)
      stop
      ;;
status)
      status
      ;;
restart)
      stop
      start
      ;;
*)
    echo "Usage: emperor {start|stop|restart}"
    exit 1
esac

 Setup app user & environment

$ useradd foobar
$ usermod -md /srv/foobar foobar
$ chmod 755 /srv/foobar
$ sudo su - foobar
foobar@local~$ virtualenv --python=python2.7 ~/venv
foobar@local~$ mkdir www
foobar@local~$ mkdir logs
foobar@local~$ touch logs/uwsgi.log
foobar@local~$ touch uwsgi.ini
foobar@local~$ echo "source ~/venv/bin/activate" >> ~/.bashrc
foobar@local~$ source ~/venv/bin/activate
(venv)foobar@local~$ vim uwsgi.ini
[uwsgi]
master = true
processes = 2
socket = /tmp/foobar.sock

chdir = /srv/foobar/www
virtualenv = /srv/foobar/venv
module = app:app

uid = foobar
chown-socket = foobar:nginx
chmod-socket = 660
vacuum = true

die-on-term = true
python-autoreload = 3
py-autoreload = 1
logger = file:/srv/foobar/logs/uwsgi.log

Exit from foobar user & create uwsgi symlink

(venv)foobar@local~$ exit
$ sudo ln -s /srv/foobar/uwsgi.ini /etc/uwsgi/vassels/foobar.ini

Start emperor service & setup set the startup

$ sudo service emperor start
$ sudo chkconfig emperor on

Setup uWSGI and Emperor on Centos 6.xx

uWSGI is driving me crazy when install it, every single time. I’m using Centos 6.9 to setup my python app.

This one is for setup uWSGI and make sure you already installed Python2.7 on your machine:

$ wget https://bootstrap.pypa.io/get-pip.py
$ which python2.7
$ sudo /usr/local/bin/python2.7 get-pip.py
$ which pip2.7
$ sudo /usr/local/bin/pip2.7 install uWSGI
$ which uwsgi
$ uwsgi --version

Test your app with uWSGI manually:

$ sudo /usr/local/bin/uwsgi --ini /dir/to/app/uwsgi.ini

Setup vassels from uwsgi ini file

$ sudo mkdir -p /etc/uwsgi/vassels
$ sudo ln -s /dir/to/app/uwsgi.ini /etc/uwsgi/vassels/appname.ini

Create emperor to make uWSGI as a service:

$ sudo vim /etc/init.d/emperor
#!/bin/sh
# chkconfig: 2345 99 10
# Description: Starts and stops the emperor-uwsgi
# See how we were called.

RUNEMPEROR="/usr/local/bin/uwsgi --emperor=/etc/uwsgi/vassels"

PIDFILE=/var/run/emperor-uwsgi.pid
LOGFILE=/var/log/uwsgi/emperor.log

start() {
  if [ -f /var/run/$PIDNAME ] && kill -0 $(cat /var/run/$PIDNAME); then
    echo 'Service emperor-uwsgi already running' >&2
    return 1
  fi
  echo 'Starting Emperor...' >&2
  local CMD="$RUNEMPEROR &> \"$LOGFILE\" & echo \$!"
  su -c "$CMD" > "$PIDFILE"
  echo 'Service started' >&2
}

stop() {
  if [ ! -f "$PIDFILE" ] || ! kill -0 $(cat "$PIDFILE"); then
    echo 'Service emperor-uwsgi not running' >&2
    return 1
  fi
  echo 'Stopping emperor-uwsgi' >&2
  kill -7 $(cat "$PIDFILE") && rm -f "$PIDFILE"
  echo 'Service stopped' >&2
}

status() {
    if [ ! -f "$PIDFILE" ]; then
	echo "Emperor is not running." >&2
	return 1
    else
    	echo "Emperor (pid  `cat ${PIDFILE}`) is running..."
    	ps -ef |grep `cat $PIDFILE`| grep -v grep
    fi
}

case "$1" in
start)
      start
      ;;
stop)
      stop
      ;;
status)
      status
      ;;
restart)
      stop
      start
      ;;
*)
    echo "Usage: emperor {start|stop|restart}"
    exit 1
esac
$ sudo chmod +x /etc/init.d/emperor
$ sudo /etc/init.d/emperor start
$ sudo chkconfig emperor on

Check your app if it’s already running:

$ sudo ps -ef | grep uwsgi

Post twitter using python script

$ pip install --upgrade pip
$ pip install tweepy
import tweepy

def get_api(cfg):
    auth = tweepy.OAuthHandler(cfg['consumer_key'], cfg['consumer_secret'])
    auth.set_access_token(cfg['access_token'], cfg['access_token_secret'])
    return tweepy.API(auth)

def main():
    cfg = {
        "consumer_key"        : "xxxxx",
        "consumer_secret"     : "xxxxx",
        "access_token"        : "xxxxx",
        "access_token_secret" : "xxxxx"
    }

    api = get_api(cfg)
    tweet = "my status here"
    status = api.update_status(status=tweet)

if __name__ == "__main__":
    main()

Create watermark and resize images with Python

After holiday, I have many pictures that needs to be uploaded and all of them should have watermark. Watermark in image is one thing that I could’ve done it with photoshop when I was in school. But I’m not gonna waste my time to do that one by one instead I’m doing with this simple python script. The library that I used was PIL (Python Image Library).

This one I’ve done with custom font so you can choose your favorite font in your images. This script will be run two jobs to edit every images you have in one directory. The first thing it gonna resize your image based on what I already defined inside the script, you might want to change the basewidth variable if you want and after resizing, the script will put the watermark at the corner of your image.

For the font, I was just searching for the free license in dafont.com

Make sure you download the .ttf extension, not sure whether the other extensions will work.

#!/usr/bin/env python
import PIL
from PIL import Image, ImageDraw, ImageFont

def resize():
  image = Image.open('IMG_0495.JPG')
  width, height = image.size
  basewidth = 800
  wpercent = (basewidth / float(image.size[0]))
  hsize = int((float(image.size[1]) * float(wpercent)))
  image = image.resize((basewidth, hsize), PIL.Image.ANTIALIAS)
  image.save('TES.JPG')
def watermark():
  image = Image.open('TES.JPG')
  width, height = image.size
  draw = ImageDraw.Draw(image)
  text = "Your credit written here"
  font = ImageFont.truetype('Downloads/Capture_it.ttf', 15)
  textwidth, textheight = draw.textsize(text, font)
  # calculate the x,y coordinates of the text
  margin = 5
  x = width - textwidth - margin
  y = height - textheight - margin
  # draw watermark in the bottom right corner
  draw.text((x, y), text)
  image.save('TES.JPG')
def main():
  resize()
  watermark()
main()

Make paging output from your python script

pydoc is a python library to make your console output paginates for easier reading. to use it follow the simple example below:

import pydoc

text = "...paginate me..."
pager = pydoc.ttypager(text)
print pager

you will see the output is gonna be truncated at the beginning when you start to see like when you open up a man page, then you have to scroll down with keyboard to read the rest of the text line by line. pretty cool!


Django table is marked as crashed and should be repaired

My django app is showing 500 because one of the tables in database marked crashed and should be repaired.

InternalError at /

(145, "Table './dbname/django_session' is marked as crashed and should be repaired")

I thought it was just an usual error from mysql and needed to restart but it wasn’t.

Solutions:

 Login to mysql, select your dbname, and use this command:

USE user_base;
REPAIR TABLE TABLE;

Get json results from mysql with python

This is just simple trick to retrieve data from mysql rows and turn into json.

#!/usr/bin/env python
import MySQLdb as mdb
import sys

con = mdb.connect('localhost', 'root', 'pass123', 'mydb1', charset='utf8')
cur = con.cursor(mdb.cursors.DictCursor)

def get_post():
    list = []
    gw = {}
    cur.execute("select * from tbl1_core")
    rows = cur.fetchall()
    for i in rows:
        tes = []
        # we want to fetch row to store in results
        gw["content"] = i["content"]
        tes.append(gw)
        list.append(tes)
        tes = []
        gw = {}
    return list
tes = get_post()
print tes

Install and Create Virtualenv Python in ubuntu machine

Let’s say I want to have a project called webproject, and this project has own user with the same name to access the project directory. And this is supposed to be a python project and it needs virtual environment. also I want the project has specific python version 2.7 with virtualenv. So let’s get started.

First you may have to check your python version if it is already installed python 2.7 otherwise you’re gonna have to download and install it, but if you prefer to not use python 2.7, let’s say version 2.6, or whatever. That’s fine as well.

Something related:

Install different version of Python in linux

Let’s start with install the virtualenv via apt in ubuntu

$ apt-get install python-virtualenv virtualenvwrapper

Create user and setup home directory

$ useradd -m webproject -d /srv/webproject

Change user to webproject, until you’re inside the home directory project

$ sudo su - webproject

Create virtual environment with virtualenv with python version 2.7

$ virtualenv ~/venv --python=2.7

Or, if you prefer not to use spesific python version, just run the command like this:

$ virtualenv ~/venv

Now, inside your home directory, there is a new directory named venv that contained your python binary.

You might want to put this line of code inside your .bash_history or .profile inside your home directory, so whenever you try to change user, it is automatically activate your python virtual environment.

$ vim ~/.profile
....
....
source ~/venv/bin/activate

and save the file.

Now, try to change user to webproject:

$ sudo su - webproject
(venv)webproject@localhost:~$

Create simple arg parse in python

argparse is a module to make user-friendly command-line interfaces. It’s probably the one of the most frequently used module when I create a script in python that needs to parse some arguments. Check this out.

$ vim test.py
#!/usr/bin/python
import argparse

def init_args():
  parser = argparse. ArgumentParser(description="This is the description")
  parser.add_argument("--arg1", required=True, type=str, help="This is arg1")
  parser.add_argument("--allow", required=False, action="store_true", help="Allow mode")
  return parser.parse_args()

def main(arg1):
  return "arg1: %s"%(arg1)

if __name__ == "__main__":
  args = init_args()
  main(args.arg1)

See what happens when we run it

$ ./test.py --arg1 "showme"
arg1: showme

Sometimes we want to create an argumen but only store it as a True variable. we can just create simple test like this.

$ vim test.py
#!/usr/bin/python
import argparse

def init_args():
  parser = argparse. ArgumentParser(description="This is the description")
  parser.add_argument("--arg1", required=False, action="store_true", help="Enable arg1")
  return parser.parse_args()

def main():
  return "arg1 is enabled"

if __name__ == "__main__":
  args = init_args()
  if args.arg1:
    main()
  else:
    print "arg1 parse is not enabled"

Try to run it

$ ./test.py --arg1
arg1 is enabled