1. 随机执行方法

# vi ~/multipro1.py

import multiprocessing

def spawn(num):
    print('Sprawned! {}.'.format(num))


if __name__ == "__main__":
    for i in range(50):
        p = multiprocessing.Process(target=spawn, args=(i,))

        p.start()

2. 按顺序执行方法

# vi ~/multipro2.py

import multiprocessing


def spawn(num):
    print('Sprawned! {}.'.format(num))


if __name__ == "__main__":
    for i in range(50):
        p = multiprocessing.Process(target=spawn, args=(i,))

        p.start()

        # Waiting to finish One by one
        p.join()

Tip: 这里if __name__ == "__main__":的作用:

1. 当直接执行该脚本时, 内建变量__name__的值被赋予"__main__", 所以按照if逻辑, python解释器可以继续执行接下来的代码.

2. 当其他脚本去调用(import)该脚本时, __name__的值被赋予当前脚本名"multipro1", 而不是"__main__", 所以按照if逻辑, 保证if下面的代码不会被其他脚本调用.

所以在我们日常编写脚本的时候, 可以推荐按照如下结构, 在自己使用正常的同时, 保证在其他python脚本在调用test.py中的test1(), test2(), test3()的同时, 不会去执行if条件下的方法.

# vi test.py

def test1():
    ....

def test2():
    ....

def test3():
    ....

if __name__ == "__main__":
    ....

3. 多线程处理返回值.

# vi multipro3.py

from multiprocessing import pool

def job(num):
	return num * 2

if __name__ == '__main__':
	p = Pool(processes=20)
	data = p.map(job, range(5))
	data2 = p.map(job, [5, 2])
	p.close()

4.多线程爬取随机网站URL

# vi ~/multipro4.py

from multiprocessing import Pool
import bs4 as bs
import random
import requests
import string

# Return Four digits domain URL
def random_starting_url():
    starting = ''.join(random.SystemRandom().choice(
        string.ascii_lowercase) for _ in range(4))
    url = ''.join(['http://', starting, '.com'])
    return url


# Correct URL if it is relative
def handle_local_links(url, link):
    if link.startswith('/'):
        return ''.join([url, link])
    else:
        return link


# Get URL from "a" tag of 'body' tag
def get_links(url):
    try:
        # Request the URL
        resp = requests.get(url)
        # return the HTML
        soup = bs.BeautifulSoup(resp.text, 'lxml')
        # Get 'body' tag of the HTML
        body = soup.body
        # Get URL from "a" tag of 'body' tag
        links = [link.get('href') for link in body.find_all('a')]
        # Correct URL if it is relative
        links = [handle_local_links(url, link) for link in links]
        # Encoding the URL to ascii
        links = [str(link.encode('ascii')) for link in links]

        return links

    except TypeError as e:
        print(e)
        print('Got a TypeError, probably got a None that we tried to iterate over')
        return([])
    except IndexError as e:
        print(e)
        print('No valid link found, return a empty list')
        return([])
    except AttributeError as e:
        print(e)
        print('Likely got None for links, so we are throwing this')
        return([])
    except Exception as e:
        print(str(e))
        return([])


def main():
    # CPU process
    process = 5
    # The site intends to scrap
    site = 3
    p = Pool(processes=process)
    # Get random URL list
    parse_us = [random_starting_url() for _ in range(site)]
    # Multiprocessing the URL and parse the result
    data = p.map(get_links, parse_us)
    # Get each URL in a list
    data = [url for url_list in data for url in url_list]
    p.close()
    # Write to txt file
    with open('urls.txt', 'w') as f:
        f.write(str(data))


if __name__ == '__main__':
    main()

Beautiful Soup主要从解析好的HTML源码中抓取我们所需要的关键内容

Pandas与Beautiful Soup类似, 不过它侧重去抓取源码中的表格信息

pyqt5这里的作用是模拟浏览器去解析源码中的Javasript, 并最终抓取JS实际的返回值.

这里我在我的Flask env下创建了一个测试页面, 用这些模块进行一些简单的页面爬虫测试.

http://flask.showerlee.com/scrapingtest/

安装环境

OS:       Windows 7 x64   
Python:   Python3.6.2
Git Bash: Git-2.15.1.2-64-bit

一. 环境配置:

1. 安装并运行Git bash

2. 安装并测试python版本

# python -V

Python 3.6.2

3. 安装相关爬虫模块

# python -m pip install beautifulsoup4 lxml pandas html5lib pyqt5 selenum

二. Beautiful Soup演示

# vi ~/scrap1.py

Tip: 这里首先去调用io和sys模块是为了改变默认的标准输出为utf-8, 这么做是为了保证无论我们抓取的源页面是什么格式, 都不会在用BS解析时报UnicodeEncodeError.

import io
import sys
import bs4 as bs
import urllib.request

# 改变标准输出的默认编码为utf-8
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')

# 获取该页面编码并解码成utf-8
sauce = urllib.request.urlopen(
    'http://flask.showerlee.com/scrapingtest/').read().decode('utf-8')

# 利用BS抓取页面源代码,并利用lxml规范格式
soup = bs.BeautifulSoup(sauce, 'lxml')

# 获取页面源代码
print(soup)

# 获取titile标签源代码
print(soup.title)

# 获取titile标签name
print(soup.title.name)

# 获取titile标签字符
print(soup.title.string)
print(soup.title.text)

# 获取第一个p标签源代码
print(soup.p)

# 获取p标签源代码
print(soup.find_all('p'))

# 获取p标签所有内容
for paragraph in soup.find_all('p'):
    print(paragraph.text)

# 获取页面所有内容
print(soup.get_text())

# 获取a标签所有内容
for url in soup.find_all('a'):
    print(url.text)

# 获取a标签所有href链接
for url in soup.find_all('a'):
    print(url.get('href'))

# 获取nav标签源代码
nav = soup.nav
print(nav)

# 获取nav标签URL
for url in nav.find_all('a'):
    print(url.get('href'))

# 获取body标签内容
body = soup.body
for paragraph in body.find_all('p'):
    print(paragraph.text)

# 获取div标签下body下的内容
for div in soup.find_all('div', class_='body'):
    print(div.text)

# 获取table标签源代码
table = soup.table
# table = soup.find('table')
print(table)

# 获取table每行内容
table_rows = table.find_all('tr')

for tr in table_rows:
    td = tr.find_all('td')
    row = [i.text for i in td]
    print(row)

	

# python scrap1.py

......

三. Pandas演示

# vi ~/scrap2.py

import pandas as pd

dfs = pd.read_html(
    'http://flask.showerlee.com/scrapingtest/', header=0)

for df in dfs:
    print(df)

# python scrap2.py

  Program Name  Internet Points    Kittens?
0       Python        932914021  Definitely
1       Pascal              532    Unlikely
2         Lisp             1522   Uncertain
3           D#               12    Possibly
4        Cobol                3         No.
5      Fortran            52124        Yes.
6      Haskell               24        lol.

四. pyqt5演示

这里我们首先不解析JS, 直接利用Beautiful Soup去抓取p标签下class=jstest的内容

# vi ~/scrap3.py

import io
import sys
import bs4 as bs
import urllib.request

# 改变标准输出的默认编码为utf-8
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')

# 获取该页面编码并解码成utf-8
sauce = urllib.request.urlopen(
    'http://flask.showerlee.com/scrapingtest/').read().decode('utf-8')

# 利用BS抓取页面源代码,并利用lxml规范格式
soup = bs.BeautifulSoup(sauce, 'lxml')

js_test = soup.find('p', class_='jstest')

print(js_test.text)

# python scrap3.py

No js loaded

可以看到实际抓取的为未被JS处理的标签内容

这里利用pyqt5去抓取p标签下class=jstest的内容

# vi ~/scrap4.py

import bs4 as bs
import sys
from PyQt5.QtWebEngineWidgets import QWebEnginePage
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QUrl


class Page(QWebEnginePage):
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebEnginePage.__init__(self)
        self.html = ''
        self.loadFinished.connect(self._on_load_finished)
        self.load(QUrl(url))
        self.app.exec_()

    def _on_load_finished(self):
        self.html = self.toHtml(self.Callable)
        # print('Load finished')

    def Callable(self, html_str):
        self.html = html_str
        self.app.quit()


def main():
    page = Page('http://flask.showerlee.com/scrapingtest/')
    soup = bs.BeautifulSoup(page.html, 'html.parser')
    js_test = soup.find('p', class_='jstest')
    print(js_test.text)


if __name__ == '__main__':
    main()

# python scrap4.py

js loaded successfully

JS解析成功.

四. selenum演示

首先我们需要从官网下载chrome driver, 并放到脚本同路径的driver目录里.

这里需要查找匹配你当前chrome浏览器版本的driver版本. 这边我的chrome版本为62.0, 所以选择driver版本为2.35.

# vi selenum.py

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from bs4 import BeautifulSoup
import time, sys, io

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')

def scrape():

	chromedriver = r".\driver\chromedriver.exe"
	URL = "http://flask.showerlee.com/scrapingtest/"

	try:
		driver = webdriver.Chrome(chromedriver)
		driver.set_window_position(-10000, 0)
		driver.get(URL)
		time.sleep(10)
		result = driver.execute_script("return document.body.innerHTML").encode('utf-8')
	except TimeoutException as e:
		print(e)

	soup = BeautifulSoup(result, "lxml")

	print(soup)

	driver.close()

scrape()

# python selenum.py

...

更多文档:

https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/

http://pandas.pydata.org/pandas-docs/stable/

http://pyqt.sourceforge.net/Docs/PyQt5/

https://sites.google.com/a/chromium.org/chromedriver/downloads

https://chromedriver.storage.googleapis.com/index.html

最近在业余时间去学习python web开发, 并用Flask做了一个BLOG小程序放到我的个人仓库和Github仓库, 这里把自己的文档分享给大家, 仅供参考

https://github.com/showerlee/Flaskdev

https://git.showerlee.com/showerlee/Flaskdev

安装环境

OS(virtualbox): CentOS 6.9 x64 (flask.example.com)

OS(localhost):  Windows 7 x64   

Python: Python3.4/Python2.6.6

Flask: Flask 0.12.x

nWSGI:  uwsgi-2.0.15

Nginx:  nginx- 1.10.2-1.el6

一. 系统环境配置

1.关闭iptables和selinux

# su - root

# service iptables stop

# setenforce 0

# vi /etc/sysconfig/selinux

修改

SELINUX=disabled

2.添加本地host DNS

# vi /etc/hosts

127.0.0.1    flask.example.com

二. Python配置

1.安装python3.4源及依赖包

# yum install epel-release -y

# yum groupinstall "Development tools" -y

# yum install zlib-devel bzip2-devel openssl-devel ncurses-devel zx-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel -y

2.编译安装python3.4以及pip package manager

# wget https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tar.xz --no-check-certificate

# tar xf Python-3.4.3.tar.xz

# cd Python-3.4.3

# ./configure --prefix=/usr/local --with-ensurepip=install

# make && make install

3.安装virtualenv

# pip install --upgrade pip

# pip install virtualenv

三. Nginx配置

1. 安装nginx package

# yum install nginx -y

2.配置nginx with nWSGI

# vi /etc/nginx/conf.d/flask.conf

server {
    listen	 80;
    server_name  flask.example.com;  

    charset utf-8;

    access_log  /var/log/nginx/flask_access.log  main;
    error_log   /var/log/nginx/flask_error.log;


    location / { 
        try_files $uri @yourapplication; 
        client_max_body_size 32M;
    }
    location @yourapplication {
        include uwsgi_params;
        uwsgi_pass unix:/tmp/uwsgi.sock;       
        uwsgi_read_timeout 30s;
        uwsgi_send_timeout 30s;
    }
}

四. Flask+uWSGI配置

1. uWSGI配置

# mkdir -p /etc/uwsgi

# vi /etc/uwsgi/uwsgi.ini

[uwsgi]
base = /usr/share/nginx/html/flask
 
python-path = %(base)
app = hello
module = %(app)
callable = app
 
pidfile = /tmp/uwsgi-master.pid
touch-reload = /etc/uwsgi/bin/reload
 
master = true
processes = 5
enable-threads = true
 
limit-as = 512
 
# use unix socket because it is more secure and faster than TCP socket
socket = /tmp/uwsgi.sock
chmod-socket = 660
uid = nginx
gid = nginx
 
vacuum = true
die-on-term = true
emperor = true
 
logto = /var/log/nginx/uwsgi.log

2. 配置Flask base folder

# cd /usr/share/nginx/html

# mkdir flask

# cd flask

# virtualenv -p /usr/local/bin/python3 .py3env

3. 开启virtualenv python3环境

# source .py3env/bin/activate

4. 在此环境安装Flask相关模块

# pip install flask uwsgi PyMySQL flask-wtf passlib

5. 写入Flask测试文件

# vi hello.py

from flask import Flask
app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello World!"

if __name__ == "__main__":
    app.run()

6. 创建uWSGI启动脚本

# mkdir -p /etc/uwsgi/bin

# vi /etc/uwsgi/bin/uwsgi.sh

#!/bin/bash
 
# Get function from functions library
. /etc/init.d/functions
 
BASE=/usr/share/nginx/html/flask
UWSGI=$BASE/.py3env/bin
INI=/etc/uwsgi
 
start() {
  echo -n "Starting server ..." 
  touch $INI/bin/reload
  $UWSGI/uwsgi --ini $INI/uwsgi.ini >/dev/null 2>&1 &
  success $"Started"
  echo
}
 
reload() {  
  $UWSGI/uwsgi --reload /tmp/uwsgi-master.pid
  success $ "Reloaded"
  echo
}
 
stop() {
  echo -n "Stopping server ..."
  $UWSGI/uwsgi --stop /tmp/uwsgi-master.pid
  success $ "Stopped"
  echo
}
 
### main logic ###
case "$1" in
  start) 
    start
    ;;
  stop)
    stop
    ;;
  reload)
    reload
    ;;
  restart)
    stop
    start
    ;;
  *)
    echo $"Usage: $0 {start|stop|restart|reload}"
    exit 1
 
esac
exit 0

7. 修改权限(可执行并保持与nginx启动user一致)

# chmod -R 755 /etc/uwsgi

# chown -R nginx:nginx /etc/uwsgi

# chmod -R 755 /usr/share/nginx/html/flask

# chown -R nginx:nginx /usr/share/nginx/html/flask

8.启动nginx+uwsgi

# service nginx start

# /etc/uwsgi/bin/uwsgi.sh start

展示效果(保证Windows本地host文件能够解析flask.example.com)

Finished...

Trouble shooting:

1.查看uwsgi log, 发现循环输出如下warning.

# tail -f /var/log/nginx/uwsgi.log

chdir(): Permission denied [core/emperor.c line 1499]
chdir(): Permission denied [core/emperor.c line 1499]
chdir(): Permission denied [core/emperor.c line 1499]
chdir(): Permission denied [core/emperor.c line 1499]
...

Solution:

# cd /usr/share/nginx/html

# source .py3env/bin/activate

# python hellp.py

 * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
 * Restarting with stat
 * Debugger is active!
 * Debugger PIN: 126-450-249

若无报错提示并显示如上输出, ctrl+C 强制退出. 并重启服务, 该warning即停止.

Tip:  该方法亦可测试你的主方法文件是否有语法格式错误.

# /etc/uwsgi/bin/uwsgi.sh start

该错误会偶尔在开发过程中添加新方法后出现, 暂不清楚原因, 有经验的朋友可以留言.

My Flask APP for the reference:

https://git.showerlee.com/showerlee/Flaskdev

# yum install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel wget xz -y

2.编译安装python2.7

# cd ~ 

# wget --no-check-certificate https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tar.xz

# tar xvfJ Python-2.7.9.tar.xz 

# cd Python-2.7.9

# ./configure --prefix=/usr/local LDFLAGS="-Wl,-rpath /usr/local/lib"

# make && make altinstall

# ln -s /usr/local/bin/python2.7 /usr/local/bin/python

退出SSH并重新登录

# python -V

Python 2.7.9

# which python

/usr/local/bin/python

3.安装python setuptools

# python ez_setup.py

# which easy_install

/usr/local/bin/easy_install

# /usr/local/bin/easy_install-2.7 pip

# ll /usr/local/bin

total 6136
-rwxr-xr-x 1 root root     101 Oct 29 02:34 2to3
-rwxr-xr-x 1 root root     323 Oct 29 02:42 easy_install
-rwxr-xr-x 1 root root     331 Oct 29 02:42 easy_install-2.7
-rwxr-xr-x 1 root root      99 Oct 29 02:34 idle
-rwxr-xr-x 1 root root     287 Oct 29 02:48 pip
-rwxr-xr-x 1 root root     289 Oct 29 02:48 pip2
-rwxr-xr-x 1 root root     293 Oct 29 02:48 pip2.7
-rwxr-xr-x 1 root root      84 Oct 29 02:34 pydoc
lrwxrwxrwx 1 root root      24 Oct 29 02:36 python -> /usr/local/bin/python2.7
-rwxr-xr-x 1 root root 6224569 Oct 29 02:34 python2.7
-rwxr-xr-x 1 root root    1687 Oct 29 02:34 python2.7-config
-rwxr-xr-x 1 root root   18547 Oct 29 02:34 smtpd.py

升级完毕

# vi diff.py

#!/usr/bin/env python

import difflib

text1 = '''text1:
This mudule provides classes and fuctions for comparing sequences.
including HTML and context and unified diffs.
difflib document v7.4
add string
'''
text1_lines = text1.splitlines()

text2 = '''text2:
This mudule provides classes and fuctions for Comparing sequences.
including HTML and  context and unified diffs.
difflib document v7.5
''' 
text2_lines = text2.splitlines()

#d = difflib.Differ()
#diff = d.compare(text1_lines, text2_lines)

#print '\n'.join(list(diff))

d = difflib.HtmlDiff()
print d.make_file(text1_lines, text2_lines)

运行脚本并重定向为一个html静态页面

# chmod 755 diff.py

# ./diff.py  > diff.html

浏览器打开该页面

如图:

2. 比对两个配置文件的差异.

# vi diff02.py

#!/usr/bin/env python

import difflib
import sys

try:
	textfile1 = sys.argv[1]
	textfile2 = sys.argv[2]
except Exception, e:
	print "Error: " + str(e)
	print "Usage: %s filename1 filename2" %sys.argv[0]
	sys.exit()

def readfile(filename):
	try:
		with open(filename, 'rb') as fileHandle:
			text = fileHandle.read().splitlines()
		return text
	except IOError as error:
		print ('Read file Error:' + str(error))
		sys.exit()

if not textfile1 or not textfile2:
	print "Usage: %s filename1 filename2" %sys.argv[0]
	sys.exit()

text1_lines = readfile(textfile1)
text2_lines = readfile(textfile2)

d = difflib.HtmlDiff()

print d.make_file(text1_lines, text2_lines)

# ./diff02.py nginx.conf.v1 nginx.conf.v2 >diff02.html

QQ邮箱 POP3端口: 995 SMTP端口: 587

密码使用QQ邮箱独立密码

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import smtplib  
import email.encoders
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
import getpass

# Build a new attach instance
msg = MIMEMultipart()

# Mail configuration
Mail_subject = 'Python test mail'
Mail_content = 'Send with pic attachment'
Recipient_list = ['XXX@126.com', 'XXX@qq.com']
SMTP_server = "smtp.qq.com"  
SMTP_port = "587"
Username = raw_input('Please input your Username:')  
Password = getpass.getpass("Please input your Password: ")   
mail_postfix = "qq.com"  

# Mail attachment
def attach(file_path, file_name, type, postfix):
    with open(file_path + "/" + file_name, 'rb') as f:
        # 设置附件的MIME和文件名，这里是png类型:
        mime = MIMEBase(type, postfix, filename = file_name)
        # 加上必要的头信息:
        mime.add_header('Content-Disposition', 'attachment', filename = file_name)
        mime.add_header('Content-ID', '<0>')
        mime.add_header('X-Attachment-Id', '0')
        # 把附件的内容读进来:
        mime.set_payload(f.read())
        # 用Base64编码:
        email.encoders.encode_base64(mime)
        # 添加到MIMEMultipart:
        msg.attach(mime)

  
def send_mail(recipient, title, content): 
    # Mail info
    author = "%s<%s@%s>" %(Username, Username, mail_postfix)
    msg['Subject'] = title  
    msg['From'] = author  
    msg['To'] = ";".join(recipient)

    # Send attachment
    msg.attach(MIMEText(content, 'plain', 'utf-8'))
    attach('/Users/XXX/Pictures/com.tencent.ScreenCapture','QQ20150827-1.png', 'image', 'png')
    attach('/Users/XXX/Work/Python','test01.py', 'txt', 'py')
    attach('/Users/XXX/Work/Python','test.zip', 'zip', 'zip')

    try:  
        server = smtplib.SMTP(SMTP_server, SMTP_port)
        server.starttls()
        server.set_debuglevel(1)   
        server.login(Username, Password)  
        server.sendmail(author, recipient, msg.as_string())  
        server.quit()  
        return True  
    except Exception, e:
        print str(e)
        return False


if __name__ == '__main__': 
    if send_mail(Recipient_list, Mail_subject, Mail_content): 
        print "Sent Successfully"  
    else: 
        print "Sent Failure" 

QQ邮箱利用的是加密STMP, 需要加密版本的童鞋请关注随后的更新.

TIPS: 我的本地环境是MAC系统, Windows环境需要修改相应的字符编码.

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import smtplib  
import email.encoders
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
import getpass

# Build a new attach instance
msg = MIMEMultipart()

# Mail configuration
Mail_subject = 'Python test mail'
Mail_content = 'Send with attachments'
Recipient_list = ['XXX@126.com', 'XXX@qq.com']
SMTP_server = "smtp.126.com"  
Username = raw_input('Please input your Username:')  
Password = getpass.getpass("Please input your Password: ")   
mail_postfix = "126.com"  

# Mail attachment
def attach(file_path, file_name, type, postfix):
    with open(file_path + "/" + file_name, 'rb') as f:
        # 设置附件的MIME和文件名，这里是png类型:
        mime = MIMEBase(type, postfix, filename = file_name)
        # 加上必要的头信息:
        mime.add_header('Content-Disposition', 'attachment', filename = file_name)
        mime.add_header('Content-ID', '<0>')
        mime.add_header('X-Attachment-Id', '0')
        # 把附件的内容读进来:
        mime.set_payload(f.read())
        # 用Base64编码:
        email.encoders.encode_base64(mime)
        # 添加到MIMEMultipart:
        msg.attach(mime)

  
def send_mail(recipient, title, content): 
    # Mail info
    author = "%s<%s@%s>" %(Username, Username, mail_postfix)
    msg['Subject'] = title  
    msg['From'] = author  
    msg['To'] = ";".join(recipient)

    # send attachment
    msg.attach(MIMEText(content, 'plain', 'utf-8'))
    attach('/Users/XXX/Pictures/com.tencent.ScreenCapture','QQ20150827-1.png', 'image', 'png')
    attach('/Users/XXX/Work/Python','test01.py', 'txt', 'py')
    attach('/Users/XXX/Work/Python','test.zip', 'zip', 'zip')

    try:  
        server = smtplib.SMTP()
        server.set_debuglevel(1) 
        server.connect(SMTP_server)  
        server.login(Username, Password)  
        server.sendmail(author, recipient, msg.as_string())  
        server.quit()  
        return True  
    except Exception, e:
        print str(e)
        return False


if __name__ == '__main__': 
    if send_mail(Recipient_list, Mail_subject, Mail_content): 
        print "Sent Successfully"  
    else: 
        print "Sent Failure" 

  正则表达式使用反斜杠" \ "来代表特殊形式或用作转义字符，这里跟Python的语法冲突，因此，Python用" \\\\ "表示正则表达式中的" \ "，因为正则表达式中如果要匹配" \ "，需要用\来转义，变成" \\ "，而Python语法中又需要对字符串中每一个\进行转义，所以就变成了" \\\\ "。
上面的写法是不是觉得很麻烦，为了使正则表达式具有更好的可读性，Python特别设计了原始字符串(raw string)，需要提醒你的是，在写文件路径的时候就不要使用raw string了，这里存在陷阱。raw string就是用'r'作为字符串的前缀，如 r"\n"：表示两个字符"\"和"n"，而不是换行符了。Python中写正则表达式时推荐使用这种形式。

二、正则表达式元字符说明：

.    匹配除换行符以外的任意字符

^    匹配字符串的开始

$    匹配字符串的结束

[]   用来匹配一个指定的字符类别

？   匹配前一个字符字符0次到1次

*    匹配前一个字符0次到无穷次

+    匹配前一个字符1次到无穷次

{}   对于前一个字符重复m次

{m，n} 对前一个字符重复为m到n次

\d   匹配数字，相当于[0-9]

\D   匹配任何非数字字符，相当于[^0-9]

\s   匹配任意的空白符，相当于[ fv]

\S   匹配任何非空白字符，相当于[^ fv]

\w   匹配任何字母数字字符，相当于[a-zA-Z0-9_]

\W   匹配任何非字母数字字符，相当于[^a-zA-Z0-9_]

\b   匹配单词的开始或结束

三、导入正则表达式模块

3.1、导入正则表达式模块

>>> import  re

3.2、查看正则表达式模块方法

>>> dir(re)

['DEBUG', 'DOTALL', 'I', 'IGNORECASE', 'L', 'LOCALE', 'M', 'MULTILINE', 'S', 'Scanner', 'T', 'TEMPLATE', 'U', 'UNICODE', 'VERBOSE', 'X', '_MAXCACHE', '__all__', '__builtins__', '__doc__', '__file__', '__name__', '__package__', '__version__', '_alphanum', '_cache', '_cache_repl', '_compile', '_compile_repl', '_expand', '_pattern_type', '_pickle', '_subx', 'compile', 'copy_reg', 'error', 'escape', 'findall', 'finditer', 'match', 'purge', 'search', 'split', 'sre_compile', 'sre_parse', 'sub', 'subn', 'sys', 'template']
>>>

四、常用的正则表达式处理函数

4.1、re.search

     re.search 函数会在字符串内查找模式匹配，只到找到第一个匹配然后返回，如果字符串没有匹配，则返回None。

    提示：当我们不会用模块方法的时候用help

>>> help(re.search)

search(pattern, string, flags=0)

    第一个参数：规则

    第二个参数：表示要匹配的字符串

    第三个参数：标致位，用于控制正则表达式的匹配方式

    实例：下面的例子kuangl 

>>> name="Hello,My name is kuangl,nice to meet you..."

>>> k=re.search(r'k(uan)gl',name)

>>> if k:

...     print k.group(0),k.group(1)

... else:

...     print "Sorry,not search!"

...

kuangl uan

4.2、re.match

    re.match 尝试从字符串的开始匹配一个模式，也等于说是匹配第一个单词

>>> help(re.match)

match(pattern, string, flags=0)

    第一个参数：规则

    第二个参数：表示要匹配的字符串

    第三个参数：标致位，用于控制正则表达式的匹配方式

    实例：下面的例子匹配Hello单词

>>> name="Hello,My name is kuangl,nice to meet you..."

>>> k=re.match(r"(\H....)",name)

>>> if k:

...      print k.group(0),'\n',k.group(1)

... else:

...     print "Sorry,not match!"

...

Hello

Hello

>>>

     re.match与re.search的区别：re.match只匹配字符串的开始，如果字符串开始不符合正则表达式，则匹配失败，函数返回None；而re.search匹配整个字串，直到找到一个匹配。

4.3、re.findall

    re.findall 在目标字符串查找符合规则的字符串

>>> help(re.findall)

findall(pattern, string, flags=0)

   第一个参数：规则

   第二个参数：目标字符串

   但三个参数：后面还可以跟一个规则选择项

   返回的结果是一个列表，建中存放的是符合规则的字符串，如果没有符合规则的字符串呗找到，就会返回一个空值。

   实例：查找邮件账号

>>> mail='<user01@mail.com> <user02@mail.com> user04@mail.com' #第3个故意没有尖括号

>>> re.findall(r'(\w+@m....[a-z]{3})',mail)

['user01@mail.com', 'user02@mail.com', 'user04@mail.com']

4.4、re.sub

   re.sub 用于替换字符串的匹配项

>>> help(re.sub)

sub(pattern, repl, string, count=0)

   第一个参数：规则

   第二个参数：替换后的字符串

   第三个参数：字符串

   第四个参数：替换个数。默认为0，表示每个匹配项都替换

   实例：将空白处替换成-

>>> test="Hi, nice to meet you where are you from?"

>>> re.sub(r'\s','-',test)

'Hi,-nice-to-meet-you-where-are-you-from?'

>>> re.sub(r'\s','-',test,5)                      #替换至第5个

'Hi,-nice-to-meet-you-where are you from?'

>>>

4.5、re.split

   re.split 用于来分割字符串

>>> help(re.split)

split(pattern, string, maxsplit=0)

   第一个参数：规则

   第二个参数：字符串

   第三个参数：最大分割字符串，默认为0，表示每个匹配项都分割

   实例：分割所有的字符串

>>> test="Hi, nice to meet you where are you from?"

>>> re.split(r"\s+",test)

['Hi,', 'nice', 'to', 'meet', 'you', 'where', 'are', 'you', 'from?']

>>> re.split(r"\s+",test,3)                  #分割前三个

['Hi,', 'nice', 'to', 'meet you where are you from?']

>>>

4.6、re.compile

   re.compile 可以把正则表达式编译成一个正则对象

>>> help(re.compile)

compile(pattern, flags=0)

   第一个参数：规则

   第二个参数：标志位 

   实例：

>>> test="Hi, nice to meet you where are you from?"

>>> k=re.compile(r'\w*o\w*') #匹配带o的字符串

>>> dir(k)

['__copy__', '__deepcopy__', 'findall', 'finditer', 'match', 'scanner', 'search', 'split', 'sub', 'subn']

>>> print k.findall(test)     #显示所有包涵o的字符串

['to', 'you', 'you', 'from']

>>> print k.sub(lambda m: '[' + m.group(0) + ']',test)  # 将字符串中含有o的单词用[]括起来

Hi, nice [to] meet [you] where are [you] [from]?

>>>

五、用urllib2、re、os 模块下载文件的脚本

#!/usr/bin/env python

import urllib2
import re
import os
URL='http://image.baidu.com/channel/wallpaper'
read=urllib2.urlopen(URL).read()
pat =  re.compile(r'src="http://.+?.js">')
urls=re.findall(pat,read)
for i in urls:
    url= i.replace('src="','').replace('">','')
try:
    iread=urllib2.urlopen(url).read()
    name=os.path.basename(url)
    with open(name,'wb') as jsname:
    jsname.write(iread)
except:
    print url,"url error"

在与array的比较中，值得注意的是，列表会使用更多的存储空间相对于array。

通过import array导入python的数组类型，就可以使用array类型了。

例如：

from array import array  

node=array('H')  

node.append(12)  

python的array和C的数组很类似，它只能存储同样地数据类型的数据。它所占的存储空间的大小就是数据的大小。如果你相对一致类型的数据进行数学运算，那么，array将是一个很好的选择。并且它可以用于和C的代码做接口。或者可以说array就是披上了python外衣的C的数组。
下面介绍一下常见操作：

创建列表

sample_list = ['a',1,('a','b')]

Python 列表操作

sample_list = ['a','b',0,1,3]

得到列表中的某一个值

value_start = sample_list[0]

end_value = sample_list[-1]

删除列表的第一个值

del sample_list[0]

在列表中插入一个值

sample_list[0:0] = ['sample value']

得到列表的长度

list_length = len(sample_list)

列表遍历

for element in sample_list:

print(element)

Python 列表高级操作/技巧

产生一个数值递增列表

num_inc_list = range(30)

#will return a list [0,1,2,...,29]

用某个固定值初始化列表

initial_value = 0

list_length = 5

sample_list = [ initial_value for i in range(10)]

sample_list = [initial_value]*list_length

# sample_list ==[0,0,0,0,0]

附：python内置类型

1、list：列表（即动态数组，C++标准库的vector，但可含不同类型的元素于一个list中）

a = ["I","you","he","she"] ＃元素可为任何类型。

下标：按下标读写，就当作数组处理

以0开始，有负下标的使用

0第一个元素，-1最后一个元素，

-len第一个元 素，len-1最后一个元素

取list的元素数量

len(list) #list的长度。实际该方法是调用了此对象的__len__(self)方法。

创建连续的list

L = range(1,5) #即 L=[1,2,3,4],不含最后一个元素

L = range(1, 10, 2) #即 L=[1, 3, 5, 7, 9]

list的方法

L.append(var) #追加元素

L.insert(index,var)

L.pop(var) #返回最后一个元素，并从list中删除之

L.remove(var) #删除第一次出现的该元素

L.count(var) #该元素在列表中出现的个数

L.index(var) #该元素的位置,无则抛异常

L.extend(list) #追加list，即合并list到L上

L.sort() #排序

L.reverse() #倒序

list 操作符:,+,*，关键字del

a[1:] #片段操作符，用于子list的提取

[1,2]+[3,4] #为[1,2,3,4]。同extend()

[2]*4 #为[2,2,2,2]

del L[1] #删除指定下标的元素

del L[1:3] #删除指定下标范围的元素

list的复制

L1 = L #L1为L的别名，用C来说就是指针地址相同，对L1操作即对L操作。函数参数就是这样传递的

L1 = L[:] #L1为L的克隆，即另一个拷贝。

list comprehension

[ <expr1> for k in L if <expr2> ]

2、dictionary： 字典（即C++标准库的map）

dict = {‘ob1′:’computer’, ‘ob2′:’mouse’, ‘ob3′:’printer’}

每一个元素是pair，包含key、value两部分。key是Integer或string类型，value 是任意类型。

键是唯一的，字典只认最后一个赋的键值。

dictionary的方法

D.get(key, 0) #同dict[key]，多了个没有则返回缺省值，0。[]没有则抛异常

D.has_key(key) #有该键返回TRUE，否则FALSE

D.keys() #返回字典键的列表

D.values()

D.items()

D.update(dict2) #增加合并字典

D.popitem() #得到一个pair，并从字典中删除它。已空则抛异常

D.clear() #清空字典，同del dict

D.copy() #拷贝字典

D.cmp(dict1,dict2) #比较字典，(优先级为元素个数、键大小、键值大小)

#第一个大返回1，小返回-1，一样返回0

dictionary的复制

dict1 = dict #别名

dict2=dict.copy() #克隆，即另一个拷贝。

3、tuple：元组（即常量数组）

tuple = (‘a’, ‘b’, ‘c’, ‘d’, ‘e’)

可以用list的 [],:操作符提取元素。就是不能直接修改元素。

4、string： 字符串（即不能修改的字符list）

str = “Hello My friend”

字符串是一个整 体。如果你想直接修改字符串的某一部分，是不可能的。但我们能够读出字符串的某一部分。

子字符串的提取

str[:6]

字符串包含 判断操作符：in，not in

“He” in str

“she” not in str

string模块，还提供了很多方法，如

S.find(substring, [start [,end]]) #可指范围查找子串，返回索引值，否则返回-1

S.rfind(substring,[start [,end]]) #反向查找

S.index(substring,[start [,end]]) #同find，只是找不到产生ValueError异常

S.rindex(substring,[start [,end]])#同上反向查找

S.count(substring,[start [,end]]) #返回找到子串的个数

S.lowercase()

S.capitalize() #首字母大写

S.lower() #转小写

S.upper() #转大写

S.swapcase() #大小写互换

S.split(str, ‘ ‘) #将string转list，以空格切分

S.join(list, ‘ ‘) #将list转string，以空格连接

处理字符串的内置函数

len(str) #串长度

cmp(“my friend”, str) #字符串比较。第一个大，返回1

max(‘abcxyz’) #寻找字符串中最大的字符

min(‘abcxyz’) #寻找字符串中最小的字符

string的转换

oat(str) #变成浮点数，float(“1e-1″) 结果为0.1

int(str) #变成整型， int(“12″) 结果为12

int(str,base) #变成base进制整型数，int(“11″,2) 结果为2

long(str) #变成长整型，

long(str,base) #变成base进制长整型，

字符串的格式化（注意其转义字符，大多如C语言的，略）

str_format % (参数列表)

Target:

column A 是DB内存储的若干Project name,需要通过查询数据库,获取这些project name在DB对应的Project key的具体值,并插入column A后保存.

#!/usr/bin/env python
# encoding: utf-8

import MySQLdb
import csv
import sys

# Define csv list
Csv_content = []
Csv_content_edited = []

# Define project list
Project_names = []

# Define db list
Db_content = []

# Define the file that needs to be handled.
try:
	file_name = sys.argv[1]
	new_file_name = file_name.split('.')[0] + '_new.' + file_name.split('.')[1]
except IndexError:
	pass

def selectDB():
	# Open db connection
	db = MySQLdb.connect("test.com", "testuser", "testuser", "testdb")

	# Use cursor() fuction to get current db cursor
	cursor = db.cursor()

	Csv_content_edited = readContent(file_name)

	for c in range(1,len(Csv_content_edited)):
		Project_names.append(Csv_content_edited[c][0])

	Project_name_string = ",".join(['"' + p + '"' for p in Project_names])

	# SQL "SELECT" statement
	sql = 'select pname,pkey from project where pname in (%s)' %Project_name_string
	
	try:
		# Execute SQL
		cursor.execute(sql)
		# Obtain all the record list
		results = cursor.fetchall()

		for row in results:
			#lower_user_name = row[3]
			Db_content.append(row)

		return Db_content
	except:
		print "Error: unable to fecth data"

	# Close connection
	db.close()


def readContent(file_name):
	# Read the csv file,then put it into list.
	with open(file_name, 'r') as csvfile:
		csv_reader = csv.reader(csvfile, delimiter=',')
		for row in csv_reader:			 
			if row[0]:
				Csv_content.append(row)
	return Csv_content


def insert_col():
	Csv_content_edited = readContent(file_name)
	# Insert null value to each components of "Csv_content_edited" afterward.
	for i in range(0,len(Csv_content_edited)):
		Csv_content_edited[i].insert(1,'')

	# Define the second inserted column title.
	Csv_content_edited[0][1] = "Pkey"
	# Grab the users data from db.
	Db_content = selectDB()
	# print Db_content
	
	for d in range(0,len(Db_content)):
		Pkey = Db_content[d][1]
		Pname = Db_content[d][0]
		for c in range(0,len(Csv_content_edited)):  
			if Csv_content_edited[c][0] == Pname:
				Csv_content_edited[c][1] = Pkey
	# print Csv_content_edited

	pname_list = []
	Csv_content_edited_new = []
	
	for c in range(0,len(Csv_content_edited)):
		if not Csv_content_edited[c][0] in pname_list:
			pname_list.append(Csv_content_edited[c][0])
			Csv_content_edited_new.append(Csv_content_edited[c])
	# print Csv_content_edited_new
	
	return Csv_content_edited_new



# Write the csv file. 
def writeContent():
	with open(new_file_name,'wb') as csvfile:
		csv_writer = csv.writer(csvfile)
		csv_writer.writerows(insert_col())


# Execute the finnal function.	
if __name__ == '__main__':
	try:
		writeContent()
	except (IOError,NameError,IndexError):
		print "Please type the correct file name. e.g: '" + sys.argv[0] + " testfile.csv'"
	else:
		print 'The result file is: %s' %new_file_name

Result:

Before the change:

after the change: