scrapy爬虫并保存到数据库

#/usr/bin/python2.7 # -*- coding: utf-8 -*- #mysql import MySQLdb import ConfigParser import sys reload(sys) sys.setdefaultencoding("utf-8") class MyDbHelper(object): def __init__(self, conf): self._parser_conf(conf) self._connect_db() def _parser_conf(self, conf): config = ConfigParser.ConfigParser() config.readfp(open(conf)) self._host = config.get("db", "host") self._user = config.get("db", "user") self._passwd = config.get("db", "passwd") self._db = config.get("db", "database") try: self._port = int(config.get("db", "port")) except: self._port = 3306 def _connect_db(self): try: self._conn = MySQLdb.connect(self._host, self._user, self._passwd, self._db, self._port) except MySQLdb.Error, e: print "mysql connect error info: %s" % str(e) return except Exception, e: print "error: %s" % str(e) return self._trans = None self._cursor = self._conn.cursor() self._cursor.execute("set autocommit = 1") self._conn.set_character_set('utf8') self._cursor.execute("SET NAMES utf8;") self._cursor.execute("SET CHARACTER SET utf8;") self._cursor.execute("SET character_set_connection=utf8;") """ function: delete, update, insert """ def query(self, sql, param): try: return self._cursor.execute(sql, param) except Exception, e: print "mysql execute error: %s" % str(e) return 0 def query_many_sql(self, sql, param): try: self._cursor.executemany(sql, param) return self._cursor.fetchall() except Exception, e: print "mysql many sql execute error: %s" % str(e) """ function: select """ def select(self, sql, param): try: self._cursor.execute(sql, param) except Exception, e: print "mysql select error %s" % str(e) return () if self._cursor.rawcount == 0: return () return self._cursor.fetchall() def ping(self): try: _ret = self._conn.ping() except Exception, e: _ret = e if _ret: self.close() self._connect_db() def close(self): if self._conn != None: self._conn.close() self._conn = None self._cursor = None self._trans = None """ transaction """ def begin(self): #If already start transaction, donn't restart it. maybe commit the last trans if self._trans: return self._conn.autocommit(False) self._cursor.execute("BEGIN") self._trans = True def rollback(self): if not self._trans: return self._conn.rollback() self._trans = False self._conn.autocommit(True) def commit(self): if not self._trans: return self._conn.commit() self._trans = False self._conn.autocommit(True) def __del__(self): self._cursor.close() self._conn.close()