import MySQLdb
import MySQLdb.cursors
class MysqlPipeline(object):
采用同步的机制写入mysql
def init(self):
self.conn = MySQLdb.connect(‘192.168.0.106’, ‘root’, ‘root’, ‘article_spider’, charset=”utf8″, use_unicode=True)
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
insert_sql = “””
insert into jobbole_article(title, url, create_date, fav_nums)
VALUES (%s, %s, %s, %s)
“””
self.cursor.execute(insert_sql, (item[“title”], item[“url”], item[“create_date”], item[“fav_nums”]))
self.conn.commit()
class MysqlTwistedPipline(object):
def init(self, dbpool):
self.dbpool = dbpool
@classmethod
def from_settings(cls, settings):
dbparms = dict(
host = settings[“MYSQL_HOST”],
db = settings[“MYSQL_DBNAME”],
user = settings[“MYSQL_USER”],
passwd = settings[“MYSQL_PASSWORD”],
charset=’utf8′,
cursorclass=MySQLdb.cursors.DictCursor,
use_unicode=True,
dbpool = adbapi.ConnectionPool(“MySQLdb”, **dbparms)
return cls(dbpool)
def process_item(self, item, spider):
使用twisted将mysql插入变成异步执行
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) #处理异常
def handle_error(self, failure, item, spider):
处理异步插入的异常
print (failure)
def do_insert(self, cursor, item):
执行具体的插入
根据不同的item 构建不同的sql语句并插入到mysql中
insert_sql, params = item.get_insert_sql()
print (insert_sql, params)
cursor.execute(insert_sql, params)
Original: https://blog.csdn.net/weixin_42373893/article/details/113238795
Author: 微尘一笑
Title: scrapy mysql pipeline_scrapy MysqlPipeline 同步和异步
原创文章受到原创版权保护。转载请注明出处：https://www.johngo689.com/792523/
转载文章受原作者版权保护。转载请注明原作者出处！