Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!0 e& O( F( i N" g* s8 j
- #!/usr/bin/env python
! a; H: }- I2 j/ M6 X - # -*- encoding: utf-8 -*-) z& R( l& E1 T/ B- ^ F- x
- # Created on 2019-05-05 21:43:11
$ ]* W$ W% U/ K6 y1 x% e - # Project: XiaoShuo8 O+ h% w- A+ T9 q/ C1 P
- 4 c+ ^! B9 E+ ^# o, ~5 Y J
- from pyspider.libs.base_handler import *
$ C( c% i6 ]( ^% \* M: { - import pymysql
( y- V* _& b4 Q- s" h - import random
( u% R* y: Q9 _6 @: }; h - import datetime
* X& Z- O7 R9 z) E- ^ - import urllib2,HTMLParser,re5 N p3 ]7 Q* T9 C
- import os
4 g. R- t8 K" B0 Z# W y" y( D - import sys' J! ~4 A) R) c9 y
- import re& `: I$ R/ {, \( k
- import codecs; a2 j0 Q+ a! P( N% Z9 u
- import requests
* X5 _9 L4 u+ \; d# _% a) C - import json7 _. L4 }2 ]" }. `
-
2 k% g M. W3 z6 w0 U0 Z6 c - class Handler(BaseHandler):) L, H+ o2 o; j! _1 J5 a
- global Datos
6 X/ O ]' I# _. f2 x; F5 p" t - global P_dir
0 i$ q- @- x# X, s# ~ - P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径, @5 L7 n4 ?4 M4 A" I+ V
- global Datos
p6 Y2 c* U' D" W+ R+ z - Datos = {}
' c: q# Q0 j2 F; A- F4 W& O - headers= {* }( x! o- d5 k$ Z5 J
- 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
# Q3 ]1 T( p* A - 'Accept-Encoding':'gzip, deflate, sdch',
& e! p, F( @$ M' o G8 H& z - 'Accept-Language':'zh-CN,zh;q=0.8',
7 N* \9 ~4 P! @) H8 h) `% ^9 \3 { - 'Cache-Control':'max-age=0',8 j1 ]3 Q0 a( y5 E
- 'Connection':'keep-alive',
( ~2 D/ W9 k' _3 D4 d - 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36': ? _0 e$ @9 [
- }
: A- F* y* }7 `, C" G; _& R* s - crawl_config = {
8 }. m; p# y5 M" i5 g- _ - 'headers' : headers,* H* q1 {( @; q9 ]' J- [' I
- 'timeout' : 300
% a4 F. S" F. ~9 Z8 d* F - }$ K& a5 O' Q' W
- def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):! s0 v O2 i: `0 [9 J$ j
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
9 u9 S) S* V: t' e6 @1 e - try:
) o8 p U, p U- p0 H; K/ P z4 ^3 b - cursor = db.cursor()
" Q5 h, [9 C0 Y; u! O$ I - #注意此处字符串的占位符要加双引号"%s"1 `% Q- |& a! I$ f2 X# W( r
- sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);+ d$ O9 x |3 j6 Q/ ?- S. T
- # print(sql)/ G: a" ?2 x# D, D' r) R/ {, @$ s
- cursor.execute(sql)
3 h1 o' g$ u) j9 T. n -
. U/ ]. g$ A+ |' F# E# p - #qid = cursor.lastrowid4 j; x; O6 z* b# C+ O4 @
- #print(qid)! w6 @8 y: A- j- h& K0 V w
- 7 y: T7 d v& R2 U8 a8 s3 O
- db.commit()
2 r! B$ |) N4 D. t* o - except Exception as err:
' A6 P. l0 `! t& ` - print("Error %s for execute sql: %s" % (err, sql))
5 @9 |, i& u" \: b$ N - db.rollback()
5 f/ u2 Z# ^) x+ h. |& P8 n5 d6 \+ `% j - def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):
' c) B5 q7 z! R6 c, \& @" F - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")* B* t: G+ c/ B
- try:( q# s5 I% V2 V
- cursor = db.cursor()7 F! v- I) B* X2 j7 s
- #注意此处字符串的占位符要加双引号"%s") G& v- E* v0 a& w
- sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);
- k+ g5 M6 s! T) @ - # print(sql)5 ?* P6 l9 ?$ H/ N) a( A
- cursor.execute(sql)- e3 O& ?6 u" l& o) D$ E$ k( c
- / e6 R, v+ R, b, d- j8 z
- #qid = cursor.lastrowid- H2 U* v' L( N N) E+ Y+ R
- #print(qid)/ i2 N1 o' V) Q& u( [: E9 t
-
5 Q$ R6 x; K w: h - db.commit()
7 S, Q5 {, b& U - except Exception as err:
7 @7 Y4 R2 L( ^7 f - print("Error %s for execute sql: %s" % (err, sql))
5 \# M/ V3 Z9 L1 @. x) f5 a - db.rollback()" c' s3 Q& i: F3 q4 b1 L
- def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):9 a, ]7 j# w8 P. i( E
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
- S4 ?; ~* G; I8 A$ _5 b7 G1 u- ~ - try:
( X8 m8 Z/ l% w/ k. V - cursor = db.cursor() k& j3 }- a( }2 H3 Z+ w
- #注意此处字符串的占位符要加双引号"%s"9 Q/ s# R4 G& t' A+ g) Q
- sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);
( [" {" v( N4 i - print(sql)+ W2 s% f y- E9 O8 `) T- K
- cursor.execute(sql)+ H: y* B2 ^$ ~9 C1 @
- print(cursor.lastrowid)' G8 P. B6 b* r) d/ ] w" R( _
- db.commit()
# N7 T' n! V) r! v V; e - except Exception as err:
- {7 e4 z3 a. ~8 k' ]' I; h - # except:$ V- H: Q7 [/ h
- # print('Failed')
4 J4 G' Q2 T/ n9 y; O5 \ - print("Error %s for execute sql: %s" % (err, sql))- R* Q) p" T8 }
- db.rollback()4 |9 C- C; L: O: r! F
- 0 P) i& c% N) X" P% f% t- Y" U
- def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover):
: G; w6 x q0 L, x' _' F$ ~/ P7 n - reload(sys)+ M* w4 n! r3 c4 e, o/ |
- sys.setdefaultencoding("gbk")/ x- m2 ]' A4 t5 d+ s3 U% g" n
- locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址7 \: D% K+ Y0 ^; p! w
- locoy_data = {5 ~, U, e5 i% Y# E X9 C
- 'my_u':'用户名', #后台用户名+ ~5 r. d) \$ }4 S5 j; |5 e/ h
- 'my_p':'密码', #后台密码
0 S" t" a+ \( `& B - 'subject_669977_net':Bookname.encode('gbk', 'ignore'),% C! G, ?) Y" u& _, h
- 'caid':Cater_Name.encode('gbk', 'ignore'),
1 d6 e4 x5 f( d2 x+ z0 H& ^ - 'title_669977_net':Booktitle.encode('gbk', 'ignore'),
: K7 `( E# q% {9 D - 'article':BookConte.encode('gbk', 'ignore'),2 l1 x% A3 o0 {3 y) F7 B# w+ U
- 'author':Book_author.encode('gbk', 'ignore'),0 J0 r5 E0 I% B3 ?2 t+ b9 p, B! y
- 'ready_1':Book_Palabras.encode('gbk', 'ignore'),2 y3 U* C+ y- ?% e
- 'thumb':Book_img,
/ f. y; h9 `, S( {( \7 {8 ^ - 'content':Book_Introduction.encode('gbk', 'ignore'),1 P% m9 h0 Z/ T) T: C
- 'abover':abover.encode('gbk', 'ignore') . h: O% h: l7 B. a# K* Y- Q
- }
1 O! c0 v& V8 O( @ - res = requests.post(locoy_url, data=locoy_data)
+ `+ J- C0 C* V7 T2 j# d% ?& S - print res.text2 B, w P, @1 U3 l$ P2 {9 d$ c
- print res.content% U$ ^0 V5 \7 I" J
- # print Dsd
) P3 H f) I( u q7 x* I- { - return res2 S8 b2 F2 u3 Q( U
- . V, h, u' L4 x* b0 g% C
- def __init__(self):
0 \9 M# {* c+ I7 J* _% z' I - self.base_url1 = 'https://www.****.cc/'
& M" g( E* |3 | - self.base_url2 = '/'
! ~+ ^/ m" \2 @ ^' J r# p - self.CaterId = []/ p5 _. u- ]( f5 S) d
- self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']
, ^- [4 T' i& |/ W: r7 X - self.page_num = 1
3 v/ t9 X$ e1 p& ?' X - self.total_num = 200
( r6 H6 i' x; e4 A$ S! ~ -
% l6 [" e' P# S. i! o7 Q8 f7 k/ ^; C1 a - @every(minutes=8 * 60)! f% X d4 E4 c1 r" C' o4 k6 ^- X
- def on_start(self):
5 I; t- K1 D7 q! S' s/ G - global Cater_Name: o" \9 {2 E2 z% ]9 t# @4 }
- Cater_Name = []! H$ K2 }; @- G4 T& J: R& ^( b; h
- while self.page_num <= self.total_num: 7 ]+ ]. i! ]8 R4 f
- for self.CaterId in self.CaterIds:
' B5 h! T( q, [3 E$ P+ G+ y4 O - if self.CaterId == 'xuanhuan':1 |2 v1 D9 h2 U0 Q& s
- Cater_Name = '玄幻'" x+ O) F, i! `& ]6 W& C& p/ ?* T
- if self.CaterId == 'wuxia':
% j8 I0 k( G( U& H) | - Cater_Name = '武侠'
6 I9 b/ e9 ~9 p$ f7 a ~ - if self.CaterId == 'lishi':3 q4 ]) z2 m6 _8 E3 j
- Cater_Name = '历史' % |5 a" I8 I+ g) R
- if self.CaterId == 'yanqing':
- X6 m. f3 G; ^6 g. `$ H+ D. e - Cater_Name = '都市' 5 L5 P7 F6 s% [
- if self.CaterId == 'nvsheng':
2 ~1 R0 G) W( H - Cater_Name = '都市' . \* m* Q# ~% c/ l
- if self.CaterId == 'kehuan':
* }) L$ U. g4 H9 }5 O - Cater_Name = '科幻' 2 M# X& O" }+ `" t1 ]9 `4 s
- if self.CaterId == 'kongbu': w; y& R4 Y5 u& o1 M
- Cater_Name = '游戏'
: o, c% w" V9 Q$ ? - print self.CaterId; B! p2 j8 ?6 d+ _+ ~9 q5 ~/ N
- url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/" & K; x# N% a/ v/ H3 s. m( b4 t7 }0 q
- self.crawl(url, callback=self.list_Caterg,save=Cater_Name)' R6 j) u% s1 F$ |) M
- self.page_num += 1 ( u H* z! M! i4 u) T# A
-
3 t4 L* Y4 j9 n' m - def list_Caterg(self, response):9 F5 H8 i0 ?4 E v8 }6 M! [6 [
- Cater_Name = response.save# a, l( G- p6 M8 X
- for each in response.doc('.pic-list a[href^="http"]').items():# w* y B5 V- U8 W$ d8 t
- self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)$ a |$ _2 Q3 Z. o3 \
- 2 M- V. h# i/ y9 ?% ]0 t
- def list_Caterg_detail(self, response):
- m) A2 f8 U9 u' A3 F0 V7 P K - Cater_Name = response.save
! ^" B. ]4 W2 B, e - # print Cater_Name
. k6 i0 Y$ d! n6 j9 A - Bookname = response.doc('h1').text()
Q& j! f. t" p - print Bookname
3 L. b. y8 [! v$ _/ }/ h - Book_author = response.doc('.authorname > a').text()3 g4 \2 K. w% f) K
- # print Book_author. z' |+ n/ j* }& t: ?7 m
- Book_Introduction = response.doc('.book-intro > div').text(), j# r7 T/ M. M' i# S+ n* r" o9 f
- # print Book_Introduction
, m g" N6 f: t - Book_Synopsis = response.doc('b').eq(1).text(); y* `& B* W; c' I3 B6 X
- # print Book_Synopsis
" N9 L" O8 a- G - Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]5 ^" d. ]1 N- S( G5 ` t7 {. M0 X
- # print Book_Palabras Y$ H+ x6 K k$ P6 I
- BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID
5 d% a" { G! D A - # print BookIDs6 u5 x4 ]- Y! R7 D' {) ~
- Book_Dates = str(datetime.datetime.now())
( `, @' b* m1 n' ~ - for imgs in response.doc('.bigpic > img[src^="http"]').items():
6 ?8 `3 b0 |& U2 o H* a - img = imgs.attr.src
7 V& K5 |- }* @7 x( Z - print img% P: s5 T8 O5 V7 {4 S& }( d( u
- #小说封面下载1 ], B/ I s0 z! m0 S' [: J7 x/ Z
- extension = self.getExtension(img)
4 Y* @* Z1 N& T% U1 ? - name = self.getname(img)9 @6 q$ b; D. G. u6 A! l
- file_name = name + "." + extension+ n7 X$ ]' t. {
- imgDir = P_dir + name
, f' S4 b. b* E' u- M r3 v - Locaimg = imgDir + "/" + file_name- w- Z) T! h1 N
- print Locaimg" j8 X4 C, f6 C3 T! Y0 ~: z
- if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地, K; p/ t8 c5 k: o k9 k
- print('attachment url is ' + img) #
$ F( k# U ~! P$ A - Datos = {
0 G# N6 [- r, \7 h8 v - "Cater_Name":Cater_Name,# I( m: N2 M1 z3 H$ F* A8 p
- "Book_author":Book_author,
0 t; \5 j# L' } x+ f( Q - "Book_Introduction":Book_Introduction,
& Z" h) X( M: _4 y* ?1 V - "Book_Synopsis":Book_Synopsis,
! i$ B8 P0 Z5 p! k - "Book_Palabras":Book_Palabras,
+ }2 f0 R" s2 I/ a& t - "img":img,) W- b% O/ }0 t9 P7 {6 _
- }& }9 i6 G" y v) W( ~
- self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布
& `. r3 s$ O* J6 l% E& { - for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():2 n% K' p/ r, _
- self.crawl(each.attr.href, callback=self.index_page,save=Datos)9 I C; H A9 x, v& }" w/ q1 X
-
0 R* r1 \2 _8 g- [- ~ - @config(age=8 * 60 * 60)
" C" S( v0 G6 B* k2 n5 |- G - def index_page(self, response):
( M, Y: H3 e# X: D: n. f - Datos = {
% W7 ?4 `7 ^3 U5 ] - "Cater_Name":response.save['Cater_Name'],) Y; O9 @/ ]3 i$ c/ B. r& |
- "Book_author":response.save['Book_author'], [& I D: G4 q8 f+ y; W9 p
- "Book_Introduction":response.save['Book_Introduction'],! }" s5 H+ e0 L! Q) {
- "Book_Synopsis":response.save['Book_Synopsis'],
3 F8 o9 o/ \1 r - "Book_Palabras":response.save['Book_Palabras'],
' F3 J- e+ G0 E# L$ c, a! e8 w' ` - "img":response.save['img'],
" u7 J6 \* L; q1 [# j9 F& E1 f# K2 E - }
. k8 g2 s. {0 y0 B! d% K$ I - for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():7 h `2 ?5 D& G. B' h) p
- # for each in response.doc('.chapter-list a[href^="http"]').items(): & C- s7 \5 F! [% _+ m
- self.crawl(each.attr.href, callback=self.detail_page,save=Datos). l2 _( H0 M' R" _
- @config(priority=2)1 w9 d$ T) T1 `. p. N- l
- @catch_status_code_error
: n3 l, H/ N: A1 }; s/ U6 ~. V) o - def detail_page(self, response):
- a% W6 a4 B5 @/ e& V, y5 Z - NewRe1 = u'哈书'
8 N) Q! E. l. s) v - NewRe2 = u'huhjsd.CC'
* v4 G, `' u% ]# I" g W - NewRe3 = r'^\\n\\n'7 Z D9 O0 @8 w- n/ a' J% _
- NewRe5 = u'小说网'! g' v n/ D! F6 [
- NewRe6 = u'fgdfgf'
$ O; h. _# _8 y+ K7 ~7 t4 k - NewRe7 = u'fgfgf'6 V7 ?5 ~0 F' b" b/ y
- NewRe8 = u'ffhgf'
8 s f# F! ^7 \2 o' o - NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'' ?* y( K# p) k# |% Q- v
- ReC1 = u'静思'
! m6 M4 `1 D( a3 \ - ReC2 = u'aghgf.com'
3 \: G9 K# P" _. _ @& |! I/ u - ReC3 = u'aghgfh.com'
' r# H' U/ X5 m3 g; m2 b3 C - ReC4 = u''# \) _' f2 A' P# N$ c6 P
- ReC5 = u'文学网'
( s% h6 D8 q; u9 \ - ReC6 = r'<BR>'
5 Y8 t0 R8 Y! J - Bookname = response.doc('.readlocation a').eq(2).text() #小说名称2 x# ~- z2 C( B' G
- print Bookname
3 E H( ~' I" f, m: P; F% T7 Y: c; z - Cater_Name = response.save['Cater_Name'] # 小说分类
G/ I4 \# l8 q - Book_author = response.save['Book_author'] #小说作者
4 x8 Z" I. [/ a1 Y1 H8 k - Book_Introduction1 = response.save['Book_Introduction'] #小说简介
9 n+ [9 R' q1 \: D/ K/ J2 | - Book_Synopsis = response.save['Book_Synopsis'] #最近更新: I" X- ?0 ]0 o
- Book_Palabras = response.save['Book_Palabras'] #小说字数
9 @8 S5 r \/ f - Bookurl = response.url #小说网址
) q, d+ B5 A# f- d - Booktitle = response.doc('.article-title').text() #章节名称
c$ T1 o) w6 ?' _7 \# u+ V* H! V5 E - BookID = response.doc('.readset-r span').text() #小说ID* p% S$ M6 T3 r# [. u& v, M$ P
- BookConte1 = response.doc('.article-con').text() #小说章节内容! B. d: e% U5 r: {& ^: }. @3 M
- abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)8 t$ f/ {; T; K
- Book_Date = str(datetime.datetime.now()) # 采集时间
* d1 u# w- P% z( p3 F' Q - BookConte2 = BookConte1.replace(NewRe1 , ReC1)5 T ~( y2 F B3 n5 d
- BookConte3 = BookConte2.replace(NewRe2 , ReC2)
6 K7 H' ~, U) N/ q3 O) t' V) O - BookConte5 = BookConte3.replace(NewRe5 , ReC5)0 e1 p3 G: z% }1 z/ o
- BookConte6 = BookConte5.replace(NewRe6 , ReC2): l6 k* d4 H9 t' a
- BookConte7 = BookConte6.replace(NewRe7 , ReC2)# j- y2 M. M3 q: B
- BookConte8 = BookConte7.replace(NewRe3 , ReC6)
4 Y( Y, D/ J8 _9 N: c3 } T3 p) f! b - BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)& v( x7 d( ^6 m$ \' a
- BookConte = BookConte4.replace("\n\n","<br>")" |6 m% |6 l; e; e" w
- print BookConte
& K! z( T8 g( j6 A- m' m% } - Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)- U( V2 P. h) b0 }; z
- Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)
; K3 N3 t( ^4 Z i. [ - Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3)7 M+ ]7 E& ^9 ^, X4 H H
- Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)
4 r0 |" B0 l5 l8 G - Titleid = response.url.split(BookID + "/")[-1].split("/")[0]
/ t' I3 x( [8 x" y; ^- K2 @* a3 Y - Book_img = response.save['img'], #小说图片
4 i7 n3 H$ X3 i3 z) _ -
4 F' y0 x e6 d) ~ - #insert into MySQL 小说入库+ ?9 O5 z: r* ?# b" Q( ]4 C
- self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布: L6 {3 O! `% J) u. }% M$ ]1 `3 ]
- self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布7 v$ |2 I- q- y. q. F# r' D" Y" t* f
- #post提交发布1 C2 T8 S* E5 b
- self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消, a+ j$ N }/ o
- Datos = {# {: j0 O# k, `7 Y
- "Cater_Name":response.save['Cater_Name'],0 e9 F. V% Z) G* \9 I+ K! A
- "Book_author":response.save['Book_author'],
& `$ M4 G8 ^3 \; K7 W# T - "Book_Introduction":response.save['Book_Introduction'],; J' ]2 |% g6 V+ U
- "Book_Synopsis":response.save['Book_Synopsis'],8 g. L! F# N |
- "Book_Palabras":response.save['Book_Palabras'],* |( b4 E: x: ]! v C) k
- "img":response.save['img'],$ Y6 i# [, j) [# P2 x' j
- }# ~, q# {8 h+ P3 W4 o1 J0 y
- for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():0 A1 V! n9 ^: _" ]0 H
- self.crawl(each.attr.href, callback=self.detail_page,save=Datos)
0 ~' g) D M2 j9 w9 h0 e - return {
: a. M! z: `; Y0 j, _ - "Cater_Name":Cater_Name,
9 G7 K* A3 f" \% R. d - "Bookname":Bookname,5 C+ {- z6 ?2 k$ P' O% R) |
- "Book_author":Book_author,( Z: b- M- k. v* {# _, n% H
- "Book_Introduction":Book_Introduction,
4 g( ?2 p: m* i8 o) h$ s# ] - "Book_Synopsis":Book_Synopsis,& Q: ~" p! z, e* o) ?7 L! U- z
- "Book_Palabras":Book_Palabras,
8 i) P6 y; a+ a! r# @$ N6 {. H* s( F - "Book_img":Book_img,
, z A C8 }! X/ Z. \: e - "Bookurl": response.url,
1 t/ }$ U" E) T7 } e, C) B - "Booktitle": Booktitle,
5 }5 Q2 M! C. H( ]$ {* w" Z- @3 M+ q - "BookID": BookID,
6 h- h2 X$ U6 W, o$ ~ - "BookConte": BookConte,0 H+ V% B! y" ^$ E! r, |
- "Titleid": Titleid,
) J! {7 N+ S+ z8 x* i* J7 M - "abover":abover,$ D3 _$ S- G; D1 l4 u2 t9 s* Y8 h; M
- # "Book_Date" = str(datetime.datetime.now()),% I+ f s( R7 H
- }
# v3 h, [* W* z+ j# y0 j - def download(self, P_dir, imgDir, file_name, Book_img):
# ~9 h8 ^, k8 k3 j. G9 g - if not os.path.exists(imgDir):
$ y3 D/ M* g; }& S* k1 h - os.makedirs(imgDir)5 C# @4 u' J5 |" T5 x) M# b( i
- file = imgDir + "/" + file_name! {7 W' X5 n* [/ L' t* R
- # print file. W6 i, o3 S1 `
- f = open(file, 'wb+')8 X4 ]; j( H6 Y' O! ?
- imag = requests.get(Book_img)
' K5 [# D, S: L0 p$ }, c - f.write(imag.content)% ]; C" p6 h: ~
- f.close()
o2 ` d; H# _# ?; {' @# w9 v/ W - #保存图片前7 L" b- E) |7 O7 ?4 I6 }
- def save_imgs(self,response):
$ z" o) M8 y# n: `" {* N - content = response.content
! t, ~6 {& ^6 u6 F2 |6 B+ B; T* j/ p4 x - file_name = response.save["file_name"]. t/ j- F: t* _; ?; O
- imgDir = response.save["imgDir"]+ K/ }+ p! I8 Z+ ~( Z
- file_path = imgDir + file_name
4 t5 C! L) q' s - self.save_img(content,imgDir,file_path), T/ t+ ^3 @$ u! t( A' ^% ?
- #保存图片
: g" c9 q, O# N3 ~ f - def save_img(self,content,imgDir,path):5 R$ {3 u4 A, w
- if not os.path.exists(imgDir):
* S+ U( D0 v }, E% Q0 C0 X: A - os.makedirs(imgDir)( T+ ?$ D/ I, k
- f = open(path,"wb" )
3 P5 b: J, e/ b8 A - f.write(content)/ n& ? f% k' Y' W
- f.close()5 \7 V4 A: y/ ^. Y6 ?+ h" [' u
- #获取url后缀名
& R. H8 x& V+ z. [. L7 @ - def getExtension(self,url):
1 C B4 l8 t1 ?; X - extension = url.split(".")[-1]0 y; X' I9 Y2 c9 @; A! g1 m3 Z
- return extension
3 K7 N* G- i) t5 K' _ -
3 l8 W- p4 F7 ~ p2 `# @, Y - #获取图片名1 }, n* K9 l" H: S8 o: i
- def getname(self,url):
3 A3 a8 Q4 G7 J1 c8 J - name=url.split("/")[-1].split(".")[0]
7 ~# h( K. g1 `# |$ h) T5 T - return name
复制代码 , B3 Y: U' J* c8 j. s5 }$ i+ J3 _
6 f, V% H) C2 J$ J- l: G
|