x技术-JZTXT

# -*- coding: utf-8 -*-
# @Time : 2023/05/16 0016 18:25
# @Author : 张鑫
# @File : xnews
# @Project : PyCharm 2023.1.1

import hashlib
import random
import requests
import time
from lxml import etree
import pymysql
# 把唯一值进行MD5加密
def get_md5(parmStr):
# 1、参数必须是utf8
# 2、python3所有字符都是unicode形式，已经不存在unicode关键字
# 3、python3 str 实质上就是unicode
if isinstance(parmStr, str):
# 如果是unicode先转utf-8
parmStr = parmStr.encode("utf-8")
m = hashlib.md5()
m.update(parmStr)
return m.hexdigest()

# 链接mysql数据库
connect = pymysql.Connect(
host='127.0.0.1',
port=3306,
user='root',
passwd='123456',
db='xnews',
charset='utf8mb4'
)
for pages in range(1,20):
time.sleep(random.randint(3,10))

# 抓取网页源码
url = f'https://xjishu.com/zhuanli/index_{pages}.html'
print(f'第{pages}页：{url}*****************')
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
'referer': 'https://xjishu.com/',
'cookie': 'ASP.NET_SessionId=fbpowcx5hlle0egc1pmgxrke; Hm_lvt_c211ded9353d076e967627b1a8fe9b09=1684232564; Hm_lpvt_c211ded9353d076e967627b1a8fe9b09=1684232574',
}
html = requests.get(url=url, headers=headers).content.decode()
tree = etree.HTML(html)
second_url_list = tree.xpath('//div[@class="col-search-list"]//ul//li//div[@class="so-header"]//a//@href')
picture_url_list = []
content_list = []
wendangxuhao_list = []
laiyuan_list = []
for second_url in second_url_list:
print(f'详情页链接：{second_url}')
try:
html_second = requests.get(url=second_url, headers=headers).content.decode()
except:
html_second = requests.get(url=second_url, headers=headers).content.decode('gbk')
tree_second = etree.HTML(html_second)
# 标题
title=''.join(tree_second.xpath('//h1[@class="title"]//text()')).replace(' ','').replace('\r','')
# 发布时间
publish_time=tree_second.xpath('//div[@class="art-meta"]//span[2]//text()')[0].replace('发布日期：','')+':00'
# 图片链接
try:
picture_url = tree_second.xpath('//div[@class="art-body"]//img//@src')[0]
except:
picture_url='暂无图片'
picture_url_list.append(picture_url)
# 内容
contents =( tree_second.xpath('//div[@class="art-body"]//text()'))
print(f'contents:{contents}')
neirong = []
for content in contents[1:-1]:
neirong.append(''.join(content).replace('.', '').replace('"','*') + '\n')
neirongs=''.join(neirong)
content_list.append(neirongs)
# 文档序号
wendangxuhao = tree_second.xpath('//div[@class="art-meta"]//span[1]//text()')[0].replace('文档序号：','')
wendangxuhao_list.append(wendangxuhao)
# 来源
laiyuan = tree_second.xpath('//div[@class="art-meta"]//span[4]//text()')[0].replace('来源：','')
laiyuan_list.append(laiyuan)
# # 对字段进行加密和去重
ir_md5 = get_md5(wendangxuhao)
cursor = connect.cursor()
sql1 = 'select ir_md5 from zhuanli'
connect.ping(reconnect=True)
cursor.execute(sql1)
ir_md5_dec = cursor.fetchall()
# 把需要查询的数据转变为元组
md5 = tuple([ir_md5], )

if md5 in ir_md5_dec:
pass
print('数据已存在')
else:
try:
# print(ir_md5,title,publish_time,second_url,picture_url,neirong,wendangxuhao,laiyuan)
sql = 'INSERT IGNORE INTO zhuanli (ir_md5,title,publish_time,second_url,picture_url,neirongs,wendangxuhao,laiyuan)' \
'VALUES("%s","%s","%s","%s","%s","%s","%s","%s")'\
% (ir_md5,title,publish_time,second_url,picture_url,neirongs,wendangxuhao,laiyuan)
connect.ping(reconnect=True)
cursor.execute(sql)
ir_idd = int(connect.insert_id())
print('数据库自增id', ir_idd, '数据')
# 提交数据库
connect.commit()
print("zhuanli表数据存储成功！", )
print('提交成功')
except:
open('错误网站.txt','a').write(second_url+'/n')
continue

connect.close()
# data = {
# '标题': title,
# '文档序号': wendangxuhao_list,
# '来源': laiyuan_list,
# '详情页链接': second_url_list,
# '图片链接': picture_url_list,
# '文章内容': content_list,
# '发布时间': publish_time,
#
# }
# s = pd.DataFrame(data)
# s.to_excel('某技术文档.xlsx')