2023数据采集与融合技术实践作业一

import requests
from bs4 import BeautifulSoup
import bs4

#将url信息从网络中爬取下来，并且将它其中的html页面返回给其他的函数
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "getHTMLText(url) error"

#提取html信息中关键的数据，并且填到一个列表中
def fillUnivList(ulist,html):
    soup = BeautifulSoup(html,"html.parser")

    for tr in soup.find("tbody").children:

        if isinstance(tr,bs4.element.Tag):
            #print(tr)
            tds = tr('td')
            tds1 = tr('a')
            #print(tds[3].text)
            #print(tds1[0].string)
            ulist.append([tds[0].string.strip(),tds1[0].string.strip(),tds[2].text.strip(),tds[3].text.strip(),tds[4].string.strip()])


#格式化打印学校列表信息
def printUnivList(ulist,num):
    tplt="{0:^8}{1:{5}^10}{2:{6}^10}{3:{7}^10}{4:^6}"
    print(tplt.format("排名 ","学校名称 "," 省市 ","学校类型","总分 ",chr(12288),chr(12288),chr(12288)))
    for i in range(num):
        u=ulist[i]
        # print(u[0])
        print(tplt.format(u[0],u[1],u[2],u[3],u[4],chr(12288),chr(12288),chr(12288)))

def main():
    uinfo = []
    url = 'http://www.shanghairanking.cn/rankings/bcur/2020'
    html = getHTMLText(url)
    fillUnivList(uinfo,html)
    printUnivList(uinfo,30)
main()

import requests
import re
from bs4 import BeautifulSoup
import urllib.request
import urllib.parse

def parsePage(ilt,html):
    try:
        plt = re.findall(r'\&yen\;[\d\.]*',html)
        # print(plt)
        tlt = re.findall(r'alt\=\'.*?\'',html)
        # print(tlt)
        for i in range(len(plt)):
            price = eval(plt[i].split(';')[1])
            # print(price)
            title = eval(tlt[i].split('=')[1])
            # print(title)
            ilt.append([price,title])
            # print(ilt)
    except:
        print("")

def printGoodsList(ilt):
    tplt="{:4}\t{:8}\t{:16}"
    print(tplt.format("  序号","    价格","         商品名"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count,g[0],g[1]))
def main():
    # 更改头部信息模拟浏览器访问
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
    }
    # 设置搜索关键词
    wood = '书包'
    # 将中文转换为url编码
    urlwood = urllib.parse.quote(wood, safe='/', encoding='utf-8')
    # 自定义爬取页数
    page = 3
    # 初始化下载页数显示
    a = 1
    # 循环遍历每一页
    infoList = []
    for i in range(1, page):
        try:
            # 获取url内容
            r = requests.get(f"https://search.dangdang.com/?key={urlwood}&act=input&page_index={i}", headers=headers)
            r.raise_for_status()
            r.encoding = r.apparent_encoding
            html = r.text
            # print(html)
            parsePage(infoList,html)
        except:
            continue
    printGoodsList(infoList)

main()

import requests
from bs4 import BeautifulSoup
import urllib.request
def getPhotoImg():
    url=requests.get("https://xcb.fzu.edu.cn/info/1071/4481.htm")
    htm=url.text
    soup = BeautifulSoup(htm,'html.parser')
    jpeg = soup.find_all('img')
    a = 1
    for i in jpeg:
        imgsrc = 'https://xcb.fzu.edu.cn'+i.get('src')
        # print(i.get('src'))
        filename = 'D:/学习/数据采集与融合技术/爬取图片/%s.jpg'%a
        urllib.request.urlretrieve(imgsrc , filename)
        print('下载第%d张' % a)
        a += 1
    print('下载完成!')

getPhotoImg()