Biegral Blog

爬取时需要进行的操作：

    1：输入你是要爬取国内的还是海外的，1表示国内，2表示海外；
    2：然后输入你要爬取的城市名称，就可以了；

每个函数的功能：

    choose_area函数根据你输入的是国内还是海外，输出不同的区域名称
    url_list函数根据你输入的城市名称，来爬取城市有多少房源，来判断有几页数据，因为小猪短租网上面只显示13页数据，超过了的话页只显示13页的数据，所以做个判断就可以
    get_url函数根据你输入的城市名称和页码，来构建你输入的城市每页的url

    get_html函数就是获取每页的html数据
    get_zf_url函数根据每页的html数据来爬取每个租房的url链接
    get_zf_message函数，通过传入的租房url链接来获取每个租房的信息

'''
遇到不懂的问题？Python学习交流群：1136201545满足你的需求，资料都已经上传群文件，可以自行下载！
'''
import  re
import requests

url = 'http://jci.xiaozhustatic1.com/e17061601/xzjs?k=Front_Search&httphost=bj.xiaozhu.com'     #获取城市名称的链接

ser = input('输入你要查找的地区（1：国内;2：海外）：')            #输入你你要查找是国内的短租房还是海外的短租房
html = requests.get(url).text       #通过上面提供的url来爬取每个省份的拼音

def choose_area():      #判断你需要查找的是国内的还是海外的，并输出相应的内容
    city_tup = re.compile('citys[[0-9]\d*]=new Array(.*?);').findall(html)
    #print(city_tup)
    for city_name in city_tup[29:]:
        #print(city_name)
        city_time = re.compile('[0-9]\d*:[0-9]\d*').findall(city_name)
        if ser == '1':
            if len(city_time) == 0:
                city = re.compile('[\u4E00-\u9FA5]+').findall(city_name)[0]     #城市名称
                city_jc = re.compile('[a-z]\w*').findall(city_name)[1]      #城市拼音
                city_zf = re.compile('[0-9]\d*').findall(city_name)[0]      #城市租房数量
                city_dic = {city:[city_jc,city_zf]}
                yield city_dic
            else:
                pass
        elif ser == '2':
            if len(city_time) != 0:
                city = re.compile('[\u4E00-\u9FA5]+').findall(city_name)[0]
                city_jc = re.compile('[a-z]\w*').findall(city_name)[1]
                city_zf = re.compile('[0-9]\d*').findall(city_name)[0]
                city_dic = {city: [city_jc, city_zf]}
                yield city_dic
            else:
                pass

def get_url(city_jc,page):  #提供省份的名称和页码来构建需要爬取的url
    url = 'http://{}.xiaozhu.com/search-duanzufang-p{}-0/'.format(city_jc,page)
    return url

#根据你提供的省份名称来判断，这个省份有多少房源，但是每个省份的房源只显示13页的数据，所有做个判断，超过了的话就只显示13页，没有超过的话就有几页就显示几页
def url_list(city_name):
    #city_name = input('输入你要查找的城市名称:')
    for city in choose_area():
        if city_name in city.keys():
            if int(int(city[city_name][1])/24) > 13:
                for page in range(1,14):
                    url = get_url(city[city_name][0],page)
                    yield url
            elif int(int(city[city_name][1])/24) <= 13:
                sum_page = int(int(city[city_name][1])/24) <= 13
                for page in range(1,sum_page + 1):
                    url = get_url(city[city_name][0], page)
                    yield url

import requests
from lxml import etree
import re
'''
遇到不懂的问题？Python学习交流群：1136201545满足你的需求，资料都已经上传群文件，可以自行下载！
'''
headers = {
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36'
}

def get_html(url):      #获取网页的html内容
    response = requests.get(url,headers=headers)
    if response.status_code == 200:
        response.encoding = response.apparent_encoding
        html = response.text
        return html
    else:
        print('没有获取到HTML')

def get_zf_url(url):  #根据网页的内容来获取每个短租房的url，并传入get_zf_message函数来获取每个短租房的信息
    html = get_html(url)
    links = etree.HTML(html).xpath('//*[@id="page_list"]/ul/li/a/@href')
    return links    #返回是一个列表

def get_zf_message(zf_url):
    html = get_html(zf_url)
    area = etree.HTML(html).xpath('//div[@class="pho_info"]/p/@title')[0]
    h_image = etree.HTML(html).xpath('//*[@id="curBigImage"]/@src')[0]
    #//*[@id="floatRightBox"]/div[3]/div[3]/h6/a
    #因为用lxml获取不到房东姓名，不知道为什么，但是用正则就可以
    fd_name = re.compile('<a class="lorder_name" href=".*?" title="(.*?)" target="_blank">.*?</a>').findall(html)[0]
    #fd_name = etree.HTML(html).xpath('//a[class="lorder_name"]/text()')
    fd_link = re.compile('<a class="lorder_name" href="(.*?)" title=".*?" target="_blank">.*?</a>').findall(html)[0]
    zf_price = etree.HTML(html).xpath('//*[@id="pricePart"]/div[1]/span/text()')[0]
    zf_title = etree.HTML(html).xpath('//div[@class="pho_info"]/h4/em/text()')[0]
    data = {
        '标题': zf_title,
        '价格': zf_price,
        '地址': area,
        '图片': h_image,
        '房东': fd_name,
        '房东链接': fd_link,
    }
    return data

from spider_ziaozu import *
from get_url import *

'''
遇到不懂的问题？Python学习交流群：1136201545满足你的需求，资料都已经上传群文件，可以自行下载！
'''
if __name__ == '__main__':
    city = input('输入你想爬取的城市名称:')
    for url in url_list(city):      #通过传入上海的，获取上海的短租房的所有页面url
        print(url)
        zf_list = get_zf_url(url)       #把上海的短租房的所有页面url,传给get_zf_url来获取每个短租房url,返回的也是一个列表
        #print(zf_list)
        for zf_url in zf_list:          #通过遍历每个短租房的url,并传入给get_zf_message，来获取每个租房的信息
            print(zf_url)
            try:
                zf_message = get_zf_message(zf_url)
                print(zf_message)
            except Exception as e:
                print(e)

下面是城市的数据：

domestic_list = [
{'北京': ['beijing', '8221']},
{'上海': ['shanghai', '6996']},
{'广州': ['guangzhou', '2727']},
{'成都': ['chengdu', '5369']},
{'深圳': ['shenzhen', '2522']},
{'西安': ['xian', '2562']},
{'南京': ['nanjing', '1675']},
{'杭州': ['hangzhou', '2455']},
{'重庆': ['chongqing', '3171']},
{'武汉': ['wuhan', '1901']},
{'苏州': ['suzhou', '1603']},
{'无锡': ['wuxi', '240']},
{'青岛': ['qingdao', '3712']},
{'厦门': ['xiamen', '1548']},
{'三亚': ['sanya', '2384']},
{'大连': ['dalian', '1034']},
{'哈尔滨': ['haerbin', '790']},
{'秦皇岛': ['qinhuangdao', '1924']},
{'天津': ['tianjin', '485']},
{'昆明': ['kunming', '819']},
{'香港': ['xianggang', '164']},
{'长春': ['changchun', '350']},
{'沈阳': ['shenyang', '562']},
{'合肥': ['hefei', '533']},
{'郑州': ['zhengzhou', '370']},
{'太原': ['taiyuan', '470']},
{'威海': ['weihai', '821']},
{'丽江': ['lijiang', '632']},
{'大理': ['dali', '576']},
{'桂林': ['guilin', '518']},
{'澳门': ['aomen', '33']},
{'福州': ['fuzhou', '410']},
{'宁波': ['ningbo', '233']},
{'珠海': ['zhuhai', '595']},
{'长沙': ['changsha', '727']},
{'石家庄': ['shijiazhuang', '288']},
{'拉萨': ['lasa', '47']},
{'常州': ['changzhou', '137']},
{'扬州': ['yangzhou', '224']},
{'东莞': ['dongguan', '70']},
{'海口': ['haikou', '518']},
{'兰州': ['lanzhou', '198']},
{'洛阳': ['luoyang', '195']},
{'乌鲁木齐': ['wulumuqi', '213']},
{'徐州': ['xuzhou', '51']},
{'贵阳': ['guiyang', '503']},
{'呼和浩特': ['huhehaote', '82']},
{'济南': ['jinan', '375']},
{'唐山': ['tangshan', '102']},
{'保定': ['baoding', '83']},
{'南昌': ['nanchang', '206']},
{'邯郸': ['handan', '12']},
{'南宁': ['nanning', '168']},
{'潍坊': ['weifang', '65']},
{'锦州': ['jinzhou', '54']},
{'日照': ['rizhao', '508']},
{'临沂': ['linyi', '41']},
{'鞍山': ['anshan', '23']},
{'廊坊': ['langfang', '101']},
{'大庆': ['daqing', '29']},
{'北海': ['beihai', '436']},
{'中山': ['zhongshan', '70']},
{'西宁': ['xining', '362']},
{'金华': ['jinhua', '71']},
{'丹东': ['dandong', '181']},
{'承德': ['chengde', '437']},
{'盘锦': ['panjin', '35']},
{'淄博': ['zibo', '30']},
{'株洲': ['zhuzhou', '17']},
{'佛山': ['foshan', '127']},
{'吉林': ['jilinshi', '50']},
{'邢台': ['xingtai', '9']},
{'齐齐哈尔': ['qiqihaer', '8']},
{'宜昌': ['yichang', '42']},
{'大同': ['datong', '83']},
{'烟台': ['yantai', '803']},
{'银川': ['yinchuan', '76']},
{'温州': ['wenzhou', '52']},
{'淮安': ['huaian', '37']},
{'绵阳': ['mianyang', '121']},
{'包头': ['baotou', '40']},
{'抚顺': ['fushun', '5']},
{'泰安': ['taian', '103']},
{'济宁': ['jining', '11']},
{'连云港': ['lianyungang', '33']},
{'泉州': ['quanzhou', '95']},
{'安阳': ['anyang', '24']},
{'惠州': ['huizhou', '537']},
{'葫芦岛': ['huludao', '595']},
{'嘉兴': ['jiaxing', '405']},
{'南通': ['nantong', '143']},
{'攀枝花': ['panzhihua', '15']},
{'柳州': ['liuzhou', '19']},
{'东营': ['dongying', '1']},
{'佳木斯': ['jiamusi', '5']},
{'通辽': ['tongliao', '5']},
{'德州': ['dezhou', '22']},
{'赣州': ['ganzhou', '6']},
{'滨州': ['binzhou', '3']},
{'咸阳': ['xianyang', '23']},
{'江门': ['jiangmen', '17']},
{'漳州': ['zhangzhou', '84']},
{'新乡': ['xinxiang', '8']},
{'襄樊': ['xiangfan', '4']},
{'南充': ['nanchong', '29']},
{'聊城': ['liaocheng', '17']},
{'张家口': ['zhangjiakou', '196']},
{'沧州': ['cangzhou', '22']},
{'石河子': ['shihezi', '4']},
{'宝鸡': ['baoji', '5']},
{'赤峰': ['chifeng', '22']},
{'湛江': ['zhanjiang', '41']},
{'商丘': ['shangqiu', '5']},
{'平顶山': ['pingdingshan', '4']},
{'信阳': ['xinyang', '13']},
{'九江': ['jiujiang', '29']},
{'营口': ['yingkou', '500']},
{'本溪': ['benxi', '6']},
{'钦州': ['qinzhou', '2']},
{'衡阳': ['hengyang', '19']},
{'汕头': ['shantou', '63']},
{'芜湖': ['wuhu', '18']},
{'呼伦贝尔': ['hulunbeier', '124']},
{'湘潭': ['xiangtan', '11']},
{'朝阳市': ['chaoyang', '2']},
{'清远': ['qingyuan', '137']},
{'遂宁': ['suining', '6']},
{'泰州': ['jstaizhou', '6']},
{'莆田': ['putian', '3']},
{'枣庄': ['zaozhuang', '14']},
{'泸州': ['luzhou', '52']},
{'舟山': ['zhoushan', '304']},
{'镇江': ['zhenjiang', '19']},
{'开封': ['kaifeng', '130']},
{'鄂尔多斯': ['eerduosi', '3']},
{'十堰': ['shiyan', '14']},
{'延边': ['yanbian', '75']},
{'淮北': ['huaibei', '5']},
{'临汾': ['linfen', '21']},
{'常德': ['changde', '5']},
{'荆州': ['jingzhou', '3']},
{'郴州': ['chenzhou', '46']},
{'德阳': ['deyang', '13']},
{'绍兴': ['shaoxing', '33']},
{'南阳': ['nanyang', '7']},
{'菏泽': ['heze', '1']},
{'台州': ['zjtaizhou', '36']},
{'遵义': ['zunyi', '7']},
{'阜新': ['fuxin', '2']},
{'盐城': ['yancheng', '6']},
{'宿迁': ['suqian', '2']},
{'焦作': ['jiaozuo', '16']},
{'长治': ['changzhi', '26']},
{'吉安': ['jian', '14']},
{'驻马店': ['zhumadian', '1']},
{'汉中': ['hanzhong', '28']},
{'河源': ['heyuan', '11']},
{'铁岭': ['tieling', '2']},
{'晋中': ['jinzhong', '63']},
{'安康': ['ankang', '4']},
{'岳阳': ['yueyang', '13']},
{'肇庆': ['zhaoqing', '15']},
{'衡水': ['hengshui', '21']},
{'牡丹江': ['mudanjiang', '24']},
{'安庆': ['anqing', '11']},
{'黄冈': ['huanggang', '2']},
{'娄底': ['loudi', '3']},
{'乐山': ['leshan', '187']},
{'蚌埠': ['bengbu', '14']},
{'昌吉': ['changji', '1']},
{'韶关': ['shaoguan', '28']},
{'阳江': ['yangjiang', '87']},
{'潮州': ['chaozhou', '15']},
{'张家界': ['zhangjiajie', '171']},
{'怀化': ['huaihua', '2']},
{'西双版纳': ['xishuangbanna', '141']},
{'三明': ['sanming', '9']},
{'运城': ['yuncheng', '15']},
{'眉山': ['meishan', '17']},
{'许昌': ['xuchang', '11']},
{'防城港': ['fangchenggang', '16']},
{'永州': ['yongzhou', '1']},
{'益阳': ['yiyang', '5']},
{'上饶': ['shangrao', '45']},
{'衢州': ['quzhou', '1']},
{'六盘水': ['liupanshui', '10']},
{'白山': ['baishan', '37']},
{'六安': ['luan', '1']},
{'铜陵': ['tongling', '1']},
{'池州': ['chizhou', '5']},
{'晋城': ['jincheng', '3']},
{'黄石': ['huangshi', '10']},
{'湘西': ['xiangxi', '24']},
{'宜春': ['jxyichun', '18']},
{'茂名': ['maoming', '1']},
{'梅州': ['meizhou', '2']},
{'凉山': ['liangshan', '330']},
{'宜宾': ['yibin', '22']},
{'湖州': ['huzhou', '296']},
{'海拉尔': ['hailaer', '2']},
{'延安': ['yanan', '5']},
{'内江': ['neijiang', '9']},
{'南平': ['nanping', '11']},
{'三门峡': ['sanmenxia', '2']},
{'松原': ['songyuan', '5']},
{'阜阳': ['fuyang', '3']},
{'黄山': ['huangshan', '105']},
{'巴彦淖尔': ['bayannaoer', '1']},
{'渭南': ['weinan', '8']},
{'咸宁': ['xianning', '12']},
{'恩施': ['enshi', '29']},
{'抚州': ['jxfuzhou', '5']},
{'龙岩': ['longyan', '13']},
{'通化': ['tonghua', '18']},
{'莱芜': ['laiwu', '1']},
{'宣城': ['xuancheng', '8']},
{'锡林郭勒': ['xilinguole', '18']},
{'景德镇': ['jingdezhen', '21']},
{'曲靖': ['qujing', '3']},
{'广元': ['guangyuan', '9']},
{'巴中': ['bazhong', '5']},
{'济源': ['jiyuan', '3']},
{'鹤岗': ['hegang', '2']},
{'黑河': ['heihe', '6']},
{'吕梁': ['lvliang', '3']},
{'天水': ['tianshui', '11']},
{'榆林': ['sxyulin', '4']},
{'萍乡': ['pingxiang', '4']},
{'哈密': ['hami', '7']},
{'自贡': ['zigong', '23']},
{'阿坝': ['aba', '260']},
{'宁德': ['ningde', '20']},
{'马鞍山': ['maanshan', '1']},
{'阿拉善': ['alashan', '12']},
{'阳泉': ['yangquan', '2']},
{'新余': ['xinyu', '1']},
{'喀什': ['kashi', '2']},
{'黔西南': ['qianxinan', '14']},
{'鸡西': ['jixi', '1']},
{'伊春': ['hljyichun', '24']},
{'大兴安岭': ['daxinganling', '5']},
{'宿州': ['ahsuzhou', '2']},
{'梧州': ['wuzhou', '2']},
{'阿克苏': ['akesu', '1']},
{'汕尾': ['shanwei', '12']},
{'广安': ['guangan', '4']},
{'资阳': ['ziyang', '3']},
{'安顺': ['anshun', '7']},
{'黔东南': ['qiandongnan', '15']},
{'七台河': ['qitaihe', '1']},
{'河池': ['hechi', '4']},
{'张掖': ['zhangye', '27']},
{'酒泉': ['jiuquan', '113']},
{'陇南': ['longnan', '4']},
{'神农架': ['shennongjia', '12']},
{'克拉玛依': ['kelamayi', '4']},
{'伊犁': ['yili', '19']},
{'雅安': ['yaan', '8']},
{'甘孜': ['ganzi', '94']},
{'丽水': ['lishui', '39']},
{'瓦房店': ['wafangdian', '2']},
{'武夷山': ['wuyishan', '34']},
{'亳州': ['bozhou', '1']},
{'贺州': ['hezhou', '6']},
{'石嘴山': ['shizuishan', '1']},
{'中卫': ['zhongwei', '20']},
{'平凉': ['pingliang', '1']},
{'铜川': ['tongchuan', '3']},
{'昭通': ['zhaotong', '1']},
{'巴音郭楞': ['bayinguoleng', '2']},
{'日喀则': ['rikaze', '4']},
{'铜仁': ['tongren', '6']},
{'忻州': ['xinzhou', '15']},
{'吴忠': ['wuzhong', '1']},
{'玉树': ['yushu', '1']},
{'海西': ['haixi', '11']},
{'玉溪': ['yuxi', '11']},
{'红河': ['honghe', '7']},
{'德宏': ['dehong', '8']},
{'吐鲁番': ['tulufan', '2']},
{'黔南': ['qiannan', '9']},
{'张北': ['zhangbei', '3']},
{'鹤壁': ['hebi', '1']},
{'五指山': ['wuzhishan', '4']},
{'兴安': ['xingan', '6']},
{'嘉峪关': ['jiayuguan', '20']},
{'商洛': ['shangluo', '1']},
{'海东': ['haidong', '5']},
{'海北': ['haibei', '17']},
{'随州': ['suizhou', '1']},
{'保山': ['baoshan', '25']},
{'楚雄': ['chuxiong', '2']},
{'普洱': ['puer', '3']},
{'文山': ['wenshan', '1']},
{'迪庆': ['diqing', '14']},
{'和田': ['hetian', '1']},
{'阿拉尔': ['alaer', '1']},
{'文昌': ['wenchang', '39']},
{'琼海': ['qionghai', '30']},
{'儋州': ['danzhou', '1']},
{'万宁': ['wanning', '17']},
{'东方': ['dongfang', '5']},
{'安定': ['anding', '1']},
{'澄迈': ['chengmai', '7']},
{'临高': ['lingao', '1']},
{'白沙': ['baisha', '2']},
{'昌江': ['changjiang', '10']},
{'乐东': ['ledong', '5']},
{'陵水': ['lingshui', '60']},
{'保亭': ['baoting', '2']},
{'琼中': ['qiongzhong', '1']},
{'长白山': ['changbaishan', '113']},
{'台北': ['taibei', '14']},
{'新北': ['xinbei', '4']},
{'台中': ['taizhong', '9']},
{'高雄': ['gaoxiong', '2']},
{'新竹': ['xinzhu', '1']},
{'嘉义': ['jiayi', '1']},
{'花莲乡': ['hualianxiang', '20']},
{'台东县': ['taidongxian', '1']},
{'澎湖县': ['penghuxian', '1']},
]

overseas_list = [
{'墨尔本': ['moerben', '4']},
{'悉尼': ['xini', '3']},
{'维多利亚': ['weiduoliya', '2']},
{'巴黎': ['bali', '856']},
{'巴厘岛': ['balidaobalidao', '1']},
{'佛罗伦萨': ['foluolunsa', '1']},
{'名古屋': ['nagoya', '2']},
{'福冈': ['fugang', '1']},
{'吉隆坡': ['jilongpo', '19']},
{'马累': ['malei', '1']},
{'奥克兰': ['aokelan', '4']},
{'圣彼得堡': ['shengbidebao', '1']},
{'普吉岛': ['phuket', '5']},
{'芭堤雅': ['badiya', '10']},
{'伦敦': ['lundun', '1']},
{'旧金山': ['jiujinshan', '1']},
{'拉斯维加斯': ['lasiweijiasi', '1']},
{'文莱': ['wenlai', '4']},
{'弗雷德里顿': ['fuleidelidun', '1']},
{'杜塞尔多夫': ['dusaierduofu', '3']},
{'雅加达': ['yajiada', '1']},
{'埼玉': ['qiyu', '2']},
{'广岛': ['guangdao', '1']},
{'千叶': ['qianye', '2']},
{'堺': ['jie', '1']},
{'相模原': ['xiangmoyuan', '1']},
{'船桥': ['chuanqiao', '1']},
{'东大阪': ['dongdaban', '12']},
{'暹粒': ['xianli', '3']},
{'哥打基纳巴鲁': ['gedajinabalu', '4']},
{'奥兰多': ['aolanduo', '3']},
{'圣何塞': ['shenghs', '1']},
{'立川': ['lichuan', '1']},
{'调布': ['diaobu', '1']},
{'日野': ['riye', '1']},
{'马塔兰': ['mataram', '1']},
]