标签：name Python text region community 爬取 url 链家 data
一、前言：
安居客、链家和房天下是目前网上可以获取小区数据较为精准的网站，之前已经发过链家和房天下的部分区域（仅浦东）获取攻略。这次因为工作原因，需要获取整个上海的所有小区数据（仅别墅和住宅），所以过年这几天在不断的数据分析、获取、清洗和验证。特此记录一下，也把代码和各位分享。
二、爬取思路：
不管是安居客、链家还是房天下，获取数据的思路都是一致的：
1、获取不同行政区的网址
2、获取不同行政区下不同商圈/街镇的网址
3、获取不同行政区下每一个商圈/街镇中所有小区的网址
4、根据3中获得的网址，把需要的页面元素爬下来
三、安居客、房天下和链家对比：
我把三个网站的数据都爬下来了，不过最后只用了安居客的数据
四、链家代码
1 import requests
2 from bs4 import BeautifulSoup
3 import re
4 import time
5 import traceback
6 import math
7
8 headers = {
9 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
10 'Host': 'sh.lianjia.com',
11 'Cookie': ''
12 }
13
14 def read_Lregion_dict():
15 '''读取行政区域的文件，并输出为字典'''
16 with open('行政区url.txt', 'r') as f:
17 large_region_list = f.readlines()
18 large_region_dict = {}
19 for ele in large_region_list:
20 url, region = ele.split(' ')
21 region = region.replace('\n', '')
22 large_region_dict[url] = region
23 return large_region_dict
24
25 def get_jiezhen_urls():
26 '''获取街镇的url'''
27 large_region_dict = read_Lregion_dict()
28 small_region_dict = {}
29 for k, v in large_region_dict.items():
30 if v != '上海周边':
31 url = 'https://sh.lianjia.com' + k
32 r = requests.get(url=url, headers=headers)
33 soup = BeautifulSoup(r.text, 'lxml')
34 a = soup.find(name='div', attrs={'data-role': 'ershoufang'})
35 esf_urls = a.find_all(name='a')
36 for ele in esf_urls:
37 href = ele.attrs['href']
38 name = ele.string
39 if name in large_region_dict.values():
40 continue
41 else:
42 small_region_dict[href] = name
43 with open('街镇url.txt', 'a', encoding='utf-8') as file:
44 file.write(','.join([v, name, href]))
45 file.write('\n')
46 print(v, name, href)
47
48 def region_total(url):
49 '''获取该区域的小区数量'''
50 url = r"https://sh.lianjia.com" + url + '?from=rec'
51 r = requests.get(url=url, headers=headers)
52 soup = BeautifulSoup(r.text, 'lxml')
53 total_find = soup.find(name='h2', attrs={'class': 'total fl'})
54 total_num = int(total_find.find(name='span').string.strip())
55 return total_num
56
57 def get_all_urls():
58 '''获取所有小区名字和链接'''
59 with open('街镇url.txt', 'r', encoding='utf-8') as f:
60 small_region_list = f.readlines()
61 for ele in small_region_list:
62 l_region, s_region, url = ele.split(',')
63 url = url.replace('\n', '')
64 total_num = region_total(url)
65 pages = int(math.ceil(int(total_num)/30))
66 for i in range(1, pages+1):
67 if i == 1:
68 i = ""
69 else:
70 i = 'pg' + str(i)
71 tmp_url = r"https://sh.lianjia.com" + url + i
72 r = requests.get(url=tmp_url, headers=headers)
73 soup = BeautifulSoup(r.text, 'lxml')
74 for j in soup.find_all(name='div', attrs={'class': 'title'}):
75 community = str(j)
76 if '''target="_blank"''' in community:
77 community_list = re.search('''<a href="(.*?)" target="_blank">(.*?)</a>.*?''', community)
78 community_url = community_list.group(1)
79 community_name = community_list.group(2)
80 with open('小区url.txt', 'a', encoding='utf-8') as file:
81 file.write(','.join([l_region, s_region, community_name, community_url]))
82 file.write('\n')
83 time.sleep(1)
84 print('{}, {}总共有{}个小区，共有{}页，已全部url爬取完成！'.format(l_region, s_region, total_num, pages))
85
86 def get_communityInfo(l_region, s_region, community_name, community_url):
87 '''获取某个小区的信息'''
88 r = requests.get(url=community_url, headers=headers)
89 soup = BeautifulSoup(r.text, 'lxml')
90 try:
91 unitPrice = soup.find(name='span', attrs={'class': 'xiaoquUnitPrice'}).string #小区均价
92 except:
93 unitPrice = '空'
94 try:
95 address = soup.find(name='div', attrs={'class': 'detailDesc'}).string #小区地址
96 address = '"' + address + '"'
97 except:
98 address = '空'
99 try:
100 xiaoquInfo = soup.find_all(name='span', attrs={'class': 'xiaoquInfoContent'}) #小区信息
101 xiaoquInfo_list = [l_region, s_region]
102 community_name = '"' + community_name + '"'
103 xiaoquInfo_list.append(community_name)
104 xiaoquInfo_list.append(address)
105 xiaoquInfo_list.append(unitPrice)
106 for info in xiaoquInfo:
107 xiaoquInfo_list.append(info.string)
108 xiaoquInfo_list.pop()
109 export_communityInfo(xiaoquInfo_list)
110 time.sleep(1)
111 print('已爬取{},{}的{}信息'.format(l_region, s_region, community_name))
112 except:
113 print('{},{}的{}爬取错误，url是{}'.format(l_region, s_region, community_name, community_url))
114
115 def export_communityInfo(xiaoquInfo_list):
116 '''导出小区信息'''
117 with open('上海地区小区信息.txt', 'a', encoding='utf-8') as file:
118 file.write(','.join(xiaoquInfo_list))
119 file.write('\n')
120
121 if __name__ == "__main__":
122 # get_jiezhen_urls() #获取街镇的url
123 # get_all_urls() #获取所有小区名字和链接
124 with open('小区url.csv', 'r') as f:
125 xiaoqu_list = f.readlines()
126 for ele in xiaoqu_list:
127 l_region, s_region, community_name, community_url = ele.split(',')
128 community_url = community_url.replace('\n', '')
129 try:
130 get_communityInfo(l_region, s_region, community_name, community_url)
131 except:
132 traceback.print_exc()
133 break
View Code
五、房天下代码
1 import requests
2 from bs4 import BeautifulSoup
3 import pandas as pd
4 import time
5 import traceback
6
7 headers = {
8 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
9 'cookie': ''''''
10 }
11
12 def get_true_url(old_url):
13 '''获得正确的url'''
14 r = requests.get(url=old_url, headers=headers)
15 if r'<title>跳转...</title>' in r.text:
16 soup = BeautifulSoup(r.text, 'lxml')
17 new_url = soup.find(name='a', attrs={'class': 'btn-redir'}).attrs['href']
18 return new_url
19 return old_url
20
21 def get_region_urls():
22 '''获得上海行政区中不同街镇的url和名称'''
23 sh_dict = {'浦东': '25', '嘉定': '29', '宝山': '30', '闵行': '18', '松江': '586', '普陀': '28',
24 '静安': '21', '黄浦': '24', '虹口': '23', '青浦': '31', '奉贤': '32', '金山': '35',
25 '杨浦': '26', '徐汇': '19', '长宁': '20', '崇明': '996'}
26 for l_region_name, l_region_url in sh_dict.items():
27 url = r"https://sh.esf.fang.com/housing/" + l_region_url + '__0_3_0_0_1_0_0_0/'
28 true_url = get_true_url(url)
29 r = requests.get(url=true_url, headers=headers)
30 soup = BeautifulSoup(r.text, 'lxml')
31 a = soup.find(name='p', attrs={'id': 'shangQuancontain', 'class': 'contain'})
32 for i in a.find_all(name='a'):
33 if i.string != '不限':
34 this_url = r"https://sh.esf.fang.com" + i.attrs['href']
35 this_url_list = get_region_url(this_url)
36 with open('上海地区街镇url.txt', 'a', encoding='utf-8') as file:
37 for tmp_url in this_url_list:
38 file.write(','.join([l_region_name, i.string, tmp_url]))
39 file.write('\n')
40 print('{}已完成'.format(l_region_name))
41
42 def get_region_url(old_url):
43 '''获得这个区域的其它page_url'''
44 true_url = get_true_url(old_url)
45 r = requests.get(url=true_url, headers=headers)
46 soup = BeautifulSoup(r.text, 'lxml')
47 page_url = soup.find(name='div', attrs={'class': 'fanye gray6'})
48 page_url_list = []
49 page_url_list.append(old_url)
50 for j in page_url.find_all(name='a'):
51 if 'href' in j.attrs:
52 temp_url = r'https://sh.esf.fang.com/' + j.attrs['href'][1:]
53 if temp_url not in page_url_list:
54 page_url_list.append(temp_url)
55 return page_url_list
56
57 def get_xiaoqu_url(bigregion, smallregion, old_url):
58 '''获得某区域某一页的小区信息和url'''
59 true_url = get_true_url(old_url)
60 r = requests.get(url=true_url, headers=headers)
61 soup = BeautifulSoup(r.text, 'lxml')
62 j = 0
63 for i in soup.find_all(name='a', attrs={'class': 'plotTit', 'target': '_blank'}):
64 xiaoqu_type = soup.find('a', text=i.string, attrs={'class': 'plotTit', 'target': '_blank'}).parent.find('span', attrs={'class':'plotFangType'}).string
65 xiaoqu_name = i.string
66 xiaoqu_url = 'https://sh.esf.fang.com/' + i.attrs['href'][1:]
67 xiaoqu_url = xiaoqu_url.replace('.htm', '/housedetail.htm')
68 print(bigregion, smallregion, xiaoqu_name, xiaoqu_type, xiaoqu_url)
69 j += 1
70 with open('上海地区小区url.txt', 'a', encoding='utf-8') as file:
71 file.write(','.join([bigregion, smallregion, xiaoqu_name, xiaoqu_type, xiaoqu_url]))
72 file.write('\n')
73 time.sleep(1)
74 print(bigregion, smallregion, old_url, '所有小区url获取完毕，共有{}条数据'.format(j))
75 print('-'*100)
76
77 def get_all_urls(last_url=None):
78 '''获得所有小区的URL'''
79 '''获得结果后还需要清洗一下，因为有些小区跨区域，所以会有重复'''
80 with open('上海地区街镇url.txt', 'r', encoding='utf-8') as f:
81 region_list = f.readlines()
82 event_tracking = False
83 for i in range(len(region_list)):
84 l_region, s_region, url = region_list[i].split(',')
85 url = url.replace('\n', '')
86 if last_url == url:
87 event_tracking = True
88 if event_tracking:
89 print(l_region, s_region, url)
90 get_xiaoqu_url(l_region, s_region, url)
91
92 def get_total_informations(l_region, s_region, community_name, community_type, community_url):
93 '''爬取某个小区的有用信息'''
94 r = requests.get(url=community_url, headers=headers)
95 soup = BeautifulSoup(r.text, 'lxml')
96 informations = soup.find(name='div', attrs={'class': 'village_info base_info'})
97 if not informations:
98 print('{}, {}, {}, {}爬取失败！'.format(l_region, s_region, community_name, community_url))
99 return None
100 else:
101 all_info = [l_region, s_region, community_name, community_type]
102 for ele in ['本月均价', '小区地址', '产权描述', '环线位置', '建筑年代', '建筑面积', '占地面积', '房屋总数', '楼栋总数', '绿 化 率', '容 积 率', '物 业 费', '开 发 商', '物业公司']:
103 try:
104 all_info.append(informations.find('span', text=ele).parent.find(name='p').text.strip().replace('\r', '').replace('\n', '、').replace('\t', '').replace(',', '，'))
105 except:
106 try:
107 all_info.append(informations.find('span', text=ele).parent.find(name='a').text.strip().replace('\r', '').replace('\n', '、').replace('\t', '').replace(',', '，'))
108 except:
109 all_info.append('')
110 return all_info
111
112 def get_data(last_url=None):
113 '''主程序，爬所有小区信息'''
114 with open('上海地区小区url.txt', 'r', encoding='utf-8') as f:
115 village_list = f.readlines()
116 error_count = 0
117 if last_url == None:
118 event_tracking = True
119 else:
120 event_tracking = False
121 for i in range(len(village_list)):
122 l_region, s_region, community_name, community_type, community_url = village_list[i].split(',')
123 community_url = community_url.replace('\n', '')
124 if last_url == community_url:
125 event_tracking = True
126 if event_tracking == True:
127 if community_type=='住宅' or community_type=='别墅':
128 # print(l_region, s_region, community_name, community_type,community_url)
129 try:
130 with open('上海小区数据.txt', 'a', encoding='utf-8') as file:
131 back = get_total_informations(l_region, s_region, community_name, community_type, community_url)
132 if not back:
133 if error_count>=2:
134 break
135 else:
136 error_count +=1
137 time.sleep(1)
138 continue
139 else:
140 error_count = 0
141 file.write(','.join(back))
142 file.write('\n')
143 print('{}, {}, {}, {}爬取成功！'.format(l_region, s_region, community_name, community_type, community_url))
144 time.sleep(1)
145 except:
146 print('{}, {}, {}, {}爬取失败！'.format(l_region, s_region, community_name, community_url))
147 traceback.print_exc()
148 break
149 else:
150 continue
151
152 if __name__ == "__main__":
153 get_region_urls() #得上海行政区中不同街镇的url和名称
154 get_xiaoqu_url() #获得某区域某一页的小区信息和url，这里应该是遍历，代码不完全
155 get_data() #爬取所有小区信息
View Code
六、安居客代码
1 import requests
2 from bs4 import BeautifulSoup
3 import re
4 import time
5 import traceback
6
7 headers = {
8 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
9 'Cookie': ''
10 }
11
12 def get_jiezhen_urls():
13 '''获取所有街镇的url'''
14 lregion_dict = {'浦东': 'pudong', '闵行': 'minhang', '松江': 'songjiang', '宝山': 'baoshan', '嘉定':'jiading',
15 '徐汇':'xuhui', '青浦':'qingpu', '静安':'jingan', '普陀':'putuo', '杨浦':'yangpu',
16 '奉贤': 'fengxian', '黄浦':'huangpu', '虹口':'hongkou', '长宁':'changning','金山':'jinshan',
17 '崇明':'chongming'}
18 for k, v in lregion_dict.items():
19 url = 'https://shanghai.anjuke.com/community/' + v + '/'
20 r = requests.get(url=url, headers=headers)
21 soup = BeautifulSoup(r.text, 'lxml')
22 a = soup.find_all('li', attrs={'class': 'region-item'})
23 for i in range(19, len(a)):
24 temp = a[i].find('a')
25 with open('街镇url.txt', 'a', encoding='utf-8') as file:
26 file.write(','.join([k, temp.text, temp.attrs['href']]))
27 file.write('\n')
28 print('{}区域的url都爬取完毕!'.format(k))
29 time.sleep(1)
30
31 def region_total(url):
32 '''获取该区域的小区数量'''
33 r = requests.get(url=url, headers=headers)
34 soup = BeautifulSoup(r.text, 'lxml')
35 # print(soup)
36 total_find = soup.find(name='span', attrs={'class': 'total-info'})
37 total_num = int(total_find.text.replace('共找到 ', '').replace(' 个小区', ''))
38 return total_num
39
40 def get_all_urls():
41 '''获取所有小区名字和链接'''
42 with open('街镇url.txt', 'r', encoding='utf-8') as f:
43 small_region_list = f.readlines()
44 for ele in small_region_list:
45 l_region, s_region, url = ele.split(',')
46 url = url.replace('\n', '')
47 total_num = region_total(url)
48 pages = int(math.ceil(int(total_num)/25))
49 for i in range(1, pages+1):
50 i = 'p' + str(i) + '/'
51 tmp_url = url + i
52 r = requests.get(url=tmp_url, headers=headers)
53 soup = BeautifulSoup(r.text, 'lxml')
54 a = soup.find_all('div', attrs={'class': 'li-info'})
55 for ele in a:
56 xiaoqu_name = ele.find('div', attrs={'class': 'li-title'}).text.strip()
57 xiaoqu_address = ele.find('div', attrs={'class': 'props nowrap'}).text.split(' - ')[-1].strip()
58 xiaoqu_tag = ele.find_all('span', attrs={'class': 'prop-tag'})
59 xiaoqu_url = ele.find('span', text='小区解读').parent.find('a').attrs['href']
60 xiaoqu_url = xiaoqu_url.replace('props/sale', 'view')
61 tag_list = []
62 for tag in xiaoqu_tag:
63 if 'display:none' in str(tag):
64 continue
65 else:
66 tag_list.append(tag.text)
67 with open('小区url.txt', 'a', encoding='utf-8') as file:
68 file.write('$'.join([l_region, s_region, xiaoqu_name, xiaoqu_address, str(tag_list), xiaoqu_url]))
69 file.write('\n')
70 time.sleep(1)
71 print('{}, {}总共有{}个小区，共有{}页，已全部url爬取完成！'.format(l_region, s_region, total_num, pages))
72
73 def get_communityInfo(l_region, s_region, community_name, community_address, community_tag, community_url):
74 '''获取某个小区的信息'''
75 r = requests.get(url=community_url, headers=headers)
76 soup = BeautifulSoup(r.text, 'lxml')
77 # print(soup)
78 if '访问验证-ajk' in str(soup):
79 print('触发反爬机制了！url是', community_url)
80 exit()
81 else:
82 # print('pa虫运行正常！')
83 try:
84 unitPrice = soup.find(name='span', attrs={'class': 'average'}).string #小区均价
85 except:
86 unitPrice = '暂无均价'
87 xiaoquInfo = soup.find_all(name='div', attrs={'class': 'hover-inner'}) #小区信息
88 xiaoquInfo_list = [l_region, s_region, community_name, community_address, community_tag, unitPrice]
89 for info in xiaoquInfo:
90 temp = info.find('div', attrs={'class': 'hover-value'})
91 if temp:
92 xiaoquInfo_list.append(temp.text.replace('\n', '').strip())
93 export_communityInfo(xiaoquInfo_list)
94 time.sleep(0.5)
95 print('已pa取{},{}的{}信息'.format(l_region, s_region, community_name))
96
97 def export_communityInfo(xiaoquInfo_list):
98 '''导出小区信息'''
99 with open('上海地区小区信息.txt', 'a', encoding='utf-8') as file:
100 file.write('&'.join(xiaoquInfo_list))
101 file.write('\n')
102
103 if __name__ == "__main__":
104 # get_jiezhen_urls() #获取所有街镇的url
105 # get_all_urls() #获取所有小区名字和链接
106 with open('小区url.txt', 'r', encoding='utf-8') as f:
107 xiaoqu_list = f.readlines()
108 last_url = 'https://shanghai.anjuke.com/community/view/8338/'
109 stop_place = False
110 for ele in xiaoqu_list:
111 l_region, s_region, community_name, community_address, community_tag, community_url = ele.split('$')
112 community_url = community_url.replace('\n', '')
113 if community_url == last_url or last_url == '':
114 stop_place = True
115 if stop_place:
116 try:
117 get_communityInfo(l_region, s_region, community_name, community_address, community_tag, community_url)
118 except:
119 print('{}爬取失败，url是：{}'.format(community_name, community_url))
120 traceback.print_exc()
121 break
View Code
七、数据清洗和特征工程
获取的数据很脏，有重复值需要剔重，有异常值需要修正（比如明显是外环的数据被归纳为内环）；需要根据业务场景，区分小区是否高档；需要根据需要，与内部数据结合…
我这边就举例几种场景，供大家参考（以安居客数据为例）：
1、从标签中判断小区是否靠近地铁
1 data['是否靠近地铁'] = data['标签'].apply(lambda x: '是' if '近地铁' in str(x) or '号线' in str(x) else '否')
2、从标签中判断环线位置
1 def huanxian_position(text): 2 '''环线位置''' 3 if '内环以内' in str(text): 4 return '内环以内' 5 elif '内中环之间' in str(text): 6 return '内中环之间' 7 elif '郊环以外' in str(text): 8 return '郊环以外' 9 elif '外郊环之间' in str(text): 10 return '外郊环之间' 11 elif '中外环之间' in str(text): 12 return '中外环之间' 13 else: 14 return np.nan 15 16 data['环线位置'] = data['标签'].apply(huanxian_position)
3、纠正环线位置
1 data_pivot = data.pivot_table(index='所属商圈', columns='环线位置', values='名称', aggfunc='count').reset_index()
2 data_pivot['环线位置2'] = ''
3 for i in range(data_pivot.shape[0]):
4 huan_dict = {}
5 huan_dict['中外环之间'] = data_pivot.iloc[i,1]
6 huan_dict['内中环之间'] = data_pivot.iloc[i,2]
7 huan_dict['内环以内'] = data_pivot.iloc[i,3]
8 huan_dict['外郊环之间'] = data_pivot.iloc[i,4]
9 huan_dict['郊环以外'] = data_pivot.iloc[i,5]
10 best_answer = ''
11 best_v = 0
12 for k,v in huan_dict.items():
13 if v == np.nan:
14 continue
15 elif v >= best_v:
16 best_answer = k
17 else:
18 continue
19 data_pivot.iloc[i,6] = best_answer
20
21 huan_dict = {}
22 for k,v in zip(data_pivot['所属商圈'].values, data_pivot['环线位置2'].values):
23 huan_dict[k] = v
24
25 data['环线位置'] = data['所属商圈'].map(huan_dict)
4、根据竣工时间判断小区年龄
1 def new_age(text):
2 '''竣工时间推导小区年龄'''
3 if str(text) != 'nan':
4 text = 2022 - int(text.split('、')[0].replace('年',''))
5 return text
6 else:
7 return np.nan
8
9 data['小区年龄'] = data['竣工时间'].apply(new_age)
5、判断是否商务楼宇、园区等（链家）
1 def if_business(text): 2 '''判断是否商务楼宇、园区等''' 3 for ele in ['商务', '园区', '大厦', '写字楼', '广场']: 4 if ele in text: 5 return '是' 6 else: 7 return '否' 8 9 data['是否商务楼宇等'] = data['小区名称'].apply(if_business)
6、提取物业费上下限（链家）
1 def wuyefei_down(text):
2 '''输出物业费下限'''
3 if text is np.nan:
4 return np.nan
5 elif '至' not in text:
6 return text.replace('元/平米/月','')
7 else:
8 down, up = text.split('至')
9 return down.replace('元/平米/月','')
10
11 def wuyefei_up(text):
12 '''输出物业费上限'''
13 if text is np.nan:
14 return np.nan
15 elif '至' not in text:
16 return text.replace('元/平米/月','')
17 else:
18 down, up = text.split('至')
19 return up.replace('元/平米/月','')
20
21 data['物业费下限'] = data['物业费'].apply(wuyefei_down)
22 data['物业费上限'] = data['物业费'].apply(wuyefei_up)
7、判断小区名字是否有地址
1 def if_number(text): 2 '''判断小区名称里是否有数字''' 3 if bool(re.search(r'\d', text)): 4 return '是' 5 else: 6 return '否' 7 8 data['小区名称里是否有数字'] = data['名称'].apply(if_number)
8、匹配百度经纬度
1 from urllib.request import urlopen, quote 2 import json 3 import math 4 from math import radians, cos, sin, asin, sqrt 5 import requests 6 7 def getjwd_bd(address): 8 '''根据地址获得经纬度（百度）''' 9 try: 10 url = 'http://api.map.baidu.com/geocoding/v3/?address=' 11 output = 'json' 12 ak = '******'#需填入自己申请应用后生成的ak 13 add = quote(address) #本文城市变量为中文，为防止乱码，先用quote进行编码 14 url2 = url+add+'&output='+output+"&ak="+ak 15 req = urlopen(url2) 16 res = req.read().decode() 17 temp = json.loads(res) 18 lng = float(temp['result']['location']['lng']) # 经度 Longitude 简写Lng 19 lat = float(temp['result']['location']['lat']) # 纬度 Latitude 简写Lat 20 return lng, lat 21 except: 22 return np.nan, np.nan 23 24 for i in tqdm(range(data.shape[0])): 25 region = data.iloc[i, 0] 26 if region=='浦东': 27 region = '上海市浦东新区' 28 else: 29 region = '上海市'+ region + '区' 30 xiaoqu_name = data.iloc[i, 2] 31 address = data.iloc[i, 3] 32 if str(data.iloc[i, 19]) !='nan': 33 continue 34 else: 35 lng1, lat1 = getjwd_bd(region+address+xiaoqu_name) 36 if 120<=lng1<=122 and 30<=lat1<=32: 37 data.iloc[i, 19] = lng1 38 data.iloc[i, 20] = lat1 39 else: 40 data.iloc[i, 19] = np.nan 41 data.iloc[i, 20] = np.nan
9、计算两个经纬度之间的距离（用于与内部数据匹配）
1 def get_distance(lng1,lat1,lng2,lat2):
2 '''计算距离'''
3 lng1, lat1, lng2, lat2 = map(radians, [float(lng1), float(lat1), float(lng2), float(lat2)]) # 经纬度转换成弧度
4 dlon = lng2 - lng1
5 dlat = lat2 - lat1
6 a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
7 distance = 2 * asin(sqrt(a)) * 6371393 # 地球平均半径，6371km
8 distance = round(distance, 0)
9 return distance
10
11 for i in tqdm(range(data.shape[0])):
12 xiaoqu_name = data.iloc[i, 2]
13 lng1 = data.iloc[i, 18]
14 lat1 = data.iloc[i, 19]
15 match_wg = data.iloc[i, 24]
16 min_distance = 9999999999
17 tmp_grid_cd = ''
18 tmp_grid_name = ''
19 # print(xiaoqu_name, lng1, lat1)
20 if str(match_wg) != 'nan':
21 print('{}已匹配，跳过'.format(xiaoqu_name))
22 continue
23 else:
24 for j in range(grid_data.shape[0]):
25 lng2 = grid_data.iloc[j, 19]
26 lat2 = grid_data.iloc[j, 20]
27 grid_cd = grid_data.iloc[j, 0]
28 grid_name = grid_data.iloc[j, 1]
29 if str(lng2) == 'nan':
30 continue
31 else:
32 tmp_distance = get_distance(lng1, lat1, lng2, lat2)
33 # print(grid_name,tmp_distance)
34 if tmp_distance == 0:
35 print('{}精确匹配的网格是{}'.format(xiaoqu_name, grid_name))
36 data.iloc[i, 24] = grid_cd
37 print('-'*100)
38 break
39 else:
40 if tmp_distance < min_distance:
41 min_distance = tmp_distance
42 tmp_grid_cd = grid_cd
43 tmp_grid_name = grid_name
44 # print(min_distance, tmp_grid_cd, tmp_grid_name)
45 else:
46 continue
47 else:
48 data.iloc[i, 24] = tmp_grid_cd
49 print('{}模糊匹配的网格是{}'.format(xiaoqu_name, tmp_grid_name))
50 print(min_distance, tmp_grid_cd, tmp_grid_name)
51 print('-'*100)
10、找出区域内top10%均价的房子
1 region_dict = data['行政区'].value_counts().to_dict() 2 top10_list = [] 3 for k, v in region_dict.items(): 4 temp_data = data[data['行政区']==k] 5 temp_data = temp_data.sort_values(by='均价', ascending=False).reset_index() 6 temp_top10 = temp_data.iloc[:int(v*0.1), :] 7 top10_index = temp_top10['index'].to_list() 8 top10_list.extend(top10_index) 9 10 data['是否区域内均价top10%'] = '否' 11 for i in top10_list: 12 data.loc[i, '是否区域内均价top10%'] = '是'
11、判断是否高档小区
1 def if_upscale(df):
2 '''判断是否高档小区'''
3 if df['物业类型'] == '别墅':
4 return '是'
5 elif df['均价'] <=30000:
6 return '否'
7 elif df['小区年龄'] <= 10 and df['环线位置'] in ('内环以内', '内中环之间', '中外环之间'):
8 return '是'
9 elif df['物业费'] >= 3:
10 return '是'
11 elif df['是否区域内均价top10%'] == '是':
12 return '是'
13 else:
14 return '否'
15
16 data['是否高档小区'] = data.apply(if_upscale, axis=1)
标签：name,Python,text,region,community,爬取,url,链家,data 来源： https://www.cnblogs.com/cyber1987/p/15865582.html
本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享； 2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关； 3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关； 4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除； 5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。