import csv
from bs4 import BeautifulSoup
import requests
href_list = []
for i in range(1,101):
url = f'https://sz.lianjia.com/ershoufang/pg{i}rs%E6%B7%B1%E5%9C%B3/'
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
}
data = requests.get(url=url, headers=headers).text
soup = BeautifulSoup(data,'lxml')
a_tags = soup.find_all('a', class_='noresultRecommend')
for a_tag in a_tags:
href = a_tag.get('href')
href_list.append(href)
with open('深圳2024年链家二手房信息_1.csv', mode='w', newline='', encoding='utf-8') as csv_file:
fieldnames = [
'title', 'area', 'community', 'position', 'total_price',
'unit_price', 'hourseType', 'hourseSize', 'direction', 'fitment'
]
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
for j in range(len(href_list)):
try:
print(href_list[j])
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
}
data = requests.get(url=href_list[j], headers=headers).text
soup = BeautifulSoup(data,'lxml')
#title
title_h1 = soup.find('h1',class_='main')
for titles in title_h1:
title = titles
#area
area_span = soup.find('span',class_='info')
area_a = area_span.find('a').text
for position_a in area_span.find_all('a')[-1]:
position = position_a
#community
community = soup.find('a',class_='info').text
#total_price
try:
total_price_div = soup.find('div',class_='price')
for total_price_span in total_price_div.find('span',class_='total'):
total_price = total_price_span
except:
total_price = None
# unit_price
try:
unit_price_span = soup.find('span',class_='unitPriceValue')
unit_price = unit_price_span.get_text(strip=True, separator=" ")[:-4]
except:
unit_price = None
#hourseType
hourseType = soup.find('div',class_='mainInfo').text
#hourseSize
hourseSize_div = soup.find('div',class_='area')
for hourseSize_divs in hourseSize_div.find('div',class_='mainInfo'):
hourseSize = hourseSize_divs
#direction
direction_div = soup.find('div',class_='type')
for direction_divs in direction_div.find('div',class_='mainInfo'):
direction = direction_divs
#fitment
fitment_div = soup.find('div',class_='type')
for fitment_divs in fitment_div.find('div',class_='subInfo'):
fitment = fitment_divs[-2:]
row_data = {
'title':title,
'area':area_a,
'community':community,
'position':position,
'total_price':total_price,
'unit_price':unit_price,
'hourseType':hourseType,
'hourseSize':hourseSize,
'direction':direction,
'fitment':fitment
}
writer.writerow(row_data)
except:
pass
