2024-05-31 02:43:15 +00:00
|
|
|
import pandas as pd
|
|
|
|
|
import csv
|
|
|
|
|
import requests
|
|
|
|
|
from fake_useragent import UserAgent
|
|
|
|
|
ua = UserAgent()
|
|
|
|
|
headers = {
|
|
|
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
|
|
|
'Accept-Encoding': 'gzip, deflate, sdch',
|
|
|
|
|
'Accept-Language': 'zh-CN,zh;q=0.8',
|
|
|
|
|
'Connection': 'keep-alive',
|
|
|
|
|
'User-Agent': ua.random
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_response(url):
|
|
|
|
|
response = requests.get(url, headers=headers)
|
|
|
|
|
return response
|
|
|
|
|
|
|
|
|
|
# 用于存储网址的一维列表
|
|
|
|
|
url_list = []
|
|
|
|
|
|
|
|
|
|
# 读取CSV文件并将网址存储到列表中
|
|
|
|
|
with open('cities_urls.csv', newline='', encoding='utf-8') as csvfile:
|
|
|
|
|
reader = csv.reader(csvfile)
|
|
|
|
|
# 跳过表头
|
|
|
|
|
next(reader, None)
|
|
|
|
|
for row in reader:
|
|
|
|
|
url_list.append(row[0])
|
|
|
|
|
|
|
|
|
|
# 爬取data_now.csv数据
|
|
|
|
|
weather_data = []
|
|
|
|
|
|
|
|
|
|
#爬取data_hour.csv数据
|
|
|
|
|
weather_hour = []
|
|
|
|
|
|
2024-05-31 03:02:23 +00:00
|
|
|
#爬取data_nextDay.csv数据
|
|
|
|
|
weather_nexts = []
|
|
|
|
|
|
2024-05-31 02:43:15 +00:00
|
|
|
for index in url_list:
|
|
|
|
|
print(index)
|
|
|
|
|
response = get_response(index)
|
|
|
|
|
data = response.json()['data']
|
|
|
|
|
city_weather = {
|
|
|
|
|
'city_index': data['real']['station']['code'],
|
|
|
|
|
'city_name' : data['predict']['station']['city'],
|
|
|
|
|
'time' : data['real']['publish_time'],
|
|
|
|
|
'temperature': data['real']['weather']['temperature'],
|
|
|
|
|
'humidity' :data['real']['weather']['humidity'],
|
|
|
|
|
'rain' : data['real']['weather']['rain'],
|
|
|
|
|
'wind_direction' : data['real']['wind']['direct'],
|
|
|
|
|
'wind_speed' : data['real']['wind']['speed'],
|
|
|
|
|
'info' : data['real']['weather']['info'],
|
|
|
|
|
'aqi': data['air']['aqi'] if 'air' in data and 'aqi' in data['air'] else None,
|
|
|
|
|
'sqiText': data['air']['text'] if 'air' in data and 'text' in data['air'] else None
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
for i in range(0,24):
|
|
|
|
|
city_hour = {
|
|
|
|
|
'city_index': data['real']['station']['code'],
|
|
|
|
|
'city_name': data['predict']['station']['city'],
|
|
|
|
|
'time' : data['passedchart'][i]['time'],
|
|
|
|
|
'temperature': data['passedchart'][i]['temperature'],
|
|
|
|
|
'humidity': data['passedchart'][i]['humidity'],
|
|
|
|
|
'rain1h' : data['passedchart'][i]['rain1h'],
|
|
|
|
|
'pressure' : data['passedchart'][i]['pressure'],
|
|
|
|
|
'windDirection' : data['passedchart'][i]['windDirection'],
|
|
|
|
|
'windSpeed' : data['passedchart'][i]['windSpeed']
|
|
|
|
|
}
|
|
|
|
|
weather_hour.append(city_hour)
|
|
|
|
|
|
2024-05-31 03:02:23 +00:00
|
|
|
for j in range(0,14):
|
|
|
|
|
city_next = {
|
|
|
|
|
'city_index': data['real']['station']['code'],
|
|
|
|
|
'city_name': data['predict']['station']['city'],
|
|
|
|
|
'time' : data['tempchart'][j]['time'],
|
|
|
|
|
'maxtemp': data['tempchart'][j]['max_temp'],
|
|
|
|
|
'mintemp': data['tempchart'][j]['min_temp'],
|
|
|
|
|
'day_text' : data['tempchart'][j]['day_text'],
|
|
|
|
|
'night_text' : data['tempchart'][j]['night_text']
|
|
|
|
|
}
|
|
|
|
|
weather_nexts.append(city_next)
|
2024-05-31 02:43:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
weather_data.append(city_weather)
|
|
|
|
|
# 将列表数据转换为 DataFrame
|
|
|
|
|
df1 = pd.DataFrame(weather_data)
|
|
|
|
|
# 将 DataFrame 写入 CSV 文件并包含表头
|
|
|
|
|
df1.to_csv('data_now.csv', index=False)
|
|
|
|
|
|
|
|
|
|
df2 = pd.DataFrame(weather_hour)
|
2024-05-31 03:02:23 +00:00
|
|
|
df2.to_csv('data_hour.csv', index=False)
|
|
|
|
|
|
|
|
|
|
df3 = pd.DataFrame(weather_nexts)
|
|
|
|
|
df3.to_csv('data_next.csv', index=False)
|
