From faf212844ba2e0fcd7ba70c1851055dcdcf5a83b Mon Sep 17 00:00:00 2001 From: xtaodada Date: Thu, 27 Apr 2023 16:44:27 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20support:=20some=20icons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/python.yml | 1 + fix_data.py | 13 +++++ func/README.md | 3 ++ func/client.py | 2 +- func/fetch_avatars.py | 1 - func/fetch_light_cones.py | 7 ++- models/avatar.py | 2 - models/avatar_config.py | 24 +++++++++ models/light_cone_config.py | 12 +++++ res_func/README.md | 3 ++ res_func/__init__.py | 0 res_func/avatar.py | 96 ++++++++++++++++++++++++++++++++++++ res_func/client.py | 3 ++ res_func/light_cone.py | 61 +++++++++++++++++++++++ res_func/url.py | 10 ++++ 15 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 fix_data.py create mode 100644 func/README.md create mode 100644 models/avatar_config.py create mode 100644 models/light_cone_config.py create mode 100644 res_func/README.md create mode 100644 res_func/__init__.py create mode 100644 res_func/avatar.py create mode 100644 res_func/client.py create mode 100644 res_func/light_cone.py create mode 100644 res_func/url.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ec7fc15..7d492eb 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -25,6 +25,7 @@ jobs: - name: Fetch Remote Files run: | python main.py + python fix_data.py - name: Commit changes uses: EndBug/add-and-commit@v9 diff --git a/fix_data.py b/fix_data.py new file mode 100644 index 0000000..0a08504 --- /dev/null +++ b/fix_data.py @@ -0,0 +1,13 @@ +import asyncio + +from res_func.avatar import fix_avatar_config +from res_func.light_cone import fix_light_cone_config + + +async def main(): + await fix_avatar_config() + await fix_light_cone_config() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/func/README.md b/func/README.md new file mode 100644 index 0000000..82bae62 --- /dev/null +++ b/func/README.md @@ -0,0 +1,3 @@ +# wiki 解析 + +通过米游社官方 wiki 解析部分数据 diff --git a/func/client.py b/func/client.py index 4d00d5f..fdf7efc 100644 --- a/func/client.py +++ b/func/client.py @@ -16,4 +16,4 @@ headers = { 'sec-fetch-site': 'same-site', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', } -client = AsyncClient(headers=headers, timeout=30) +client = AsyncClient(headers=headers, timeout=120.0) diff --git a/func/fetch_avatars.py b/func/fetch_avatars.py index 5934f96..f7b6246 100644 --- a/func/fetch_avatars.py +++ b/func/fetch_avatars.py @@ -27,7 +27,6 @@ async def fetch_avatars(data: Children): avatar = Avatar( id=content.content_id, name=content.title, - icon=content.icon, quality=m_quality, element=m_element, destiny=m_destiny, diff --git a/func/fetch_light_cones.py b/func/fetch_light_cones.py index aa9ae1c..ffe38c9 100644 --- a/func/fetch_light_cones.py +++ b/func/fetch_light_cones.py @@ -1,7 +1,7 @@ import asyncio import re from pathlib import Path -from typing import List +from typing import List, Dict import aiofiles import ujson @@ -15,6 +15,7 @@ from models.light_cone import LightCone, LightConePromote, LightConeItem from models.wiki import Children all_light_cones: List[LightCone] = [] +all_light_cones_name: Dict[str, LightCone] = {} async def fetch_light_cones(data: Children): @@ -32,6 +33,7 @@ async def fetch_light_cones(data: Children): promote=[], ) all_light_cones.append(light_cone) + all_light_cones_name[light_cone.name] = light_cone def parse_promote(light_cone: LightCone, soup: BeautifulSoup) -> None: @@ -109,6 +111,9 @@ async def dump_light_cones(path: Path): async def read_light_cones(path: Path): async with aiofiles.open(path, "r", encoding="utf-8") as f: data = ujson.loads(await f.read()) + all_light_cones.clear() + all_light_cones_name.clear() for light_cone in data: m = LightCone(**light_cone) all_light_cones.append(m) + all_light_cones_name[m.name] = m diff --git a/models/avatar.py b/models/avatar.py index e36cbb2..f069d08 100644 --- a/models/avatar.py +++ b/models/avatar.py @@ -44,8 +44,6 @@ class Avatar(BaseModel): """角色ID""" name: str """名称""" - icon: str - """图标""" quality: Quality """品质""" destiny: Destiny diff --git a/models/avatar_config.py b/models/avatar_config.py new file mode 100644 index 0000000..19d77d2 --- /dev/null +++ b/models/avatar_config.py @@ -0,0 +1,24 @@ +from typing import List + +from pydantic import BaseModel + + +class AvatarName(BaseModel): + Hash: int + + +class AvatarConfig(BaseModel): + name: str = "" + AvatarID: int + AvatarName: AvatarName + AvatarVOTag: str + Release: bool + + +class AvatarIcon(BaseModel): + id: int + """角色ID""" + name: str + """名称""" + icon: List[str] + """图标（从小到大）""" diff --git a/models/light_cone_config.py b/models/light_cone_config.py new file mode 100644 index 0000000..f8698c2 --- /dev/null +++ b/models/light_cone_config.py @@ -0,0 +1,12 @@ +from typing import List + +from pydantic import BaseModel + + +class LightConeIcon(BaseModel): + id: int + """光锥ID""" + name: str + """名称""" + icon: List[str] + """图标（从小到大）""" diff --git a/res_func/README.md b/res_func/README.md new file mode 100644 index 0000000..ea2915f --- /dev/null +++ b/res_func/README.md @@ -0,0 +1,3 @@ +# 素材解析 + +通过 **** 和 https://starrailstation.com 解析图片素材 diff --git a/res_func/__init__.py b/res_func/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/res_func/avatar.py b/res_func/avatar.py new file mode 100644 index 0000000..b65597e --- /dev/null +++ b/res_func/avatar.py @@ -0,0 +1,96 @@ +import asyncio +from pathlib import Path +from typing import Dict, List + +import aiofiles +import ujson +from bs4 import BeautifulSoup, Tag + +from func.fetch_avatars import read_avatars, all_avatars_name, dump_avatars +from .client import client +from .url import avatar_config, text_map, base_station_url, avatar_url +from models.avatar_config import AvatarConfig, AvatarIcon + + +async def fetch_text_map() -> Dict[str, str]: + res = await client.get(text_map) + return res.json() + + +async def fetch_config(text_map: Dict[str, str]) -> List[AvatarConfig]: + res = await client.get(avatar_config) + data = res.json() + datas = [] + for i in data.values(): + a = AvatarConfig(**i) + a.name = text_map[str(a.AvatarName.Hash)] + datas.append(a) + return datas + + +async def parse_station(datas, tag: Tag, avatar: AvatarConfig): + second_pic = f'{base_station_url}{tag.find("img").get("src")}' + html = await client.get(f'{base_station_url}{tag.get("href")}') + soup = BeautifulSoup(html.text, "lxml") + text = soup.find("div", {"class": "a6678 a4af5"}).get("style") + third_pic = f'{base_station_url}{text[text.find("(") + 2:text.find(")") - 1]}' if text else "" + first_pic = f'{base_station_url}{soup.find("img", {"class": "ac39b a6602"}).get("src")}' + datas.append( + AvatarIcon( + id=avatar.AvatarID, + name=avatar.name, + icon=[first_pic, second_pic, third_pic], + ) + ) + + +async def dump_icons(path: Path, datas: List[AvatarIcon]): + data = [icon.dict() for icon in datas] + data.sort(key=lambda x: x["id"]) + async with aiofiles.open(path, "w", encoding="utf-8") as f: + await f.write(ujson.dumps(data, indent=4, ensure_ascii=False)) + + +async def fetch_station(configs_map: Dict[str, AvatarConfig]) -> List[AvatarIcon]: + print("开始获取角色素材") + html = await client.get(avatar_url) + soup = BeautifulSoup(html.text, "lxml") + avatars = soup.find_all("a", {"class": "char-entry-select-option"}) + tasks = [] + datas: List[AvatarIcon] = [] + ktz = False + for avatar in avatars: + name = avatar.find("span").get_text().strip() + if name == "开拓者" and ktz: + continue + if avatar_model := configs_map.get(name): + tasks.append(parse_station(datas, avatar, avatar_model)) + if name == "开拓者": + ktz = True + else: + print(f"未找到角色 {name} 的数据") + await asyncio.gather(*tasks) + return datas + + +async def fix_avatar_config(): + text_map_data = await fetch_text_map() + configs = await fetch_config(text_map_data) + configs_map: Dict[str, AvatarConfig] = {config.name: config for config in configs} + configs_map["开拓者"] = configs_map["{NICKNAME}"] + print(f"读取到原始数据：{list(configs_map.keys())}") + data_path = Path("data") + await read_avatars(data_path / "avatars.json") + for key, value in all_avatars_name.items(): + if key.startswith("开拓者"): + config = configs_map["{NICKNAME}"] + config.name = "开拓者" + else: + config = configs_map.get(key) + if config is None: + print(f"错误：未找到角色 {key} 的配置") + continue + value.id = config.AvatarID + icons = await fetch_station(configs_map) + await dump_icons(data_path / "avatar_icons.json", icons) + await dump_avatars(data_path / "avatars.json") diff --git a/res_func/client.py b/res_func/client.py new file mode 100644 index 0000000..dcf44f3 --- /dev/null +++ b/res_func/client.py @@ -0,0 +1,3 @@ +from httpx import AsyncClient + +client = AsyncClient(timeout=120.0) diff --git a/res_func/light_cone.py b/res_func/light_cone.py new file mode 100644 index 0000000..73398c4 --- /dev/null +++ b/res_func/light_cone.py @@ -0,0 +1,61 @@ +import asyncio +from pathlib import Path +from typing import Dict, List + +import aiofiles +import ujson +from bs4 import BeautifulSoup, Tag + +from func.fetch_light_cones import read_light_cones, all_light_cones_name, dump_light_cones +from .client import client +from .url import base_station_url, light_cone_url +from models.light_cone_config import LightConeIcon + + +async def parse_station(icon: LightConeIcon, tag: Tag): + html = await client.get(f'{base_station_url}{tag.get("href")}') + soup = BeautifulSoup(html.text, "lxml") + first_pic = f'{base_station_url}{soup.find("img", {"class": "standard-icon a6602"}).get("src")}' + second_pic = f'{base_station_url}{soup.find("img", {"class": "a2b16 mobile-only-elem ab8c3"}).get("src")}' + icon.icon = [first_pic, second_pic] + + +async def dump_icons(path: Path, datas: List[LightConeIcon]): + data = [icon.dict() for icon in datas] + data.sort(key=lambda x: x["id"]) + async with aiofiles.open(path, "w", encoding="utf-8") as f: + await f.write(ujson.dumps(data, indent=4, ensure_ascii=False)) + + +async def fetch_station() -> List[LightConeIcon]: + print("开始获取光锥素材") + html = await client.get(light_cone_url) + soup = BeautifulSoup(html.text, "lxml") + light_cones = soup.find_all("a", {"class": "a4041 ae026 a5abc"}) + tasks = [] + datas: List[LightConeIcon] = [] + for light_cone in light_cones: + name = light_cone.find("span").get_text().strip() + url = light_cone.get("href") + nid = int(url.split("/")[-1]) + if light_cone_model := all_light_cones_name.get(name): + light_cone_model.id = nid + else: + print(f"wiki 未找到光锥数据 {name} ，修复 id 失败") + icon = LightConeIcon( + id=nid, + name=name, + icon=[] + ) + datas.append(icon) + tasks.append(parse_station(icon, light_cone)) + await asyncio.gather(*tasks) + return datas + + +async def fix_light_cone_config(): + data_path = Path("data") + await read_light_cones(data_path / "light_cones.json") + icons = await fetch_station() + await dump_icons(data_path / "light_cone_icons.json", icons) + await dump_light_cones(data_path / "light_cones.json") diff --git a/res_func/url.py b/res_func/url.py new file mode 100644 index 0000000..0ad2229 --- /dev/null +++ b/res_func/url.py @@ -0,0 +1,10 @@ +import base64 + +base_data_url = base64.b64decode( + "aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL0RpbWJyZWF0aC9TdGFyUmFpbERhdGEvbWFzdGVyLw==" +).decode("utf-8") +avatar_config = f"{base_data_url}ExcelOutput/AvatarConfig.json" +text_map = f"{base_data_url}TextMap/TextMapCN.json" +base_station_url = "https://starrailstation.com" +avatar_url = f"{base_station_url}/cn/characters" +light_cone_url = f"{base_station_url}/cn/equipment"