From 35f5d248772cb3cd2b7b7d70917d34d31be55408 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 31 May 2023 02:20:45 +0800 Subject: [PATCH] Add: Switching dungeon nav --- dev_tools/keyword_extract.py | 2 +- module/base/base.py | 30 ++++- module/base/button.py | 33 ++++- module/ocr/keyword.py | 10 +- module/ui/draggable_list.py | 197 ++++++++++++++++++++++++++++++ tasks/dungeon/keywords/classes.py | 3 + tasks/dungeon/keywords/nav.py | 9 +- tasks/dungeon/ui.py | 6 +- 8 files changed, 280 insertions(+), 10 deletions(-) create mode 100644 module/ui/draggable_list.py diff --git a/dev_tools/keyword_extract.py b/dev_tools/keyword_extract.py index 20ac9495f..6f1016737 100644 --- a/dev_tools/keyword_extract.py +++ b/dev_tools/keyword_extract.py @@ -97,7 +97,7 @@ class KeywordExtract: def generate(): ex = KeywordExtract() - ex.load_keywords(['模拟宇宙', '拟造花萼（金）', '拟造花萼（赤）', '凝滞虚影', '侵蚀隧洞', '忘却之庭']) + ex.load_keywords(['模拟宇宙', '拟造花萼（金）', '拟造花萼（赤）', '凝滞虚影', '侵蚀隧洞', '历战余响', '忘却之庭']) ex.write_keywords(keyword_class='DungeonNav', output_file='./tasks/dungeon/keywords/nav.py') ex.load_keywords(['行动摘要', '生存索引', '每日实训']) ex.write_keywords(keyword_class='DungeonTab', output_file='./tasks/dungeon/keywords/tab.py') diff --git a/module/base/base.py b/module/base/base.py index 27fd0653d..c44c758c5 100644 --- a/module/base/base.py +++ b/module/base/base.py @@ -1,4 +1,4 @@ -from module.base.button import Button, ButtonWrapper, ClickButton +from module.base.button import Button, ButtonWrapper, ClickButton, match_template from module.base.timer import Timer from module.base.utils import * from module.config.config import AzurLaneConfig @@ -127,6 +127,30 @@ class ModuleBase: self.device.click(button) return appear + def wait_until_stable(self, button, timer=Timer(0.3, count=1), timeout=Timer(5, count=10)): + """ + A terrible method, don't rely too much on it. + """ + logger.info(f'Wait until stable: {button}') + prev_image = self.image_crop(button) + timer.reset() + timeout.reset() + while 1: + self.device.screenshot() + + if timeout.reached(): + logger.warning(f'wait_until_stable({button}) timeout') + break + + image = self.image_crop(button) + if match_template(image, prev_image): + if timer.reached(): + logger.info(f'{button} stabled') + break + else: + prev_image = image + timer.reset() + def image_crop(self, button): """Extract the area from image. @@ -135,7 +159,9 @@ class ModuleBase: """ if isinstance(button, Button): return crop(self.device.image, button.area) - if isinstance(button, ButtonWrapper): + elif isinstance(button, ButtonWrapper): + return crop(self.device.image, button.area) + elif hasattr(button, 'area'): return crop(self.device.image, button.area) else: return crop(self.device.image, button) diff --git a/module/base/button.py b/module/base/button.py index cf9d1ee0e..34ec111d0 100644 --- a/module/base/button.py +++ b/module/base/button.py @@ -182,21 +182,29 @@ class ButtonWrapper(Resource): return self._matched_button @property - def area(self): + def area(self) -> tuple[int, int, int, int]: return self.matched_button.area @property - def search(self): + def search(self) -> tuple[int, int, int, int]: return self.matched_button.search @property - def color(self): + def color(self) -> tuple[int, int, int]: return self.matched_button.color @property - def button(self): + def button(self) -> tuple[int, int, int, int]: return self.matched_button.button + @property + def width(self) -> int: + return area_size(self.area)[0] + + @property + def height(self) -> int: + return area_size(self.area)[1] + class ClickButton: def __init__(self, button, name='CLICK_BUTTON'): @@ -216,3 +224,20 @@ class ClickButton: def __bool__(self): return True + + +def match_template(image, template, similarity=0.85): + """ + Args: + image (np.ndarray): Screenshot + template (np.ndarray): + area (tuple): Crop area of image. + offset (int, tuple): Detection area offset. + similarity (float): 0-1. Similarity. Lower than this value will return float(0). + + Returns: + bool: + """ + res = cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED) + _, sim, _, point = cv2.minMaxLoc(res) + return sim > similarity diff --git a/module/ocr/keyword.py b/module/ocr/keyword.py index 599045cca..9ee2bcc2d 100644 --- a/module/ocr/keyword.py +++ b/module/ocr/keyword.py @@ -84,7 +84,7 @@ class Keyword: return [self.jp_parsed] else: return [self.jp] - case 'tw': + case 'cht': if ignore_punctuation: return [self.cht_parsed] else: @@ -107,6 +107,14 @@ class Keyword: """ Class attributes and methods + + Note that dataclasses inherited `Keyword` must override `instances` attribute, + or `instances` will still be a class attribute of base class. + ``` + @dataclass + class DungeonNav(Keyword): + instances: ClassVar = {} + ``` """ instances: ClassVar = {} diff --git a/module/ui/draggable_list.py b/module/ui/draggable_list.py new file mode 100644 index 000000000..b87b064ad --- /dev/null +++ b/module/ui/draggable_list.py @@ -0,0 +1,197 @@ +from typing import Optional + +import numpy as np + +from module.base.base import ModuleBase +from module.base.button import ButtonWrapper +from module.base.timer import Timer +from module.base.utils import area_size +from module.logger import logger +from module.ocr.keyword import Keyword +from module.ocr.ocr import OcrResultButton + + +class DraggableList: + """ + A wrapper to handle draggable lists like + - Simulated Universe + - Calyx (Golden) + - Calyx (Crimson) + - Stagnant Shadow + - Cavern of Corrosion + """ + + def __init__( + self, + name, + keyword_class, + ocr_class, + search_button: ButtonWrapper, + ): + """ + Args: + name: + keyword_class: Keyword + search_button: + """ + self.name = name + self.keyword_class = keyword_class + self.ocr_class = ocr_class + self.known_rows = list(keyword_class.instances.values()) + self.search_button = search_button + + self.row_min = 1 + self.row_max = len(self.known_rows) + self.cur_min = 1 + self.cur_max = 1 + self.cur_buttons: list[OcrResultButton] = [] + + self.drag_vector = (0.65, 0.85) + + def __str__(self): + return f'DraggableList({self.name})' + + __repr__ = __str__ + + def __eq__(self, other): + return str(self) == str(other) + + def __hash__(self): + return hash(self.name) + + def keyword2index(self, row: Keyword) -> int: + try: + return self.known_rows.index(row) + 1 + except ValueError: + logger.warning(f'Row "{row}" does not belong to {self}') + return 0 + + def keyword2button(self, row: Keyword) -> Optional[OcrResultButton]: + for button in self.cur_buttons: + if button == row: + return button + + logger.warning(f'Keyword {row} is not in current rows of {self}') + logger.warning(f'Current rows: {self.cur_buttons}') + return None + + def load_rows(self, main: ModuleBase): + """ + Parse current rows to get list position. + """ + self.cur_buttons = self.ocr_class(self.search_button) \ + .matched_ocr(main.device.image, keyword_class=self.keyword_class) + # Get indexes + indexes = [self.keyword2index(row.matched_keyword) for row in self.cur_buttons] + indexes = [index for index in indexes if index] + # Check row order + if len(indexes) >= 2: + if not np.all(np.diff(indexes) > 0): + logger.warning(f'Rows given to {self} are not ascending sorted') + if not indexes: + logger.warning(f'No valid rows loaded into {self}') + return + + self.cur_min = min(indexes) + self.cur_max = max(indexes) + logger.attr(self.name, f'{self.cur_min} - {self.cur_max}') + + def _page_drag(self, direction: str, main: ModuleBase): + vector = np.random.uniform(*self.drag_vector) + width, height = area_size(self.search_button.button) + if direction == 'down': + vector = (0, vector * height) + elif direction == 'up': + vector = (0, -vector * height) + elif direction == 'left': + vector = (-vector * width, 0) + elif direction == 'right': + vector = (vector * width, 0) + else: + logger.warning(f'Unknown drag direction: {direction}') + return + main.device.swipe_vector( + vector, box=self.search_button.button, random_range=(-10, -10, 10, 10), name=f'{self.name}_DRAG') + + def insight_row(self, row: Keyword, main: ModuleBase, skip_first_screenshot=True) -> bool: + """ + Args: + row: + main: + skip_first_screenshot: + + Returns: + If success + """ + row_index = self.keyword2index(row) + if not row_index: + logger.warning(f'Insight row {row} but index unknown') + return False + + logger.info(f'Insight row: {row}, index={row_index}') + while 1: + if skip_first_screenshot: + skip_first_screenshot = False + else: + main.device.screenshot() + + self.load_rows(main=main) + + # End + if self.cur_min <= row_index <= self.cur_max: + break + + # Drag pages + if row_index < self.cur_min: + self._page_drag('down', main=main) + elif self.cur_max < row_index: + self._page_drag('up', main=main) + # Wait for bottoming out + main.wait_until_stable(self.search_button, timer=Timer(0, count=0), timeout=Timer(1.5, count=5)) + skip_first_screenshot = True + + return True + + def is_row_selected(self, row: Keyword, main: ModuleBase) -> bool: + button = self.keyword2button(row) + if not button: + return False + + # Having gold letters + if main.image_color_count(button, color=(190, 175, 124), threshold=221, count=50): + return True + + return False + + def select_row(self, row: Keyword, main: ModuleBase, skip_first_screenshot=True): + """ + Args: + row: + main: + skip_first_screenshot: + + Returns: + If success + """ + result = self.insight_row(row, main=main, skip_first_screenshot=skip_first_screenshot) + if not result: + return False + + logger.info(f'Select row: {row}') + skip_first_screenshot = True + interval = Timer(5) + while 1: + if skip_first_screenshot: + skip_first_screenshot = False + else: + main.device.screenshot() + + # End + if self.is_row_selected(row, main=main): + logger.info('Row selected') + break + + # Click + if interval.reached(): + main.device.click(self.keyword2button(row)) + interval.reset() diff --git a/tasks/dungeon/keywords/classes.py b/tasks/dungeon/keywords/classes.py index e237cd8f8..daf2be228 100644 --- a/tasks/dungeon/keywords/classes.py +++ b/tasks/dungeon/keywords/classes.py @@ -1,13 +1,16 @@ from dataclasses import dataclass +from typing import ClassVar from module.ocr.keyword import Keyword @dataclass class DungeonNav(Keyword): + instances: ClassVar = {} pass @dataclass class DungeonTab(Keyword): + instances: ClassVar = {} pass diff --git a/tasks/dungeon/keywords/nav.py b/tasks/dungeon/keywords/nav.py index cbc743073..f3ffe90fc 100644 --- a/tasks/dungeon/keywords/nav.py +++ b/tasks/dungeon/keywords/nav.py @@ -38,8 +38,15 @@ Cavern_of_Corrosion = DungeonNav( en='Cavern of Corrosion', jp='侵蝕トンネル', ) -Forgotten_Hall = DungeonNav( +Echo_of_War = DungeonNav( id=6, + cn='历战余响', + cht='歷戰餘響', + en='Echo of War', + jp='歴戦余韻', +) +Forgotten_Hall = DungeonNav( + id=7, cn='忘却之庭', cht='忘卻之庭', en='Forgotten Hall', diff --git a/tasks/dungeon/ui.py b/tasks/dungeon/ui.py index dce21e63b..668f0c3b7 100644 --- a/tasks/dungeon/ui.py +++ b/tasks/dungeon/ui.py @@ -3,11 +3,13 @@ import numpy as np from module.base.timer import Timer from module.base.utils import get_color from module.logger import logger +from module.ocr.ocr import Ocr +from module.ui.draggable_list import DraggableList from module.ui.switch import Switch from tasks.base.page import page_guide from tasks.base.ui import UI from tasks.dungeon.assets.assets_dungeon_ui import * -from tasks.dungeon.keywords import DungeonTab, KEYWORDS_DUNGEON_TAB +from tasks.dungeon.keywords import DungeonNav, DungeonTab, KEYWORDS_DUNGEON_NAV, KEYWORDS_DUNGEON_TAB class DungeonTabSwitch(Switch): @@ -38,6 +40,8 @@ SWITCH_DUNGEON_TAB.add_state( check_button=SURVIVAL_INDEX_CHECK, click_button=SURVIVAL_INDEX_CLICK ) +DUNGEON_NAV_LIST = DraggableList( + 'DungeonNavList', keyword_class=DungeonNav, ocr_class=Ocr, search_button=OCR_DUNGEON_NAV) class DungeonUI(UI):