From e3c65ffa4653434230cb4a62cce02596d32f569e Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Fri, 10 Nov 2023 21:36:53 +0800 Subject: [PATCH] Add: Switch to ranged characters --- .../switch/OCR_MAP_CHARACTERS.png} | Bin tasks/base/assets/assets_base_main_page.py | 10 - tasks/base/main_page.py | 162 +------------- .../assets/assets_character_switch.py | 15 ++ tasks/character/keywords/__init__.py | 32 ++- tasks/character/keywords/classes.py | 8 +- tasks/character/switch.py | 198 ++++++++++++++++++ tasks/rogue/route/loader.py | 7 +- 8 files changed, 258 insertions(+), 174 deletions(-) rename assets/share/{base/main_page/OCR_CHARACTERS.png => character/switch/OCR_MAP_CHARACTERS.png} (100%) create mode 100644 tasks/character/assets/assets_character_switch.py create mode 100644 tasks/character/switch.py diff --git a/assets/share/base/main_page/OCR_CHARACTERS.png b/assets/share/character/switch/OCR_MAP_CHARACTERS.png similarity index 100% rename from assets/share/base/main_page/OCR_CHARACTERS.png rename to assets/share/character/switch/OCR_MAP_CHARACTERS.png diff --git a/tasks/base/assets/assets_base_main_page.py b/tasks/base/assets/assets_base_main_page.py index bc511da61..3a50ad9ed 100644 --- a/tasks/base/assets/assets_base_main_page.py +++ b/tasks/base/assets/assets_base_main_page.py @@ -3,16 +3,6 @@ from module.base.button import Button, ButtonWrapper # This file was auto-generated, do not modify it manually. To generate: # ``` python -m dev_tools.button_extract ``` -OCR_CHARACTERS = ButtonWrapper( - name='OCR_CHARACTERS', - share=Button( - file='./assets/share/base/main_page/OCR_CHARACTERS.png', - area=(1041, 142, 1205, 436), - search=(1021, 122, 1225, 456), - color=(123, 130, 143), - button=(1041, 142, 1205, 436), - ), -) OCR_MAP_NAME = ButtonWrapper( name='OCR_MAP_NAME', share=Button( diff --git a/tasks/base/main_page.py b/tasks/base/main_page.py index ae9ba7611..7fbd26e7f 100644 --- a/tasks/base/main_page.py +++ b/tasks/base/main_page.py @@ -1,21 +1,14 @@ import re -import cv2 -import numpy as np -from scipy import signal - import module.config.server as server -from module.base.timer import Timer -from module.base.utils import area_center, crop, rgb2luma from module.config.server import VALID_LANG from module.exception import RequestHumanTakeover, ScriptError from module.logger import logger -from module.ocr.ocr import OcrResultButton, OcrWhiteLetterOnComplexBackground -from tasks.base.assets.assets_base_main_page import OCR_CHARACTERS, OCR_MAP_NAME, ROGUE_LEAVE_FOR_NOW +from module.ocr.ocr import OcrWhiteLetterOnComplexBackground +from tasks.base.assets.assets_base_main_page import OCR_MAP_NAME, ROGUE_LEAVE_FOR_NOW from tasks.base.assets.assets_base_page import CLOSE, MAP_EXIT from tasks.base.page import Page, page_gacha, page_main from tasks.base.popup import PopupHandler -from tasks.character.keywords import CharacterList from tasks.daily.assets.assets_daily_trial import START_TRIAL from tasks.map.keywords import KEYWORDS_MAP_PLANE, MapPlane @@ -63,30 +56,13 @@ class OcrPlaneName(OcrWhiteLetterOnComplexBackground): return super().after_process(result) -class OcrCharacterName(OcrWhiteLetterOnComplexBackground): - merge_thres_x = 20 - merge_thres_y = 20 - - def after_process(self, result): - result = result.replace('蛆', '妲') - - return super().after_process(result) - - class MainPage(PopupHandler): # Same as BigmapPlane class # Current plane plane: MapPlane = KEYWORDS_MAP_PLANE.Herta_ParlorCar - character_buttons: list[OcrResultButton] = [] - character_current: CharacterList | None = None _lang_checked = False - @property - def characters(self) -> list[CharacterList]: - characters = [button.matched_keyword for button in self.character_buttons] - return characters - def update_plane(self, lang=None) -> MapPlane | None: """ Pages: @@ -174,140 +150,6 @@ class MainPage(PopupHandler): self.handle_lang_check(page=page_main) return True - def update_characters(self) -> list[CharacterList]: - ocr = OcrCharacterName(OCR_CHARACTERS) - self.character_buttons = ocr.matched_ocr(self.device.image, keyword_classes=CharacterList) - characters = self.characters - logger.attr('Characters', characters) - self.character_current = self._convert_selected_to_character(self._update_current_character()) - return characters - - def _update_current_character(self) -> list[int]: - """ - Returns: - list[int]: Selected index, 1 to 4. - """ - # 50px-width area starting from the right edge of HP bars - area = (1101, 151, 1151, 459) - # Y coordinates where the color peaks should be when character is selected - expected_peaks = np.array([201, 279, 357, 435]) - expected_peaks_in_area = expected_peaks - area[1] - # Use Luminance to fit H264 video stream - image = rgb2luma(crop(self.device.image, area)) - # Remove character names - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) - image = cv2.erode(image, kernel) - # To find peaks along Y - line = cv2.reduce(image, 1, cv2.REDUCE_AVG).flatten().astype(int) - - # Find color peaks - parameters = { - 'height': (60, 255), - 'prominence': 30, - 'distance': 5, - } - peaks, _ = signal.find_peaks(line, **parameters) - # Remove smooth peaks - parameters = { - 'height': (5, 255), - 'prominence': 5, - 'distance': 5, - } - diff = -np.diff(line) - diff_peaks, _ = signal.find_peaks(diff, **parameters) - - def is_steep_peak(y, threshold=5): - return np.abs(diff_peaks - y).min() <= threshold - - def peak_to_selected(y, threshold=5): - distance = np.abs(expected_peaks_in_area - y) - return np.argmin(distance) + 1 if distance.min() < threshold else 0 - - selected = [peak_to_selected(peak) for peak in peaks if peak_to_selected(peak) and is_steep_peak(peak)] - logger.attr('CharacterSelected', selected) - return selected - - def _convert_selected_to_character(self, selected: list[int]) -> CharacterList | None: - expected_peaks = np.array([201, 279, 357, 435]) - if not selected: - logger.warning(f'No current character') - logger.attr('CurrentCharacter', None) - return None - elif len(selected) == 1: - selected = selected[0] - else: - logger.warning(f'Too many current characters: {selected}, using first') - selected = selected[0] - - expected_y = expected_peaks[selected - 1] - for button in self.character_buttons: - y = area_center(button.area)[1] - if expected_y - 78 < y < expected_y: - logger.attr('CurrentCharacter', button.matched_keyword) - return button.matched_keyword - - logger.warning(f'Current character: {selected} does not belong to any detected character') - logger.attr('CurrentCharacter', None) - return None - - def character_switch(self, character: CharacterList | str | int, skip_first_screenshot=True) -> bool: - """ - Args: - character: CharacterList object, or character name, or select index from 1 to 4. - skip_first_screenshot: - - Returns: - bool: If chose - """ - logger.info(f'Character choose: {character}') - characters = self.characters - if isinstance(character, int): - character = self._convert_selected_to_character([character]) - if character is None: - return False - try: - index = characters.index(character) + 1 - except IndexError: - logger.warning(f'Cannot choose character {character} as it was not detected') - return False - else: - if isinstance(character, str): - character = CharacterList.find(character) - try: - index = characters.index(character) + 1 - except IndexError: - logger.warning(f'Cannot choose character {character} as it was not detected') - return False - - button = self.character_buttons[index - 1] - interval = Timer(1, count=3) - count = 0 - while 1: - if skip_first_screenshot: - skip_first_screenshot = False - else: - self.device.screenshot() - - # End - selected = self._update_current_character() - if index in selected: - logger.info('Character chose') - return True - if count > 3: - logger.warning('Failed to choose character, assume chose') - return False - - try: - is_in_main = self.is_in_main - except AttributeError: - logger.critical('Method ui_goto() not found, class MainPage must be inherited by class UI') - raise ScriptError - - if interval.reached() and is_in_main(): - self.device.click(button) - interval.reset() - count += 1 - def ui_leave_special(self): """ Leave from: diff --git a/tasks/character/assets/assets_character_switch.py b/tasks/character/assets/assets_character_switch.py new file mode 100644 index 000000000..7a7fbdd99 --- /dev/null +++ b/tasks/character/assets/assets_character_switch.py @@ -0,0 +1,15 @@ +from module.base.button import Button, ButtonWrapper + +# This file was auto-generated, do not modify it manually. To generate: +# ``` python -m dev_tools.button_extract ``` + +OCR_MAP_CHARACTERS = ButtonWrapper( + name='OCR_MAP_CHARACTERS', + share=Button( + file='./assets/share/character/switch/OCR_MAP_CHARACTERS.png', + area=(1041, 142, 1205, 436), + search=(1021, 122, 1225, 456), + color=(123, 130, 143), + button=(1041, 142, 1205, 436), + ), +) diff --git a/tasks/character/keywords/__init__.py b/tasks/character/keywords/__init__.py index 83cecbfaf..3ee0bd3dd 100644 --- a/tasks/character/keywords/__init__.py +++ b/tasks/character/keywords/__init__.py @@ -1,2 +1,32 @@ import tasks.character.keywords.character_list as KEYWORD_CHARACTER_LIST -from tasks.character.keywords.classes import CharacterList \ No newline at end of file +from tasks.character.keywords.character_list import * +from tasks.character.keywords.classes import CharacterList + +DICT_SORTED_RANGES = { + # Mage, hit instantly, no trajectories + 'Mage': [ + DanHengImbibitorLunae, + Welt, + FuXuan, + ], + # Mage, but character moved after attack + 'MageSecondary': [ + Yanqing, + ], + # Archer + 'Archer': [ + Yukong, + TopazandNumby, + March7th, + Bronya, + Asta, + Pela, + Qingque, + ], + # Archer, but her parabolic trajectory has 0% accuracy on moving targets + 'ArcherSecondary': [ + Natasha, + ], + # Melee + # rest of the characters are classified as melee and will not be switched to +} diff --git a/tasks/character/keywords/classes.py b/tasks/character/keywords/classes.py index 6bd84c3d6..925e90908 100644 --- a/tasks/character/keywords/classes.py +++ b/tasks/character/keywords/classes.py @@ -1,8 +1,14 @@ from dataclasses import dataclass from typing import ClassVar +from module.base.decorator import cached_property from module.ocr.keyword import Keyword + @dataclass(repr=False) class CharacterList(Keyword): - instances: ClassVar = {} \ No newline at end of file + instances: ClassVar = {} + + @cached_property + def is_trailblazer(self) -> bool: + return 'Trailblazer' in self.name diff --git a/tasks/character/switch.py b/tasks/character/switch.py new file mode 100644 index 000000000..a7b237eb1 --- /dev/null +++ b/tasks/character/switch.py @@ -0,0 +1,198 @@ +import cv2 +import numpy as np +from scipy import signal + +from module.base.timer import Timer +from module.base.utils import area_center, crop, rgb2luma +from module.logger import logger +from module.ocr.ocr import OcrResultButton, OcrWhiteLetterOnComplexBackground +from tasks.base.ui import UI +from tasks.character.assets.assets_character_switch import OCR_MAP_CHARACTERS +from tasks.character.keywords import CharacterList, DICT_SORTED_RANGES + + +class OcrCharacterName(OcrWhiteLetterOnComplexBackground): + merge_thres_x = 20 + merge_thres_y = 20 + + def after_process(self, result): + result = result.replace('蛆', '妲') + + return super().after_process(result) + + +class CharacterSwitch(UI): + characters: list[CharacterList] = [] + character_current: CharacterList | None = None + character_buttons: list[OcrResultButton] = [] + + def character_update(self) -> list[CharacterList]: + """ + The following properties will be updated: + - self.characters + - self.character_current + - self.character_buttons + """ + ocr = OcrCharacterName(OCR_MAP_CHARACTERS) + self.character_buttons = ocr.matched_ocr(self.device.image, keyword_classes=CharacterList) + self.characters = [button.matched_keyword for button in self.character_buttons] + logger.attr('Characters', self.characters) + self.character_current = self._convert_selected_to_character(self._update_current_character()) + return self.characters + + def _update_current_character(self) -> list[int]: + """ + Returns: + list[int]: Selected index, 1 to 4. + """ + # 50px-width area starting from the right edge of HP bars + area = (1101, 151, 1151, 459) + # Y coordinates where the color peaks should be when character is selected + expected_peaks = np.array([201, 279, 357, 435]) + expected_peaks_in_area = expected_peaks - area[1] + # Use Luminance to fit H264 video stream + image = rgb2luma(crop(self.device.image, area)) + # Remove character names + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) + image = cv2.erode(image, kernel) + # To find peaks along Y + line = cv2.reduce(image, 1, cv2.REDUCE_AVG).flatten().astype(int) + + # Find color peaks + parameters = { + 'height': (60, 255), + 'prominence': 30, + 'distance': 5, + } + peaks, _ = signal.find_peaks(line, **parameters) + # Remove smooth peaks + parameters = { + 'height': (5, 255), + 'prominence': 5, + 'distance': 5, + } + diff = -np.diff(line) + diff_peaks, _ = signal.find_peaks(diff, **parameters) + + def is_steep_peak(y, threshold=5): + return np.abs(diff_peaks - y).min() <= threshold + + def peak_to_selected(y, threshold=5): + distance = np.abs(expected_peaks_in_area - y) + return np.argmin(distance) + 1 if distance.min() < threshold else 0 + + selected = [peak_to_selected(peak) for peak in peaks if peak_to_selected(peak) and is_steep_peak(peak)] + logger.attr('CharacterSelected', selected) + return selected + + def _convert_selected_to_character(self, selected: list[int]) -> CharacterList | None: + expected_peaks = np.array([201, 279, 357, 435]) + if not selected: + logger.warning(f'No current character') + logger.attr('CurrentCharacter', None) + return None + elif len(selected) == 1: + selected = selected[0] + else: + logger.warning(f'Too many current characters: {selected}, using first') + selected = selected[0] + + expected_y = expected_peaks[selected - 1] + for button in self.character_buttons: + y = area_center(button.area)[1] + if expected_y - 78 < y < expected_y: + logger.attr('CurrentCharacter', button.matched_keyword) + return button.matched_keyword + + logger.warning(f'Current character: {selected} does not belong to any detected character') + logger.attr('CurrentCharacter', None) + return None + + def character_switch(self, character: CharacterList | str | int, skip_first_screenshot=True) -> bool: + """ + character_update() must be called before switching. + + Args: + character: CharacterList object, or character name, or select index from 1 to 4. + skip_first_screenshot: + + Returns: + bool: If chose + """ + logger.info(f'Character choose: {character}') + if isinstance(character, int): + character = self._convert_selected_to_character([character]) + if character is None: + return False + try: + index = self.characters.index(character) + 1 + except IndexError: + logger.warning(f'Cannot choose character {character} as it was not detected') + return False + else: + if isinstance(character, str): + character = CharacterList.find(character) + try: + index = self.characters.index(character) + 1 + except IndexError: + logger.warning(f'Cannot choose character {character} as it was not detected') + return False + + button = self.character_buttons[index - 1] + interval = Timer(1, count=3) + count = 0 + while 1: + if skip_first_screenshot: + skip_first_screenshot = False + else: + self.device.screenshot() + + # End + selected = self._update_current_character() + if index in selected: + logger.info('Character chose') + return True + if count > 3: + logger.warning('Failed to choose character, assume chose') + return False + + if interval.reached() and self.is_in_main(): + self.device.click(button) + interval.reset() + count += 1 + + def _get_ranged_character(self) -> CharacterList | bool: + # Check if it's using a ranged character already + for level, character_list in DICT_SORTED_RANGES.items(): + if self.character_current in character_list: + logger.info(f'Already using a ranged character: {self.character_current}, range={level}') + return True + # Check if there is a ranged character in team + for level, character_list in DICT_SORTED_RANGES.items(): + for ranged_character in character_list: + if ranged_character in self.characters: + logger.info(f'Use ranged character: {ranged_character}, range={level}') + return ranged_character + # No ranged characters + logger.info('No ranged characters in team') + return False + + def character_switch_to_ranged(self, update=True) -> bool: + """ + Args: + update: If update characters before switching + + Returns: + bool: If using a ranged character now + """ + logger.hr('Character switch to ranged') + if update: + self.character_update() + + character = self._get_ranged_character() + if character is True: + return True + elif character is False: + return False + else: + return self.character_switch(character) diff --git a/tasks/rogue/route/loader.py b/tasks/rogue/route/loader.py index eed2f867e..c17192aac 100644 --- a/tasks/rogue/route/loader.py +++ b/tasks/rogue/route/loader.py @@ -5,7 +5,7 @@ import numpy as np from module.base.decorator import cached_property from module.base.timer import Timer from module.logger import logger -from tasks.base.main_page import MainPage +from tasks.character.switch import CharacterSwitch from tasks.map.keywords import MapPlane from tasks.map.keywords.plane import ( Herta_MasterControlZone, @@ -75,7 +75,7 @@ class MinimapWrapper: return self.all_minimap[route.plane_floor] -class RouteLoader(RogueUI, MinimapWrapper, RouteLoader_, MainPage): +class RouteLoader(RogueUI, MinimapWrapper, RouteLoader_, CharacterSwitch): def position_find_known(self, image, force_return=False) -> Optional[RogueRouteModel]: """ Try to find from known route spawn point @@ -217,6 +217,9 @@ class RouteLoader(RogueUI, MinimapWrapper, RouteLoader_, MainPage): logger.hr(f'Route run: {count}', level=1) base.clear_blessing() + if count == 1: + self.character_switch_to_ranged(update=True) + self.route_run() # if not success: # self.device.image_save()