Fix: AdaptiveScroll to handle transparent scroll on complex background (#391)

module/ui/scroll.py

@@ -1,9 +1,10 @@
 import numpy as np
+from scipy import signal
 
 from module.base.base import ModuleBase
 from module.base.button import Button, ButtonWrapper
 from module.base.timer import Timer
-from module.base.utils import color_similarity_2d, random_rectangle_point
+from module.base.utils import color_similarity_2d, random_rectangle_point, rgb2gray
 from module.logger import logger
 
 
@@ -46,7 +47,7 @@ class Scroll:
         Returns:
             np.ndarray: Shape (n,), dtype bool.
         """
-        image = main.image_crop(self.area)
+        image = main.image_crop(self.area, copy=False)
         image = color_similarity_2d(image, color=self.color)
         mask = np.max(image, axis=1 if self.is_vertical else 0) > self.color_threshold
         self.length = np.sum(mask)
@@ -210,3 +211,49 @@ class Scroll:
 
     def prev_page(self, main, page=0.8, random_range=(-0.01, 0.01), skip_first_screenshot=True):
         return self.drag_page(-page, main=main, random_range=random_range, skip_first_screenshot=skip_first_screenshot)
+
+
+class AdaptiveScroll(Scroll):
+    def __init__(self, area, parameters: dict = None, background=5, is_vertical=True, name='Scroll'):
+        """
+        Args:
+            area (Button, tuple): A button or area of the whole scroll.
+            prominence (dict): Parameters passing to scipy.find_peaks
+            background (int):
+            is_vertical (bool): True if vertical, false if horizontal.
+            name (str):
+        """
+        if parameters is None:
+            parameters = {}
+        self.parameters = parameters
+        self.background = background
+        super().__init__(area, color=(255, 255, 255), is_vertical=is_vertical, name=name)
+
+    def match_color(self, main):
+        if self.is_vertical:
+            area = (self.area[0] - self.background, self.area[1], self.area[2] + self.background, self.area[3])
+            image = main.image_crop(area, copy=False)
+            image = rgb2gray(image)
+            image = image.flatten()
+            wlen = area[2] - area[0]
+        else:
+            area = (self.area[0], self.area[1] - self.background, self.area[2], self.area[3] + self.background)
+            image = main.image_crop(area, copy=False)
+            image = rgb2gray(image)
+            image = image.flatten('F')
+            wlen = area[3] - area[1]
+
+        parameters = {
+            'height': 128,
+            'prominence': 30,
+            'wlen': wlen,
+            'width': 2,
+        }
+        parameters.update(self.parameters)
+        peaks, _ = signal.find_peaks(image, **parameters)
+        peaks //= wlen
+
+        self.length = len(peaks)
+        mask = np.zeros((self.total,), dtype=np.bool_)
+        mask[peaks] = 1
+        return mask

tasks/combat/support.py

@@ -1,12 +1,11 @@
 import cv2
 import numpy as np
-from scipy import signal
 
 from module.base.button import ClickButton
 from module.base.timer import Timer
-from module.base.utils import area_offset, area_size, crop, load_image, rgb2luma
+from module.base.utils import area_offset, crop, load_image
 from module.logger import logger
-from module.ui.scroll import Scroll
+from module.ui.scroll import AdaptiveScroll
 from tasks.base.assets.assets_base_popup import POPUP_CANCEL
 from tasks.base.ui import UI
 from tasks.combat.assets.assets_combat_support import COMBAT_SUPPORT_ADD, COMBAT_SUPPORT_LIST, \
@@ -116,41 +115,6 @@ class NextSupportCharacter:
         return SUPPORT_SELECTED.match_template(image, similarity=0.75, direct_match=True)
 
 
-class SupportListScroll(Scroll):
-    def cal_position(self, main):
-        """
-        Args:
-            main (ModuleBase):
-
-        Returns:
-            float: 0 to 1.
-        """
-        image = main.device.image
-
-        temp_area = list(self.area)
-        temp_area[0] = int(temp_area[0] * 0.98)
-        temp_area[2] = int(temp_area[2] * 1.02)
-
-        line = rgb2luma(crop(image, temp_area)).flatten()
-        width = area_size(temp_area)[0]
-        parameters = {
-            "height": 180,
-            "prominence": 30,
-            "distance": width * 0.75,
-        }
-        peaks, _ = signal.find_peaks(line, **parameters)
-        peaks //= width
-        self.length = len(peaks)
-        middle = np.mean(peaks)
-
-        position = (middle - self.length / 2) / (self.total - self.length)
-        position = position if position > 0 else 0.0
-        position = position if position < 1 else 1.0
-        logger.attr(
-            self.name, f"{position:.2f} ({middle}-{self.length / 2})/({self.total}-{self.length})")
-        return position
-
-
 class CombatSupport(UI):
     def support_set(self, support_character_name: str = "FirstCharacter"):
         """
@@ -212,8 +176,8 @@ class CombatSupport(UI):
             out: COMBAT_SUPPORT_LIST
         """
         logger.hr("Combat support search")
-        scroll = SupportListScroll(area=COMBAT_SUPPORT_LIST_SCROLL.area, color=(194, 196, 205),
+        scroll = AdaptiveScroll(area=COMBAT_SUPPORT_LIST_SCROLL.area,
-                                   name=COMBAT_SUPPORT_LIST_SCROLL.name)
+                                name=COMBAT_SUPPORT_LIST_SCROLL.name)
         if scroll.appear(main=self):
             if not scroll.at_bottom(main=self):
                 # Dropdown to load the entire support list, so large threshold is acceptable
@@ -311,8 +275,8 @@ class CombatSupport(UI):
             out: COMBAT_SUPPORT_LIST
         """
         skip_first_screenshot = True
-        scroll = SupportListScroll(area=COMBAT_SUPPORT_LIST_SCROLL.area, color=(194, 196, 205),
+        scroll = AdaptiveScroll(area=COMBAT_SUPPORT_LIST_SCROLL.area,
-                                   name=COMBAT_SUPPORT_LIST_SCROLL.name)
+                                name=COMBAT_SUPPORT_LIST_SCROLL.name)
         interval = Timer(1)
         next_support = None
         if scroll.appear(main=self):

tasks/daily/assets/assets_daily_synthesize_consumable.py

@@ -141,9 +141,9 @@ SYNTHESIZE_SCROLL = ButtonWrapper(
     name='SYNTHESIZE_SCROLL',
     share=Button(
         file='./assets/share/daily/synthesize_consumable/SYNTHESIZE_SCROLL.png',
-        area=(458, 80, 464, 628),
+        area=(460, 90, 464, 618),
-        search=(438, 60, 484, 648),
+        search=(440, 70, 484, 638),
-        color=(134, 130, 144),
+        color=(167, 165, 177),
-        button=(458, 80, 464, 628),
+        button=(460, 90, 464, 618),
     ),
 )

tasks/daily/synthesize.py

@@ -1,5 +1,5 @@
 from module.ocr.ocr import *
-from module.ui.scroll import Scroll
+from module.ui.scroll import AdaptiveScroll, Scroll
 from tasks.base.assets.assets_base_page import MENU_CHECK, MENU_SCROLL, SYNTHESIZE_CHECK
 from tasks.base.assets.assets_base_popup import POPUP_CONFIRM
 from tasks.base.page import Page, page_menu, page_synthesize
@@ -32,7 +32,7 @@ class SynthesizeUI(UI):
                 scroll = Scroll(MENU_SCROLL.button, color=(191, 191, 191), name=MENU_SCROLL.name)
             case page_synthesize.name:
                 check_image = SYNTHESIZE_CHECK
-                scroll = Scroll(SYNTHESIZE_SCROLL.button, color=(210, 210, 210), name=SYNTHESIZE_SCROLL.name)
+                scroll = AdaptiveScroll(SYNTHESIZE_SCROLL.button, name=SYNTHESIZE_SCROLL.name)
             case _:
                 logger.info(f'No page matched, just skip')
                 return
@@ -109,7 +109,7 @@ class SynthesizeUI(UI):
             else self.__class__.default_candidate_items
 
         # Search target button from top to bottom
-        scroll = Scroll(SYNTHESIZE_SCROLL.button, color=(210, 210, 210), name=SYNTHESIZE_SCROLL.name)
+        scroll = AdaptiveScroll(SYNTHESIZE_SCROLL.button, name=SYNTHESIZE_SCROLL.name)
         if scroll.appear(main=self):
             skip_first_screenshot = True
             while 1:

tasks/item/assets/assets_item_consumable_usage.py

@@ -7,10 +7,10 @@ ITEM_CONSUMABLE_SCROLL = ButtonWrapper(
     name='ITEM_CONSUMABLE_SCROLL',
     share=Button(
         file='./assets/share/item/consumable_usage/ITEM_CONSUMABLE_SCROLL.png',
-        area=(837, 89, 843, 615),
+        area=(838, 90, 842, 614),
-        search=(817, 69, 863, 635),
+        search=(818, 70, 862, 634),
-        color=(118, 117, 121),
+        color=(141, 141, 153),
-        button=(837, 89, 843, 615),
+        button=(838, 90, 842, 614),
     ),
 )
 SIMPLE_PROTECTIVE_GEAR = ButtonWrapper(

tasks/item/consumable_usage.py

@@ -1,5 +1,5 @@
 from module.ocr.ocr import *
-from module.ui.scroll import Scroll
+from module.ui.scroll import AdaptiveScroll
 from tasks.base.assets.assets_base_popup import POPUP_CONFIRM
 from tasks.base.page import page_item
 from tasks.daily.assets.assets_daily_synthesize_consumable import \
@@ -54,7 +54,7 @@ class ConsumableUsageUI(ItemUI):
 
         # Determine if there is a scroll bar. If there is a scroll bar,
         # pull it down and check if the consumable to be used can be found
-        scroll = Scroll(area=ITEM_CONSUMABLE_SCROLL.button, color=(200, 200, 200), name=ITEM_CONSUMABLE_SCROLL.name)
+        scroll = AdaptiveScroll(area=ITEM_CONSUMABLE_SCROLL.button, name=ITEM_CONSUMABLE_SCROLL.name)
         if scroll.appear(main=self):
             if not scroll.at_top(main=self):
                 scroll.set_top(main=self)