From f0069157d3cb94a4ec6da78ece663f1a4a9446f3 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:20:13 +0800 Subject: [PATCH 1/4] Add: [ALAS] Screenshot method ldopengl --- module/device/connection_attr.py | 5 + module/device/method/ldopengl.py | 339 +++++++++++++++++++++++++++++++ module/device/screenshot.py | 4 +- 3 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 module/device/method/ldopengl.py diff --git a/module/device/connection_attr.py b/module/device/connection_attr.py index 57fdd8468..823910efd 100644 --- a/module/device/connection_attr.py +++ b/module/device/connection_attr.py @@ -147,6 +147,11 @@ class ConnectionAttr: # 127.0.0.1:16384 + 32*n return self.serial == '127.0.0.1:7555' or self.is_mumu12_family + @cached_property + def is_ldplayer_bluestacks_family(self): + # Note that LDPlayer and BlueStacks have the same serial range + return self.serial.startswith('emulator-') or 5555 <= self.port <= 5587 + @cached_property def is_nox_family(self): return 62001 <= self.port <= 63025 diff --git a/module/device/method/ldopengl.py b/module/device/method/ldopengl.py new file mode 100644 index 000000000..cd7af4c59 --- /dev/null +++ b/module/device/method/ldopengl.py @@ -0,0 +1,339 @@ +import ctypes +import os +import subprocess +import typing as t +from dataclasses import dataclass +from functools import wraps + +import cv2 +import numpy as np + +from module.base.decorator import cached_property +from module.device.method.utils import RETRY_TRIES, get_serial_pair, retry_sleep +from module.device.platform import Platform +from module.exception import RequestHumanTakeover +from module.logger import logger + + +class LDOpenGLIncompatible(Exception): + pass + + +class LDOpenGLError(Exception): + pass + + +def bytes_to_str(b: bytes) -> str: + for encoding in ['utf-8', 'gbk']: + try: + return b.decode(encoding) + except UnicodeDecodeError: + pass + return str(b) + + +@dataclass +class DataLDPlayerInfo: + # Emulator instance index, starting from 0 + index: int + # Instance name + name: str + # Handle of top window + topWnd: int + # Handle of bind window + bndWnd: int + # If instance is running, 1 for True, 0 for False + sysboot: int + # PID of the instance process, or -1 if instance is not running + playerpid: int + # PID of the vbox process, or -1 if instance is not running + vboxpid: int + # Resolution + width: int + height: int + dpi: int + + def __post_init__(self): + self.index = int(self.index) + self.name = bytes_to_str(self.name) + self.topWnd = int(self.topWnd) + self.bndWnd = int(self.bndWnd) + self.sysboot = int(self.sysboot) + self.playerpid = int(self.playerpid) + self.vboxpid = int(self.vboxpid) + self.width = int(self.width) + self.height = int(self.height) + self.dpi = int(self.dpi) + + +class LDConsole: + def __init__(self, ld_folder: str): + """ + Args: + ld_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/LDPlayer9 + which should have a `ldconsole.exe` in it. + """ + self.ld_console = os.path.abspath(os.path.join(ld_folder, './ldconsole.exe')) + + def subprocess_run(self, cmd, timeout=10): + """ + Args: + cmd (list): + timeout (int): + + Returns: + bytes: + """ + cmd = [self.ld_console] + cmd + logger.info(f'Execute: {cmd}') + + try: + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=False) + except FileNotFoundError as e: + logger.warning(f'warning when calling {cmd}, {str(e)}') + raise LDOpenGLIncompatible(f'ld_folder does not have ldconsole.exe') + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + logger.warning(f'TimeoutExpired when calling {cmd}, stdout={stdout}, stderr={stderr}') + return stdout + + def list2(self) -> t.List[DataLDPlayerInfo]: + """ + > ldconsole.exe list2 + 0,雷电模拟器,28053900,42935798,1,59776,36816,1280,720,240 + 1,雷电模拟器-1,0,0,0,-1,-1,1280,720,240 + """ + out = [] + data = self.subprocess_run(['list2']) + for row in data.strip().split(b'\n'): + info = row.strip().split(b',') + info = DataLDPlayerInfo(*info) + out.append(info) + return out + + +class IScreenShotClass: + def __init__(self, ptr): + self.ptr = ptr + + # Define in class since ctypes.WINFUNCTYPE is windows only + cap_type = ctypes.WINFUNCTYPE(ctypes.c_void_p) + release_type = ctypes.WINFUNCTYPE(None) + self.class_cap = cap_type(1, "IScreenShotClass_Cap") + # Keep reference count + # so __del__ won't have an empty IScreenShotClass_Cap + self.class_release = release_type(2, "IScreenShotClass_Release") + + def cap(self): + return self.class_cap(self.ptr) + + def __del__(self): + self.class_release(self.ptr) + + +def retry(func): + @wraps(func) + def retry_wrapper(self, *args, **kwargs): + """ + Args: + self (NemuIpcImpl): + """ + init = None + for _ in range(RETRY_TRIES): + try: + if callable(init): + retry_sleep(_) + init() + return func(self, *args, **kwargs) + # Can't handle + except RequestHumanTakeover: + break + # Can't handle + except LDOpenGLIncompatible as e: + logger.error(e) + break + # NemuIpcError + except LDOpenGLError as e: + logger.error(e) + + def init(): + pass + # Unknown, probably a trucked image + except Exception as e: + logger.exception(e) + + def init(): + pass + + logger.critical(f'Retry {func.__name__}() failed') + raise RequestHumanTakeover + + return retry_wrapper + + +class LDOpenGLImpl: + def __init__(self, ld_folder: str, instance_id: int): + """ + Args: + ld_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/LDPlayer9 + instance_id: Emulator instance ID, starting from 0 + """ + ldopengl_dll = os.path.abspath(os.path.join(ld_folder, './ldopengl64.dll')) + logger.info( + f'LDOpenGL init, ' + f'ld_folder={ld_folder}, ' + f'ldopengl_dll={ldopengl_dll}, ' + f'instance_id={instance_id}' + ) + # Load dll + try: + self.lib = ctypes.WinDLL(ldopengl_dll) + except OSError as e: + logger.error(e) + if not os.path.exists(ldopengl_dll): + raise LDOpenGLIncompatible( + f'ldopengl_dll={ldopengl_dll} does not exist, ' + f'ldopengl requires LDPlayer >= 9.0.78, please check your version' + ) + else: + raise LDOpenGLIncompatible( + f'ldopengl_dll={ldopengl_dll} exist, ' + f'but cannot be loaded' + ) + # Get info after loading DLL, so DLL existence can act as a version check + self.console = LDConsole(ld_folder) + self.info = self.get_player_info_by_index(instance_id) + + self.lib.CreateScreenShotInstance.restype = ctypes.c_void_p + + # Get screenshot instance + instance_ptr = ctypes.c_void_p(self.lib.CreateScreenShotInstance(instance_id, self.info.playerpid)) + self.screenshot_instance = IScreenShotClass(instance_ptr) + + def get_player_info_by_index(self, instance_id: int): + """ + Args: + instance_id: + + Returns: + DataLDPlayerInfo: + + Raises: + LDOpenGLError: + """ + for info in self.console.list2(): + if info.index == instance_id: + logger.info(f'Match LDPlayer instance: {info}') + if not info.sysboot: + raise LDOpenGLError('Trying to connect LDPlayer instance but emulator is not running') + return info + raise LDOpenGLError(f'No LDPlayer instance with index {instance_id}') + + @retry + def screenshot(self): + """ + Returns: + np.ndarray: Image array in BGR color space + Note that image is upside down + """ + width, height = self.info.width, self.info.height + + img_ptr = self.screenshot_instance.cap() + # ValueError: NULL pointer access + if img_ptr is None: + raise LDOpenGLError('Empty image pointer') + + img = ctypes.cast(img_ptr, ctypes.POINTER(ctypes.c_ubyte * (height * width * 3))).contents + + image = np.ctypeslib.as_array(img).reshape((height, width, 3)) + return image + + @staticmethod + def serial_to_id(serial: str): + """ + Predict instance ID from serial + E.g. + "127.0.0.1:5555" -> 0 + "127.0.0.1:5557" -> 1 + "emulator-5554" -> 0 + + Returns: + int: instance_id, or None if failed to predict + """ + serial, _ = get_serial_pair(serial) + if serial is None: + return None + try: + port = int(serial.split(':')[1]) + except (IndexError, ValueError): + return None + if 5555 <= port <= 5555 + 32: + return int((port - 5555) // 2) + return None + + +class LDOpenGL(Platform): + @cached_property + def ldopengl(self): + """ + Initialize a ldopengl implementation + """ + # Try existing settings first + if self.config.EmulatorInfo_path: + folder = os.path.abspath(os.path.join(self.config.EmulatorInfo_path, '../')) + index = LDOpenGLImpl.serial_to_id(self.serial) + if index is not None: + try: + return LDOpenGLImpl( + ld_folder=folder, + instance_id=index, + ) + except (LDOpenGLIncompatible, LDOpenGLError) as e: + logger.error(e) + logger.error('Emulator info incorrect') + + # Search emulator instance + # with E:/ProgramFiles/LDPlayer9/dnplayer.exe + # installation path is E:/ProgramFiles/LDPlayer9 + if self.emulator_instance is None: + logger.error('Unable to use LDOpenGL because emulator instance not found') + raise RequestHumanTakeover + try: + return LDOpenGLImpl( + ld_folder=self.emulator_instance.emulator.abspath('./'), + instance_id=self.emulator_instance.LDPlayer_id, + ) + except (LDOpenGLIncompatible, LDOpenGLError) as e: + logger.error(e) + logger.error('Unable to initialize LDOpenGL') + raise RequestHumanTakeover + + def ldopengl_available(self) -> bool: + if not self.is_ldplayer_bluestacks_family: + return False + + try: + _ = self.ldopengl + except RequestHumanTakeover: + return False + return True + + def screenshot_ldopengl(self): + image = self.ldopengl.screenshot() + + image = cv2.flip(image, 0) + cv2.cvtColor(image, cv2.COLOR_BGR2RGB, dst=image) + return image + + +if __name__ == '__main__': + ld = LDOpenGLImpl('E:/ProgramFiles/LDPlayer9', instance_id=1) + for _ in range(5): + import time + + start = time.time() + ld.screenshot() + print(time.time() - start) diff --git a/module/device/screenshot.py b/module/device/screenshot.py index b3bc69b6f..209a8af16 100644 --- a/module/device/screenshot.py +++ b/module/device/screenshot.py @@ -13,6 +13,7 @@ from module.base.utils import get_color, image_size, limit_in, save_image from module.device.method.adb import Adb from module.device.method.ascreencap import AScreenCap from module.device.method.droidcast import DroidCast +from module.device.method.ldopengl import LDOpenGL from module.device.method.nemu_ipc import NemuIpc from module.device.method.scrcpy import Scrcpy from module.device.method.wsa import WSA @@ -20,7 +21,7 @@ from module.exception import RequestHumanTakeover, ScriptError from module.logger import logger -class Screenshot(Adb, WSA, DroidCast, AScreenCap, Scrcpy, NemuIpc): +class Screenshot(Adb, WSA, DroidCast, AScreenCap, Scrcpy, NemuIpc, LDOpenGL): _screen_size_checked = False _screen_black_checked = False _minicap_uninstalled = False @@ -40,6 +41,7 @@ class Screenshot(Adb, WSA, DroidCast, AScreenCap, Scrcpy, NemuIpc): 'DroidCast_raw': self.screenshot_droidcast_raw, 'scrcpy': self.screenshot_scrcpy, 'nemu_ipc': self.screenshot_nemu_ipc, + 'ldopengl': self.screenshot_ldopengl, } @cached_property From b0da726cbf1beb6e6216edafc147be3532a3d9ac Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:22:17 +0800 Subject: [PATCH 2/4] Add: Screenshot method ldopengl to settings --- module/config/argument/args.json | 3 ++- module/config/argument/argument.yaml | 1 + module/config/config_generated.py | 2 +- module/config/i18n/en-US.json | 3 ++- module/config/i18n/es-ES.json | 3 ++- module/config/i18n/ja-JP.json | 3 ++- module/config/i18n/zh-CN.json | 3 ++- module/config/i18n/zh-TW.json | 3 ++- 8 files changed, 14 insertions(+), 7 deletions(-) diff --git a/module/config/argument/args.json b/module/config/argument/args.json index 916f3a9c1..030dfd471 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -49,7 +49,8 @@ "DroidCast", "DroidCast_raw", "scrcpy", - "nemu_ipc" + "nemu_ipc", + "ldopengl" ], "display": "hide" }, diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index 8703d3130..6ef88345a 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -40,6 +40,7 @@ Emulator: DroidCast_raw, scrcpy, nemu_ipc, + ldopengl, ] ControlMethod: value: MaaTouch diff --git a/module/config/config_generated.py b/module/config/config_generated.py index 3bcc92056..6e2c0808b 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -20,7 +20,7 @@ class GeneratedConfig: Emulator_GameClient = 'android' # android, cloud_android Emulator_PackageName = 'auto' # auto, CN-Official, CN-Bilibili, OVERSEA-America, OVERSEA-Asia, OVERSEA-Europe, OVERSEA-TWHKMO Emulator_GameLanguage = 'auto' # auto, cn, en - Emulator_ScreenshotMethod = 'auto' # auto, ADB, ADB_nc, uiautomator2, aScreenCap, aScreenCap_nc, DroidCast, DroidCast_raw, scrcpy, nemu_ipc + Emulator_ScreenshotMethod = 'auto' # auto, ADB, ADB_nc, uiautomator2, aScreenCap, aScreenCap_nc, DroidCast, DroidCast_raw, scrcpy, nemu_ipc, ldopengl Emulator_ControlMethod = 'MaaTouch' # minitouch, MaaTouch Emulator_AdbRestart = False diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index 458559256..59d720824 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -140,7 +140,8 @@ "DroidCast": "DroidCast", "DroidCast_raw": "DroidCast_raw", "scrcpy": "scrcpy", - "nemu_ipc": "nemu_ipc" + "nemu_ipc": "nemu_ipc", + "ldopengl": "ldopengl" }, "ControlMethod": { "name": "Control Method", diff --git a/module/config/i18n/es-ES.json b/module/config/i18n/es-ES.json index 3d0189934..46f29b6a7 100644 --- a/module/config/i18n/es-ES.json +++ b/module/config/i18n/es-ES.json @@ -140,7 +140,8 @@ "DroidCast": "DroidCast", "DroidCast_raw": "DroidCast_raw", "scrcpy": "scrcpy", - "nemu_ipc": "nemu_ipc" + "nemu_ipc": "nemu_ipc", + "ldopengl": "ldopengl" }, "ControlMethod": { "name": "Método de control", diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index 8992aedcf..bdbd3a51a 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -140,7 +140,8 @@ "DroidCast": "DroidCast", "DroidCast_raw": "DroidCast_raw", "scrcpy": "scrcpy", - "nemu_ipc": "nemu_ipc" + "nemu_ipc": "nemu_ipc", + "ldopengl": "ldopengl" }, "ControlMethod": { "name": "Emulator.ControlMethod.name", diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index c10684f8a..2bc232c54 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -140,7 +140,8 @@ "DroidCast": "DroidCast", "DroidCast_raw": "DroidCast_raw", "scrcpy": "scrcpy", - "nemu_ipc": "nemu_ipc" + "nemu_ipc": "nemu_ipc", + "ldopengl": "ldopengl" }, "ControlMethod": { "name": "模拟器控制方案", diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index 54526740a..f9f0735c8 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -140,7 +140,8 @@ "DroidCast": "DroidCast", "DroidCast_raw": "DroidCast_raw", "scrcpy": "scrcpy", - "nemu_ipc": "nemu_ipc" + "nemu_ipc": "nemu_ipc", + "ldopengl": "ldopengl" }, "ControlMethod": { "name": "模擬器控制方案", From a5c9724ef3a3f947cf7969d305df987cee661376 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:31:00 +0800 Subject: [PATCH 3/4] Add: Use ldopengl if available --- module/daemon/benchmark.py | 4 ++++ module/device/screenshot.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/module/daemon/benchmark.py b/module/daemon/benchmark.py index e2efdc26b..f59212a4a 100644 --- a/module/daemon/benchmark.py +++ b/module/daemon/benchmark.py @@ -192,6 +192,8 @@ class Benchmark(DaemonBase): click = ['ADB', 'Hermit', 'MaaTouch'] if self.device.nemu_ipc_available(): screenshot.append('nemu_ipc') + if self.device.ldopengl_available(): + screenshot.append('ldopengl') scene = self.config.Benchmark_TestScene if 'screenshot' not in scene: @@ -232,6 +234,8 @@ class Benchmark(DaemonBase): screenshot = remove('ADB_nc', 'aScreenCap_nc') if self.device.nemu_ipc_available(): screenshot.append('nemu_ipc') + if self.device.ldopengl_available(): + screenshot.append('ldopengl') screenshot = tuple(screenshot) self.TEST_TOTAL = 3 diff --git a/module/device/screenshot.py b/module/device/screenshot.py index 209a8af16..292d3869b 100644 --- a/module/device/screenshot.py +++ b/module/device/screenshot.py @@ -51,6 +51,10 @@ class Screenshot(Adb, WSA, DroidCast, AScreenCap, Scrcpy, NemuIpc, LDOpenGL): logger.attr('nemu_ipc_available', available) if available: return 'nemu_ipc' + available = self.ldopengl_available() + logger.attr('ldopengl_available', available) + if available: + return 'ldopengl' return '' def screenshot(self): From a8a33625311a4df70f01b86c98475433dda26ec7 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:32:37 +0800 Subject: [PATCH 4/4] Fix: [ALAS] Skip trying ldopengl if it is not a LDPlayer --- module/device/method/ldopengl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/module/device/method/ldopengl.py b/module/device/method/ldopengl.py index cd7af4c59..4e4687173 100644 --- a/module/device/method/ldopengl.py +++ b/module/device/method/ldopengl.py @@ -314,6 +314,9 @@ class LDOpenGL(Platform): def ldopengl_available(self) -> bool: if not self.is_ldplayer_bluestacks_family: return False + logger.attr('EmulatorInfo_Emulator', self.config.EmulatorInfo_Emulator) + if self.config.EmulatorInfo_Emulator not in ['LDPlayer9']: + return False try: _ = self.ldopengl