From 7b8f30598d05211b8ce2535559e9f4a00521f45c Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Fri, 27 May 2022 18:44:44 -0700 Subject: [PATCH 1/7] only resets rune_active if successfully solved rune, start/stop bot still resets rune alert b/c that means user already at computer --- src/modules/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/bot.py b/src/modules/bot.py index 3d8f5e20..8dd7c344 100644 --- a/src/modules/bot.py +++ b/src/modules/bot.py @@ -144,10 +144,10 @@ def _solve_rune(self, model): target = tuple(round(rune_buff_pos[i] + config.capture.window[i]) for i in range(2)) click(target, button='right') + self.rune_active = False break elif len(solution) == 4: inferences.append(solution) - self.rune_active = False def load_commands(self, file): """Prompts the user to select a command module to import. Updates config's command book.""" From 68d2aa32f4eeeebcfe982b910019ea989d522f0b Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Sat, 28 May 2022 14:05:28 -0700 Subject: [PATCH 2/7] disabling all routine menu options before command book is loaded --- resources | 2 +- src/gui/menu/file.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/resources b/resources index c87c40af..40cc0b6f 160000 --- a/resources +++ b/resources @@ -1 +1 @@ -Subproject commit c87c40af1372d2f3a1f80f4d9687527ef28bf57a +Subproject commit 40cc0b6f0a422f6579b09e106dad56a8b932547a diff --git a/src/gui/menu/file.py b/src/gui/menu/file.py index d0bd77fa..d82db545 100644 --- a/src/gui/menu/file.py +++ b/src/gui/menu/file.py @@ -11,8 +11,16 @@ def __init__(self, parent, **kwargs): super().__init__(parent, 'File', **kwargs) # parent.add_cascade(label='File', menu=self) - self.add_command(label='New Routine', command=utils.async_callback(self, File._new_routine)) - self.add_command(label='Save Routine', command=utils.async_callback(self, File._save_routine)) + self.add_command( + label='New Routine', + command=utils.async_callback(self, File._new_routine), + state=tk.DISABLED + ) + self.add_command( + label='Save Routine', + command=utils.async_callback(self, File._save_routine), + state=tk.DISABLED + ) self.add_separator() self.add_command(label='Load Command Book', command=utils.async_callback(self, File._load_commands)) self.add_command( @@ -22,6 +30,8 @@ def __init__(self, parent, **kwargs): ) def enable_routine_state(self): + self.entryconfig('New Routine', state=tk.NORMAL) + self.entryconfig('Save Routine', state=tk.NORMAL) self.entryconfig('Load Routine', state=tk.NORMAL) @staticmethod From f0e76fab9e4673337714992a2d0fe73f9d3b730a Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Sat, 28 May 2022 14:35:53 -0700 Subject: [PATCH 3/7] supports importing changes to command book while program is running --- src/modules/bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/bot.py b/src/modules/bot.py index 8dd7c344..52a4d377 100644 --- a/src/modules/bot.py +++ b/src/modules/bot.py @@ -5,6 +5,7 @@ import git import cv2 import inspect +import importlib from os.path import splitext, basename from src.common import config, utils from src.detection import detection @@ -168,7 +169,8 @@ def load_commands(self, file): # Import the desired command book file module_name = splitext(basename(file))[0] target = '.'.join(['resources', 'command_books', module_name]) - module = __import__(target, fromlist=['']) + module = importlib.import_module(target) + module = importlib.reload(module) # Check if the 'step' function has been implemented step_found = False @@ -210,10 +212,8 @@ def load_commands(self, file): config.gui.view.status.set_cb(basename(file)) config.routine.clear() print(f" ~ Successfully loaded command book '{module_name}'.") - return True else: print(f" ! Command book '{module_name}' was not loaded.") - return False def update_submodules(self, force=False): print('\n[~] Retrieving latest submodules:') From b3f5af310fbe6a4edfa7ee90bff8885e2bfbffa8 Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Sat, 28 May 2022 14:49:25 -0700 Subject: [PATCH 4/7] printing error trace during command book load --- src/modules/bot.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/modules/bot.py b/src/modules/bot.py index 52a4d377..a4b891b9 100644 --- a/src/modules/bot.py +++ b/src/modules/bot.py @@ -6,6 +6,7 @@ import cv2 import inspect import importlib +import traceback from os.path import splitext, basename from src.common import config, utils from src.detection import detection @@ -169,8 +170,17 @@ def load_commands(self, file): # Import the desired command book file module_name = splitext(basename(file))[0] target = '.'.join(['resources', 'command_books', module_name]) - module = importlib.import_module(target) - module = importlib.reload(module) + try: + module = importlib.import_module(target) + module = importlib.reload(module) + except ImportError: # Display errors in the target Command Book + print(' ! Errors during compilation:\n') + for line in traceback.format_exc().split('\n'): + line = line.rstrip() + if line: + print(' ' * 4 + line) + print(f"\n ! Command book '{module_name}' was not loaded") + return # Check if the 'step' function has been implemented step_found = False @@ -202,7 +212,7 @@ def load_commands(self, file): if not step_found and not movement_found: print(f" ! Error: Must either implement both 'Move' and 'Adjust' commands, " - f"or the function 'step'.") + f"or the function 'step'") if required_found and (step_found or movement_found): self.module_name = module_name self.command_book = new_cb @@ -211,9 +221,9 @@ def load_commands(self, file): config.gui.menu.file.enable_routine_state() config.gui.view.status.set_cb(basename(file)) config.routine.clear() - print(f" ~ Successfully loaded command book '{module_name}'.") + print(f" ~ Successfully loaded command book '{module_name}'") else: - print(f" ! Command book '{module_name}' was not loaded.") + print(f" ! Command book '{module_name}' was not loaded") def update_submodules(self, force=False): print('\n[~] Retrieving latest submodules:') From d1dc0b08a5c1648c064ed0bb51b79ec9a27d0b04 Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Sat, 28 May 2022 15:58:13 -0700 Subject: [PATCH 5/7] switched back to mss, much faster performance, still debugging resolution change errors --- src/modules/bot.py | 6 +- src/modules/capture.py | 178 +++++++++++++++++++++++------------------ 2 files changed, 105 insertions(+), 79 deletions(-) diff --git a/src/modules/bot.py b/src/modules/bot.py index a4b891b9..baeccde5 100644 --- a/src/modules/bot.py +++ b/src/modules/bot.py @@ -143,8 +143,10 @@ def _solve_rune(self, model): threshold=0.9) if rune_buff: rune_buff_pos = min(rune_buff, key=lambda p: p[0]) - target = tuple(round(rune_buff_pos[i] + config.capture.window[i]) - for i in range(2)) + target = ( + round(rune_buff_pos[0] + config.capture.window['left']), + round(rune_buff_pos[1] + config.capture.window['top']) + ) click(target, button='right') self.rune_active = False break diff --git a/src/modules/capture.py b/src/modules/capture.py index 21cf4016..e3ff130f 100644 --- a/src/modules/capture.py +++ b/src/modules/capture.py @@ -4,10 +4,12 @@ import cv2 import threading import ctypes +import mss +import mss.windows import numpy as np from src.common import config, utils from ctypes import wintypes -from PIL import ImageGrab +# from PIL import ImageGrab user32 = ctypes.windll.user32 user32.SetProcessDPIAware() @@ -16,7 +18,7 @@ MINIMAP_TOP_BORDER = 5 # The thickness of the other three borders of the minimap -MINIMAP_BOTTOM_BORDER = 8 +MINIMAP_BOTTOM_BORDER = 9 # Offset in pixels to adjust for windowed mode WINDOWED_OFFSET_TOP = 36 @@ -50,8 +52,14 @@ def __init__(self): self.minimap = {} self.minimap_ratio = 1 self.minimap_sample = None - self.window = (0, 0, 1366, 768) - self.scale = 1.0 + self.sct = None + # self.window = (0, 0, 1366, 768) + self.window = { + 'left': 0, + 'top': 0, + 'width': 1366, + 'height': 768 + } self.ready = False self.calibrated = False @@ -67,81 +75,97 @@ def start(self): def _main(self): """Constantly monitors the player's position and in-game events.""" - while True: - if not self.calibrated: - handle = user32.FindWindowW(None, 'MapleStory') - rect = wintypes.RECT() - user32.GetWindowRect(handle, ctypes.pointer(rect)) - rect = (rect.left, rect.top, rect.right, rect.bottom) - rect = tuple(max(0, x) for x in rect) - - # Preliminary window to template match minimap - self.scale = ctypes.windll.shcore.GetScaleFactorForDevice(0) / 100 - self.window = ( - rect[0], - rect[1], - max(rect[2], rect[0] + MMT_WIDTH), # Make room for minimap templates - max(rect[3], rect[1] + MMT_HEIGHT) - ) - - # Calibrate by finding the bottom right corner of the minimap + mss.windows.CAPTUREBLT = 0 + with mss.mss() as self.sct: + while True: + if not self.calibrated: + handle = user32.FindWindowW(None, 'MapleStory') + rect = wintypes.RECT() + user32.GetWindowRect(handle, ctypes.pointer(rect)) + rect = (rect.left, rect.top, rect.right, rect.bottom) + rect = tuple(max(0, x) for x in rect) + + # Preliminary window to template match minimap + # self.window = ( + # rect[0], + # rect[1], + # max(rect[2], rect[0] + MMT_WIDTH), # Make room for minimap templates + # max(rect[3], rect[1] + MMT_HEIGHT) + # ) + self.window['left'] = rect[0] + self.window['top'] = rect[1] + self.window['width'] = max(rect[2] - rect[0], MMT_WIDTH) + self.window['height'] = max(rect[3] - rect[1], MMT_HEIGHT) + + # Calibrate by finding the bottom right corner of the minimap + self.frame = self.screenshot() + if self.frame is None: + continue + # self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) + tl, _ = utils.single_match(self.frame, MM_TL_TEMPLATE) + _, br = utils.single_match(self.frame, MM_BR_TEMPLATE) + mm_tl = ( + tl[0] + MINIMAP_BOTTOM_BORDER, + tl[1] + MINIMAP_TOP_BORDER + ) + mm_br = ( + max(mm_tl[0] + PT_WIDTH, br[0] - MINIMAP_BOTTOM_BORDER), + max(mm_tl[1] + PT_HEIGHT, br[1] - MINIMAP_BOTTOM_BORDER) + ) + + # Resize window to encompass minimap if needed + # self.window = ( + # rect[0], + # rect[1], + # max(rect[2], mm_br[0]), + # max(rect[3], mm_br[1]) + # ) + + self.minimap_ratio = (mm_br[0] - mm_tl[0]) / (mm_br[1] - mm_tl[1]) + self.minimap_sample = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] + self.calibrated = True + + # Take screenshot self.frame = self.screenshot() if self.frame is None: continue - self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) - tl, _ = utils.single_match(self.frame, MM_TL_TEMPLATE) - _, br = utils.single_match(self.frame, MM_BR_TEMPLATE) - mm_tl = ( - tl[0] + MINIMAP_BOTTOM_BORDER, - tl[1] + MINIMAP_TOP_BORDER - ) - mm_br = ( - max(mm_tl[0] + PT_WIDTH, br[0] - MINIMAP_BOTTOM_BORDER), - max(mm_tl[1] + PT_HEIGHT, br[1] - MINIMAP_BOTTOM_BORDER - 1) - ) - - # Resize window to encompass minimap if needed - self.window = ( - rect[0], - rect[1], - max(rect[2], mm_br[0]), - max(rect[3], mm_br[1]) - ) - self.minimap_ratio = (mm_br[0] - mm_tl[0]) / (mm_br[1] - mm_tl[1]) - self.minimap_sample = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] - self.calibrated = True - - # Take screenshot - self.frame = self.screenshot() - if self.frame is None: - continue - self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) - - # Crop the frame to only show the minimap - minimap = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] - - # Determine the player's position - player = utils.multi_match(minimap, PLAYER_TEMPLATE, threshold=0.8) - if player: - config.player_pos = utils.convert_to_relative(player[0], minimap) - - # Package display information to be polled by GUI - self.minimap = { - 'minimap': minimap, - 'rune_active': config.bot.rune_active, - 'rune_pos': config.bot.rune_pos, - 'path': config.path, - 'player_pos': config.player_pos - } - - if not self.ready: - self.ready = True - time.sleep(0.001) + # self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) + + # Crop the frame to only show the minimap + minimap = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] + + # Determine the player's position + player = utils.multi_match(minimap, PLAYER_TEMPLATE, threshold=0.8) + if player: + config.player_pos = utils.convert_to_relative(player[0], minimap) + + # Package display information to be polled by GUI + self.minimap = { + 'minimap': minimap, + 'rune_active': config.bot.rune_active, + 'rune_pos': config.bot.rune_pos, + 'path': config.path, + 'player_pos': config.player_pos + } + + if not self.ready: + self.ready = True + time.sleep(0.001) + + # def screenshot(self, delay=1): + # try: + # return np.array(ImageGrab.grab(self.window)) + # except OSError: + # print(f'\n[!] Error while taking screenshot, retrying in {delay} second' + # + ('s' if delay != 1 else '')) + # time.sleep(delay) def screenshot(self, delay=1): - try: - return np.array(ImageGrab.grab(self.window)) - except OSError: - print(f'\n[!] Error while taking screenshot, retrying in {delay} second' - + ('s' if delay != 1 else '')) - time.sleep(delay) + if self.sct is not None: + try: + return np.array(self.sct.grab(self.window)) + except mss.exception.ScreenShotError: + print(f'\n[!] Error while taking screenshot, retrying in {delay} second' + + ('s' if delay != 1 else '')) + print(self.window) + time.sleep(delay) From 6483969957da18463d6ccddbe1120aec80ea9139 Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Sat, 28 May 2022 16:09:33 -0700 Subject: [PATCH 6/7] restarting mss every calibration, fixed resolution change errors --- src/modules/capture.py | 156 +++++++++++++++++------------------------ 1 file changed, 66 insertions(+), 90 deletions(-) diff --git a/src/modules/capture.py b/src/modules/capture.py index e3ff130f..9a626bb4 100644 --- a/src/modules/capture.py +++ b/src/modules/capture.py @@ -9,7 +9,6 @@ import numpy as np from src.common import config, utils from ctypes import wintypes -# from PIL import ImageGrab user32 = ctypes.windll.user32 user32.SetProcessDPIAware() @@ -53,7 +52,6 @@ def __init__(self): self.minimap_ratio = 1 self.minimap_sample = None self.sct = None - # self.window = (0, 0, 1366, 768) self.window = { 'left': 0, 'top': 0, @@ -76,96 +74,74 @@ def _main(self): """Constantly monitors the player's position and in-game events.""" mss.windows.CAPTUREBLT = 0 - with mss.mss() as self.sct: - while True: - if not self.calibrated: - handle = user32.FindWindowW(None, 'MapleStory') - rect = wintypes.RECT() - user32.GetWindowRect(handle, ctypes.pointer(rect)) - rect = (rect.left, rect.top, rect.right, rect.bottom) - rect = tuple(max(0, x) for x in rect) - - # Preliminary window to template match minimap - # self.window = ( - # rect[0], - # rect[1], - # max(rect[2], rect[0] + MMT_WIDTH), # Make room for minimap templates - # max(rect[3], rect[1] + MMT_HEIGHT) - # ) - self.window['left'] = rect[0] - self.window['top'] = rect[1] - self.window['width'] = max(rect[2] - rect[0], MMT_WIDTH) - self.window['height'] = max(rect[3] - rect[1], MMT_HEIGHT) - - # Calibrate by finding the bottom right corner of the minimap + while True: + # Calibrate screen capture + handle = user32.FindWindowW(None, 'MapleStory') + rect = wintypes.RECT() + user32.GetWindowRect(handle, ctypes.pointer(rect)) + rect = (rect.left, rect.top, rect.right, rect.bottom) + rect = tuple(max(0, x) for x in rect) + + self.window['left'] = rect[0] + self.window['top'] = rect[1] + self.window['width'] = max(rect[2] - rect[0], MMT_WIDTH) + self.window['height'] = max(rect[3] - rect[1], MMT_HEIGHT) + + # Calibrate by finding the bottom right corner of the minimap + with mss.mss() as self.sct: + self.frame = self.screenshot() + if self.frame is None: + continue + tl, _ = utils.single_match(self.frame, MM_TL_TEMPLATE) + _, br = utils.single_match(self.frame, MM_BR_TEMPLATE) + mm_tl = ( + tl[0] + MINIMAP_BOTTOM_BORDER, + tl[1] + MINIMAP_TOP_BORDER + ) + mm_br = ( + max(mm_tl[0] + PT_WIDTH, br[0] - MINIMAP_BOTTOM_BORDER), + max(mm_tl[1] + PT_HEIGHT, br[1] - MINIMAP_BOTTOM_BORDER) + ) + self.minimap_ratio = (mm_br[0] - mm_tl[0]) / (mm_br[1] - mm_tl[1]) + self.minimap_sample = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] + self.calibrated = True + + with mss.mss() as self.sct: + while True: + if not self.calibrated: + break + + # Take screenshot self.frame = self.screenshot() if self.frame is None: continue - # self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) - tl, _ = utils.single_match(self.frame, MM_TL_TEMPLATE) - _, br = utils.single_match(self.frame, MM_BR_TEMPLATE) - mm_tl = ( - tl[0] + MINIMAP_BOTTOM_BORDER, - tl[1] + MINIMAP_TOP_BORDER - ) - mm_br = ( - max(mm_tl[0] + PT_WIDTH, br[0] - MINIMAP_BOTTOM_BORDER), - max(mm_tl[1] + PT_HEIGHT, br[1] - MINIMAP_BOTTOM_BORDER) - ) - - # Resize window to encompass minimap if needed - # self.window = ( - # rect[0], - # rect[1], - # max(rect[2], mm_br[0]), - # max(rect[3], mm_br[1]) - # ) - - self.minimap_ratio = (mm_br[0] - mm_tl[0]) / (mm_br[1] - mm_tl[1]) - self.minimap_sample = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] - self.calibrated = True - - # Take screenshot - self.frame = self.screenshot() - if self.frame is None: - continue - # self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) - - # Crop the frame to only show the minimap - minimap = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] - - # Determine the player's position - player = utils.multi_match(minimap, PLAYER_TEMPLATE, threshold=0.8) - if player: - config.player_pos = utils.convert_to_relative(player[0], minimap) - - # Package display information to be polled by GUI - self.minimap = { - 'minimap': minimap, - 'rune_active': config.bot.rune_active, - 'rune_pos': config.bot.rune_pos, - 'path': config.path, - 'player_pos': config.player_pos - } - - if not self.ready: - self.ready = True - time.sleep(0.001) - - # def screenshot(self, delay=1): - # try: - # return np.array(ImageGrab.grab(self.window)) - # except OSError: - # print(f'\n[!] Error while taking screenshot, retrying in {delay} second' - # + ('s' if delay != 1 else '')) - # time.sleep(delay) + + # Crop the frame to only show the minimap + minimap = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]] + + # Determine the player's position + player = utils.multi_match(minimap, PLAYER_TEMPLATE, threshold=0.8) + if player: + config.player_pos = utils.convert_to_relative(player[0], minimap) + + # Package display information to be polled by GUI + self.minimap = { + 'minimap': minimap, + 'rune_active': config.bot.rune_active, + 'rune_pos': config.bot.rune_pos, + 'path': config.path, + 'player_pos': config.player_pos + } + + if not self.ready: + self.ready = True + time.sleep(0.001) def screenshot(self, delay=1): - if self.sct is not None: - try: - return np.array(self.sct.grab(self.window)) - except mss.exception.ScreenShotError: - print(f'\n[!] Error while taking screenshot, retrying in {delay} second' - + ('s' if delay != 1 else '')) - print(self.window) - time.sleep(delay) + try: + return np.array(self.sct.grab(self.window)) + except mss.exception.ScreenShotError: + print(f'\n[!] Error while taking screenshot, retrying in {delay} second' + + ('s' if delay != 1 else '')) + print(self.window) + time.sleep(delay) From 0382df0c899b2f46c2adf2b6312e070ca12e6ac0 Mon Sep 17 00:00:00 2001 From: Jeffrey Tan Date: Sat, 28 May 2022 16:13:02 -0700 Subject: [PATCH 7/7] removed debugging prints --- src/modules/capture.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/modules/capture.py b/src/modules/capture.py index 9a626bb4..0d8bf0cf 100644 --- a/src/modules/capture.py +++ b/src/modules/capture.py @@ -143,5 +143,4 @@ def screenshot(self, delay=1): except mss.exception.ScreenShotError: print(f'\n[!] Error while taking screenshot, retrying in {delay} second' + ('s' if delay != 1 else '')) - print(self.window) time.sleep(delay)