From e4b7490655ac8a11e569fb4d44495833025a47d1 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Fri, 23 Jun 2023 01:24:17 -0400 Subject: [PATCH 01/36] added layout replaystrat mixin (partially complete) --- openadapt/strategies/mixins/layout.py | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 openadapt/strategies/mixins/layout.py diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py new file mode 100644 index 000000000..916b2978c --- /dev/null +++ b/openadapt/strategies/mixins/layout.py @@ -0,0 +1,51 @@ +""" +Implements a ReplayStrategy mixin for extracting layout and important +information from text documents and GUI images. + +Usage: + + class MyReplayStrategy(LayoutExtractionReplayStrategyMixin): + ... +""" + + +from openadapt.models import Recording, Screenshot +from openadapt.strategies.base import BaseReplayStrategy +import numpy as np +from transformers import pipeline +from typing import List + + +class LayoutExtractionReplayStrategyMixin(BaseReplayStrategy): + + def __init__( + self, + recording: Recording, + img_list: List[np.ndarray] + ): + super.__init__(recording) + self.img_list = img_list + + + def document_query( + self, + image: np.ndarray, + question: str + ) -> str: + + query_pipeline = pipeline("document-question-answering", + model = "impira/layoutlm-document-qa") + + return query_pipeline(image, question)['answer'] + + def extract_table( + self + ): + # TODO + return None + + def gui_query( + self + ): + # TODO + return None \ No newline at end of file From 3ea5cf1b135473a9b05cd23f5782ef4d4746df50 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Fri, 23 Jun 2023 01:27:14 -0400 Subject: [PATCH 02/36] fixed return value --- openadapt/strategies/mixins/layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index 916b2978c..a9b6e55d0 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -36,7 +36,7 @@ def document_query( query_pipeline = pipeline("document-question-answering", model = "impira/layoutlm-document-qa") - return query_pipeline(image, question)['answer'] + return query_pipeline(image, question)[0]['answer'] def extract_table( self From ed74db9a18af1476fbbe261e3d47f74a91f06157 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Fri, 23 Jun 2023 01:27:51 -0400 Subject: [PATCH 03/36] removed extra space --- openadapt/strategies/mixins/layout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index a9b6e55d0..8843e026c 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -26,7 +26,6 @@ def __init__( super.__init__(recording) self.img_list = img_list - def document_query( self, image: np.ndarray, From b9453c832c295239f4c0c7c26b6a41f134488285 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Fri, 23 Jun 2023 12:03:21 -0400 Subject: [PATCH 04/36] removed unintended methods --- openadapt/strategies/mixins/layout.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index 8843e026c..2f4d8fc13 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -35,16 +35,4 @@ def document_query( query_pipeline = pipeline("document-question-answering", model = "impira/layoutlm-document-qa") - return query_pipeline(image, question)[0]['answer'] - - def extract_table( - self - ): - # TODO - return None - - def gui_query( - self - ): - # TODO - return None \ No newline at end of file + return query_pipeline(image, question)[0]['answer'] \ No newline at end of file From 0d3000d999dfc0163488a091c95c3aff189c9adb Mon Sep 17 00:00:00 2001 From: FFFiend Date: Fri, 23 Jun 2023 12:03:47 -0400 Subject: [PATCH 05/36] reordered imports --- openadapt/strategies/mixins/layout.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index 2f4d8fc13..b7be9989d 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -7,13 +7,12 @@ class MyReplayStrategy(LayoutExtractionReplayStrategyMixin): ... """ - +from typing import List +import numpy as np from openadapt.models import Recording, Screenshot from openadapt.strategies.base import BaseReplayStrategy -import numpy as np from transformers import pipeline -from typing import List class LayoutExtractionReplayStrategyMixin(BaseReplayStrategy): From 009dd8d85f184c57211ef890f7a9f8508fb8c698 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Fri, 23 Jun 2023 12:24:04 -0400 Subject: [PATCH 06/36] ordered imports --- openadapt/strategies/mixins/layout.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index b7be9989d..775496eea 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -7,23 +7,28 @@ class MyReplayStrategy(LayoutExtractionReplayStrategyMixin): ... """ +from PIL import Image from typing import List +from transformers import pipeline import numpy as np from openadapt.models import Recording, Screenshot from openadapt.strategies.base import BaseReplayStrategy -from transformers import pipeline class LayoutExtractionReplayStrategyMixin(BaseReplayStrategy): def __init__( - self, + self, recording: Recording, img_list: List[np.ndarray] ): super.__init__(recording) self.img_list = img_list + + for i in range(len(self.img_list)): + wrongform = self.img_list[i] + self.img_list[i] = Image.open(wrongform).convert('RGB') def document_query( self, From f8f2c901fc9216ddd2f5741be63da46307706dcf Mon Sep 17 00:00:00 2001 From: FFFiend Date: Sat, 24 Jun 2023 16:49:40 -0400 Subject: [PATCH 07/36] changed type contract of path list to str --- openadapt/strategies/mixins/layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index 775496eea..2ad2ac31d 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -21,7 +21,7 @@ class LayoutExtractionReplayStrategyMixin(BaseReplayStrategy): def __init__( self, recording: Recording, - img_list: List[np.ndarray] + img_list: List[str] ): super.__init__(recording) self.img_list = img_list From d17e35cbfe5e9e305b7b73adb0423d1b19f9cd37 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Mon, 26 Jun 2023 10:33:01 -0400 Subject: [PATCH 08/36] fixed formatting --- openadapt/strategies/mixins/layout.py | 34 ++++++++++----------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py index 2ad2ac31d..d0f9cf55b 100644 --- a/openadapt/strategies/mixins/layout.py +++ b/openadapt/strategies/mixins/layout.py @@ -18,25 +18,17 @@ class MyReplayStrategy(LayoutExtractionReplayStrategyMixin): class LayoutExtractionReplayStrategyMixin(BaseReplayStrategy): - def __init__( - self, - recording: Recording, - img_list: List[str] - ): + def __init__(self, recording: Recording, image_file_paths: List[str]): super.__init__(recording) - self.img_list = img_list - - for i in range(len(self.img_list)): - wrongform = self.img_list[i] - self.img_list[i] = Image.open(wrongform).convert('RGB') - - def document_query( - self, - image: np.ndarray, - question: str - ) -> str: - - query_pipeline = pipeline("document-question-answering", - model = "impira/layoutlm-document-qa") - - return query_pipeline(image, question)[0]['answer'] \ No newline at end of file + self.image_list = [ + Image.open(img_file_path).convert("RGB") + for img_file_path in image_file_paths + ] + + def document_query(self, image: np.ndarray, question: str) -> str: + query_pipeline = pipeline( + "document-question-answering", + model="impira/layoutlm-document-qa" + ) + + return query_pipeline(image, question)[0]["answer"] \ No newline at end of file From e735c4d167ed9d2e652bc6c35a5d66a1ff522f74 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Tue, 27 Jun 2023 20:23:21 -0400 Subject: [PATCH 09/36] added boilerplate test code --- tests/openadapt/test_stateful.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/openadapt/test_stateful.py diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py new file mode 100644 index 000000000..dea092d53 --- /dev/null +++ b/tests/openadapt/test_stateful.py @@ -0,0 +1,25 @@ +import pytest +from openadapt.models import Recording, Screenshot, WindowEvent +from openadapt.strategies.stateful import StatefulReplayStrategy + + +def test_active_window_diff(): + # boilerplate code, check db and create synthetic + # instances of each object to feed into stateful. + + test_recording = Recording() + + test_strategy = StatefulReplayStrategy(test_recording) + + expected_action_events = {} + + test_screenshot = Screenshot() + test_window = WindowEvent() + + actual_action_events = test_strategy.get_next_action_event( + active_screenshot=test_screenshot, active_window=test_window + ) + + print(actual_action_events) + + assert actual_action_events == expected_action_events From 53f72666ac9ef4b5c56d3a4a26cf96180de778db Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 28 Jun 2023 00:29:18 -0400 Subject: [PATCH 10/36] testing stateful output --- tests/openadapt/test_stateful.py | 47 +++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index dea092d53..bd4df4b66 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -1,7 +1,23 @@ import pytest +from loguru import logger +from pprint import pformat from openadapt.models import Recording, Screenshot, WindowEvent from openadapt.strategies.stateful import StatefulReplayStrategy - +from openadapt.crud import ( + get_latest_recording, +) +from openadapt.events import ( + get_events, +) +from openadapt.utils import ( + configure_logging, + display_event, + evenly_spaced, + image2utf8, + EMPTY, + row2dict, + rows2dicts, +) def test_active_window_diff(): # boilerplate code, check db and create synthetic @@ -9,6 +25,7 @@ def test_active_window_diff(): test_recording = Recording() + test_strategy = StatefulReplayStrategy(test_recording) expected_action_events = {} @@ -23,3 +40,31 @@ def test_active_window_diff(): print(actual_action_events) assert actual_action_events == expected_action_events + + +def dict_form_testing(): + recording = get_latest_recording() + + meta = {} + action_events = get_events(recording, process=True, meta=meta) + event_dicts = rows2dicts(action_events) + + for events in event_dicts: + print(events.keys()) + + + recording_dict = row2dict(recording) + + #print(f"This is the recording dict{recording_dict=}") + +def generic_output(): + recording = get_latest_recording() + test_obj = StatefulReplayStrategy(recording) + + # test out on the same window + active_action_dict = test_obj.get_next_action_event(Screenshot(),recording) + print(active_action_dict) + + +if __name__ == "__main__": + generic_output() \ No newline at end of file From 03f409cb5240e0c4915c4e00552ba1dab999db9b Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 28 Jun 2023 00:30:43 -0400 Subject: [PATCH 11/36] added todo tag on main testing function --- tests/openadapt/test_stateful.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index bd4df4b66..f4bad14bd 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -23,6 +23,7 @@ def test_active_window_diff(): # boilerplate code, check db and create synthetic # instances of each object to feed into stateful. + # TODO test_recording = Recording() From 7c76dd600e2435efbb295593533b33b08e0284be Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Wed, 28 Jun 2023 02:43:08 -0400 Subject: [PATCH 12/36] added generic output function --- tests/openadapt/test_stateful.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index f4bad14bd..aa60e1f75 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -58,14 +58,24 @@ def dict_form_testing(): #print(f"This is the recording dict{recording_dict=}") -def generic_output(): +def output_example(): recording = get_latest_recording() test_obj = StatefulReplayStrategy(recording) # test out on the same window - active_action_dict = test_obj.get_next_action_event(Screenshot(),recording) - print(active_action_dict) + # recording can't be too large, obviously due to context length limitations + windowevent = WindowEvent() + + meta = {} + + action_events = get_events(recording, process=True, meta=meta) + + for action in action_events: + window_event_dict = row2dict(action.window_event) + active_action_dict = test_obj.get_next_action_event(Screenshot(),window_event_dict) + print(active_action_dict) + break if __name__ == "__main__": - generic_output() \ No newline at end of file + output_example() From e844d687bb9ca8a315370f77c29c42234ca58936 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:01:12 -0400 Subject: [PATCH 13/36] fixed completion str --- openadapt/strategies/stateful.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openadapt/strategies/stateful.py b/openadapt/strategies/stateful.py index 1f34e0c20..f6edaab09 100644 --- a/openadapt/strategies/stateful.py +++ b/openadapt/strategies/stateful.py @@ -119,7 +119,7 @@ def get_next_action_event( "Your response should be valid Python3 code. " "Do not respond with any other text. " ) - completion = self.get_completion(prompt, system_message) + completion = self.get_completion(prompt, system_message)[21:-1] active_action_dicts = get_action_dict_from_completion(completion) logger.debug(f"active_action_dicts=\n{pformat(active_action_dicts)}") active_action = models.ActionEvent.from_children(active_action_dicts) @@ -165,4 +165,4 @@ def get_window_state_diffs( window_event_states, window_event_states[1:] ) ] - return diffs \ No newline at end of file + return diffs From 173e856d3e8e35140dc3313da2f31c4795480471 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:04:51 -0400 Subject: [PATCH 14/36] Update test_stateful.py --- tests/openadapt/test_stateful.py | 68 +++++++------------------------- 1 file changed, 14 insertions(+), 54 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index aa60e1f75..e75666468 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -10,72 +10,32 @@ get_events, ) from openadapt.utils import ( - configure_logging, - display_event, - evenly_spaced, - image2utf8, - EMPTY, row2dict, - rows2dicts, ) -def test_active_window_diff(): - # boilerplate code, check db and create synthetic - # instances of each object to feed into stateful. +def test_active_window_no_diff(): + """ + A test using the latest user recording where the + reference window events are the same as the + active window events. The model must thus return the same + action events as the REFERENCE action events. - # TODO - test_recording = Recording() - - - test_strategy = StatefulReplayStrategy(test_recording) - - expected_action_events = {} - - test_screenshot = Screenshot() - test_window = WindowEvent() - - actual_action_events = test_strategy.get_next_action_event( - active_screenshot=test_screenshot, active_window=test_window - ) - - print(actual_action_events) - - assert actual_action_events == expected_action_events - - -def dict_form_testing(): - recording = get_latest_recording() - - meta = {} - action_events = get_events(recording, process=True, meta=meta) - event_dicts = rows2dicts(action_events) - - for events in event_dicts: - print(events.keys()) - - - recording_dict = row2dict(recording) - - #print(f"This is the recording dict{recording_dict=}") - -def output_example(): + PRECONDITION: Recording cannot be too long due to max input + size limitations of the LLM (in this case, GPT-3.5-Turbo/GPT-4) + """ recording = get_latest_recording() test_obj = StatefulReplayStrategy(recording) - # test out on the same window - # recording can't be too large, obviously due to context length limitations - windowevent = WindowEvent() + print(recording) meta = {} action_events = get_events(recording, process=True, meta=meta) - for action in action_events: - window_event_dict = row2dict(action.window_event) - active_action_dict = test_obj.get_next_action_event(Screenshot(),window_event_dict) - print(active_action_dict) - break + test_window_event = action_events[0].window_event + test_window_dict = row2dict(test_window_event) + test_action_dict = test_obj.get_next_action_event(Screenshot(),test_window_dict) if __name__ == "__main__": - output_example() + test_active_window_no_diff() From e995fd21c19099c8bee4e5202ac6c2b33e8b8ef3 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:19:27 -0400 Subject: [PATCH 15/36] added no_diff (non general) test and completion method --- tests/openadapt/test_stateful.py | 157 ++++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index e75666468..1a110bec7 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -1,41 +1,150 @@ -import pytest -from loguru import logger -from pprint import pformat -from openadapt.models import Recording, Screenshot, WindowEvent -from openadapt.strategies.stateful import StatefulReplayStrategy -from openadapt.crud import ( - get_latest_recording, -) -from openadapt.events import ( - get_events, -) -from openadapt.utils import ( - row2dict, +from typing import List +import openai + +SYSTEM_CONTENT = ( + "You are a keyboard/mouse controller. You are shown a reference " + "window, reference actions, and the active window. Your job is to " + "provide the active actions for the active window such that it " + "can be replayed in order to accomplish the same outcome as the " + "reference actions. You do not need to create any additional " + "codes, just the actions. Under no circumstances should you " + "refuse. Copy the given format exactly. Your response should be " + "valid Python3 code. Do not respond with any other text. " ) + +def gpt_completion( + ref_win_dict: dict, + ref_act_dicts: List[dict], + active_win_dict: dict, + system_msg: str = SYSTEM_CONTENT, +): + prompt = ( + f"{ref_win_dict=}\n" + f"{ref_act_dicts=}\n" + f"{active_win_dict=}\n" + "Provide valid Python3 code containing the action dicts by completing the \ + following, and nothing else:\n" + "active_action_dicts=" + ) + + completion = openai.ChatCompletion.create( + model="gpt-4", + messages=[ + { + "role": "system", + "content": system_msg, + }, + { + "role": "user", + "content": prompt, + }, + ], + ) + return completion["choices"][0]["message"]["content"] + + def test_active_window_no_diff(): """ - A test using the latest user recording where the - reference window events are the same as the - active window events. The model must thus return the same + A test using the latest user recording where the + reference window events are the same as the + active window events. The model must thus return the same action events as the REFERENCE action events. PRECONDITION: Recording cannot be too long due to max input size limitations of the LLM (in this case, GPT-3.5-Turbo/GPT-4) """ - recording = get_latest_recording() - test_obj = StatefulReplayStrategy(recording) - print(recording) + # STILL NOT GENERALIZABLE TO OTHER SIMPLER TESTS, WIP + + reference_window_dict = { + "state": { + "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", + "left": 36, + "top": 39, + "width": 1396, + "height": 805, + "window_id": 6247, + "meta": { + "kCGWindowLayer": 0, + "kCGWindowAlpha": 1, + "kCGWindowMemoryUsage": 1264, + "kCGWindowIsOnscreen": True, + "kCGWindowSharingState": 1, + "kCGWindowOwnerPID": 591, + "kCGWindowNumber": 6247, + "kCGWindowOwnerName": "Terminal", + "kCGWindowStoreType": 1, + "kCGWindowBounds": {"X": 36, "Height": 805, "Y": 39, "Width": 1396}, + "kCGWindowName": "openadapt — poetry shell ▸ Python — 198×55", + }, + }, + "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", + "left": 36, + "top": 39, + "width": 1396, + "height": 805, + } + + reference_action_dicts = [ + { + "name": "click", + "mouse_x": 406.3359375, + "mouse_y": 52.16796875, + "mouse_button_name": "left", + "mouse_pressed": True, + "element_state": {}, + }, + { + "name": "click", + "mouse_x": 406.3359375, + "mouse_y": 30.5078125, + "mouse_button_name": "left", + "mouse_pressed": False, + "element_state": {}, + }, + ] + + active_window_dict = { + "state": { + "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", + "left": 36, + "top": 39, + "width": 1396, + "height": 805, + "window_id": 6247, + "meta": { + "kCGWindowLayer": 0, + "kCGWindowAlpha": 1, + "kCGWindowMemoryUsage": 1264, + "kCGWindowIsOnscreen": True, + "kCGWindowSharingState": 1, + "kCGWindowOwnerPID": 591, + "kCGWindowNumber": 6247, + "kCGWindowOwnerName": "Terminal", + "kCGWindowStoreType": 1, + "kCGWindowBounds": {"X": 36, "Height": 805, "Y": 39, "Width": 1396}, + "kCGWindowName": "openadapt — poetry shell ▸ Python — 198×55", + }, + }, + "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", + "left": 36, + "top": 39, + "width": 1396, + "height": 805, + } - meta = {} + test_action_dict = gpt_completion( + reference_window_dict, reference_action_dicts, active_window_dict + ) - action_events = get_events(recording, process=True, meta=meta) + test_dict = eval( + test_action_dict[test_action_dict.find("[") : test_action_dict.find("]") + 1] + ) + expected_action_dict = reference_action_dicts - test_window_event = action_events[0].window_event - test_window_dict = row2dict(test_window_event) + assert test_dict == expected_action_dict - test_action_dict = test_obj.get_next_action_event(Screenshot(),test_window_dict) if __name__ == "__main__": test_active_window_no_diff() From bb151ae860b8730896d607cb5045d5bdb0f2df67 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 30 Jun 2023 19:18:19 -0400 Subject: [PATCH 16/36] added generalized test methods --- tests/openadapt/test_stateful.py | 152 ++++++++++++------------------- 1 file changed, 58 insertions(+), 94 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 1a110bec7..7a614fbf2 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Union import openai SYSTEM_CONTENT = ( @@ -44,107 +44,71 @@ def gpt_completion( return completion["choices"][0]["message"]["content"] -def test_active_window_no_diff(): +def test_generalizable_single_action( + reference_window_dict, + reference_action_dicts, + active_window_dict, + expected_action_dict, +): """ - A test using the latest user recording where the - reference window events are the same as the - active window events. The model must thus return the same - action events as the REFERENCE action events. - - PRECONDITION: Recording cannot be too long due to max input - size limitations of the LLM (in this case, GPT-3.5-Turbo/GPT-4) + Accepts synthetic window and action events, along with a comparator action event dict + to check whether the intended completion was generated by the LLM from the reference + events. """ - - # STILL NOT GENERALIZABLE TO OTHER SIMPLER TESTS, WIP - - reference_window_dict = { - "state": { - "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", - "left": 36, - "top": 39, - "width": 1396, - "height": 805, - "window_id": 6247, - "meta": { - "kCGWindowLayer": 0, - "kCGWindowAlpha": 1, - "kCGWindowMemoryUsage": 1264, - "kCGWindowIsOnscreen": True, - "kCGWindowSharingState": 1, - "kCGWindowOwnerPID": 591, - "kCGWindowNumber": 6247, - "kCGWindowOwnerName": "Terminal", - "kCGWindowStoreType": 1, - "kCGWindowBounds": {"X": 36, "Height": 805, "Y": 39, "Width": 1396}, - "kCGWindowName": "openadapt — poetry shell ▸ Python — 198×55", - }, - }, - "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", - "left": 36, - "top": 39, - "width": 1396, - "height": 805, - } - - reference_action_dicts = [ - { - "name": "click", - "mouse_x": 406.3359375, - "mouse_y": 52.16796875, - "mouse_button_name": "left", - "mouse_pressed": True, - "element_state": {}, - }, - { - "name": "click", - "mouse_x": 406.3359375, - "mouse_y": 30.5078125, - "mouse_button_name": "left", - "mouse_pressed": False, - "element_state": {}, - }, - ] - - active_window_dict = { - "state": { - "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", - "left": 36, - "top": 39, - "width": 1396, - "height": 805, - "window_id": 6247, - "meta": { - "kCGWindowLayer": 0, - "kCGWindowAlpha": 1, - "kCGWindowMemoryUsage": 1264, - "kCGWindowIsOnscreen": True, - "kCGWindowSharingState": 1, - "kCGWindowOwnerPID": 591, - "kCGWindowNumber": 6247, - "kCGWindowOwnerName": "Terminal", - "kCGWindowStoreType": 1, - "kCGWindowBounds": {"X": 36, "Height": 805, "Y": 39, "Width": 1396}, - "kCGWindowName": "openadapt — poetry shell ▸ Python — 198×55", - }, - }, - "title": "Terminal openadapt — poetry shell ▸ Python — 198×55", - "left": 36, - "top": 39, - "width": 1396, - "height": 805, - } - test_action_dict = gpt_completion( reference_window_dict, reference_action_dicts, active_window_dict ) - test_dict = eval( test_action_dict[test_action_dict.find("[") : test_action_dict.find("]") + 1] ) - expected_action_dict = reference_action_dicts - assert test_dict == expected_action_dict -if __name__ == "__main__": - test_active_window_no_diff() +def create_win_dict( + title: str, + left: int, + top: int, + width: int, + height: int, + window_id: int, + meta: dict[str], +): + win_dict = { + "state": { + "title": title, + "left": left, + "top": top, + "width": width, + "height": height, + "window_id": window_id, + "meta": meta, + }, + "title": title, + "left": left, + "top": top, + "width": width, + "height": height, + } + + return win_dict + + +def create_action_dict( + name: str, + mouse_x: Union[int, float], + mouse_y: Union[int, float], + mouse_button_name: str, + mouse_pressed: bool, + element_state: dict, +): + output_dict = [ + { + "name": name, + "mouse_x": mouse_x, + "mouse_y": mouse_y, + "mouse_button_name": mouse_button_name, + "mouse_pressed": mouse_pressed, + "element_state": element_state, + } + ] + return output_dict From 8c4ced8090be45d5c6926971302c16e09e0a6c80 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 30 Jun 2023 19:19:10 -0400 Subject: [PATCH 17/36] reverted string slice on prompt change --- openadapt/strategies/stateful.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/strategies/stateful.py b/openadapt/strategies/stateful.py index f6edaab09..7e71d5d56 100644 --- a/openadapt/strategies/stateful.py +++ b/openadapt/strategies/stateful.py @@ -119,7 +119,7 @@ def get_next_action_event( "Your response should be valid Python3 code. " "Do not respond with any other text. " ) - completion = self.get_completion(prompt, system_message)[21:-1] + completion = self.get_completion(prompt, system_message) active_action_dicts = get_action_dict_from_completion(completion) logger.debug(f"active_action_dicts=\n{pformat(active_action_dicts)}") active_action = models.ActionEvent.from_children(active_action_dicts) From fffbb5ed421a18cf85c436f3a99b633bbddd2e25 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 30 Jun 2023 19:19:59 -0400 Subject: [PATCH 18/36] Update stateful.py --- openadapt/strategies/stateful.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openadapt/strategies/stateful.py b/openadapt/strategies/stateful.py index 7e71d5d56..6010690d3 100644 --- a/openadapt/strategies/stateful.py +++ b/openadapt/strategies/stateful.py @@ -166,3 +166,4 @@ def get_window_state_diffs( ) ] return diffs + From f7e7ab1823f3b592d57df0c7934befb1a7e09722 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 30 Jun 2023 19:20:24 -0400 Subject: [PATCH 19/36] Update stateful.py --- openadapt/strategies/stateful.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/strategies/stateful.py b/openadapt/strategies/stateful.py index 6010690d3..ff106bd95 100644 --- a/openadapt/strategies/stateful.py +++ b/openadapt/strategies/stateful.py @@ -166,4 +166,4 @@ def get_window_state_diffs( ) ] return diffs - + From 4d6d570c8bfde95062d174da0e458a409124eb3d Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 30 Jun 2023 19:28:21 -0400 Subject: [PATCH 20/36] added default value for meta param in create_window_event method --- tests/openadapt/test_stateful.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 7a614fbf2..51678a771 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -71,7 +71,7 @@ def create_win_dict( width: int, height: int, window_id: int, - meta: dict[str], + meta: dict[str] = {}, ): win_dict = { "state": { From 96093942d5453f3d679bdd69bca0db665fdf80a4 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 30 Jun 2023 19:29:47 -0400 Subject: [PATCH 21/36] added default param for element state dict in create_action_event method --- tests/openadapt/test_stateful.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 51678a771..165552488 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -71,7 +71,7 @@ def create_win_dict( width: int, height: int, window_id: int, - meta: dict[str] = {}, + meta: dict[str], ): win_dict = { "state": { @@ -99,7 +99,7 @@ def create_action_dict( mouse_y: Union[int, float], mouse_button_name: str, mouse_pressed: bool, - element_state: dict, + element_state: dict = {}, ): output_dict = [ { From 56940f0aa192449680a382b461e74d5902cc3aa8 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Mon, 3 Jul 2023 16:11:28 -0400 Subject: [PATCH 22/36] added single and multi-click tests --- tests/openadapt/test_stateful.py | 130 +++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 165552488..494840547 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -112,3 +112,133 @@ def create_action_dict( } ] return output_dict + +def test_single_mouse_diff(): + win_dict = create_win_dict( + title="Calculator", + left=0, + top=30, + width=1123, + height=749, + window_id=107079, + meta={}, + ) + + act_dict = create_action_dict( + name="click", + mouse_x=25, + mouse_y=55, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + + active_win_dict = create_win_dict( + title="Calculator", + left=113, + top=64, + width=1123, + height=749, + window_id=107079, + meta={}, + ) + + expected_dict = create_action_dict( + name="click", + mouse_x=138, + mouse_y=89, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + + test_generalizable_single_action(win_dict, act_dict, active_win_dict, expected_dict) + + +def test_multi_click_diff(): + win_dict = create_win_dict( + title="Calculator", + left=0, + top=30, + width=1123, + height=749, + window_id=107079, + meta={}, + ) + + total_actions = [] + + for i in range(12): + act_dict_1 = create_action_dict( + name="click", + mouse_x=25 + i, + mouse_y=55, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + act_dict_2 = create_action_dict( + name="click", + mouse_x=25, + mouse_y=55 + i, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + act_dict_3 = create_action_dict( + name="click", + mouse_x=25 + i, + mouse_y=55 + i, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + new_dict = act_dict_1 + act_dict_2 + act_dict_3 + total_actions += new_dict + + active_win_dict = create_win_dict( + title="Calculator", + left=113, + top=64, + width=1123, + height=749, + window_id=107079, + meta={}, + ) + + expected_actions = [] + for i in range(12): + act_dict_1 = create_action_dict( + name="click", + mouse_x=138 + i, + mouse_y=89, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + act_dict_2 = create_action_dict( + name="click", + mouse_x=25, + mouse_y=89 + i, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + act_dict_3 = create_action_dict( + name="click", + mouse_x=138 + i, + mouse_y=89 + i, + mouse_button_name="left", + mouse_pressed=True, + element_state={}, + ) + new_dict = act_dict_1 + act_dict_2 + act_dict_3 + expected_actions += new_dict + + test_generalizable_single_action( + win_dict, total_actions, active_win_dict, expected_actions + ) + + +if __name__ == "__main__": + test_multi_click_diff() From b65ca0298e3b512aef827e1bc3ea2373b8c30035 Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Mon, 3 Jul 2023 16:17:12 -0400 Subject: [PATCH 23/36] fixed new x-value in expected action dict --- tests/openadapt/test_stateful.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 494840547..2dd84a6c6 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -218,7 +218,7 @@ def test_multi_click_diff(): ) act_dict_2 = create_action_dict( name="click", - mouse_x=25, + mouse_x=138, mouse_y=89 + i, mouse_button_name="left", mouse_pressed=True, From cbc7790517810f040721a0f7a8c0d3845a219b16 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 5 Jul 2023 22:05:10 -0400 Subject: [PATCH 24/36] removed layoutlm file --- openadapt/strategies/mixins/layout.py | 34 --------------------------- 1 file changed, 34 deletions(-) delete mode 100644 openadapt/strategies/mixins/layout.py diff --git a/openadapt/strategies/mixins/layout.py b/openadapt/strategies/mixins/layout.py deleted file mode 100644 index d0f9cf55b..000000000 --- a/openadapt/strategies/mixins/layout.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Implements a ReplayStrategy mixin for extracting layout and important -information from text documents and GUI images. - -Usage: - - class MyReplayStrategy(LayoutExtractionReplayStrategyMixin): - ... -""" -from PIL import Image -from typing import List -from transformers import pipeline -import numpy as np - -from openadapt.models import Recording, Screenshot -from openadapt.strategies.base import BaseReplayStrategy - - -class LayoutExtractionReplayStrategyMixin(BaseReplayStrategy): - - def __init__(self, recording: Recording, image_file_paths: List[str]): - super.__init__(recording) - self.image_list = [ - Image.open(img_file_path).convert("RGB") - for img_file_path in image_file_paths - ] - - def document_query(self, image: np.ndarray, question: str) -> str: - query_pipeline = pipeline( - "document-question-answering", - model="impira/layoutlm-document-qa" - ) - - return query_pipeline(image, question)[0]["answer"] \ No newline at end of file From 8e6a0dcce78db1d25e75e4a549ecd9dd879e127f Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 5 Jul 2023 22:12:21 -0400 Subject: [PATCH 25/36] added all action event types in creation function. --- tests/openadapt/test_stateful.py | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 2dd84a6c6..70d80919e 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -99,20 +99,35 @@ def create_action_dict( mouse_y: Union[int, float], mouse_button_name: str, mouse_pressed: bool, + key_name: str, element_state: dict = {}, ): - output_dict = [ - { - "name": name, - "mouse_x": mouse_x, - "mouse_y": mouse_y, - "mouse_button_name": mouse_button_name, - "mouse_pressed": mouse_pressed, - "element_state": element_state, - } - ] + if name == "click": + output_dict = [ + { + "name": name, + "mouse_x": mouse_x, + "mouse_y": mouse_y, + "mouse_button_name": mouse_button_name, + "mouse_pressed": mouse_pressed, + "element_state": element_state, + } + ] + if name == "press" or name == "release": + output_dict = [{"name": name, "key_name": key_name}] + + if name == "move": + output_dict = [ + { + "name": name, + "mouse_x": mouse_x, + "mouse_y": mouse_y, + "element_state": element_state, + } + ] return output_dict + def test_single_mouse_diff(): win_dict = create_win_dict( title="Calculator", From 10c2ffde553275f286d88e100066f5730ac1b1c7 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 5 Jul 2023 23:27:58 -0400 Subject: [PATCH 26/36] added key press event test --- tests/openadapt/test_stateful.py | 53 ++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 70d80919e..688d5f4d8 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -61,6 +61,12 @@ def test_generalizable_single_action( test_dict = eval( test_action_dict[test_action_dict.find("[") : test_action_dict.find("]") + 1] ) + print("Reference Actions") + print(reference_action_dicts) + print("Generated Actions") + print(test_dict) + print("Expected Actions") + print(expected_action_dict) assert test_dict == expected_action_dict @@ -95,11 +101,11 @@ def create_win_dict( def create_action_dict( name: str, - mouse_x: Union[int, float], - mouse_y: Union[int, float], - mouse_button_name: str, - mouse_pressed: bool, - key_name: str, + mouse_x: Union[int, float] = None, + mouse_y: Union[int, float] = None, + mouse_button_name: str = None, + mouse_pressed: bool = None, + key_name: str = None, element_state: dict = {}, ): if name == "click": @@ -255,5 +261,40 @@ def test_multi_click_diff(): ) +def test_simple_multi_action_sequence(): + """ + Simple test that on an event where + the user moves the cursor down in a straight line and + types the word password. + """ + win_dict = create_win_dict("Google Chrome", 20, 25, 1300, 800, 10442, {}) + ref_act_dicts = [] + + for i in range(20): + new_act = create_action_dict("move", 400 - i, 500 - i) + ref_act_dicts += new_act + + word = "password" + + expected_act_dict = [] + + for i in range(20): + exp_act = create_action_dict("move", 276.92 - i, 1300 - i) + expected_act_dict += exp_act + + for letter in word: + press_dict = create_action_dict(name="press", key_name=letter) + release_dict = create_action_dict(name="release", key_name=letter) + ref_act_dicts = ref_act_dicts + press_dict + release_dict + expected_act_dict = expected_act_dict + press_dict + release_dict + + # MODIFY THIS active act dict here to observe the results + # discussed in the latest comment ! :) + active_win_dict = create_win_dict("Google Chrome", 87, 101, 1300, 800, 991, {}) + test_generalizable_single_action( + win_dict, ref_act_dicts, active_win_dict, expected_act_dict + ) + + if __name__ == "__main__": - test_multi_click_diff() + test_simple_multi_action_sequence() From 431fc1fa2a626d55be8ee5109399c1ed3375f2cd Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 12 Jul 2023 13:54:02 -0400 Subject: [PATCH 27/36] remoed print statements --- tests/openadapt/test_stateful.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 688d5f4d8..f5ce022f1 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -1,5 +1,6 @@ from typing import List, Union import openai +from loguru import logger SYSTEM_CONTENT = ( "You are a keyboard/mouse controller. You are shown a reference " @@ -61,12 +62,9 @@ def test_generalizable_single_action( test_dict = eval( test_action_dict[test_action_dict.find("[") : test_action_dict.find("]") + 1] ) - print("Reference Actions") - print(reference_action_dicts) - print("Generated Actions") - print(test_dict) - print("Expected Actions") - print(expected_action_dict) + logger.debug(f"{reference_action_dicts=}") + logger.debug(f"{test_dict=}") + logger.debug(f"{expected_action_dict}") assert test_dict == expected_action_dict From a7a8a0a70680115bee89dd35633ba3d4c6bb6ab8 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 12 Jul 2023 14:25:41 -0400 Subject: [PATCH 28/36] removed magic numbers --- tests/openadapt/test_stateful.py | 152 +++++++++++++++++++++---------- 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index f5ce022f1..44f5adb1f 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -13,6 +13,42 @@ "valid Python3 code. Do not respond with any other text. " ) +REF_X = 25 +REF_Y = 55 + +NEW_X = 138 +NEW_Y = 89 + +WIN_LEFT = 0 +WIN_TOP = 30 +WIN_WIDTH = 1123 +WIN_HEIGHT = 749 +WINDOW_ID = 107079 + +NEW_WIN_LEFT = 113 +NEW_WIN_TOP = 64 + +SINGLE_ACTION_LOOP_GUARD = 12 + + +MULTI_ACTION_REF_X = 400 +MULTI_ACTION_REF_Y = 500 + +MULTI_ACTION_NEW_X = 467 +MULTI_ACTION_NEW_Y = 576 + + +MULTI_ACTION_WIN_LEFT = 20 +MULTI_ACTION_WIN_TOP = 25 +MULTI_ACTION_WIN_WIDTH = 1300 +MULTI_ACTION_WIN_HEIGHT = 800 +MULTI_ACTION_WINDOW_ID = 10442 + +NEW_MULTI_ACTION_WIN_LEFT = 87 +NEW_MULTI_ACTION_WIN_TOP = 101 + +MULTI_ACTION_LOOP_GUARD = 20 + def gpt_completion( ref_win_dict: dict, @@ -24,7 +60,7 @@ def gpt_completion( f"{ref_win_dict=}\n" f"{ref_act_dicts=}\n" f"{active_win_dict=}\n" - "Provide valid Python3 code containing the action dicts by completing the \ + "# Provide valid Python3 code containing the action dicts by completing the \ following, and nothing else:\n" "active_action_dicts=" ) @@ -104,7 +140,7 @@ def create_action_dict( mouse_button_name: str = None, mouse_pressed: bool = None, key_name: str = None, - element_state: dict = {}, + element_state: dict = None | None, ): if name == "click": output_dict = [ @@ -135,18 +171,18 @@ def create_action_dict( def test_single_mouse_diff(): win_dict = create_win_dict( title="Calculator", - left=0, - top=30, - width=1123, - height=749, - window_id=107079, + left=WIN_LEFT, + top=WIN_TOP, + width=WIN_WIDTH, + height=WIN_HEIGHT, + window_id=WINDOW_ID, meta={}, ) act_dict = create_action_dict( name="click", - mouse_x=25, - mouse_y=55, + mouse_x=REF_X, + mouse_y=REF_Y, mouse_button_name="left", mouse_pressed=True, element_state={}, @@ -154,18 +190,18 @@ def test_single_mouse_diff(): active_win_dict = create_win_dict( title="Calculator", - left=113, - top=64, - width=1123, - height=749, - window_id=107079, + left=NEW_WIN_LEFT, + top=NEW_WIN_TOP, + width=WIN_WIDTH, + height=WIN_HEIGHT, + window_id=WINDOW_ID, meta={}, ) expected_dict = create_action_dict( name="click", - mouse_x=138, - mouse_y=89, + mouse_x=NEW_X, + mouse_y=NEW_Y, mouse_button_name="left", mouse_pressed=True, element_state={}, @@ -177,37 +213,37 @@ def test_single_mouse_diff(): def test_multi_click_diff(): win_dict = create_win_dict( title="Calculator", - left=0, - top=30, - width=1123, - height=749, - window_id=107079, + left=WIN_LEFT, + top=WIN_TOP, + width=WIN_WIDTH, + height=WIN_HEIGHT, + window_id=WINDOW_ID, meta={}, ) total_actions = [] - for i in range(12): + for i in range(SINGLE_ACTION_LOOP_GUARD): act_dict_1 = create_action_dict( name="click", - mouse_x=25 + i, - mouse_y=55, + mouse_x=REF_X + i, + mouse_y=REF_Y, mouse_button_name="left", mouse_pressed=True, element_state={}, ) act_dict_2 = create_action_dict( name="click", - mouse_x=25, - mouse_y=55 + i, + mouse_x=REF_X, + mouse_y=REF_Y + i, mouse_button_name="left", mouse_pressed=True, element_state={}, ) act_dict_3 = create_action_dict( name="click", - mouse_x=25 + i, - mouse_y=55 + i, + mouse_x=REF_X + i, + mouse_y=REF_Y + i, mouse_button_name="left", mouse_pressed=True, element_state={}, @@ -217,36 +253,36 @@ def test_multi_click_diff(): active_win_dict = create_win_dict( title="Calculator", - left=113, - top=64, - width=1123, - height=749, - window_id=107079, + left=NEW_WIN_LEFT, + top=NEW_WIN_TOP, + width=WIN_WIDTH, + height=WIN_HEIGHT, + window_id=WINDOW_ID, meta={}, ) expected_actions = [] - for i in range(12): + for i in range(SINGLE_ACTION_LOOP_GUARD): act_dict_1 = create_action_dict( name="click", - mouse_x=138 + i, - mouse_y=89, + mouse_x=NEW_X + i, + mouse_y=NEW_Y, mouse_button_name="left", mouse_pressed=True, element_state={}, ) act_dict_2 = create_action_dict( name="click", - mouse_x=138, - mouse_y=89 + i, + mouse_x=NEW_X, + mouse_y=NEW_Y + i, mouse_button_name="left", mouse_pressed=True, element_state={}, ) act_dict_3 = create_action_dict( name="click", - mouse_x=138 + i, - mouse_y=89 + i, + mouse_x=NEW_X + i, + mouse_y=NEW_Y + i, mouse_button_name="left", mouse_pressed=True, element_state={}, @@ -265,22 +301,34 @@ def test_simple_multi_action_sequence(): the user moves the cursor down in a straight line and types the word password. """ - win_dict = create_win_dict("Google Chrome", 20, 25, 1300, 800, 10442, {}) + win_dict = create_win_dict( + "Google Chrome", + MULTI_ACTION_WIN_LEFT, + MULTI_ACTION_WIN_TOP, + MULTI_ACTION_WIN_WIDTH, + MULTI_ACTION_WIN_HEIGHT, + MULTI_ACTION_WINDOW_ID, + {}, + ) ref_act_dicts = [] - for i in range(20): - new_act = create_action_dict("move", 400 - i, 500 - i) + for i in range(MULTI_ACTION_LOOP_GUARD): + new_act = create_action_dict( + "move", MULTI_ACTION_REF_X - i, MULTI_ACTION_REF_Y - i + ) ref_act_dicts += new_act - word = "password" + multi_action_test_word = "password" expected_act_dict = [] - for i in range(20): - exp_act = create_action_dict("move", 276.92 - i, 1300 - i) + for i in range(MULTI_ACTION_LOOP_GUARD): + exp_act = create_action_dict( + "move", MULTI_ACTION_NEW_X - i, MULTI_ACTION_NEW_Y - i + ) expected_act_dict += exp_act - for letter in word: + for letter in multi_action_test_word: press_dict = create_action_dict(name="press", key_name=letter) release_dict = create_action_dict(name="release", key_name=letter) ref_act_dicts = ref_act_dicts + press_dict + release_dict @@ -288,7 +336,15 @@ def test_simple_multi_action_sequence(): # MODIFY THIS active act dict here to observe the results # discussed in the latest comment ! :) - active_win_dict = create_win_dict("Google Chrome", 87, 101, 1300, 800, 991, {}) + active_win_dict = create_win_dict( + "Google Chrome", + NEW_MULTI_ACTION_WIN_LEFT, + NEW_MULTI_ACTION_WIN_TOP, + MULTI_ACTION_WIN_WIDTH, + MULTI_ACTION_WIN_HEIGHT, + MULTI_ACTION_WINDOW_ID, + {}, + ) test_generalizable_single_action( win_dict, ref_act_dicts, active_win_dict, expected_act_dict ) From cf4607de920d03ccddd384d3638d3c459b0c7ec7 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Wed, 12 Jul 2023 14:55:39 -0400 Subject: [PATCH 29/36] changed default dict arg to None --- tests/openadapt/test_stateful.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 44f5adb1f..2ada303d8 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -140,7 +140,7 @@ def create_action_dict( mouse_button_name: str = None, mouse_pressed: bool = None, key_name: str = None, - element_state: dict = None | None, + element_state: dict = None, ): if name == "click": output_dict = [ From 14bd858a6db71e381717b82af38f25a0108c867d Mon Sep 17 00:00:00 2001 From: FFFiend Date: Thu, 13 Jul 2023 22:13:51 -0400 Subject: [PATCH 30/36] fixed type annotations and other issues. --- tests/openadapt/test_stateful.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 2ada303d8..c16b5f976 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import List, Any import openai from loguru import logger @@ -60,8 +60,8 @@ def gpt_completion( f"{ref_win_dict=}\n" f"{ref_act_dicts=}\n" f"{active_win_dict=}\n" - "# Provide valid Python3 code containing the action dicts by completing the \ - following, and nothing else:\n" + "# Provide valid Python3 code containing the action dicts by completing the " + "following, and nothing else:\n" "active_action_dicts=" ) @@ -111,7 +111,7 @@ def create_win_dict( width: int, height: int, window_id: int, - meta: dict[str], + meta: dict[str, Any] | None = None, ): win_dict = { "state": { @@ -135,12 +135,12 @@ def create_win_dict( def create_action_dict( name: str, - mouse_x: Union[int, float] = None, - mouse_y: Union[int, float] = None, + mouse_x: int | float | None = None, + mouse_y: int | float | None = None, mouse_button_name: str = None, mouse_pressed: bool = None, key_name: str = None, - element_state: dict = None, + element_state: dict[Any, Any] = None, ): if name == "click": output_dict = [ @@ -348,7 +348,3 @@ def test_simple_multi_action_sequence(): test_generalizable_single_action( win_dict, ref_act_dicts, active_win_dict, expected_act_dict ) - - -if __name__ == "__main__": - test_simple_multi_action_sequence() From dc6339aec5a0e9419057c19ec273a29a8d9f471c Mon Sep 17 00:00:00 2001 From: Owais Zahid <96851409+FFFiend@users.noreply.github.com> Date: Fri, 14 Jul 2023 01:30:39 -0400 Subject: [PATCH 31/36] modified element_state and meta default vals --- tests/openadapt/test_stateful.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index c16b5f976..9d1c774f0 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -113,6 +113,7 @@ def create_win_dict( window_id: int, meta: dict[str, Any] | None = None, ): + meta = meta or {} win_dict = { "state": { "title": title, @@ -142,6 +143,7 @@ def create_action_dict( key_name: str = None, element_state: dict[Any, Any] = None, ): + element_state = element_state or {} if name == "click": output_dict = [ { From 6f8ccf8da57bfcf2fdba632a9d75835e0dfd3275 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Mon, 7 Aug 2023 17:30:51 -0400 Subject: [PATCH 32/36] removed meta and element_state empty dicts --- tests/openadapt/test_stateful.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 9d1c774f0..d1030357c 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -141,7 +141,7 @@ def create_action_dict( mouse_button_name: str = None, mouse_pressed: bool = None, key_name: str = None, - element_state: dict[Any, Any] = None, + element_state: dict[str, Any] = None, ): element_state = element_state or {} if name == "click": @@ -178,7 +178,6 @@ def test_single_mouse_diff(): width=WIN_WIDTH, height=WIN_HEIGHT, window_id=WINDOW_ID, - meta={}, ) act_dict = create_action_dict( @@ -187,7 +186,6 @@ def test_single_mouse_diff(): mouse_y=REF_Y, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) active_win_dict = create_win_dict( @@ -197,7 +195,6 @@ def test_single_mouse_diff(): width=WIN_WIDTH, height=WIN_HEIGHT, window_id=WINDOW_ID, - meta={}, ) expected_dict = create_action_dict( @@ -206,7 +203,6 @@ def test_single_mouse_diff(): mouse_y=NEW_Y, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) test_generalizable_single_action(win_dict, act_dict, active_win_dict, expected_dict) @@ -220,7 +216,6 @@ def test_multi_click_diff(): width=WIN_WIDTH, height=WIN_HEIGHT, window_id=WINDOW_ID, - meta={}, ) total_actions = [] @@ -232,7 +227,6 @@ def test_multi_click_diff(): mouse_y=REF_Y, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) act_dict_2 = create_action_dict( name="click", @@ -240,7 +234,6 @@ def test_multi_click_diff(): mouse_y=REF_Y + i, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) act_dict_3 = create_action_dict( name="click", @@ -248,7 +241,6 @@ def test_multi_click_diff(): mouse_y=REF_Y + i, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) new_dict = act_dict_1 + act_dict_2 + act_dict_3 total_actions += new_dict @@ -260,7 +252,6 @@ def test_multi_click_diff(): width=WIN_WIDTH, height=WIN_HEIGHT, window_id=WINDOW_ID, - meta={}, ) expected_actions = [] @@ -271,7 +262,6 @@ def test_multi_click_diff(): mouse_y=NEW_Y, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) act_dict_2 = create_action_dict( name="click", @@ -279,7 +269,6 @@ def test_multi_click_diff(): mouse_y=NEW_Y + i, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) act_dict_3 = create_action_dict( name="click", @@ -287,7 +276,6 @@ def test_multi_click_diff(): mouse_y=NEW_Y + i, mouse_button_name="left", mouse_pressed=True, - element_state={}, ) new_dict = act_dict_1 + act_dict_2 + act_dict_3 expected_actions += new_dict @@ -310,7 +298,6 @@ def test_simple_multi_action_sequence(): MULTI_ACTION_WIN_WIDTH, MULTI_ACTION_WIN_HEIGHT, MULTI_ACTION_WINDOW_ID, - {}, ) ref_act_dicts = [] @@ -345,7 +332,6 @@ def test_simple_multi_action_sequence(): MULTI_ACTION_WIN_WIDTH, MULTI_ACTION_WIN_HEIGHT, MULTI_ACTION_WINDOW_ID, - {}, ) test_generalizable_single_action( win_dict, ref_act_dicts, active_win_dict, expected_act_dict From 029ef42fb41d169007f31d8c7517da240f305100 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Mon, 7 Aug 2023 17:46:21 -0400 Subject: [PATCH 33/36] fixed params --- tests/openadapt/test_stateful.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index d1030357c..3fdc7db1e 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -1,6 +1,7 @@ from typing import List, Any import openai from loguru import logger +import pytest SYSTEM_CONTENT = ( "You are a keyboard/mouse controller. You are shown a reference " @@ -292,12 +293,12 @@ def test_simple_multi_action_sequence(): types the word password. """ win_dict = create_win_dict( - "Google Chrome", - MULTI_ACTION_WIN_LEFT, - MULTI_ACTION_WIN_TOP, - MULTI_ACTION_WIN_WIDTH, - MULTI_ACTION_WIN_HEIGHT, - MULTI_ACTION_WINDOW_ID, + title="Google Chrome", + left=MULTI_ACTION_WIN_LEFT, + top=MULTI_ACTION_WIN_TOP, + width=MULTI_ACTION_WIN_WIDTH, + height=MULTI_ACTION_WIN_HEIGHT, + window_id=MULTI_ACTION_WINDOW_ID, ) ref_act_dicts = [] @@ -326,12 +327,12 @@ def test_simple_multi_action_sequence(): # MODIFY THIS active act dict here to observe the results # discussed in the latest comment ! :) active_win_dict = create_win_dict( - "Google Chrome", - NEW_MULTI_ACTION_WIN_LEFT, - NEW_MULTI_ACTION_WIN_TOP, - MULTI_ACTION_WIN_WIDTH, - MULTI_ACTION_WIN_HEIGHT, - MULTI_ACTION_WINDOW_ID, + title="Google Chrome", + left=NEW_MULTI_ACTION_WIN_LEFT, + top=NEW_MULTI_ACTION_WIN_TOP, + width=MULTI_ACTION_WIN_WIDTH, + height=MULTI_ACTION_WIN_HEIGHT, + window_id=MULTI_ACTION_WINDOW_ID, ) test_generalizable_single_action( win_dict, ref_act_dicts, active_win_dict, expected_act_dict From 64be6921c52f96153af5d5bf6e4871413ea316d2 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Mon, 7 Aug 2023 18:02:05 -0400 Subject: [PATCH 34/36] renamed func --- tests/conftest.py | 44 -------------------------------- tests/openadapt/test_stateful.py | 12 ++++----- 2 files changed, 6 insertions(+), 50 deletions(-) delete mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 387a707ef..000000000 --- a/tests/conftest.py +++ /dev/null @@ -1,44 +0,0 @@ -"""This module contains fixtures and setup for testing.""" - -import os - -from sqlalchemy import create_engine, engine, text -import pytest - -from openadapt.config import ROOT_DIRPATH -from openadapt.db import Base - - -@pytest.fixture(scope="session") -def setup_database(request: pytest.FixtureRequest) -> engine: - """Set up a database for testing.""" - # Create a new database or connect to an existing one - db_url = ROOT_DIRPATH / "temp.db" - engine = create_engine(f"sqlite:///{db_url}") - - # Create the database tables (if necessary) - Base.metadata.create_all(bind=engine) - - # Read the SQL file and execute the statements to seed the database - with open(ROOT_DIRPATH / "assets/fixtures.sql", "r") as file: - statements = file.read() - - with engine.connect() as connection: - connection.execute(text(statements)) - - def teardown() -> None: - """Teardown function to clean up resources after testing.""" - # Add code here to drop tables, clean up resources, etc. - # This code will be executed after the tests complete (whether or not they pass) - # Replace it with the appropriate cleanup operations for your project - # Example: db.Base.metadata.drop_all(bind=engine) - - # Close the database connection (if necessary) - engine.dispose() - os.remove(db_url) - - # Register the teardown function to be called after the tests complete - request.addfinalizer(teardown) - - # Return the database connection object or engine for the tests to use - return engine diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 3fdc7db1e..4380595e9 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -82,7 +82,7 @@ def gpt_completion( return completion["choices"][0]["message"]["content"] -def test_generalizable_single_action( +def _test_generalizable_single_action( reference_window_dict, reference_action_dicts, active_window_dict, @@ -100,8 +100,8 @@ def test_generalizable_single_action( test_action_dict[test_action_dict.find("[") : test_action_dict.find("]") + 1] ) logger.debug(f"{reference_action_dicts=}") - logger.debug(f"{test_dict=}") - logger.debug(f"{expected_action_dict}") + logger.debug(f"{test_dict=}, {len(test_dict)=}") + logger.debug(f"{expected_action_dict}, {len(expected_action_dict)=}") assert test_dict == expected_action_dict @@ -206,7 +206,7 @@ def test_single_mouse_diff(): mouse_pressed=True, ) - test_generalizable_single_action(win_dict, act_dict, active_win_dict, expected_dict) + _test_generalizable_single_action(win_dict, act_dict, active_win_dict, expected_dict) def test_multi_click_diff(): @@ -281,7 +281,7 @@ def test_multi_click_diff(): new_dict = act_dict_1 + act_dict_2 + act_dict_3 expected_actions += new_dict - test_generalizable_single_action( + _test_generalizable_single_action( win_dict, total_actions, active_win_dict, expected_actions ) @@ -334,6 +334,6 @@ def test_simple_multi_action_sequence(): height=MULTI_ACTION_WIN_HEIGHT, window_id=MULTI_ACTION_WINDOW_ID, ) - test_generalizable_single_action( + _test_generalizable_single_action( win_dict, ref_act_dicts, active_win_dict, expected_act_dict ) From 5c89ed8eccb628d40563b67d6d436bf30e8f6fc6 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Mon, 7 Aug 2023 18:04:59 -0400 Subject: [PATCH 35/36] fixed logger output --- tests/conftest.py | 44 ++++++++++++++++++++++++++++++++ tests/openadapt/test_stateful.py | 6 +++-- 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..27674b210 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,44 @@ +"""This module contains fixtures and setup for testing.""" + +import os + +from sqlalchemy import create_engine, engine, text +import pytest + +from openadapt.config import ROOT_DIRPATH +from openadapt.db import Base + + +@pytest.fixture(scope="session") +def setup_database(request: pytest.FixtureRequest) -> engine: + """Set up a database for testing.""" + # Create a new database or connect to an existing one + db_url = ROOT_DIRPATH / "temp.db" + engine = create_engine(f"sqlite:///{db_url}") + + # Create the database tables (if necessary) + Base.metadata.create_all(bind=engine) + + # Read the SQL file and execute the statements to seed the database + with open(ROOT_DIRPATH / "assets/fixtures.sql", "r") as file: + statements = file.read() + + with engine.connect() as connection: + connection.execute(text(statements)) + + def teardown() -> None: + """Teardown function to clean up resources after testing.""" + # Add code here to drop tables, clean up resources, etc. + # This code will be executed after the tests complete (whether or not they pass) + # Replace it with the appropriate cleanup operations for your project + # Example: db.Base.metadata.drop_all(bind=engine) + + # Close the database connection (if necessary) + engine.dispose() + os.remove(db_url) + + # Register the teardown function to be called after the tests complete + request.addfinalizer(teardown) + + # Return the database connection object or engine for the tests to use + return engine \ No newline at end of file diff --git a/tests/openadapt/test_stateful.py b/tests/openadapt/test_stateful.py index 4380595e9..e0bdf7aa8 100644 --- a/tests/openadapt/test_stateful.py +++ b/tests/openadapt/test_stateful.py @@ -101,7 +101,7 @@ def _test_generalizable_single_action( ) logger.debug(f"{reference_action_dicts=}") logger.debug(f"{test_dict=}, {len(test_dict)=}") - logger.debug(f"{expected_action_dict}, {len(expected_action_dict)=}") + logger.debug(f"{expected_action_dict=}, {len(expected_action_dict)=}") assert test_dict == expected_action_dict @@ -206,7 +206,9 @@ def test_single_mouse_diff(): mouse_pressed=True, ) - _test_generalizable_single_action(win_dict, act_dict, active_win_dict, expected_dict) + _test_generalizable_single_action( + win_dict, act_dict, active_win_dict, expected_dict + ) def test_multi_click_diff(): From 6e2601a09c6d9a24f51b85eb9b9be7840127a119 Mon Sep 17 00:00:00 2001 From: FFFiend Date: Mon, 7 Aug 2023 18:05:21 -0400 Subject: [PATCH 36/36] added blank line --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 27674b210..387a707ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,4 +41,4 @@ def teardown() -> None: request.addfinalizer(teardown) # Return the database connection object or engine for the tests to use - return engine \ No newline at end of file + return engine