From 38ad07d8742dd1a3106af68f6a612c6abedea5ca Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Wed, 4 Dec 2024 17:21:20 +0100 Subject: [PATCH 01/17] Allow to communicate with DUT via standard input (#36687) * Allow to communicate with DUT via standard input * Use fabric-sync-app.py stdin instead dedicated pipe * Drop pipe stdin forwarder in fabric-sync-app.py * Restyled by autopep8 * Wait for thread to stop * Fix referencing not-created variable --------- Co-authored-by: Restyled.io --- docs/testing/python.md | 5 ++ .../fabric-admin/scripts/fabric-sync-app.py | 37 +-------------- scripts/tests/run_python_test.py | 46 +++++++++++++++++-- src/python_testing/TC_BRBINFO_4_1.py | 3 +- src/python_testing/TC_CCTRL_2_1.py | 3 +- src/python_testing/TC_CCTRL_2_2.py | 3 +- src/python_testing/TC_CCTRL_2_3.py | 3 +- src/python_testing/TC_ECOINFO_2_1.py | 3 +- src/python_testing/TC_ECOINFO_2_2.py | 3 +- src/python_testing/TC_MCORE_FS_1_1.py | 3 +- src/python_testing/TC_MCORE_FS_1_2.py | 3 +- src/python_testing/TC_MCORE_FS_1_3.py | 13 +++--- src/python_testing/TC_MCORE_FS_1_4.py | 13 +++--- src/python_testing/TC_MCORE_FS_1_5.py | 3 +- .../chip/testing/metadata.py | 2 + 15 files changed, 82 insertions(+), 61 deletions(-) diff --git a/docs/testing/python.md b/docs/testing/python.md index a7bec6d07e17c3..9bda585726d409 100644 --- a/docs/testing/python.md +++ b/docs/testing/python.md @@ -722,6 +722,11 @@ for that run, e.g.: - Example: `"Manual pairing code: \\[\\d+\\]"` +- `app-stdin-pipe`: Specifies the path to the named pipe that the test runner + might use to send input to the application. + + - Example: `/tmp/app-fifo` + - `script-args`: Specifies the arguments to be passed to the test script. - Example: diff --git a/examples/fabric-admin/scripts/fabric-sync-app.py b/examples/fabric-admin/scripts/fabric-sync-app.py index 3de85b9f672887..a44a2a2d7ae543 100755 --- a/examples/fabric-admin/scripts/fabric-sync-app.py +++ b/examples/fabric-admin/scripts/fabric-sync-app.py @@ -16,7 +16,6 @@ import asyncio import contextlib -import os import shutil import signal import sys @@ -41,26 +40,6 @@ async def forward_f(prefix: bytes, f_in: asyncio.StreamReader, f_out.flush() -async def forward_pipe(pipe_path: str, f_out: asyncio.StreamWriter): - """Forward named pipe to f_out. - - Unfortunately, Python does not support async file I/O on named pipes. This - function performs busy waiting with a short asyncio-friendly sleep to read - from the pipe. - """ - fd = os.open(pipe_path, os.O_RDONLY | os.O_NONBLOCK) - while True: - try: - data = os.read(fd, 1024) - if data: - f_out.write(data) - await f_out.drain() - if not data: - await asyncio.sleep(0.1) - except BlockingIOError: - await asyncio.sleep(0.1) - - async def forward_stdin(f_out: asyncio.StreamWriter): """Forward stdin to f_out.""" loop = asyncio.get_event_loop() @@ -175,9 +154,6 @@ async def main(args): storage = TemporaryDirectory(prefix="fabric-sync-app") storage_dir = Path(storage.name) - if args.stdin_pipe and not args.stdin_pipe.exists(): - os.mkfifo(args.stdin_pipe) - admin, bridge = await asyncio.gather( run_admin( args.app_admin, @@ -206,8 +182,6 @@ def terminate(): admin.terminate() with contextlib.suppress(ProcessLookupError): bridge.terminate() - if args.stdin_pipe: - args.stdin_pipe.unlink(missing_ok=True) loop.remove_signal_handler(signal.SIGINT) loop.remove_signal_handler(signal.SIGTERM) @@ -249,17 +223,12 @@ def terminate(): await admin.send(f"pairing open-commissioning-window {bridge_node_id} {cw_endpoint_id}" f" {cw_option} {cw_timeout} {cw_iteration} {cw_discriminator}") - def get_input_forwarder(): - if args.stdin_pipe: - return forward_pipe(args.stdin_pipe, admin.p.stdin) - return forward_stdin(admin.p.stdin) - try: # Wait for any of the tasks to complete. _, pending = await asyncio.wait([ asyncio.create_task(admin.wait()), asyncio.create_task(bridge.wait()), - asyncio.create_task(get_input_forwarder()), + asyncio.create_task(forward_stdin(admin.p.stdin)), ], return_when=asyncio.FIRST_COMPLETED) # Cancel the remaining tasks. for task in pending: @@ -285,8 +254,6 @@ def get_input_forwarder(): help="fabric-admin RPC server port") parser.add_argument("--app-bridge-rpc-port", metavar="PORT", type=int, help="fabric-bridge RPC server port") - parser.add_argument("--stdin-pipe", metavar="PATH", type=Path, - help="read input from a named pipe instead of stdin") parser.add_argument("--storage-dir", metavar="PATH", type=Path, help=("directory to place storage files in; by default " "volatile storage is used")) @@ -309,7 +276,5 @@ def get_input_forwarder(): parser.error("fabric-admin executable not found in PATH. Use '--app-admin' argument to provide it.") if args.app_bridge is None or not args.app_bridge.exists(): parser.error("fabric-bridge-app executable not found in PATH. Use '--app-bridge' argument to provide it.") - if args.stdin_pipe and args.stdin_pipe.exists() and not args.stdin_pipe.is_fifo(): - parser.error("given stdin pipe exists and is not a named pipe") with contextlib.suppress(KeyboardInterrupt): asyncio.run(main(args)) diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index 0c40f9ac30f10b..267fb513952437 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import datetime import glob import io @@ -22,9 +23,12 @@ import os.path import pathlib import re +import select import shlex import sys +import threading import time +import typing import click import coloredlogs @@ -68,6 +72,23 @@ def process_test_script_output(line, is_stderr): return process_chip_output(line, is_stderr, TAG_PROCESS_TEST) +def forward_fifo(path: str, f_out: typing.BinaryIO, stop_event: threading.Event): + """Forward the content of a named pipe to a file-like object.""" + if not os.path.exists(path): + with contextlib.suppress(OSError): + os.mkfifo(path) + with open(os.open(path, os.O_RDONLY | os.O_NONBLOCK), 'rb') as f_in: + while not stop_event.is_set(): + if select.select([f_in], [], [], 0.5)[0]: + line = f_in.readline() + if not line: + break + f_out.write(line) + f_out.flush() + with contextlib.suppress(OSError): + os.unlink(path) + + @click.command() @click.option("--app", type=click.Path(exists=True), default=None, help='Path to local application to use, omit to use external apps.') @@ -79,6 +100,8 @@ def process_test_script_output(line, is_stderr): help='The extra arguments passed to the device. Can use placeholders like {SCRIPT_BASE_NAME}') @click.option("--app-ready-pattern", type=str, default=None, help='Delay test script start until given regular expression pattern is found in the application output.') +@click.option("--app-stdin-pipe", type=str, default=None, + help='Path for a standard input redirection named pipe to be used by the test script.') @click.option("--script", type=click.Path(exists=True), default=os.path.join(DEFAULT_CHIP_ROOT, 'src', 'controller', @@ -94,7 +117,8 @@ def process_test_script_output(line, is_stderr): help="Do not print output from passing tests. Use this flag in CI to keep GitHub log size manageable.") @click.option("--load-from-env", default=None, help="YAML file that contains values for environment variables.") def main(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args: str, - app_ready_pattern: str, script: str, script_args: str, script_gdb: bool, quiet: bool, load_from_env): + app_ready_pattern: str, app_stdin_pipe: str, script: str, script_args: str, + script_gdb: bool, quiet: bool, load_from_env): if load_from_env: reader = MetadataReader(load_from_env) runs = reader.parse_script(script) @@ -106,6 +130,7 @@ def main(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args: app=app, app_args=app_args, app_ready_pattern=app_ready_pattern, + app_stdin_pipe=app_stdin_pipe, script_args=script_args, script_gdb=script_gdb, ) @@ -128,11 +153,13 @@ def main(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args: for run in runs: logging.info("Executing %s %s", run.py_script_path.split('/')[-1], run.run) main_impl(run.app, run.factory_reset, run.factory_reset_app_only, run.app_args or "", - run.app_ready_pattern, run.py_script_path, run.script_args or "", run.script_gdb, run.quiet) + run.app_ready_pattern, run.app_stdin_pipe, run.py_script_path, + run.script_args or "", run.script_gdb, run.quiet) def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args: str, - app_ready_pattern: str, script: str, script_args: str, script_gdb: bool, quiet: bool): + app_ready_pattern: str, app_stdin_pipe: str, script: str, script_args: str, + script_gdb: bool, quiet: bool): app_args = app_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) script_args = script_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) @@ -154,6 +181,8 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a pathlib.Path(match.group("path")).unlink(missing_ok=True) app_process = None + app_stdin_forwarding_thread = None + app_stdin_forwarding_stop_event = threading.Event() app_exit_code = 0 app_pid = 0 @@ -172,7 +201,13 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a f_stdout=stream_output, f_stderr=stream_output) app_process.start(expected_output=app_ready_pattern, timeout=30) - app_process.p.stdin.close() + if app_stdin_pipe: + logging.info("Forwarding stdin from '%s' to app", app_stdin_pipe) + app_stdin_forwarding_thread = threading.Thread( + target=forward_fifo, args=(app_stdin_pipe, app_process.p.stdin, app_stdin_forwarding_stop_event)) + app_stdin_forwarding_thread.start() + else: + app_process.p.stdin.close() app_pid = app_process.p.pid script_command = [script, "--paa-trust-store-path", os.path.join(DEFAULT_CHIP_ROOT, MATTER_DEVELOPMENT_PAA_ROOT_CERTS), @@ -204,6 +239,9 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a if app_process: logging.info("Stopping app with SIGTERM") + if app_stdin_forwarding_thread: + app_stdin_forwarding_stop_event.set() + app_stdin_forwarding_thread.join() app_process.terminate() app_exit_code = app_process.returncode diff --git a/src/python_testing/TC_BRBINFO_4_1.py b/src/python_testing/TC_BRBINFO_4_1.py index 8c929d39b94f09..9caa513906427e 100644 --- a/src/python_testing/TC_BRBINFO_4_1.py +++ b/src/python_testing/TC_BRBINFO_4_1.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_CCTRL_2_1.py b/src/python_testing/TC_CCTRL_2_1.py index b656973f6afe98..24ebd19c5291fa 100644 --- a/src/python_testing/TC_CCTRL_2_1.py +++ b/src/python_testing/TC_CCTRL_2_1.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_CCTRL_2_2.py b/src/python_testing/TC_CCTRL_2_2.py index 3f60fd9e382bac..01a4fc42cc5708 100644 --- a/src/python_testing/TC_CCTRL_2_2.py +++ b/src/python_testing/TC_CCTRL_2_2.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_CCTRL_2_3.py b/src/python_testing/TC_CCTRL_2_3.py index 83c25290cf6207..26a758bea01679 100644 --- a/src/python_testing/TC_CCTRL_2_3.py +++ b/src/python_testing/TC_CCTRL_2_3.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_ECOINFO_2_1.py b/src/python_testing/TC_ECOINFO_2_1.py index a0adf75ac4b8e2..d86200d320859e 100644 --- a/src/python_testing/TC_ECOINFO_2_1.py +++ b/src/python_testing/TC_ECOINFO_2_1.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_ECOINFO_2_2.py b/src/python_testing/TC_ECOINFO_2_2.py index 6ce1e490d53841..ce98a806cef785 100644 --- a/src/python_testing/TC_ECOINFO_2_2.py +++ b/src/python_testing/TC_ECOINFO_2_2.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_MCORE_FS_1_1.py b/src/python_testing/TC_MCORE_FS_1_1.py index 780089b807ecfe..8428a998782601 100755 --- a/src/python_testing/TC_MCORE_FS_1_1.py +++ b/src/python_testing/TC_MCORE_FS_1_1.py @@ -24,8 +24,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_MCORE_FS_1_2.py b/src/python_testing/TC_MCORE_FS_1_2.py index 97b5b87017676d..ffcbe006a3a50a 100644 --- a/src/python_testing/TC_MCORE_FS_1_2.py +++ b/src/python_testing/TC_MCORE_FS_1_2.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/TC_MCORE_FS_1_3.py b/src/python_testing/TC_MCORE_FS_1_3.py index 4b732a1b3a8995..b4685f175d0fe5 100644 --- a/src/python_testing/TC_MCORE_FS_1_3.py +++ b/src/python_testing/TC_MCORE_FS_1_3.py @@ -26,8 +26,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json @@ -71,11 +72,11 @@ def setup_class(self): self.storage = None # Get the path to the TH_SERVER_NO_UID app from the user params. - th_server_app = self.user_params.get("th_server_no_uid_app_path", None) - if not th_server_app: + th_server_no_uid_app = self.user_params.get("th_server_no_uid_app_path", None) + if not th_server_no_uid_app: asserts.fail("This test requires a TH_SERVER_NO_UID app. Specify app path with --string-arg th_server_no_uid_app_path:") - if not os.path.exists(th_server_app): - asserts.fail(f"The path {th_server_app} does not exist") + if not os.path.exists(th_server_no_uid_app): + asserts.fail(f"The path {th_server_no_uid_app} does not exist") # Create a temporary storage directory for keeping KVS files. self.storage = tempfile.TemporaryDirectory(prefix=self.__class__.__name__) @@ -94,7 +95,7 @@ def setup_class(self): # Start the TH_SERVER_NO_UID app. self.th_server = AppServerSubprocess( - th_server_app, + th_server_no_uid_app, storage_dir=self.storage.name, port=self.th_server_port, discriminator=self.th_server_discriminator, diff --git a/src/python_testing/TC_MCORE_FS_1_4.py b/src/python_testing/TC_MCORE_FS_1_4.py index c8f3e764d5ce72..fb64378750cbf1 100644 --- a/src/python_testing/TC_MCORE_FS_1_4.py +++ b/src/python_testing/TC_MCORE_FS_1_4.py @@ -26,8 +26,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json @@ -129,11 +130,11 @@ def setup_class(self): asserts.fail(f"The path {th_fsa_bridge_path} does not exist") # Get the path to the TH_SERVER_NO_UID app from the user params. - th_server_app = self.user_params.get("th_server_no_uid_app_path", None) - if not th_server_app: + th_server_no_uid_app = self.user_params.get("th_server_no_uid_app_path", None) + if not th_server_no_uid_app: asserts.fail("This test requires a TH_SERVER_NO_UID app. Specify app path with --string-arg th_server_no_uid_app_path:") - if not os.path.exists(th_server_app): - asserts.fail(f"The path {th_server_app} does not exist") + if not os.path.exists(th_server_no_uid_app): + asserts.fail(f"The path {th_server_no_uid_app} does not exist") # Create a temporary storage directory for keeping KVS files. self.storage = tempfile.TemporaryDirectory(prefix=self.__class__.__name__) @@ -171,7 +172,7 @@ def setup_class(self): # Start the TH_SERVER_NO_UID app. self.th_server = AppServerSubprocess( - th_server_app, + th_server_no_uid_app, storage_dir=self.storage.name, port=self.th_server_port, discriminator=self.th_server_discriminator, diff --git a/src/python_testing/TC_MCORE_FS_1_5.py b/src/python_testing/TC_MCORE_FS_1_5.py index fc2eca6ee05cea..bd2f40a2ead49e 100755 --- a/src/python_testing/TC_MCORE_FS_1_5.py +++ b/src/python_testing/TC_MCORE_FS_1_5.py @@ -22,8 +22,9 @@ # test-runner-runs: # run1: # app: examples/fabric-admin/scripts/fabric-sync-app.py -# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --stdin-pipe=dut-fsa-stdin --discriminator=1234 +# app-args: --app-admin=${FABRIC_ADMIN_APP} --app-bridge=${FABRIC_BRIDGE_APP} --discriminator=1234 # app-ready-pattern: "Successfully opened pairing window on the device" +# app-stdin-pipe: dut-fsa-stdin # script-args: > # --PICS src/app/tests/suites/certification/ci-pics-values # --storage-path admin_storage.json diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py b/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py index 2d40d792acbd34..3ec286dc779169 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py @@ -27,6 +27,7 @@ class Metadata: app: str = "" app_args: Optional[str] = None app_ready_pattern: Optional[str] = None + app_stdin_pipe: Optional[str] = None script_args: Optional[str] = None factory_reset: bool = False factory_reset_app_only: bool = False @@ -148,6 +149,7 @@ def parse_script(self, py_script_path: str) -> List[Metadata]: app=attr.get("app", ""), app_args=attr.get("app-args"), app_ready_pattern=attr.get("app-ready-pattern"), + app_stdin_pipe=attr.get("app-stdin-pipe"), script_args=attr.get("script-args"), factory_reset=attr.get("factory-reset", False), quiet=attr.get("quiet", True), From 71dc879501d0000fa9eadf734e8d1644871f104b Mon Sep 17 00:00:00 2001 From: Boris Zbarsky Date: Wed, 4 Dec 2024 15:44:10 -0500 Subject: [PATCH 02/17] Update Darwin availability annotations. (#36721) --- src/darwin/Framework/CHIP/MTRAttributeValueWaiter.h | 4 ++-- src/darwin/Framework/CHIP/MTRDevice.h | 6 +++--- .../Framework/CHIP/XPC Protocol/MTRXPCServerProtocol.h | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/darwin/Framework/CHIP/MTRAttributeValueWaiter.h b/src/darwin/Framework/CHIP/MTRAttributeValueWaiter.h index 98ee86a800da09..d1f40910f56025 100644 --- a/src/darwin/Framework/CHIP/MTRAttributeValueWaiter.h +++ b/src/darwin/Framework/CHIP/MTRAttributeValueWaiter.h @@ -19,7 +19,7 @@ NS_ASSUME_NONNULL_BEGIN -MTR_NEWLY_AVAILABLE +MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)) @interface MTRAttributeValueWaiter : NSObject - (instancetype)init NS_UNAVAILABLE; + (instancetype)new NS_UNAVAILABLE; @@ -31,7 +31,7 @@ MTR_NEWLY_AVAILABLE */ - (void)cancel; -@property (readonly, nonatomic) NSUUID * UUID; +@property (readonly, nonatomic) NSUUID * UUID MTR_NEWLY_AVAILABLE; @end diff --git a/src/darwin/Framework/CHIP/MTRDevice.h b/src/darwin/Framework/CHIP/MTRDevice.h index 247223cbe43fbb..b1ca179df658f7 100644 --- a/src/darwin/Framework/CHIP/MTRDevice.h +++ b/src/darwin/Framework/CHIP/MTRDevice.h @@ -114,14 +114,14 @@ MTR_AVAILABLE(ios(16.1), macos(13.0), watchos(9.1), tvos(16.1)) * * A non-nil value if the vendor identifier has been determined from the device, nil if unknown. */ -@property (nonatomic, readonly, nullable, copy) NSNumber * vendorID MTR_NEWLY_AVAILABLE; +@property (nonatomic, readonly, nullable, copy) NSNumber * vendorID MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); /** * The Product Identifier associated with the device. * * A non-nil value if the product identifier has been determined from the device, nil if unknown. */ -@property (nonatomic, readonly, nullable, copy) NSNumber * productID MTR_NEWLY_AVAILABLE; +@property (nonatomic, readonly, nullable, copy) NSNumber * productID MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); /** * Network commissioning features supported by the device. @@ -362,7 +362,7 @@ MTR_AVAILABLE(ios(16.1), macos(13.0), watchos(9.1), tvos(16.1)) - (MTRAttributeValueWaiter *)waitForAttributeValues:(NSDictionary *> *)values timeout:(NSTimeInterval)timeout queue:(dispatch_queue_t)queue - completion:(void (^)(NSError * _Nullable error))completion MTR_NEWLY_AVAILABLE; + completion:(void (^)(NSError * _Nullable error))completion MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); @end diff --git a/src/darwin/Framework/CHIP/XPC Protocol/MTRXPCServerProtocol.h b/src/darwin/Framework/CHIP/XPC Protocol/MTRXPCServerProtocol.h index 6daec1bb9d05bf..45d19185d9184e 100644 --- a/src/darwin/Framework/CHIP/XPC Protocol/MTRXPCServerProtocol.h +++ b/src/darwin/Framework/CHIP/XPC Protocol/MTRXPCServerProtocol.h @@ -21,10 +21,10 @@ NS_ASSUME_NONNULL_BEGIN MTR_EXTERN NSString * const MTRDeviceControllerRegistrationNodeIDsKey MTR_AVAILABLE(ios(18.2), macos(15.2), watchos(11.2), tvos(18.2)); MTR_EXTERN NSString * const MTRDeviceControllerRegistrationNodeIDKey MTR_AVAILABLE(ios(18.2), macos(15.2), watchos(11.2), tvos(18.2)); MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerContextKey MTR_AVAILABLE(ios(18.2), macos(15.2), watchos(11.2), tvos(18.2)); -MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerNodeIDKey MTR_NEWLY_AVAILABLE; -MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerIsRunningKey MTR_NEWLY_AVAILABLE; -MTR_EXTERN NSString * const MTRDeviceControllerRegistrationDeviceInternalStateKey MTR_NEWLY_AVAILABLE; -MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerCompressedFabricIDKey MTR_NEWLY_AVAILABLE; +MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerNodeIDKey MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); +MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerIsRunningKey MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); +MTR_EXTERN NSString * const MTRDeviceControllerRegistrationDeviceInternalStateKey MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); +MTR_EXTERN NSString * const MTRDeviceControllerRegistrationControllerCompressedFabricIDKey MTR_AVAILABLE(ios(18.3), macos(15.3), watchos(11.3), tvos(18.3)); MTR_AVAILABLE(ios(18.2), macos(15.2), watchos(11.2), tvos(18.2)) @protocol MTRXPCServerProtocol_MTRDevice From ca772d7565fe48e01fecbb16f7a8491229dcb864 Mon Sep 17 00:00:00 2001 From: joonhaengHeo <85541460+joonhaengHeo@users.noreply.github.com> Date: Wed, 4 Dec 2024 22:07:30 +0100 Subject: [PATCH 03/17] [Android] Implement DiagnosticLog (#36591) * Implement Android DiagnosticLog * Update from comment * Restyled * Update DiagnosticLog UX (Download complete / failed) * Fix timeout issue * Fix java-controller build error * modify from comment * Add javadoc, test cases * kotlin code style * Fix from comment * Fix build error --- .github/workflows/java-tests.yaml | 11 + .../CHIPTool/app/src/main/AndroidManifest.xml | 9 + .../chip/chiptool/SelectActionFragment.kt | 5 + .../clusterclient/DiagnosticLogFragment.kt | 172 ++++++++++ .../res/layout/diagnostic_log_fragment.xml | 87 ++++++ .../res/layout/select_action_fragment.xml | 8 + .../app/src/main/res/values/strings.xml | 4 + .../app/src/main/res/xml/file_paths.xml | 4 + examples/java-matter-controller/BUILD.gn | 2 + .../java/src/com/matter/controller/Main.kt | 12 + .../commands/bdx/DownloadLogCommand.kt | 73 +++++ .../PairOnNetworkLongDownloadLogCommand.kt | 96 ++++++ kotlin-detect-config.yaml | 1 + scripts/tests/java/bdx_test.py | 87 ++++++ scripts/tests/run_java_test.py | 42 +++ .../java/AndroidLogDownloadFromNode.cpp | 294 ++++++++++++++++++ .../java/AndroidLogDownloadFromNode.h | 95 ++++++ src/controller/java/BUILD.gn | 6 + .../java/BdxDiagnosticLogsReceiver.cpp | 121 +++++++ .../java/BdxDiagnosticLogsReceiver.h | 65 ++++ .../java/CHIPDeviceController-JNI.cpp | 23 ++ .../ChipDeviceController.java | 21 ++ .../devicecontroller/DiagnosticLogType.java | 52 ++++ .../devicecontroller/DownloadLogCallback.java | 49 +++ src/protocols/bdx/BdxUri.h | 2 + 25 files changed, 1341 insertions(+) create mode 100644 examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/clusterclient/DiagnosticLogFragment.kt create mode 100644 examples/android/CHIPTool/app/src/main/res/layout/diagnostic_log_fragment.xml create mode 100644 examples/android/CHIPTool/app/src/main/res/xml/file_paths.xml create mode 100644 examples/java-matter-controller/java/src/com/matter/controller/commands/bdx/DownloadLogCommand.kt create mode 100644 examples/java-matter-controller/java/src/com/matter/controller/commands/bdx/PairOnNetworkLongDownloadLogCommand.kt create mode 100755 scripts/tests/java/bdx_test.py create mode 100644 src/controller/java/AndroidLogDownloadFromNode.cpp create mode 100644 src/controller/java/AndroidLogDownloadFromNode.h create mode 100644 src/controller/java/BdxDiagnosticLogsReceiver.cpp create mode 100644 src/controller/java/BdxDiagnosticLogsReceiver.h create mode 100644 src/controller/java/src/chip/devicecontroller/DiagnosticLogType.java create mode 100644 src/controller/java/src/chip/devicecontroller/DownloadLogCallback.java diff --git a/.github/workflows/java-tests.yaml b/.github/workflows/java-tests.yaml index 504cb865a9cc0b..473cd482357a75 100644 --- a/.github/workflows/java-tests.yaml +++ b/.github/workflows/java-tests.yaml @@ -236,6 +236,17 @@ jobs: --tool-args "onnetwork-long --nodeid 1 --setup-pin-code 20202021 --discriminator 3840 -t 1000" \ --factoryreset \ ' + - name: Run Pairing Onnetwork and get diagnostic log Test + run: | + scripts/run_in_python_env.sh out/venv \ + './scripts/tests/run_java_test.py \ + --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app \ + --app-args "--discriminator 3840 --interface-id -1 --crash_log ./crashLog.log --end_user_support_log ./enduser.log --network_diagnostics_log ./network.log" \ + --tool-path out/linux-x64-java-matter-controller \ + --tool-cluster "bdx" \ + --tool-args "onnetwork-long-downloadLog --nodeid 1 --setup-pin-code 20202021 --discriminator 3840 -t 3000 --logType CrashLogs --fileName ./crashLog.log" \ + --factoryreset \ + ' - name: Run Pairing Onnetwork Test run: | scripts/run_in_python_env.sh out/venv \ diff --git a/examples/android/CHIPTool/app/src/main/AndroidManifest.xml b/examples/android/CHIPTool/app/src/main/AndroidManifest.xml index 0526dc5a78160d..efd089817efe68 100644 --- a/examples/android/CHIPTool/app/src/main/AndroidManifest.xml +++ b/examples/android/CHIPTool/app/src/main/AndroidManifest.xml @@ -46,6 +46,15 @@ + + + diff --git a/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/SelectActionFragment.kt b/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/SelectActionFragment.kt index 69bfdc109912c9..fb8141e72d33a8 100644 --- a/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/SelectActionFragment.kt +++ b/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/SelectActionFragment.kt @@ -72,6 +72,7 @@ class SelectActionFragment : Fragment() { binding.provisionCustomFlowBtn.setOnClickListener { handleProvisionCustomFlowClicked() } binding.wildcardBtn.setOnClickListener { handleWildcardClicked() } binding.unpairDeviceBtn.setOnClickListener { handleUnpairDeviceClicked() } + binding.diagnosticLogBtn.setOnClickListener { handleDiagnosticLogClicked() } binding.groupSettingBtn.setOnClickListener { handleGroupSettingClicked() } binding.otaProviderBtn.setOnClickListener { handleOTAProviderClicked() } binding.icdBtn.setOnClickListener { handleICDClicked() } @@ -225,6 +226,10 @@ class SelectActionFragment : Fragment() { showFragment(OtaProviderClientFragment.newInstance()) } + private fun handleDiagnosticLogClicked() { + showFragment(DiagnosticLogFragment.newInstance()) + } + /** Notifies listener of provision-WiFi-credentials button click. */ private fun handleProvisionWiFiCredentialsClicked() { getCallback()?.setNetworkType(ProvisionNetworkType.WIFI) diff --git a/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/clusterclient/DiagnosticLogFragment.kt b/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/clusterclient/DiagnosticLogFragment.kt new file mode 100644 index 00000000000000..ea7d5438d247c3 --- /dev/null +++ b/examples/android/CHIPTool/app/src/main/java/com/google/chip/chiptool/clusterclient/DiagnosticLogFragment.kt @@ -0,0 +1,172 @@ +package com.google.chip.chiptool.clusterclient + +import android.content.Intent +import android.net.Uri +import android.os.Bundle +import android.os.Environment +import android.util.Log +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.ArrayAdapter +import androidx.core.content.FileProvider +import androidx.fragment.app.Fragment +import androidx.lifecycle.lifecycleScope +import chip.devicecontroller.ChipDeviceController +import chip.devicecontroller.DiagnosticLogType +import chip.devicecontroller.DownloadLogCallback +import com.google.chip.chiptool.ChipClient +import com.google.chip.chiptool.R +import com.google.chip.chiptool.databinding.DiagnosticLogFragmentBinding +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch + +class DiagnosticLogFragment : Fragment() { + private val deviceController: ChipDeviceController + get() = ChipClient.getDeviceController(requireContext()) + + private lateinit var scope: CoroutineScope + + private lateinit var addressUpdateFragment: AddressUpdateFragment + + private var _binding: DiagnosticLogFragmentBinding? = null + private val binding + get() = _binding!! + + private val timeout: Long + get() = binding.timeoutEd.text.toString().toULongOrNull()?.toLong() ?: 0L + + private val diagnosticLogTypeList = DiagnosticLogType.values() + private val diagnosticLogType: DiagnosticLogType + get() = diagnosticLogTypeList[binding.diagnosticTypeSp.selectedItemPosition] + + private var mDownloadFile: File? = null + private var mDownloadFileOutputStream: FileOutputStream? = null + + private var mReceiveFileLen = 0U + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle? + ): View { + _binding = DiagnosticLogFragmentBinding.inflate(inflater, container, false) + scope = viewLifecycleOwner.lifecycleScope + + addressUpdateFragment = + childFragmentManager.findFragmentById(R.id.addressUpdateFragment) as AddressUpdateFragment + + binding.getDiagnosticLogBtn.setOnClickListener { scope.launch { getDiagnosticLogClick() } } + + binding.diagnosticTypeSp.adapter = + ArrayAdapter( + requireContext(), + android.R.layout.simple_spinner_dropdown_item, + diagnosticLogTypeList + ) + + return binding.root + } + + override fun onDestroyView() { + super.onDestroyView() + _binding = null + } + + inner class ChipDownloadLogCallback : DownloadLogCallback { + override fun onError(fabricIndex: Int, nodeId: Long, errorCode: Long) { + Log.d(TAG, "onError: $fabricIndex, ${nodeId.toULong()}, $errorCode") + showMessage("Downloading Failed") + mDownloadFileOutputStream?.flush() ?: return + } + + override fun onSuccess(fabricIndex: Int, nodeId: Long) { + Log.d(TAG, "onSuccess: $fabricIndex, ${nodeId.toULong()}") + mDownloadFileOutputStream?.flush() ?: return + showMessage("Downloading Completed") + mDownloadFile?.let { showNotification(it) } ?: return + } + + override fun onTransferData(fabricIndex: Int, nodeId: Long, data: ByteArray): Boolean { + Log.d(TAG, "onTransferData : ${data.size}") + if (mDownloadFileOutputStream == null) { + Log.d(TAG, "mDownloadFileOutputStream or mDownloadFile is null") + return false + } + return addData(mDownloadFileOutputStream!!, data) + } + + private fun addData(outputStream: FileOutputStream, data: ByteArray): Boolean { + try { + outputStream.write(data) + } catch (e: IOException) { + Log.d(TAG, "IOException", e) + return false + } + mReceiveFileLen += data.size.toUInt() + showMessage("Receive Data Size : $mReceiveFileLen") + return true + } + } + + private fun getDiagnosticLogClick() { + mDownloadFile = + createLogFile( + deviceController.fabricIndex.toUInt(), + addressUpdateFragment.deviceId.toULong(), + diagnosticLogType + ) + mDownloadFileOutputStream = FileOutputStream(mDownloadFile) + deviceController.downloadLogFromNode( + addressUpdateFragment.deviceId, + diagnosticLogType, + timeout, + ChipDownloadLogCallback() + ) + } + + private fun isExternalStorageWritable(): Boolean { + return Environment.getExternalStorageState() == Environment.MEDIA_MOUNTED + } + + private fun createLogFile(fabricIndex: UInt, nodeId: ULong, type: DiagnosticLogType): File? { + if (!isExternalStorageWritable()) { + return null + } + val now = System.currentTimeMillis() + val fileName = "${type}_${fabricIndex}_${nodeId}_$now.txt" + mReceiveFileLen = 0U + return File(requireContext().getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS), fileName) + } + + private fun showNotification(file: File) { + val intent = + Intent(Intent.ACTION_VIEW).apply { + setDataAndType(getFileUri(file), "text/plain") + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + + requireActivity().startActivity(intent) + } + + private fun getFileUri(file: File): Uri { + return FileProvider.getUriForFile( + requireContext(), + "${requireContext().packageName}.provider", + file + ) + } + + private fun showMessage(msg: String) { + requireActivity().runOnUiThread { binding.diagnosticLogTv.text = msg } + } + + companion object { + private const val TAG = "DiagnosticLogFragment" + + fun newInstance(): DiagnosticLogFragment = DiagnosticLogFragment() + } +} diff --git a/examples/android/CHIPTool/app/src/main/res/layout/diagnostic_log_fragment.xml b/examples/android/CHIPTool/app/src/main/res/layout/diagnostic_log_fragment.xml new file mode 100644 index 00000000000000..56b770a645b442 --- /dev/null +++ b/examples/android/CHIPTool/app/src/main/res/layout/diagnostic_log_fragment.xml @@ -0,0 +1,87 @@ + + + + + + + + + + + + + +