diff --git a/pythonFiles/testing_tools/socket_manager.py b/pythonFiles/testing_tools/socket_manager.py index 372a50b5e012..b2afbf0e5a17 100644 --- a/pythonFiles/testing_tools/socket_manager.py +++ b/pythonFiles/testing_tools/socket_manager.py @@ -23,6 +23,12 @@ def __init__(self, addr): self.socket = None def __enter__(self): + return self.connect() + + def __exit__(self, *_): + self.close() + + def connect(self): self.socket = socket.socket( socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP ) @@ -35,7 +41,7 @@ def __enter__(self): return self - def __exit__(self, *_): + def close(self): if self.socket: try: self.socket.shutdown(socket.SHUT_RDWR) diff --git a/pythonFiles/tests/pytestadapter/helpers.py b/pythonFiles/tests/pytestadapter/helpers.py index 7195cfe43ea5..b534e950945a 100644 --- a/pythonFiles/tests/pytestadapter/helpers.py +++ b/pythonFiles/tests/pytestadapter/helpers.py @@ -12,6 +12,10 @@ import uuid from typing import Any, Dict, List, Optional, Tuple +script_dir = pathlib.Path(__file__).parent.parent.parent +sys.path.append(os.fspath(script_dir)) +sys.path.append(os.fspath(script_dir / "lib" / "python")) + TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" from typing_extensions import TypedDict diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 8d785be27c8b..674d92ac0545 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -29,15 +29,26 @@ def test_import_error(tmp_path): temp_dir.mkdir() p = temp_dir / "error_pytest_import.py" shutil.copyfile(file_path, p) - actual_list: Optional[List[Dict[str, Any]]] = runner( - ["--collect-only", os.fspath(p)] - ) - assert actual_list - for actual in actual_list: - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 2 + actual: Optional[List[Dict[str, Any]]] = runner(["--collect-only", os.fspath(p)]) + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + + # Ensure that 'error' is a list and then check its length + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 2 + else: + assert False def test_syntax_error(tmp_path): @@ -60,13 +71,25 @@ def test_syntax_error(tmp_path): p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) actual = runner(["--collect-only", os.fspath(p)]) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 2 + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + + # Ensure that 'error' is a list and then check its length + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 2 + else: + assert False def test_parameterized_error_collect(): @@ -76,12 +99,25 @@ def test_parameterized_error_collect(): """ file_path_str = "error_parametrize_discovery.py" actual = runner(["--collect-only", file_path_str]) - if actual: - actual = actual[0] - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 2 + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + + # Ensure that 'error' is a list and then check its length + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 2 + else: + assert False @pytest.mark.parametrize( @@ -146,13 +182,16 @@ def test_pytest_collect(file, expected_const): os.fspath(TEST_DATA_PATH / file), ] ) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "tests")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert actual["tests"] == expected_const + + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + actual_item = actual_list.pop(0) + assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + assert actual_item.get("tests") == expected_const def test_pytest_root_dir(): @@ -168,14 +207,16 @@ def test_pytest_root_dir(): ], TEST_DATA_PATH / "root", ) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "tests")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "root") + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + actual_item = actual_list.pop(0) + assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") assert ( - actual["tests"] + actual_item.get("tests") == expected_discovery_test_output.root_with_config_expected_output ) @@ -193,13 +234,15 @@ def test_pytest_config_file(): ], TEST_DATA_PATH / "root", ) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "tests")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "root") + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + actual_item = actual_list.pop(0) + assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") assert ( - actual["tests"] + actual_item.get("tests") == expected_discovery_test_output.root_with_config_expected_output ) diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index 07354b01709b..37a392f66d4b 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import os import shutil +from typing import Any, Dict, List import pytest @@ -23,14 +24,19 @@ def test_config_file(): expected_execution_test_output.config_file_pytest_expected_execution_output ) assert actual - assert len(actual) == len(expected_const) + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + assert len(actual_list) == len(expected_const) actual_result_dict = dict() - for a in actual: - assert all(item in a for item in ("status", "cwd", "result")) - assert a["status"] == "success" - assert a["cwd"] == os.fspath(new_cwd) - actual_result_dict.update(a["result"]) - assert actual_result_dict == expected_const + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "result") + ) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(new_cwd) + actual_result_dict.update(actual_item["result"]) + assert actual_result_dict == expected_const def test_rootdir_specified(): @@ -43,14 +49,19 @@ def test_rootdir_specified(): expected_execution_test_output.config_file_pytest_expected_execution_output ) assert actual - assert len(actual) == len(expected_const) + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + assert len(actual_list) == len(expected_const) actual_result_dict = dict() - for a in actual: - assert all(item in a for item in ("status", "cwd", "result")) - assert a["status"] == "success" - assert a["cwd"] == os.fspath(new_cwd) - actual_result_dict.update(a["result"]) - assert actual_result_dict == expected_const + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "result") + ) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(new_cwd) + actual_result_dict.update(actual_item["result"]) + assert actual_result_dict == expected_const def test_syntax_error_execution(tmp_path): @@ -73,13 +84,23 @@ def test_syntax_error_execution(tmp_path): p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) actual = runner(["error_syntax_discover.py::test_function"]) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 1 + assert actual + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 1 + else: + assert False def test_bad_id_error_execution(): @@ -88,13 +109,23 @@ def test_bad_id_error_execution(): The json should still be returned but the errors list should be present. """ actual = runner(["not/a/real::test_id"]) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 1 + assert actual + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 1 + else: + assert False @pytest.mark.parametrize( @@ -195,7 +226,8 @@ def test_pytest_execution(test_ids, expected_const): 3. uf_single_method_execution_expected_output: test run on a single method in a file. 4. uf_non_adjacent_tests_execution_expected_output: test run on unittests in two files with single selection in test explorer. 5. unit_pytest_same_file_execution_expected_output: test run on a file with both unittest and pytest tests. - 6. dual_level_nested_folder_execution_expected_output: test run on a file with one test file at the top level and one test file in a nested folder. + 6. dual_level_nested_folder_execution_expected_output: test run on a file with one test file + at the top level and one test file in a nested folder. 7. double_nested_folder_expected_execution_output: test run on a double nested folder. 8. parametrize_tests_expected_execution_output: test run on a parametrize test with 3 inputs. 9. single_parametrize_tests_expected_execution_output: test run on single parametrize test. @@ -205,18 +237,22 @@ def test_pytest_execution(test_ids, expected_const): Keyword arguments: test_ids -- an array of test_ids to run. expected_const -- a dictionary of the expected output from running pytest discovery on the files. - """ # noqa: E501 + """ args = test_ids actual = runner(args) assert actual - print(actual) - assert len(actual) == len(expected_const) + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + assert len(actual_list) == len(expected_const) actual_result_dict = dict() - for a in actual: - assert all(item in a for item in ("status", "cwd", "result")) - assert a["status"] == "success" - assert a["cwd"] == os.fspath(TEST_DATA_PATH) - actual_result_dict.update(a["result"]) + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "result") + ) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + actual_result_dict.update(actual_item["result"]) for key in actual_result_dict: if ( actual_result_dict[key]["outcome"] == "failure" diff --git a/pythonFiles/unittestadapter/discovery.py b/pythonFiles/unittestadapter/discovery.py index cbad40ad1838..6208d24bee9f 100644 --- a/pythonFiles/unittestadapter/discovery.py +++ b/pythonFiles/unittestadapter/discovery.py @@ -48,6 +48,13 @@ class PayloadDict(TypedDict): error: NotRequired[List[str]] +class EOTPayloadDict(TypedDict): + """A dictionary that is used to send a end of transmission post request to the server.""" + + command_type: Literal["discovery"] | Literal["execution"] + eot: bool + + def discover_tests( start_dir: str, pattern: str, top_level_dir: Optional[str], uuid: Optional[str] ) -> PayloadDict: @@ -106,17 +113,7 @@ def discover_tests( return payload -if __name__ == "__main__": - # Get unittest discovery arguments. - argv = sys.argv[1:] - index = argv.index("--udiscovery") - - start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) - - # Perform test discovery. - port, uuid = parse_discovery_cli_args(argv[:index]) - payload = discover_tests(start_dir, pattern, top_level_dir, uuid) - +def post_response(payload: PayloadDict | EOTPayloadDict, port: int, uuid: str) -> None: # Build the request data (it has to be a POST request or the Node side will not process it), and send it. addr = ("localhost", port) data = json.dumps(payload) @@ -132,3 +129,25 @@ def discover_tests( except Exception as e: print(f"Error sending response: {e}") print(f"Request data: {request}") + + +if __name__ == "__main__": + # Get unittest discovery arguments. + argv = sys.argv[1:] + index = argv.index("--udiscovery") + + start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + + # Perform test discovery. + port, uuid = parse_discovery_cli_args(argv[:index]) + # Post this discovery payload. + if uuid is not None: + payload = discover_tests(start_dir, pattern, top_level_dir, uuid) + post_response(payload, port, uuid) + # Post EOT token. + eot_payload: EOTPayloadDict = {"command_type": "discovery", "eot": True} + post_response(eot_payload, port, uuid) + else: + print("Error: no uuid provided or parsed.") + eot_payload: EOTPayloadDict = {"command_type": "discovery", "eot": True} + post_response(eot_payload, port, "") diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index f239f81c2d87..7bbc97e78f31 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import argparse +import atexit import enum import json import os @@ -17,7 +18,7 @@ sys.path.append(os.fspath(script_dir)) sys.path.insert(0, os.fspath(script_dir / "lib" / "python")) -from typing_extensions import NotRequired, TypeAlias, TypedDict +from typing_extensions import Literal, NotRequired, TypeAlias, TypedDict from testing_tools import process_json_util, socket_manager from unittestadapter.utils import parse_unittest_args @@ -168,6 +169,13 @@ class PayloadDict(TypedDict): error: NotRequired[str] +class EOTPayloadDict(TypedDict): + """A dictionary that is used to send a end of transmission post request to the server.""" + + command_type: Literal["discovery"] | Literal["execution"] + eot: bool + + # Args: start_path path to a directory or a file, list of ids that may be empty. # Edge cases: # - if tests got deleted since the VS Code side last ran discovery and the current test run, @@ -225,8 +233,11 @@ def run_tests( return payload +__socket = None +atexit.register(lambda: __socket.close() if __socket else None) + + def send_run_data(raw_data, port, uuid): - # Build the request data (it has to be a POST request or the Node side will not process it), and send it. status = raw_data["outcome"] cwd = os.path.abspath(START_DIR) if raw_data["subtest"]: @@ -236,7 +247,20 @@ def send_run_data(raw_data, port, uuid): test_dict = {} test_dict[test_id] = raw_data payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict} + post_response(payload, port, uuid) + + +def post_response(payload: PayloadDict | EOTPayloadDict, port: int, uuid: str) -> None: + # Build the request data (it has to be a POST request or the Node side will not process it), and send it. addr = ("localhost", port) + global __socket + if __socket is None: + try: + __socket = socket_manager.SocketManager(addr) + __socket.connect() + except Exception as error: + print(f"Plugin error connection error[vscode-pytest]: {error}") + __socket = None data = json.dumps(payload) request = f"""Content-Length: {len(data)} Content-Type: application/json @@ -244,11 +268,10 @@ def send_run_data(raw_data, port, uuid): {data}""" try: - with socket_manager.SocketManager(addr) as s: - if s.socket is not None: - s.socket.sendall(request.encode("utf-8")) - except Exception as e: - print(f"Error sending response: {e}") + if __socket is not None and __socket.socket is not None: + __socket.socket.sendall(request.encode("utf-8")) + except Exception as ex: + print(f"Error sending response: {ex}") print(f"Request data: {request}") @@ -312,3 +335,9 @@ def send_run_data(raw_data, port, uuid): "error": "No test ids received from buffer", "result": None, } + eot_payload: EOTPayloadDict = {"command_type": "execution", "eot": True} + if UUID is None: + print("Error sending response, uuid unknown to python server.") + post_response(eot_payload, PORT, "unknown") + else: + post_response(eot_payload, PORT, UUID) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 3d5bde44204f..86f753315604 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -1,7 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import atexit import json import os import pathlib import sys +import time import traceback import pytest @@ -301,12 +306,6 @@ def pytest_sessionfinish(session, exitstatus): 4: Pytest encountered an internal error or exception during test execution. 5: Pytest was unable to find any tests to run. """ - print( - "pytest session has finished, exit status: ", - exitstatus, - "in discovery? ", - IS_DISCOVERY, - ) cwd = pathlib.Path.cwd() if IS_DISCOVERY: if not (exitstatus == 0 or exitstatus == 1 or exitstatus == 5): @@ -352,6 +351,10 @@ def pytest_sessionfinish(session, exitstatus): exitstatus_bool, None, ) + # send end of transmission token + command_type = "discovery" if IS_DISCOVERY else "execution" + payload: EOTPayloadDict = {"command_type": command_type, "eot": True} + send_post_request(payload) def build_test_tree(session: pytest.Session) -> TestNode: @@ -603,44 +606,60 @@ class ExecutionPayloadDict(Dict): error: Union[str, None] # Currently unused need to check +class EOTPayloadDict(TypedDict): + """A dictionary that is used to send a end of transmission post request to the server.""" + + command_type: Literal["discovery"] | Literal["execution"] + eot: bool + + def get_node_path(node: Any) -> pathlib.Path: + """A function that returns the path of a node given the switch to pathlib.Path.""" return getattr(node, "path", pathlib.Path(node.fspath)) +__socket = None +atexit.register(lambda: __socket.close() if __socket else None) + + def execution_post( - cwd: str, - status: Literal["success", "error"], - tests: Union[testRunResultDict, None], + cwd: str, status: Literal["success", "error"], tests: Union[testRunResultDict, None] ): """ - Sends a post request to the server after the tests have been executed. - Keyword arguments: - cwd -- the current working directory. - session_node -- the status of running the tests - tests -- the tests that were run and their status. + Sends a POST request with execution payload details. + + Args: + cwd (str): Current working directory. + status (Literal["success", "error"]): Execution status indicating success or error. + tests (Union[testRunResultDict, None]): Test run results, if available. """ - testPort = os.getenv("TEST_PORT", 45454) - testuuid = os.getenv("TEST_UUID") + payload: ExecutionPayloadDict = ExecutionPayloadDict( cwd=cwd, status=status, result=tests, not_found=None, error=None ) if ERRORS: payload["error"] = ERRORS + send_post_request(payload) - addr = ("localhost", int(testPort)) - data = json.dumps(payload) - request = f"""Content-Length: {len(data)} -Content-Type: application/json -Request-uuid: {testuuid} -{data}""" - try: - with socket_manager.SocketManager(addr) as s: - if s.socket is not None: - s.socket.sendall(request.encode("utf-8")) - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") - print(f"[vscode-pytest] data: {request}") +def post_response(cwd: str, session_node: TestNode) -> None: + """ + Sends a POST request with test session details in payload. + + Args: + cwd (str): Current working directory. + session_node (TestNode): Node information of the test session. + """ + + payload: DiscoveryPayloadDict = { + "cwd": cwd, + "status": "success" if not ERRORS else "error", + "tests": session_node, + "error": [], + } + if ERRORS is not None: + payload["error"] = ERRORS + send_post_request(payload, cls_encoder=PathEncoder) class PathEncoder(json.JSONEncoder): @@ -652,35 +671,55 @@ def default(self, obj): return super().default(obj) -def post_response(cwd: str, session_node: TestNode) -> None: - """Sends a post request to the server. +def send_post_request( + payload: ExecutionPayloadDict | DiscoveryPayloadDict | EOTPayloadDict, + cls_encoder=None, +): + """ + Sends a post request to the server. Keyword arguments: - cwd -- the current working directory. - session_node -- the session node, which is the top of the testing tree. - errors -- a list of errors that occurred during test collection. + payload -- the payload data to be sent. + cls_encoder -- a custom encoder if needed. """ - payload: DiscoveryPayloadDict = { - "cwd": cwd, - "status": "success" if not ERRORS else "error", - "tests": session_node, - "error": [], - } - if ERRORS is not None: - payload["error"] = ERRORS - test_port: Union[str, int] = os.getenv("TEST_PORT", 45454) - test_uuid: Union[str, None] = os.getenv("TEST_UUID") - addr = "localhost", int(test_port) - data = json.dumps(payload, cls=PathEncoder) + testPort = os.getenv("TEST_PORT", 45454) + testuuid = os.getenv("TEST_UUID") + addr = ("localhost", int(testPort)) + global __socket + + if __socket is None: + try: + __socket = socket_manager.SocketManager(addr) + __socket.connect() + except Exception as error: + print(f"Plugin error connection error[vscode-pytest]: {error}") + __socket = None + + data = json.dumps(payload, cls=cls_encoder) request = f"""Content-Length: {len(data)} Content-Type: application/json -Request-uuid: {test_uuid} +Request-uuid: {testuuid} {data}""" - try: - with socket_manager.SocketManager(addr) as s: - if s.socket is not None: - s.socket.sendall(request.encode("utf-8")) - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") - print(f"[vscode-pytest] data: {request}") + + max_retries = 3 + retries = 0 + while retries < max_retries: + try: + if __socket is not None and __socket.socket is not None: + __socket.socket.sendall(request.encode("utf-8")) + # print("Post request sent successfully!") + # print("data sent", payload, "end of data") + break # Exit the loop if the send was successful + else: + print("Plugin error connection error[vscode-pytest]") + print(f"[vscode-pytest] data: {request}") + except Exception as error: + print(f"Plugin error connection error[vscode-pytest]: {error}") + print(f"[vscode-pytest] data: {request}") + retries += 1 # Increment retry counter + if retries < max_retries: + print(f"Retrying ({retries}/{max_retries}) in 2 seconds...") + time.sleep(2) # Wait for a short duration before retrying + else: + print("Maximum retry attempts reached. Cannot send post request.") diff --git a/pythonFiles/vscode_pytest/run_pytest_script.py b/pythonFiles/vscode_pytest/run_pytest_script.py index ffb4d0c55b16..0fca8208a406 100644 --- a/pythonFiles/vscode_pytest/run_pytest_script.py +++ b/pythonFiles/vscode_pytest/run_pytest_script.py @@ -53,7 +53,7 @@ buffer = b"" # Process the JSON data - print(f"Received JSON data: {test_ids_from_buffer}") + print("Received JSON data in run script") break except json.JSONDecodeError: # JSON decoding error, the complete JSON object is not yet received diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index aa9f2a541f51..5ef6695ca280 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -3,7 +3,7 @@ import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode'; import * as util from 'util'; -import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; +import { DiscoveredTestPayload, EOTTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; import { TestProvider } from '../../types'; import { traceError, traceLog } from '../../../logging'; import { Testing } from '../../../common/utils/localize'; @@ -12,6 +12,7 @@ import { sendTelemetryEvent } from '../../../telemetry'; import { EventName } from '../../../telemetry/constants'; import { splitLines } from '../../../common/stringUtils'; import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils'; +import { Deferred } from '../../../common/utils/async'; export class PythonResultResolver implements ITestResultResolver { testController: TestController; @@ -35,16 +36,30 @@ export class PythonResultResolver implements ITestResultResolver { this.vsIdToRunId = new Map(); } - public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise { - const workspacePath = this.workspaceUri.fsPath; - traceLog('Using result resolver for discovery'); - - const rawTestData = payload; - if (!rawTestData) { + public resolveDiscovery( + payload: DiscoveredTestPayload | EOTTestPayload, + deferredTillEOT: Deferred, + token?: CancellationToken, + ): Promise { + if (!payload) { // No test data is available return Promise.resolve(); } + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + traceLog('ResultResolver EOT received for discovery.'); + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + return this._resolveDiscovery(payload as DiscoveredTestPayload, token); + } + public _resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise { + const workspacePath = this.workspaceUri.fsPath; + const rawTestData = payload as DiscoveredTestPayload; // Check if there were any errors in the discovery process. if (rawTestData.status === 'error') { const testingErrorConst = @@ -87,8 +102,25 @@ export class PythonResultResolver implements ITestResultResolver { return Promise.resolve(); } - public resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise { - const rawTestExecData = payload; + public resolveExecution( + payload: ExecutionTestPayload | EOTTestPayload, + runInstance: TestRun, + deferredTillEOT: Deferred, + ): Promise { + if (payload !== undefined && 'eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + traceLog('ResultResolver EOT received for execution.'); + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + return this._resolveExecution(payload as ExecutionTestPayload, runInstance); + } + + public _resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise { + const rawTestExecData = payload as ExecutionTestPayload; if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { // Map which holds the subtest information for each test item. diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index d0a225ae5712..f59c486f7a85 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -15,7 +15,7 @@ import { traceError, traceInfo, traceLog } from '../../../logging'; import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; -import { jsonRPCHeaders, jsonRPCContent, JSONRPC_UUID_HEADER, createExecutionErrorPayload } from './utils'; +import { createEOTPayload, createExecutionErrorPayload, extractJsonPayload } from './utils'; import { createDeferred } from '../../../common/utils/async'; export class PythonTestServer implements ITestServer, Disposable { @@ -35,56 +35,22 @@ export class PythonTestServer implements ITestServer, Disposable { this.server = net.createServer((socket: net.Socket) => { let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { - try { - let rawData: string = data.toString(); - buffer = Buffer.concat([buffer, data]); - while (buffer.length > 0) { - const rpcHeaders = jsonRPCHeaders(buffer.toString()); - const uuid = rpcHeaders.headers.get(JSONRPC_UUID_HEADER); - const totalContentLength = rpcHeaders.headers.get('Content-Length'); - if (!uuid) { - traceError('On data received: Error occurred because payload UUID is undefined'); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } - if (!this.uuids.includes(uuid)) { - traceError('On data received: Error occurred because the payload UUID is not recognized'); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } - rawData = rpcHeaders.remainingRawData; - const rpcContent = jsonRPCContent(rpcHeaders.headers, rawData); - const extractedData = rpcContent.extractedJSON; - // do not send until we have the full content - if (extractedData.length === Number(totalContentLength)) { - // if the rawData includes tests then this is a discovery request - if (rawData.includes(`"tests":`)) { - this._onDiscoveryDataReceived.fire({ - uuid, - data: rpcContent.extractedJSON, - }); - // if the rawData includes result then this is a run request - } else if (rawData.includes(`"result":`)) { - this._onRunDataReceived.fire({ - uuid, - data: rpcContent.extractedJSON, - }); - } else { - traceLog( - `Error processing test server request: request is not recognized as discovery or run.`, - ); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } - // this.uuids = this.uuids.filter((u) => u !== uuid); WHERE DOES THIS GO?? - buffer = Buffer.alloc(0); - } else { + buffer = Buffer.concat([buffer, data]); // get the new data and add it to the buffer + while (buffer.length > 0) { + try { + // try to resolve data, returned unresolved data + const remainingBuffer = this._resolveData(buffer); + if (remainingBuffer.length === buffer.length) { + // if the remaining buffer is exactly the same as the buffer before processing, + // then there is no more data to process so loop should be exited. break; } + buffer = remainingBuffer; + } catch (ex) { + traceError(`Error reading data from buffer: ${ex} observed.`); + buffer = Buffer.alloc(0); + this._onDataReceived.fire({ uuid: '', data: '' }); } - } catch (ex) { - traceError(`Error processing test server request: ${ex} observe`); - this._onDataReceived.fire({ uuid: '', data: '' }); } }); }); @@ -107,6 +73,47 @@ export class PythonTestServer implements ITestServer, Disposable { }); } + savedBuffer = ''; + + public _resolveData(buffer: Buffer): Buffer { + try { + const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); + // what payload is so small it doesn't include the whole UUID think got this + if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { + // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. + traceInfo(`Firing data received event, ${extractedJsonPayload.cleanedJsonData}`); + this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); + } + buffer = Buffer.from(extractedJsonPayload.remainingRawData); + if (buffer.length === 0) { + // if the buffer is empty, then there is no more data to process so buffer should be cleared. + buffer = Buffer.alloc(0); + } + } catch (ex) { + traceError(`Error attempting to resolve data: ${ex}`); + this._onDataReceived.fire({ uuid: '', data: '' }); + } + return buffer; + } + + private _fireDataReceived(uuid: string, extractedJSON: string): void { + if (extractedJSON.includes(`"tests":`) || extractedJSON.includes(`"command_type": "discovery"`)) { + this._onDiscoveryDataReceived.fire({ + uuid, + data: extractedJSON, + }); + // if the rawData includes result then this is a run request + } else if (extractedJSON.includes(`"result":`) || extractedJSON.includes(`"command_type": "execution"`)) { + this._onRunDataReceived.fire({ + uuid, + data: extractedJSON, + }); + } else { + traceError(`Error processing test server request: request is not recognized as discovery or run.`); + this._onDataReceived.fire({ uuid: '', data: '' }); + } + } + public serverReady(): Promise { return this.ready; } @@ -208,10 +215,9 @@ export class PythonTestServer implements ITestServer, Disposable { traceLog(`Discovering unittest tests with arguments: ${args}\r\n`); } const deferred = createDeferred>(); - const result = execService.execObservable(args, spawnOptions); - runInstance?.token.onCancellationRequested(() => { + traceInfo('Test run cancelled, killing unittest subprocess.'); result?.proc?.kill(); }); @@ -226,18 +232,26 @@ export class PythonTestServer implements ITestServer, Disposable { result?.proc?.on('exit', (code, signal) => { // if the child has testIds then this is a run request if (code !== 0 && testIds) { + traceError( + `Subprocess exited unsuccessfully with exit code ${code} and signal ${signal}. Creating and sending error execution payload`, + ); // if the child process exited with a non-zero exit code, then we need to send the error payload. this._onRunDataReceived.fire({ uuid, data: JSON.stringify(createExecutionErrorPayload(code, signal, testIds, options.cwd)), }); + // then send a EOT payload + this._onRunDataReceived.fire({ + uuid, + data: JSON.stringify(createEOTPayload(true)), + }); } deferred.resolve({ stdout: '', stderr: '' }); - callback?.(); }); await deferred.promise; } } catch (ex) { + traceError(`Error while server attempting to run unittest command: ${ex}`); this.uuids = this.uuids.filter((u) => u !== uuid); this._onDataReceived.fire({ uuid, diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index 610c3d76d17b..386c397b310c 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -14,6 +14,7 @@ import { } from 'vscode'; import { ITestDebugLauncher, TestDiscoveryOptions } from '../../common/types'; import { IPythonExecutionFactory } from '../../../common/process/types'; +import { Deferred } from '../../../common/utils/async'; export type TestRunInstanceOptions = TestRunOptions & { exclude?: readonly TestItem[]; @@ -191,8 +192,18 @@ export interface ITestResultResolver { runIdToVSid: Map; runIdToTestItem: Map; vsIdToRunId: Map; - resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise; - resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise; + resolveDiscovery( + payload: DiscoveredTestPayload | EOTTestPayload, + deferredTillEOT: Deferred, + token?: CancellationToken, + ): Promise; + resolveExecution( + payload: ExecutionTestPayload | EOTTestPayload, + runInstance: TestRun, + deferredTillEOT: Deferred, + ): Promise; + _resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise; + _resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise; } export interface ITestDiscoveryAdapter { // ** first line old method signature, second line new method signature @@ -241,6 +252,11 @@ export type DiscoveredTestPayload = { error?: string[]; }; +export type EOTTestPayload = { + commandType: 'discovery' | 'execution'; + eot: boolean; +}; + export type ExecutionTestPayload = { cwd: string; status: 'success' | 'error'; diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index dd1b51551a45..572863ecdbfe 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -9,27 +9,100 @@ import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities'; -import { DiscoveredTestItem, DiscoveredTestNode, ExecutionTestPayload, ITestResultResolver } from './types'; +import { + DiscoveredTestItem, + DiscoveredTestNode, + EOTTestPayload, + ExecutionTestPayload, + ITestResultResolver, +} from './types'; +import { Deferred, createDeferred } from '../../../common/utils/async'; export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); return `${lines.join('\r\n')}\r\n`; } -export interface IJSONRPCContent { +export interface IJSONRPCData { extractedJSON: string; remainingRawData: string; } -export interface IJSONRPCHeaders { +export interface ParsedRPCHeadersAndData { headers: Map; remainingRawData: string; } +export interface ExtractOutput { + uuid: string | undefined; + cleanedJsonData: string | undefined; + remainingRawData: string; +} + export const JSONRPC_UUID_HEADER = 'Request-uuid'; export const JSONRPC_CONTENT_LENGTH_HEADER = 'Content-Length'; export const JSONRPC_CONTENT_TYPE_HEADER = 'Content-Type'; -export function jsonRPCHeaders(rawData: string): IJSONRPCHeaders { +export function createEOTDeferred(): Deferred { + return createDeferred(); +} + +export function extractJsonPayload(rawData: string, uuids: Array): ExtractOutput { + /** + * Extracts JSON-RPC payload from the provided raw data. + * @param {string} rawData - The raw string data from which the JSON payload will be extracted. + * @param {Array} uuids - The list of UUIDs that are active. + * @returns {string} The remaining raw data after the JSON payload is extracted. + */ + + const rpcHeaders: ParsedRPCHeadersAndData = parseJsonRPCHeadersAndData(rawData); + + // verify the RPC has a UUID and that it is recognized + let uuid = rpcHeaders.headers.get(JSONRPC_UUID_HEADER); + uuid = checkUuid(uuid, uuids); + + const payloadLength = rpcHeaders.headers.get('Content-Length'); + + // separate out the data within context length of the given payload from the remaining data in the buffer + const rpcContent: IJSONRPCData = ExtractJsonRPCData(payloadLength, rpcHeaders.remainingRawData); + const cleanedJsonData = rpcContent.extractedJSON; + const { remainingRawData } = rpcContent; + + // if the given payload has the complete json, process it otherwise wait for the rest in the buffer + if (cleanedJsonData.length === Number(payloadLength)) { + // call to process this data + // remove this data from the buffer + return { uuid, cleanedJsonData, remainingRawData }; + } + // wait for the remaining + return { uuid: undefined, cleanedJsonData: undefined, remainingRawData: rawData }; +} + +export function checkUuid(uuid: string | undefined, uuids: Array): string | undefined { + if (!uuid) { + // no UUID found, this could occurred if the payload is full yet so send back without erroring + return undefined; + } + if (!uuids.includes(uuid)) { + // no UUID found, this could occurred if the payload is full yet so send back without erroring + throw new Error('On data received: Error occurred because the payload UUID is not recognized'); + } + return uuid; +} + +export function parseJsonRPCHeadersAndData(rawData: string): ParsedRPCHeadersAndData { + /** + * Parses the provided raw data to extract JSON-RPC specific headers and remaining data. + * + * This function aims to extract specific JSON-RPC headers (like UUID, content length, + * and content type) from the provided raw string data. Headers are expected to be + * delimited by newlines and the format should be "key:value". The function stops parsing + * once it encounters an empty line, and the rest of the data after this line is treated + * as the remaining raw data. + * + * @param {string} rawData - The raw string containing headers and possibly other data. + * @returns {ParsedRPCHeadersAndData} An object containing the parsed headers as a map and the + * remaining raw data after the headers. + */ const lines = rawData.split('\n'); let remainingRawData = ''; const headerMap = new Map(); @@ -51,8 +124,21 @@ export function jsonRPCHeaders(rawData: string): IJSONRPCHeaders { }; } -export function jsonRPCContent(headers: Map, rawData: string): IJSONRPCContent { - const length = parseInt(headers.get('Content-Length') ?? '0', 10); +export function ExtractJsonRPCData(payloadLength: string | undefined, rawData: string): IJSONRPCData { + /** + * Extracts JSON-RPC content based on provided headers and raw data. + * + * This function uses the `Content-Length` header from the provided headers map + * to determine how much of the rawData string represents the actual JSON content. + * After extracting the expected content, it also returns any remaining data + * that comes after the extracted content as remaining raw data. + * + * @param {string | undefined} payloadLength - The value of the `Content-Length` header. + * @param {string} rawData - The raw string data from which the JSON content will be extracted. + * + * @returns {IJSONRPCContent} An object containing the extracted JSON content and any remaining raw data. + */ + const length = parseInt(payloadLength ?? '0', 10); const data = rawData.slice(0, length); const remainingRawData = rawData.slice(length); return { @@ -212,3 +298,10 @@ export function createExecutionErrorPayload( } return etp; } + +export function createEOTPayload(executionBool: boolean): EOTTestPayload { + return { + commandType: executionBool ? 'execution' : 'discovery', + eot: true, + } as EOTTestPayload; +} diff --git a/src/client/testing/testController/controller.ts b/src/client/testing/testController/controller.ts index 1550323ff8f8..0c7db5594004 100644 --- a/src/client/testing/testController/controller.ts +++ b/src/client/testing/testController/controller.ts @@ -371,6 +371,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc ); const dispose = token.onCancellationRequested(() => { + runInstance.appendOutput(`Run instance cancelled.\r\n`); runInstance.end(); }); diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index 450e2ef1edf2..bafa91847b42 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -9,9 +9,9 @@ import { SpawnOptions, } from '../../../common/process/types'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred } from '../../../common/utils/async'; +import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; -import { traceVerbose } from '../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../logging'; import { DataReceivedEvent, DiscoveredTestPayload, @@ -32,20 +32,20 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { ) {} async discoverTests(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { - const settings = this.configSettings.getSettings(uri); const uuid = this.testServer.createUUID(uri.fsPath); - const { pytestArgs } = settings.testing; - traceVerbose(pytestArgs); - const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { - this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + const deferredTillEOT: Deferred = createDeferred(); + const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived(async (e: DataReceivedEvent) => { + this.resultResolver?.resolveDiscovery(JSON.parse(e.data), deferredTillEOT); }); const disposeDataReceiver = function (testServer: ITestServer) { + traceInfo(`Disposing data receiver for ${uri.fsPath} and deleting UUID; pytest discovery.`); testServer.deleteUUID(uuid); dataReceivedDisposable.dispose(); }; try { await this.runPytestDiscovery(uri, uuid, executionFactory); } finally { + await deferredTillEOT.promise; disposeDataReceiver(this.testServer); } // this is only a placeholder to handle function overloading until rewrite is finished @@ -84,6 +84,7 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { // delete UUID following entire discovery finishing. const deferredExec = createDeferred>(); const execArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only'].concat(pytestArgs); + traceVerbose(`Running pytest discovery with command: ${execArgs.join(' ')}`); const result = execService?.execObservable(execArgs, spawnOptions); // Take all output from the subprocess and add it to the test output channel. This will be the pytest output. @@ -94,7 +95,12 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { result?.proc?.stderr?.on('data', (data) => { spawnOptions.outputChannel?.append(data.toString()); }); - result?.proc?.on('exit', () => { + result?.proc?.on('exit', (code, signal) => { + if (code !== 0) { + traceError( + `Subprocess exited unsuccessfully with exit code ${code} and signal ${signal}. Creating and sending error discovery payload`, + ); + } deferredExec.resolve({ stdout: '', stderr: '' }); deferred.resolve(); }); diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 9705374d74af..085af40375d4 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -4,8 +4,8 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred } from '../../../common/utils/async'; -import { traceError, traceInfo, traceVerbose } from '../../../logging'; +import { Deferred, createDeferred } from '../../../common/utils/async'; +import { traceError, traceInfo } from '../../../logging'; import { DataReceivedEvent, ExecutionTestPayload, @@ -42,29 +42,39 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { debugLauncher?: ITestDebugLauncher, ): Promise { const uuid = this.testServer.createUUID(uri.fsPath); - traceVerbose(uri, testIds, debugBool); + const deferredTillEOT: Deferred = utils.createEOTDeferred(); const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { - this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + const eParsed = JSON.parse(e.data); + this.resultResolver?.resolveExecution(eParsed, runInstance, deferredTillEOT); + } else { + traceError('No run instance found, cannot resolve execution.'); } }); const disposeDataReceiver = function (testServer: ITestServer) { + traceInfo(`Disposing data receiver for ${uri.fsPath} and deleting UUID; pytest execution.`); testServer.deleteUUID(uuid); dataReceivedDisposable.dispose(); }; runInstance?.token.onCancellationRequested(() => { - disposeDataReceiver(this.testServer); + traceInfo("Test run cancelled, resolving 'till EOT' deferred."); + deferredTillEOT.resolve(); }); - await this.runTestsNew( - uri, - testIds, - uuid, - runInstance, - debugBool, - executionFactory, - debugLauncher, - disposeDataReceiver, - ); + try { + this.runTestsNew( + uri, + testIds, + uuid, + runInstance, + debugBool, + executionFactory, + debugLauncher, + deferredTillEOT, + ); + } finally { + await deferredTillEOT.promise; + disposeDataReceiver(this.testServer); + } // placeholder until after the rewrite is adopted // TODO: remove after adoption. @@ -84,19 +94,16 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { debugBool?: boolean, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, - disposeDataReceiver?: (testServer: ITestServer) => void, + deferredTillEOT?: Deferred, ): Promise { - const deferred = createDeferred(); const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); - this.configSettings.isTestExecution(); const settings = this.configSettings.getSettings(uri); const { pytestArgs } = settings.testing; const cwd = settings.testing.cwd && settings.testing.cwd.length > 0 ? settings.testing.cwd : uri.fsPath; const pythonPathParts: string[] = process.env.PYTHONPATH?.split(path.delimiter) ?? []; const pythonPathCommand = [fullPluginPath, ...pythonPathParts].join(path.delimiter); - const spawnOptions: SpawnOptions = { cwd, throwOnStdErr: true, @@ -149,19 +156,19 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { }; traceInfo(`Running DEBUG pytest with arguments: ${testArgs.join(' ')}\r\n`); await debugLauncher!.launchDebugger(launchOptions, () => { - deferred.resolve(); - this.testServer.deleteUUID(uuid); + deferredTillEOT?.resolve(); }); } else { // combine path to run script with run args const scriptPath = path.join(fullPluginPath, 'vscode_pytest', 'run_pytest_script.py'); const runArgs = [scriptPath, ...testArgs]; - traceInfo(`Running pytests with arguments: ${runArgs.join(' ')}\r\n`); + traceInfo(`Running pytest with arguments: ${runArgs.join(' ')}\r\n`); const deferredExec = createDeferred>(); const result = execService?.execObservable(runArgs, spawnOptions); runInstance?.token.onCancellationRequested(() => { + traceInfo('Test run cancelled, killing pytest subprocess.'); result?.proc?.kill(); }); @@ -175,17 +182,24 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { }); result?.proc?.on('exit', (code, signal) => { + traceInfo('Test run finished, subprocess exited.'); // if the child has testIds then this is a run request if (code !== 0 && testIds) { + traceError( + `Subprocess exited unsuccessfully with exit code ${code} and signal ${signal}. Creating and sending error execution payload`, + ); // if the child process exited with a non-zero exit code, then we need to send the error payload. this.testServer.triggerRunDataReceivedEvent({ uuid, data: JSON.stringify(utils.createExecutionErrorPayload(code, signal, testIds, cwd)), }); + // then send a EOT payload + this.testServer.triggerRunDataReceivedEvent({ + uuid, + data: JSON.stringify(utils.createEOTPayload(true)), + }); } deferredExec.resolve({ stdout: '', stderr: '' }); - deferred.resolve(); - disposeDataReceiver?.(this.testServer); }); await deferredExec.promise; } diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index 9820aa89626c..440df4f94dc6 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -14,6 +14,7 @@ import { TestCommandOptions, TestDiscoveryCommand, } from '../common/types'; +import { Deferred, createDeferred } from '../../../common/utils/async'; /** * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. @@ -34,7 +35,7 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { const command = buildDiscoveryCommand(unittestArgs); const uuid = this.testServer.createUUID(uri.fsPath); - + const deferredTillEOT: Deferred = createDeferred(); const options: TestCommandOptions = { workspaceFolder: uri, command, @@ -44,7 +45,7 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { }; const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { - this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + this.resultResolver?.resolveDiscovery(JSON.parse(e.data), deferredTillEOT); }); const disposeDataReceiver = function (testServer: ITestServer) { testServer.deleteUUID(uuid); @@ -52,8 +53,10 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { }; await this.callSendCommand(options, () => { - disposeDataReceiver(this.testServer); + disposeDataReceiver?.(this.testServer); }); + await deferredTillEOT.promise; + disposeDataReceiver(this.testServer); // placeholder until after the rewrite is adopted // TODO: remove after adoption. const discoveryPayload: DiscoveredTestPayload = { diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index bc5e41d19d9d..9da0872ef601 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred } from '../../../common/utils/async'; +import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { DataReceivedEvent, @@ -15,7 +15,7 @@ import { TestCommandOptions, TestExecutionCommand, } from '../common/types'; -import { traceLog } from '../../../logging'; +import { traceError, traceInfo, traceLog } from '../../../logging'; import { startTestIdServer } from '../common/utils'; /** @@ -37,19 +37,30 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { runInstance?: TestRun, ): Promise { const uuid = this.testServer.createUUID(uri.fsPath); + const deferredTillEOT: Deferred = createDeferred(); const disposedDataReceived = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { - this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance, deferredTillEOT); + } else { + traceError('No run instance found, cannot resolve execution.'); } }); const disposeDataReceiver = function (testServer: ITestServer) { + traceInfo(`Disposing data receiver for ${uri.fsPath} and deleting UUID; unittest execution.`); testServer.deleteUUID(uuid); disposedDataReceived.dispose(); }; runInstance?.token.onCancellationRequested(() => { - disposeDataReceiver(this.testServer); + traceInfo("Test run cancelled, resolving 'till EOT' deferred."); + deferredTillEOT.resolve(); }); - await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, disposeDataReceiver); + try { + await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, deferredTillEOT); + await deferredTillEOT.promise; + disposeDataReceiver(this.testServer); + } catch (error) { + traceError(`Error in running unittest tests: ${error}`); + } const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; return executionPayload; } @@ -60,7 +71,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { uuid: string, runInstance?: TestRun, debugBool?: boolean, - disposeDataReceiver?: (testServer: ITestServer) => void, + deferredTillEOT?: Deferred, ): Promise { const settings = this.configSettings.getSettings(uri); const { unittestArgs } = settings.testing; @@ -77,15 +88,12 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { testIds, outChannel: this.outputChannel, }; - - const deferred = createDeferred(); traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); const runTestIdsPort = await startTestIdServer(testIds); await this.testServer.sendCommand(options, runTestIdsPort.toString(), runInstance, testIds, () => { - deferred.resolve(); - disposeDataReceiver?.(this.testServer); + deferredTillEOT?.resolve(); }); // placeholder until after the rewrite is adopted // TODO: remove after adoption. diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 5af7e59f6a46..b445772d6958 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -1,12 +1,12 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { TestRun, Uri } from 'vscode'; +import { TestController, TestRun, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; +import { ITestController, ITestResultResolver } from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { IPythonExecutionFactory } from '../../../client/common/process/types'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; @@ -17,16 +17,22 @@ import { traceLog } from '../../../client/logging'; import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; +import { PythonResultResolver } from '../../../client/testing/testController/common/resultResolver'; +import { TestProvider } from '../../../client/testing/types'; +import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../../../client/testing/common/constants'; suite('End to End Tests: test adapters', () => { - let resultResolver: typeMoq.IMock; - let pythonTestServer: ITestServer; + let resultResolver: ITestResultResolver; + let pythonTestServer: PythonTestServer; let pythonExecFactory: IPythonExecutionFactory; let debugLauncher: ITestDebugLauncher; let configService: IConfigurationService; - let testOutputChannel: ITestOutputChannel; let serviceContainer: IServiceContainer; let workspaceUri: Uri; + let testOutputChannel: typeMoq.IMock; + let testController: TestController; + const unittestProvider: TestProvider = UNITTEST_PROVIDER; + const pytestProvider: TestProvider = PYTEST_PROVIDER; const rootPathSmallWorkspace = path.join( EXTENSION_ROOT_DIR_FOR_TESTS, 'src', @@ -54,83 +60,97 @@ suite('End to End Tests: test adapters', () => { configService = serviceContainer.get(IConfigurationService); pythonExecFactory = serviceContainer.get(IPythonExecutionFactory); debugLauncher = serviceContainer.get(ITestDebugLauncher); - testOutputChannel = serviceContainer.get(ITestOutputChannel); - - // create mock resultResolver object - resultResolver = typeMoq.Mock.ofType(); + testController = serviceContainer.get(ITestController); // create objects that were not injected pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); await pythonTestServer.serverReady(); + + testOutputChannel = typeMoq.Mock.ofType(); + testOutputChannel + .setup((x) => x.append(typeMoq.It.isAny())) + .callback((appendVal: any) => { + traceLog('output channel - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + testOutputChannel + .setup((x) => x.appendLine(typeMoq.It.isAny())) + .callback((appendVal: any) => { + traceLog('output channel ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + }); + teardown(async () => { + pythonTestServer.dispose(); }); test('unittest discovery adapter small workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + workspaceUri = Uri.parse(rootPathSmallWorkspace); + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveDiscovery ${data}`); - actualData = data; - return Promise.resolve(); - }); // set workspace to test workspace folder and set up settings - workspaceUri = Uri.parse(rootPathSmallWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; // run unittest discovery const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); await discoveryAdapter.discoverTests(workspaceUri).finally(() => { // verification after discovery is complete - // resultResolver.verify( - // (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - // typeMoq.Times.once(), - // ); // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); - }); - await discoveryAdapter.discoverTests(Uri.parse(rootPathErrorWorkspace)).finally(() => { - // verification after discovery is complete - - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(actualData.tests, 'Expected tests to be present'); + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('unittest discovery adapter large workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveDiscovery ${data}`); - actualData = data; - return Promise.resolve(); - }); // set settings to work for the given workspace workspaceUri = Uri.parse(rootPathLargeWorkspace); @@ -139,84 +159,89 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); await discoveryAdapter.discoverTests(workspaceUri).finally(() => { - // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), - ); - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('pytest discovery adapter small workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveDiscovery ${data}`); - actualData = data; - return Promise.resolve(); - }); // run pytest discovery const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); - await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), - ); // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(actualData.error.length, 0, "Expected no errors in 'error' field"); + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors + assert.strictEqual(actualData.error?.length, 0, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('pytest discovery adapter large workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - actualData = data; - return Promise.resolve(); - }); // run pytest discovery const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); // set workspace to test workspace folder @@ -224,32 +249,42 @@ suite('End to End Tests: test adapters', () => { await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), - ); - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(actualData.error.length, 0, "Expected no errors in 'error' field"); + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors + assert.strictEqual(actualData.error?.length, 0, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('unittest execution adapter small workspace', async () => { // result resolver and saved data for assertions - let actualData: { - status: unknown; - error: string | any[]; - result: unknown; + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + let failureOccurred = false; + let failureMsg = ''; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + try { + assert.strictEqual( + payload.status, + 'success', + `Expected status to be 'success', instead status is ${payload.status}`, + ); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err ? (err as Error).toString() : ''; + failureOccurred = true; + } + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - actualData = data; - return Promise.resolve(); - }); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); @@ -258,8 +293,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -273,33 +308,34 @@ suite('End to End Tests: test adapters', () => { await executionAdapter .runTests(workspaceUri, ['test_simple.SimpleClass.test_simple_unit'], false, testRun.object) .finally(() => { - // verification after execution is complete - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), - ); - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm tests are found - assert.ok(actualData.result, 'Expected results to be present'); + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('unittest execution adapter large workspace', async () => { // result resolver and saved data for assertions - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveExecution ${data}`); - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status, can be subtest success or failure - assert( - data.status === 'subtest-success' || data.status === 'subtest-failure', - "Expected status to be 'subtest-success' or 'subtest-failure'", + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + let failureOccurred = false; + let failureMsg = ''; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + try { + const validStatuses = ['subtest-success', 'subtest-failure']; + assert.ok( + validStatuses.includes(payload.status), + `Expected status to be one of ${validStatuses.join(', ')}, but instead status is ${payload.status}`, ); - // 2. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - return Promise.resolve(); - }); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err ? (err as Error).toString() : ''; + failureOccurred = true; + } + return Promise.resolve(); + }; // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathLargeWorkspace); @@ -309,8 +345,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -323,28 +359,35 @@ suite('End to End Tests: test adapters', () => { ); await executionAdapter .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) - .finally(() => { - // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeastOnce(), - ); + .then(() => { + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 2000, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('pytest execution adapter small workspace', async () => { // result resolver and saved data for assertions - let actualData: { - status: unknown; - error: string | any[]; - result: unknown; + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + let failureOccurred = false; + let failureMsg = ''; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + try { + assert.strictEqual( + payload.status, + 'success', + `Expected status to be 'success', instead status is ${payload.status}`, + ); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err ? (err as Error).toString() : ''; + failureOccurred = true; + } + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - actualData = data; - return Promise.resolve(); - }); - // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); @@ -352,8 +395,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -372,40 +415,42 @@ suite('End to End Tests: test adapters', () => { testRun.object, pythonExecFactory, ) - .finally(() => { - // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), - ); - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(actualData.error, null, "Expected no errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(actualData.result, 'Expected results to be present'); + .then(() => { + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('pytest execution adapter large workspace', async () => { - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(data.error, null, "Expected no errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - return Promise.resolve(); - }); + // result resolver and saved data for assertions + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + let failureOccurred = false; + let failureMsg = ''; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + try { + assert.strictEqual( + payload.status, + 'success', + `Expected status to be 'success', instead status is ${payload.status}`, + ); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err ? (err as Error).toString() : ''; + failureOccurred = true; + } + return Promise.resolve(); + }; // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathLargeWorkspace); // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 200; i = i + 1) { + for (let i = 0; i < 2000; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } @@ -414,8 +459,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -426,35 +471,51 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - // resolve execution should be called 200 times since there are 200 tests run. - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(200), - ); + await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).then(() => { + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 2000, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('unittest execution adapter seg fault error handling', async () => { + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + let failureOccurred = false; + let failureMsg = ''; + resultResolver._resolveExecution = async (data, _token?) => { + // do the following asserts for each time resolveExecution is called, should be called once per test. + callCount = callCount + 1; + console.log(`unittest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); + try { + if (data.status === 'error') { + if (data.error === undefined) { + // Dereference a NULL pointer + const indexOfTest = JSON.stringify(data).search('Dereference a NULL pointer'); + assert.notDeepEqual(indexOfTest, -1, 'Expected test to have a null pointer'); + } else { + assert.ok(data.error, "Expected errors in 'error' field"); + } + } else { + const indexOfTest = JSON.stringify(data.result).search('error'); + assert.notDeepEqual( + indexOfTest, + -1, + 'If payload status is not error then the individual tests should be marked as errors. This should occur on windows machines.', + ); + } + assert.ok(data.result, 'Expected results to be present'); + // make sure the testID is found in the results + const indexOfTest = JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'); + assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); + } catch (err) { + failureMsg = err ? (err as Error).toString() : ''; + failureOccurred = true; + } + return Promise.resolve(); + }; + const testId = `test_seg_fault.TestSegmentationFault.test_segfault`; const testIds: string[] = [testId]; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'), - -1, - 'Expected testId to be present', - ); - return Promise.resolve(); - }); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathErrorWorkspace); @@ -463,8 +524,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -476,33 +537,45 @@ suite('End to End Tests: test adapters', () => { } as any), ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object).finally(() => { - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(1), - ); + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('pytest execution adapter seg fault error handling', async () => { - const testId = `${rootPathErrorWorkspace}/test_seg_fault.py::TestSegmentationFault::test_segfault`; - const testIds: string[] = [testId]; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + let callCount = 0; + let failureOccurred = false; + let failureMsg = ''; + resultResolver._resolveExecution = async (data, _token?) => { + // do the following asserts for each time resolveExecution is called, should be called once per test. + console.log(`pytest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); + callCount = callCount + 1; + try { + if (data.status === 'error') { + assert.ok(data.error, "Expected errors in 'error' field"); + } else { + const indexOfTest = JSON.stringify(data.result).search('error'); + assert.notDeepEqual( + indexOfTest, + -1, + 'If payload status is not error then the individual tests should be marked as errors. This should occur on windows machines.', + ); + } assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.py::TestSegmentationFault::test_segfault'), - -1, - 'Expected testId to be present', + // make sure the testID is found in the results + const indexOfTest = JSON.stringify(data).search( + 'test_seg_fault.py::TestSegmentationFault::test_segfault', ); - return Promise.resolve(); - }); + assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); + } catch (err) { + failureMsg = err ? (err as Error).toString() : ''; + failureOccurred = true; + } + return Promise.resolve(); + }; + + const testId = `${rootPathErrorWorkspace}/test_seg_fault.py::TestSegmentationFault::test_segfault`; + const testIds: string[] = [testId]; // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathErrorWorkspace); @@ -511,8 +584,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -524,10 +597,8 @@ suite('End to End Tests: test adapters', () => { } as any), ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(1), - ); + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); }); diff --git a/src/test/testing/common/testingPayloadsEot.test.ts b/src/test/testing/common/testingPayloadsEot.test.ts new file mode 100644 index 000000000000..227ad5fa1697 --- /dev/null +++ b/src/test/testing/common/testingPayloadsEot.test.ts @@ -0,0 +1,210 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { TestController, TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as path from 'path'; +import * as assert from 'assert'; +import * as net from 'net'; +import { Observable } from 'rxjs'; +import * as crypto from 'crypto'; +// import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; +import * as sinon from 'sinon'; +import { ITestController, ITestResultResolver } from '../../../client/testing/testController/common/types'; +import { PythonTestServer } from '../../../client/testing/testController/common/server'; +import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; +import { ITestDebugLauncher } from '../../../client/testing/common/types'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { initialize } from '../../initialize'; +import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; +import { PythonResultResolver } from '../../../client/testing/testController/common/resultResolver'; +import { PYTEST_PROVIDER } from '../../../client/testing/common/constants'; +import { MockChildProcess } from '../../mocks/mockChildProcess'; +import { + PAYLOAD_SINGLE_CHUNK, + PAYLOAD_MULTI_CHUNK, + PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY, + DataWithPayloadChunks, + PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY, +} from '../testController/payloadTestCases'; +import { traceLog } from '../../../client/logging'; + +const FAKE_UUID = 'fake-u-u-i-d'; +export interface TestCase { + name: string; + value: DataWithPayloadChunks; +} + +const testCases: Array = [ + { + name: 'single payload single chunk', + value: PAYLOAD_SINGLE_CHUNK(FAKE_UUID), + }, + { + name: 'multiple payloads per buffer chunk', + value: PAYLOAD_MULTI_CHUNK(FAKE_UUID), + }, + { + name: 'single payload across multiple buffer chunks', + value: PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(FAKE_UUID), + }, + { + name: 'two chunks, payload split and two payloads in a chunk', + value: PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(FAKE_UUID), + }, +]; + +suite('EOT tests', () => { + let resultResolver: ITestResultResolver; + let pythonTestServer: PythonTestServer; + let debugLauncher: ITestDebugLauncher; + let configService: IConfigurationService; + let serviceContainer: IServiceContainer; + let workspaceUri: Uri; + let testOutputChannel: typeMoq.IMock; + let testController: TestController; + let stubExecutionFactory: typeMoq.IMock; + let client: net.Socket; + const sandbox = sinon.createSandbox(); + // const unittestProvider: TestProvider = UNITTEST_PROVIDER; + // const pytestProvider: TestProvider = PYTEST_PROVIDER; + const rootPathSmallWorkspace = path.join('src'); + suiteSetup(async () => { + serviceContainer = (await initialize()).serviceContainer; + }); + + setup(async () => { + // create objects that were injected + configService = serviceContainer.get(IConfigurationService); + debugLauncher = serviceContainer.get(ITestDebugLauncher); + testController = serviceContainer.get(ITestController); + + // create client to act as python server which sends testing result response + client = new net.Socket(); + client.on('error', (error) => { + traceLog('Socket connection error:', error); + }); + + const mockProc = new MockChildProcess('', ['']); + const output2 = new Observable>(() => { + /* no op */ + }); + + // stub out execution service and factory so mock data is returned from client. + const stubExecutionService = ({ + execObservable: () => { + client.connect(pythonTestServer.getPort()); + return { + proc: mockProc, + out: output2, + dispose: () => { + /* no-body */ + }, + }; + }, + } as unknown) as IPythonExecutionService; + + stubExecutionFactory = typeMoq.Mock.ofType(); + stubExecutionFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(stubExecutionService)); + + // stub create UUID + + const v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); + + // create python test server + pythonTestServer = new PythonTestServer(stubExecutionFactory.object, debugLauncher); + await pythonTestServer.serverReady(); + // handles output from client + testOutputChannel = typeMoq.Mock.ofType(); + testOutputChannel + .setup((x) => x.append(typeMoq.It.isAny())) + .callback((appendVal: any) => { + traceLog('out - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + testOutputChannel + .setup((x) => x.appendLine(typeMoq.It.isAny())) + .callback((appendVal: any) => { + traceLog('outL - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + }); + teardown(async () => { + pythonTestServer.dispose(); + sandbox.restore(); + }); + testCases.forEach((testCase) => { + test(`Testing Payloads: ${testCase.name}`, async () => { + let actualCollectedResult = ''; + client.on('connect', async () => { + traceLog('socket connected, sending stubbed data'); + // payload is a string array, each string represents one line written to the buffer + const { payloadArray } = testCase.value; + for (let i = 0; i < payloadArray.length; i = i + 1) { + await (async (clientSub, payloadSub) => { + if (!clientSub.write(payloadSub)) { + // If write returns false, wait for the 'drain' event before proceeding + await new Promise((resolve) => clientSub.once('drain', resolve)); + } + })(client, payloadArray[i]); + } + client.end(); + }); + + resultResolver = new PythonResultResolver(testController, PYTEST_PROVIDER, workspaceUri); + resultResolver._resolveExecution = async (payload, _token?) => { + // the payloads that get to the _resolveExecution are all data and should be successful. + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + actualCollectedResult = actualCollectedResult + JSON.stringify(payload.result); + return Promise.resolve(); + }; + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathSmallWorkspace); + + // run pytest execution + const executionAdapter = new PytestTestExecutionAdapter( + pythonTestServer, + configService, + testOutputChannel.object, + resultResolver, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + await executionAdapter + .runTests( + workspaceUri, + [`${rootPathSmallWorkspace}/test_simple.py::test_a`], + false, + testRun.object, + stubExecutionFactory.object, + ) + .then(() => { + assert.strictEqual( + testCase.value.data, + actualCollectedResult, + "Expected collected result to match 'data'", + ); + // nervous about this not testing race conditions correctly + // client.end(); + // verify that the _resolveExecution was called once per test + // assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + }); + }); + }); +}); diff --git a/src/test/testing/testController/payloadTestCases.ts b/src/test/testing/testController/payloadTestCases.ts new file mode 100644 index 000000000000..5ddcc0edecf9 --- /dev/null +++ b/src/test/testing/testController/payloadTestCases.ts @@ -0,0 +1,149 @@ +export interface DataWithPayloadChunks { + payloadArray: string[]; + data: string; +} + +const EOT_PAYLOAD = `Content-Length: 42 +Content-Type: application/json +Request-uuid: fake-u-u-i-d + +{"command_type": "execution", "eot": true}`; + +const SINGLE_UNITTEST_SUBTEST = { + cwd: '/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace', + status: 'success', + result: { + 'test_parameterized_subtest.NumbersTest.test_even (i=0)': { + test: 'test_parameterized_subtest.NumbersTest.test_even', + outcome: 'success', + message: 'None', + traceback: null, + subtest: 'test_parameterized_subtest.NumbersTest.test_even (i=0)', + }, + }, +}; + +const SINGLE_PYTEST_PAYLOAD = { + cwd: 'path/to', + status: 'success', + result: { + 'path/to/file.py::test_funct': { + test: 'path/to/file.py::test_funct', + outcome: 'success', + message: 'None', + traceback: null, + subtest: 'path/to/file.py::test_funct', + }, + }, +}; + +const SINGLE_PYTEST_PAYLOAD_TWO = { + cwd: 'path/to/second', + status: 'success', + result: { + 'path/to/workspace/parametrize_tests.py::test_adding[3+5-8]': { + test: 'path/to/workspace/parametrize_tests.py::test_adding[3+5-8]', + outcome: 'success', + message: 'None', + traceback: null, + }, + }, +}; + +export function createPayload(uuid: string, data: unknown): string { + return `Content-Length: ${JSON.stringify(data).length} +Content-Type: application/json +Request-uuid: ${uuid} + +${JSON.stringify(data)}`; +} + +export function PAYLOAD_SINGLE_CHUNK(uuid: string): DataWithPayloadChunks { + const payload = createPayload(uuid, SINGLE_UNITTEST_SUBTEST); + + return { + payloadArray: [payload, EOT_PAYLOAD], + data: JSON.stringify(SINGLE_UNITTEST_SUBTEST.result), + }; +} + +// more than one payload (item with header) per chunk sent +// payload has 3 SINGLE_UNITTEST_SUBTEST +export function PAYLOAD_MULTI_CHUNK(uuid: string): DataWithPayloadChunks { + let payload = ''; + let result = ''; + for (let i = 0; i < 3; i = i + 1) { + payload += createPayload(uuid, SINGLE_UNITTEST_SUBTEST); + result += JSON.stringify(SINGLE_UNITTEST_SUBTEST.result); + } + return { + payloadArray: [payload, EOT_PAYLOAD], + data: result, + }; +} + +// single payload divided by an arbitrary character and split across payloads +export function PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid: string): DataWithPayloadChunks { + const payload = createPayload(uuid, SINGLE_PYTEST_PAYLOAD); + // payload length is know to be >200 + const splitPayload: Array = [ + payload.substring(0, 50), + payload.substring(50, 100), + payload.substring(100, 150), + payload.substring(150), + ]; + const finalResult = JSON.stringify(SINGLE_PYTEST_PAYLOAD.result); + splitPayload.push(EOT_PAYLOAD); + return { + payloadArray: splitPayload, + data: finalResult, + }; +} + +// here a payload is split across the buffer chunks and there are multiple payloads in a single buffer chunk +export function PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(uuid: string): DataWithPayloadChunks { + // payload1 length is know to be >200 + const payload1 = createPayload(uuid, SINGLE_PYTEST_PAYLOAD); + const payload2 = createPayload(uuid, SINGLE_PYTEST_PAYLOAD_TWO); + + // chunk 1 is 50 char of payload1, chunk 2 is 50-end of payload1 and all of payload2 + const splitPayload: Array = [payload1.substring(0, 100), payload1.substring(100).concat(payload2)]; + const finalResult = JSON.stringify(SINGLE_PYTEST_PAYLOAD.result).concat( + JSON.stringify(SINGLE_PYTEST_PAYLOAD_TWO.result), + ); + + splitPayload.push(EOT_PAYLOAD); + return { + payloadArray: splitPayload, + data: finalResult, + }; +} + +export function PAYLOAD_SPLIT_MULTI_CHUNK_RAN_ORDER_ARRAY(uuid: string): Array { + return [ + `Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}} + +Content-Length: 411 +Content-Type: application/json +Request-uuid: 9${uuid} + +{"cwd": "/home/runner/work/vscode-`, + `python/vscode-python/path with`, + ` spaces/src" + +Content-Length: 959 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-failure", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=1)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-failure", "message": "(, AssertionError('1 != 0'), )", "traceback": " File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 57, in testPartExecutor\n yield\n File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 538, in subTest\n yield\n File \"/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py\", line 16, in test_even\n self.assertEqual(i % 2, 0)\nAssertionError: 1 != 0\n", "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=1)"}}} +Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=2)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=2)"}}}`, + ]; +} diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index 43b763f56e6c..9cc428ab0a4c 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -21,6 +21,7 @@ import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/co import * as util from '../../../../client/testing/testController/common/utils'; import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; import { MockChildProcess } from '../../../mocks/mockChildProcess'; +import { traceInfo } from '../../../../client/logging'; suite('pytest test execution adapter', () => { let testServer: typeMoq.IMock; @@ -33,7 +34,7 @@ suite('pytest test execution adapter', () => { (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; let myTestPath: string; let mockProc: MockChildProcess; - let utilsStub: sinon.SinonStub; + let utilsStartServerStub: sinon.SinonStub; setup(() => { testServer = typeMoq.Mock.ofType(); testServer.setup((t) => t.getPort()).returns(() => 12345); @@ -51,6 +52,8 @@ suite('pytest test execution adapter', () => { isTestExecution: () => false, } as unknown) as IConfigurationService; + // mock out the result resolver + // set up exec service with child process mockProc = new MockChildProcess('', ['']); const output = new Observable>(() => { @@ -67,7 +70,7 @@ suite('pytest test execution adapter', () => { }, })); execFactory = typeMoq.Mock.ofType(); - utilsStub = sinon.stub(util, 'startTestIdServer'); + utilsStartServerStub = sinon.stub(util, 'startTestIdServer'); debugLauncher = typeMoq.Mock.ofType(); execFactory .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) @@ -79,13 +82,6 @@ suite('pytest test execution adapter', () => { deferred.resolve(); return Promise.resolve({ stdout: '{}' }); }); - debugLauncher - .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => { - deferred.resolve(); - return Promise.resolve(); - }); - execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); @@ -104,7 +100,7 @@ suite('pytest test execution adapter', () => { deferred2.resolve(); return Promise.resolve(execService.object); }); - utilsStub.callsFake(() => { + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); @@ -131,7 +127,7 @@ suite('pytest test execution adapter', () => { mockProc.trigger('close'); // assert - sinon.assert.calledWithExactly(utilsStub, testIds); + sinon.assert.calledWithExactly(utilsStartServerStub, testIds); }); test('pytest execution called with correct args', async () => { const deferred2 = createDeferred(); @@ -143,7 +139,7 @@ suite('pytest test execution adapter', () => { deferred2.resolve(); return Promise.resolve(execService.object); }); - utilsStub.callsFake(() => { + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); @@ -175,7 +171,6 @@ suite('pytest test execution adapter', () => { TEST_UUID: 'uuid123', TEST_PORT: '12345', }; - // execService.verify((x) => x.exec(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once()); execService.verify( (x) => x.execObservable( @@ -203,7 +198,7 @@ suite('pytest test execution adapter', () => { deferred2.resolve(); return Promise.resolve(execService.object); }); - utilsStub.callsFake(() => { + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); @@ -262,12 +257,28 @@ suite('pytest test execution adapter', () => { }); test('Debug launched correctly for pytest', async () => { const deferred3 = createDeferred(); - utilsStub.callsFake(() => { + const deferredEOT = createDeferred(); + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); + debugLauncher + .setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(async () => { + traceInfo('stubs launch debugger'); + deferredEOT.resolve(); + }); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); const testRun = typeMoq.Mock.ofType(); - testRun.setup((t) => t.token).returns(() => ({ onCancellationRequested: () => undefined } as any)); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); const uri = Uri.file(myTestPath); const uuid = 'uuid123'; testServer @@ -298,5 +309,6 @@ suite('pytest test execution adapter', () => { ), typeMoq.Times.once(), ); + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); }); }); diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts index 694fdacb8049..2078c72e8cf6 100644 --- a/src/test/testing/testController/resultResolver.unit.test.ts +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -14,6 +14,8 @@ import { import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; import * as util from '../../../client/testing/testController/common/utils'; +import { Deferred, createDeferred } from '../../../client/common/utils/async'; +import { traceLog } from '../../../client/logging'; suite('Result Resolver tests', () => { suite('Test discovery', () => { @@ -87,7 +89,8 @@ suite('Result Resolver tests', () => { const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); // call resolve discovery - resultResolver.resolveDiscovery(payload, cancelationToken); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveDiscovery(payload, deferredTillEOT, cancelationToken); // assert the stub functions were called with the correct parameters @@ -126,7 +129,8 @@ suite('Result Resolver tests', () => { const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); // call resolve discovery - resultResolver.resolveDiscovery(payload); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveDiscovery(payload, deferredTillEOT, cancelationToken); // assert the stub functions were called with the correct parameters @@ -171,7 +175,8 @@ suite('Result Resolver tests', () => { // stub out functionality of populateTestTreeStub which is called in resolveDiscovery const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); // call resolve discovery - resultResolver.resolveDiscovery(payload, cancelationToken); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveDiscovery(payload, deferredTillEOT, cancelationToken); // assert the stub functions were called with the correct parameters @@ -286,7 +291,7 @@ suite('Result Resolver tests', () => { .setup((t) => t.createTestItem(typemoq.It.isAny(), typemoq.It.isAny())) .callback((id: string) => { generatedId = id; - console.log('createTestItem function called with id:', id); + traceLog('createTestItem function called with id:', id); }) .returns(() => ({ id: 'id_this', label: 'label_this', uri: workspaceUri } as TestItem)); @@ -307,7 +312,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item assert.ok(generatedId); @@ -347,7 +353,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); @@ -386,7 +393,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.once()); @@ -425,7 +433,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.errored(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); @@ -464,7 +473,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.once()); @@ -485,7 +495,8 @@ suite('Result Resolver tests', () => { error: 'error', }; - resultResolver.resolveExecution(errorPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(errorPayload, runInstance.object, deferredTillEOT); // verify that none of these functions are called diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index 53c2b72e40f7..92a9a1135f55 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -6,58 +6,66 @@ import * as assert from 'assert'; import * as net from 'net'; import * as sinon from 'sinon'; import * as crypto from 'crypto'; -import { OutputChannel, Uri } from 'vscode'; import { Observable } from 'rxjs'; import * as typeMoq from 'typemoq'; +import { OutputChannel, Uri } from 'vscode'; import { IPythonExecutionFactory, IPythonExecutionService, + ObservableExecutionResult, Output, - SpawnOptions, } from '../../../client/common/process/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; import { Deferred, createDeferred } from '../../../client/common/utils/async'; import { MockChildProcess } from '../../mocks/mockChildProcess'; - -suite('Python Test Server', () => { - const fakeUuid = 'fake-uuid'; - - let stubExecutionFactory: IPythonExecutionFactory; - let stubExecutionService: IPythonExecutionService; +import { + PAYLOAD_MULTI_CHUNK, + PAYLOAD_SINGLE_CHUNK, + PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY, + DataWithPayloadChunks, +} from './payloadTestCases'; +import { traceLog } from '../../../client/logging'; + +const testCases = [ + { + val: () => PAYLOAD_SINGLE_CHUNK('fake-uuid'), + }, + { + val: () => PAYLOAD_MULTI_CHUNK('fake-uuid'), + }, + { + val: () => PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY('fake-uuid'), + }, +]; + +suite('Python Test Server, DataWithPayloadChunks', () => { + const FAKE_UUID = 'fake-uuid'; let server: PythonTestServer; - let sandbox: sinon.SinonSandbox; let v4Stub: sinon.SinonStub; let debugLauncher: ITestDebugLauncher; let mockProc: MockChildProcess; let execService: typeMoq.IMock; let deferred: Deferred; - let execFactory = typeMoq.Mock.ofType(); - - setup(() => { - sandbox = sinon.createSandbox(); - v4Stub = sandbox.stub(crypto, 'randomUUID'); + const sandbox = sinon.createSandbox(); - v4Stub.returns(fakeUuid); - stubExecutionService = ({ - execObservable: () => Promise.resolve({ stdout: '', stderr: '' }), - } as unknown) as IPythonExecutionService; + setup(async () => { + // set up test command options - stubExecutionFactory = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService), - } as unknown) as IPythonExecutionFactory; + v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); // set up exec service with child process mockProc = new MockChildProcess('', ['']); execService = typeMoq.Mock.ofType(); - const output = new Observable>(() => { + const outputObservable = new Observable>(() => { /* no op */ }); execService .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) .returns(() => ({ proc: mockProc, - out: output, + out: outputObservable, dispose: () => { /* no-body */ }, @@ -70,53 +78,153 @@ suite('Python Test Server', () => { server.dispose(); }); + testCases.forEach((testCase) => { + test(`run correctly`, async () => { + const testCaseDataObj: DataWithPayloadChunks = testCase.val(); + let eventData = ''; + const client = new net.Socket(); + + deferred = createDeferred(); + mockProc = new MockChildProcess('', ['']); + const output2 = new Observable>(() => { + /* no op */ + }); + const stubExecutionService2 = ({ + execObservable: () => { + client.connect(server.getPort()); + return { + proc: mockProc, + out: output2, + dispose: () => { + /* no-body */ + }, + }; + }, + } as unknown) as IPythonExecutionService; + + const stubExecutionFactory2 = ({ + createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + } as unknown) as IPythonExecutionFactory; + server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + const uuid = server.createUUID(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid, + }; + + const dataWithPayloadChunks = testCaseDataObj; + + await server.serverReady(); + + server.onRunDataReceived(({ data }) => { + try { + const resultData = JSON.parse(data).result; + eventData = eventData + JSON.stringify(resultData); + } catch (e) { + assert(false, 'Error parsing data'); + } + deferred.resolve(); + }); + client.on('connect', () => { + traceLog('Socket connected, local port:', client.localPort); + // since this test is a single payload as a single chunk there should be a single line in the payload. + for (const line of dataWithPayloadChunks.payloadArray) { + client.write(line); + } + client.end(); + }); + client.on('error', (error) => { + traceLog('Socket connection error:', error); + }); + + server.sendCommand(options); + await deferred.promise; + const expectedResult = dataWithPayloadChunks.data; + assert.deepStrictEqual(eventData, expectedResult); + }); + }); +}); + +suite('Python Test Server, Send command etc', () => { + const FAKE_UUID = 'fake-uuid'; + let server: PythonTestServer; + let v4Stub: sinon.SinonStub; + let debugLauncher: ITestDebugLauncher; + let mockProc: MockChildProcess; + let execService: typeMoq.IMock; + let deferred: Deferred; + const sandbox = sinon.createSandbox(); + + setup(async () => { + // set up test command options + + v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); + + // set up exec service with child process + mockProc = new MockChildProcess('', ['']); + execService = typeMoq.Mock.ofType(); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + }); + + teardown(() => { + sandbox.restore(); + server.dispose(); + }); test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { - const options = { - command: { - script: 'myscript', - args: ['-foo', 'foo'], - }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - const expectedSpawnOptions = { - cwd: '/foo/bar', - outputChannel: undefined, - token: undefined, - throwOnStdErr: true, - extraVariables: { - PYTHONPATH: '/foo/bar', - RUN_TEST_IDS_PORT: '56789', - }, - } as SpawnOptions; const deferred2 = createDeferred(); - execFactory = typeMoq.Mock.ofType(); + const RUN_TEST_IDS_PORT_CONST = '5678'; + execService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((_args, options2) => { + try { + assert.strictEqual( + options2.extraVariables.PYTHONPATH, + '/foo/bar', + 'Expect python path to exist as extra variable and be set correctly', + ); + assert.strictEqual( + options2.extraVariables.RUN_TEST_IDS_PORT, + RUN_TEST_IDS_PORT_CONST, + 'Expect test id port to be in extra variables and set correctly', + ); + } catch (e) { + assert(false, 'Error parsing data, extra variables do not match'); + } + return typeMoq.Mock.ofType>().object; + }); + const execFactory = typeMoq.Mock.ofType(); execFactory .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) .returns(() => { deferred2.resolve(); return Promise.resolve(execService.object); }); - server = new PythonTestServer(execFactory.object, debugLauncher); await server.serverReady(); - - server.sendCommand(options, '56789'); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid: FAKE_UUID, + }; + server.sendCommand(options, RUN_TEST_IDS_PORT_CONST); // add in await and trigger await deferred2.promise; mockProc.trigger('close'); const port = server.getPort(); - const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']; - execService.verify((x) => x.execObservable(expectedArgs, expectedSpawnOptions), typeMoq.Times.once()); + const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; + execService.verify((x) => x.execObservable(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once()); }); test('sendCommand should write to an output channel if it is provided as an option', async () => { - const output: string[] = []; + const output2: string[] = []; const outChannel = { appendLine: (str: string) => { - output.push(str); + output2.push(str); }, } as OutputChannel; const options = { @@ -126,11 +234,11 @@ suite('Python Test Server', () => { }, workspaceFolder: Uri.file('/foo/bar'), cwd: '/foo/bar', - uuid: fakeUuid, + uuid: FAKE_UUID, outChannel, }; deferred = createDeferred(); - execFactory = typeMoq.Mock.ofType(); + const execFactory = typeMoq.Mock.ofType(); execFactory .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) .returns(() => { @@ -147,329 +255,49 @@ suite('Python Test Server', () => { mockProc.trigger('close'); const port = server.getPort(); - const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); + const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); - assert.deepStrictEqual(output, [expected]); + assert.deepStrictEqual(output2, [expected]); }); test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { let eventData: { status: string; errors: string[] } | undefined; - stubExecutionService = ({ - execObservable: () => { + const deferred2 = createDeferred(); + const deferred3 = createDeferred(); + const stubExecutionService = typeMoq.Mock.ofType(); + stubExecutionService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + stubExecutionService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred3.resolve(); throw new Error('Failed to execute'); - }, - } as unknown) as IPythonExecutionService; + }); const options = { command: { script: 'myscript', args: ['-foo', 'foo'] }, workspaceFolder: Uri.file('/foo/bar'), cwd: '/foo/bar', - uuid: fakeUuid, + uuid: FAKE_UUID, }; + const stubExecutionFactory = typeMoq.Mock.ofType(); + stubExecutionFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => { + deferred2.resolve(); + return Promise.resolve(stubExecutionService.object); + }); - server = new PythonTestServer(stubExecutionFactory, debugLauncher); + server = new PythonTestServer(stubExecutionFactory.object, debugLauncher); await server.serverReady(); server.onDataReceived(({ data }) => { eventData = JSON.parse(data); }); - await server.sendCommand(options); - + server.sendCommand(options); + await deferred2.promise; + await deferred3.promise; assert.notEqual(eventData, undefined); assert.deepStrictEqual(eventData?.status, 'error'); assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); }); - - test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - - deferred = createDeferred(); - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('malformed data'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - // add in await and trigger - await deferred.promise; - mockProc.trigger('close'); - - assert.deepStrictEqual(eventData, ''); - }); - - test('If the server doesnt recognize the UUID it should ignore it', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('{"Request-uuid": "unknown-uuid"}'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, ''); - }); - - // required to have "tests" or "results" - // the heading length not being equal and yes being equal - // multiple payloads - test('Error if payload does not have a content length header', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('{"not content length": "5"}'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, ''); - }); - - const testData = [ - { - testName: 'fires discovery correctly on test payload', - payload: `Content-Length: 52 -Content-Type: application/json -Request-uuid: UUID_HERE - -{"cwd": "path", "status": "success", "tests": "xyz"}`, - expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', - }, - // Add more test data as needed - ]; - - testData.forEach(({ testName, payload, expectedResult }) => { - test(`test: ${testName}`, async () => { - // Your test logic here - let eventData: string | undefined; - const client = new net.Socket(); - - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - const uuid = server.createUUID(); - payload = payload.replace('UUID_HERE', uuid); - server.onDiscoveryDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write(payload); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, expectedResult); - }); - }); - - test('Calls run resolver if the result header is in the payload', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - await server.serverReady(); - const uuid = server.createUUID(); - server.onRunDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - const payload = `Content-Length: 87 -Content-Type: application/json -Request-uuid: ${uuid} - -{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write(payload); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - const expectedResult = - '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; - assert.deepStrictEqual(eventData, expectedResult); - }); }); diff --git a/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts new file mode 100644 index 000000000000..2aaffdda41df --- /dev/null +++ b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts @@ -0,0 +1,350 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { CancellationTokenSource, TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as sinon from 'sinon'; +import * as path from 'path'; +import { Observable } from 'rxjs'; +import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { Deferred, createDeferred } from '../../../client/common/utils/async'; +import { EXTENSION_ROOT_DIR } from '../../../client/constants'; +import { ITestDebugLauncher } from '../../../client/testing/common/types'; +import { ITestServer } from '../../../client/testing/testController/common/types'; +import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; +import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; +import { MockChildProcess } from '../../mocks/mockChildProcess'; +import * as util from '../../../client/testing/testController/common/utils'; + +suite('Execution Flow Run Adapters', () => { + let testServer: typeMoq.IMock; + let configService: IConfigurationService; + let execFactory = typeMoq.Mock.ofType(); + let adapter: PytestTestExecutionAdapter; + let execService: typeMoq.IMock; + let deferred: Deferred; + let debugLauncher: typeMoq.IMock; + (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; + let myTestPath: string; + let mockProc: MockChildProcess; + let utilsStartServerStub: sinon.SinonStub; + + setup(() => { + testServer = typeMoq.Mock.ofType(); + testServer.setup((t) => t.getPort()).returns(() => 12345); + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + configService = ({ + getSettings: () => ({ + testing: { pytestArgs: ['.'], unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, + }), + isTestExecution: () => false, + } as unknown) as IConfigurationService; + + // mock out the result resolver + + // set up exec service with child process + mockProc = new MockChildProcess('', ['']); + const output = new Observable>(() => { + /* no op */ + }); + execService = typeMoq.Mock.ofType(); + execService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + proc: mockProc, + out: output, + dispose: () => { + /* no-body */ + }, + })); + execFactory = typeMoq.Mock.ofType(); + utilsStartServerStub = sinon.stub(util, 'startTestIdServer'); + debugLauncher = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execService.object)); + deferred = createDeferred(); + execService + .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve({ stdout: '{}' }); + }); + execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + myTestPath = path.join('/', 'my', 'test', 'path', '/'); + }); + teardown(() => { + sinon.restore(); + }); + test('PYTEST cancelation token called mid-run resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + execServiceMock + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + cancellationToken.cancel(); + return { + proc: mockProc, + out: typeMoq.Mock.ofType>>().object, + dispose: () => { + /* no-body */ + }, + }; + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + adapter = new PytestTestExecutionAdapter( + testServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await adapter.runTests( + Uri.file(myTestPath), + [], + false, + testRunMock.object, + execFactoryMock.object, + debugLauncher.object, + ); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); + test('PYTEST cancelation token called mid-debug resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + debugLauncher + .setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(async () => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + adapter = new PytestTestExecutionAdapter( + testServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await adapter.runTests( + Uri.file(myTestPath), + [], + true, + testRunMock.object, + execFactoryMock.object, + debugLauncher.object, + ); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); + test('UNITTEST cancelation token called mid-run resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + + // Stub send command to then have token canceled + const stubTestServer = typeMoq.Mock.ofType(); + stubTestServer + .setup((t) => + t.sendCommand( + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + ), + ) + .returns(() => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + + stubTestServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + stubTestServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + execServiceMock + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + cancellationToken.cancel(); + return { + proc: mockProc, + out: typeMoq.Mock.ofType>>().object, + dispose: () => { + /* no-body */ + }, + }; + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + const unittestAdapter = new UnittestTestExecutionAdapter( + stubTestServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await unittestAdapter.runTests(Uri.file(myTestPath), [], false, testRunMock.object); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + stubTestServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); + test('UNITTEST cancelation token called mid-debug resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + + // Stub send command to then have token canceled + const stubTestServer = typeMoq.Mock.ofType(); + stubTestServer + .setup((t) => + t.sendCommand( + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + ), + ) + .returns(() => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + + stubTestServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + stubTestServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + debugLauncher + .setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(async () => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + const unittestAdapter = new UnittestTestExecutionAdapter( + stubTestServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await unittestAdapter.runTests(Uri.file(myTestPath), [], false, testRunMock.object); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + stubTestServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); +}); diff --git a/src/test/testing/testController/utils.unit.test.ts b/src/test/testing/testController/utils.unit.test.ts index d971c7d37c9f..9168abc7041f 100644 --- a/src/test/testing/testController/utils.unit.test.ts +++ b/src/test/testing/testController/utils.unit.test.ts @@ -6,15 +6,15 @@ import { JSONRPC_CONTENT_LENGTH_HEADER, JSONRPC_CONTENT_TYPE_HEADER, JSONRPC_UUID_HEADER, - jsonRPCContent, - jsonRPCHeaders, + ExtractJsonRPCData, + parseJsonRPCHeadersAndData, } from '../../../client/testing/testController/common/utils'; suite('Test Controller Utils: JSON RPC', () => { test('Empty raw data string', async () => { const rawDataString = ''; - const output = jsonRPCHeaders(rawDataString); + const output = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(output.headers.size, 0); assert.deepStrictEqual(output.remainingRawData, ''); }); @@ -22,20 +22,20 @@ suite('Test Controller Utils: JSON RPC', () => { test('Valid data empty JSON', async () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: 2\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n{}`; - const rpcHeaders = jsonRPCHeaders(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(rpcHeaders.headers.size, 3); assert.deepStrictEqual(rpcHeaders.remainingRawData, '{}'); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, '{}'); }); test('Valid data NO JSON', async () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: 0\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n`; - const rpcHeaders = jsonRPCHeaders(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(rpcHeaders.headers.size, 3); assert.deepStrictEqual(rpcHeaders.remainingRawData, ''); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, ''); }); @@ -45,10 +45,10 @@ suite('Test Controller Utils: JSON RPC', () => { '{"jsonrpc": "2.0", "method": "initialize", "params": {"processId": 1234, "rootPath": "/home/user/project", "rootUri": "file:///home/user/project", "capabilities": {}}, "id": 0}'; const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: ${json.length}\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n${json}`; - const rpcHeaders = jsonRPCHeaders(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(rpcHeaders.headers.size, 3); assert.deepStrictEqual(rpcHeaders.remainingRawData, json); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, json); }); @@ -58,9 +58,9 @@ suite('Test Controller Utils: JSON RPC', () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: ${json.length}\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n${json}`; const rawDataString2 = rawDataString + rawDataString; - const rpcHeaders = jsonRPCHeaders(rawDataString2); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString2); assert.deepStrictEqual(rpcHeaders.headers.size, 3); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, json); assert.deepStrictEqual(rpcContent.remainingRawData, rawDataString); }); diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py index 8c6a29adf495..a76856ebb929 100644 --- a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -4,13 +4,13 @@ import unittest -@pytest.mark.parametrize("num", range(0, 200)) +@pytest.mark.parametrize("num", range(0, 2000)) def test_odd_even(num): assert num % 2 == 0 class NumbersTest(unittest.TestCase): def test_even(self): - for i in range(0, 200): + for i in range(0, 2000): with self.subTest(i=i): self.assertEqual(i % 2, 0)