From e9083387ccbe17ef102fc249c1d9ae3b9040da52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Tue, 22 May 2018 10:42:03 -0700 Subject: [PATCH 1/6] Adding a build script - Added build.py - Updated the `src/coreclr/common.props` file to suppress .NET Core Preview warnings - Added a script to run `src/coreclr` benchmarks against CoreCLr's CoreRun - Updated src/coreclr readme --- build.py | 463 +++++++++++++++++++++++ src/coreclr/README.md | 19 + src/coreclr/common.props | 10 + src/coreclr/run-with-coreclr-corerun.cmd | 190 ++++++++++ 4 files changed, 682 insertions(+) create mode 100755 build.py create mode 100644 src/coreclr/run-with-coreclr-corerun.cmd diff --git a/build.py b/build.py new file mode 100755 index 00000000000..337f6f3c853 --- /dev/null +++ b/build.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 + +''' +Builds the CoreClr Benchmarks +''' + +from contextlib import contextmanager +from traceback import format_exc + +import argparse +import datetime +import logging +import os +import subprocess +import sys +import time + + +LAUNCH_TIME = time.time() +LOGGING_FORMATTER = logging.Formatter( + fmt='[%(asctime)s][%(levelname)s] %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + + +class FatalError(Exception): + ''' + Raised for various script errors regarding environment and build + requirements. + ''' + + +def is_supported_version() -> bool: + '''Checks if the script is running on the supported version (>=3.5).''' + return sys.version_info.major > 2 and sys.version_info.minor > 4 + + +def get_script_path() -> str: + '''Gets this script directory.''' + return sys.path[0] + # return os.path.dirname(os.path.realpath(__file__)) + + +def get_repo_root_path() -> str: + '''Gets repository root directory.''' + return get_script_path() + + +def generate_log_file_name() -> str: + '''Generates a unique log file name for the current script''' + log_dir = os.path.join(get_repo_root_path(), 'logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0] + timestamp = datetime.datetime.fromtimestamp(LAUNCH_TIME).strftime( + "%Y%m%d%H%M%S") + log_file_name = '{}-{}-pid{}.log'.format( + timestamp, script_name, os.getpid()) + return os.path.join(log_dir, log_file_name) + + +def get_logging_console_handler( + fmt: logging.Formatter, + verbose: bool) -> logging.StreamHandler: + ''' + Gets a logging console handler (logging.StreamHandler) based on the + specified formatter (logging.Formatter) and verbosity. + ''' + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO if verbose else logging.WARNING) + console_handler.setFormatter(fmt) + return console_handler + + +def get_logging_file_handler( + file: str, + fmt: logging.Formatter, + set_formatter: bool = True) -> logging.FileHandler: + ''' + Gets a logging file handler (logging.FileHandler) based on the specified + formatter (logging.Formatter). + ''' + file_handler = logging.FileHandler(file) + file_handler.setLevel(logging.INFO) + if set_formatter: + file_handler.setFormatter(fmt) + return file_handler + + +def init_logging(verbose: bool) -> str: + '''Initializes the loggers used by the script.''' + logging.getLogger().setLevel(logging.INFO) + + log_file_name = generate_log_file_name() + + for logger in ['shell', 'script']: + logging.getLogger(logger).addHandler(get_logging_console_handler( + LOGGING_FORMATTER, verbose)) + logging.getLogger(logger).addHandler(get_logging_file_handler( + log_file_name, LOGGING_FORMATTER)) + logging.getLogger(logger).setLevel(logging.INFO) + + return log_file_name + + +def log_start_message(name): + '''Used to log a start event message header.''' + start_msg = "Script started at {}".format( + str(datetime.datetime.fromtimestamp(LAUNCH_TIME))) + logging.getLogger(name).info('-' * len(start_msg)) + logging.getLogger(name).info(start_msg) + logging.getLogger(name).info('-' * len(start_msg)) + + +@contextmanager +def push_dir(path: str = None): + ''' + Adds the specified location to the top of a location stack, then changes to + the specified directory. + ''' + if path: + prev = os.getcwd() + try: + logging.getLogger('shell').info('pushd "%s"', path) + os.chdir(path) + yield + finally: + logging.getLogger('shell').info('popd') + os.chdir(prev) + else: + yield + + +class RunCommand(object): + ''' + This is a class wrapper around `subprocess.Popen` with an additional set + of logging features. + ''' + + def __init__( + self, + log_file, + cmdline: list, + success_exit_codes: list = None, + verbose: bool = False): + if not log_file: + raise TypeError('Unspecified log file.') + if cmdline is None: + raise TypeError('Unspecified command line to be executed.') + if not cmdline: + raise ValueError('Specified command line is empty.') + + self.__log_file = log_file + self.__cmdline = cmdline + self.__verbose = verbose + + if success_exit_codes is None: + self.__success_exit_codes = [0] + else: + self.__success_exit_codes = success_exit_codes + + @property + def log_file(self): + '''Log file name to write to.''' + return self.__log_file + + @property + def cmdline(self): + '''Command-line to use when starting the application.''' + return self.__cmdline + + @property + def success_exit_codes(self): + ''' + The successful exit codes that the associated process specifies when it + terminated. + ''' + return self.__success_exit_codes + + @property + def verbose(self): + '''Enables/Disables verbosity.''' + return self.__verbose + + def run(self, suffix: str = None, working_directory: str = None): + ''' + This is a function wrapper around `subprocess.Popen` with an additional + set of logging features. + ''' + should_pipe = self.verbose + with push_dir(working_directory): + quoted_cmdline = subprocess.list2cmdline(self.cmdline) + quoted_cmdline += ' > {}'.format( + os.devnull) if not should_pipe else '' + + logging.getLogger('shell').info(quoted_cmdline) + exe_name = os.path.basename(self.cmdline[0]).replace('.', '_') + + exe_log_file = self.log_file + if suffix is not None: + exe_log_file = exe_log_file.replace( + '.log', '.{}.log'.format(suffix)) + + exe_logger = logging.getLogger(exe_name) + exe_logger.handlers = [] + + file_handler = get_logging_file_handler( + exe_log_file, + LOGGING_FORMATTER, + set_formatter=(suffix is None)) + exe_logger.addHandler(file_handler) + + if suffix is not None: + log_start_message(exe_name) + + console_handler = get_logging_console_handler( + LOGGING_FORMATTER, self.verbose) + exe_logger.addHandler(console_handler) + + with open(os.devnull) as devnull: + proc = subprocess.Popen( + self.cmdline, + stdout=subprocess.PIPE if should_pipe else devnull, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + if proc.stdout is not None: + for line in iter(proc.stdout.readline, ''): + line = line.rstrip() + exe_logger.info(line) + proc.stdout.close() + + proc.wait() + if proc.returncode not in self.success_exit_codes: + exe_logger.error( + "Process exited with status %s", proc.returncode) + raise subprocess.CalledProcessError( + proc.returncode, quoted_cmdline) + + +def check_requirements(log_file: str, verbose: bool): + ''' + Checks that the requirements needs to build the CoreClr benchmarks are met. + ''' + logging.getLogger('script').info("Making sure dotnet exists...") + try: + cmdline = ['dotnet', '--info'] + RunCommand(log_file, cmdline, verbose=verbose).run('dotnet-info') + except Exception: + raise FatalError("Cannot find dotnet.") + + +class TargetFrameworkAction(argparse.Action): + ''' + Used by the ArgumentParser to represent the information needed to parse the + supported .NET Core target frameworks argument from the command line. + ''' + + def __call__(self, parser, namespace, values, option_string=None): + if values: + wrong_choices = [] + for value in values: + if value not in self.supported_target_frameworks(): + wrong_choices.append(value) + if wrong_choices: + message = ', '.join(wrong_choices) + message = 'Invalid choice(s): {}'.format(message) + raise argparse.ArgumentError(self, message) + setattr(namespace, self.dest, values) + + @staticmethod + def supported_target_frameworks() -> list: + '''List of supported .NET Core target frameworks.''' + return ['netcoreapp1.1', 'netcoreapp2.0', 'netcoreapp2.1', 'net461'] + + +def process_arguments(): + ''' + Function used to parse the command line arguments passed to this script + through the cli. + ''' + parser = argparse.ArgumentParser( + description="Builds the CoreClr benchmarks.", + ) + parser.add_argument( + '-c', '--configuration', + metavar='CONFIGURATION', + required=False, + default='Debug', + choices=['debug', 'release'], + type=str.casefold, + help='Configuration use for building the project (default "Debug").', + ) + parser.add_argument( + '-f', '--frameworks', + metavar='FRAMEWORK', + required=False, + nargs='*', + action=TargetFrameworkAction, + default=TargetFrameworkAction.supported_target_frameworks(), + help='Target frameworks to publish for (default all).', + ) + parser.add_argument( + '-v', '--verbose', + required=False, + default=False, + action='store_true', + help='Turns on verbosity (default "False")', + ) + + # --verbosity + # ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'] + + args = parser.parse_args() + return ( + args.configuration, + args.frameworks, + args.verbose + ) + + +class DotNet(object): + ''' + This is a class wrapper around the `dotnet` command line interface. + ''' + + def __init__( + self, + log_file: str, + working_directory: str, + csproj_file: str, + verbose: bool): + if not log_file: + raise TypeError('Unspecified log file.') + if not working_directory: + raise TypeError('Unspecified working directory.') + if not os.path.isdir(working_directory): + raise ValueError( + 'Specified working directory: {}, does not exist.'.format( + working_directory)) + + if os.path.isabs(csproj_file) and not os.path.exists(csproj_file): + raise ValueError( + 'Specified project file: {}, does not exist.'.format( + csproj_file)) + elif not os.path.exists(os.path.join(working_directory, csproj_file)): + raise ValueError( + 'Specified project file: {}, does not exist.'.format( + csproj_file)) + + self.__log_file = log_file + self.__working_directory = working_directory + self.__csproj_file = csproj_file + self.__verbose = verbose + + @property + def log_file(self): + '''Gets the log file name to write to.''' + return self.__log_file + + @property + def working_directory(self): + '''Gets the working directory for the dotnet process to be started.''' + return self.__working_directory + + @property + def csproj_file(self): + '''Gets the project file to run the dotnet cli against.''' + return self.__csproj_file + + @property + def verbose(self): + '''Gets a flag to whether verbosity if turned on or off.''' + return self.__verbose + + @property + def packages_path(self): + '''Gets the folder to restore packages to.''' + return os.path.join(get_repo_root_path(), 'packages') + + @property + def bin_path(self): + '''Gets the directory in which the built binaries will be placed.''' + return os.path.join(get_repo_root_path(), 'bin{}'.format(os.path.sep)) + + def restore(self): + ''' + Calls dotnet to restore the dependencies and tools of the specified + project. + ''' + cmdline = ['dotnet', 'restore', + '--packages', self.packages_path, + self.csproj_file] + RunCommand(self.log_file, cmdline, verbose=self.verbose).run( + 'dotnet-restore', self.working_directory) + + def publish(self, configuration: str, framework: str,): + ''' + Calls dotnet to pack the specified application and its dependencies + into the repo bin folder for deployment to a hosting system. + ''' + cmdline = ['dotnet', 'publish', + '--no-restore', + '--configuration', configuration, + '--framework', framework, + self.csproj_file, + '/p:BaseOutputPath={}'.format(self.bin_path)] + RunCommand(self.log_file, cmdline, verbose=self.verbose).run( + 'dotnet-publish', self.working_directory) + + +def build_coreclr( + log_file: str, + configuration: str, + frameworks: list, + verbose: bool): + '''Builds the CoreClr set of benchmarks (Code Quality).''' + working_directory = os.path.join( + get_repo_root_path(), 'src', 'coreclr', 'PerformanceHarness') + csproj_file = 'PerformanceHarness.csproj' + + dotnet = DotNet(log_file, working_directory, csproj_file, verbose) + dotnet.restore() + for framework in frameworks: + dotnet.publish(configuration, framework) + + +def main() -> int: + '''Script main entry point.''' + try: + if not is_supported_version(): + raise FatalError("Unsupported python version.") + + args = process_arguments() + configuration, frameworks, verbose = args + log_file = init_logging(verbose) + + log_start_message('script') + check_requirements(log_file, verbose) + build_coreclr(log_file, configuration, frameworks, verbose) + + return 0 + except FatalError as ex: + logging.getLogger('script').error(str(ex)) + except subprocess.CalledProcessError as ex: + logging.getLogger('script').error( + 'Command: "%s", exited with status: %s', ex.cmd, ex.returncode) + except IOError as ex: + logging.getLogger('script').error( + "I/O error (%s): %s", ex.errno, ex.strerror) + except SystemExit: # Argparse throws this exception when it exits. + pass + except Exception: + logging.getLogger('script')( + 'Unexpected error: {}'.format(sys.exc_info()[0])) + logging.getLogger('script')(format_exc()) + raise + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/coreclr/README.md b/src/coreclr/README.md index d147748e1d7..b3ed850cd7e 100644 --- a/src/coreclr/README.md +++ b/src/coreclr/README.md @@ -4,6 +4,7 @@ This is a collection of micro benchmarks, ported from [CoreClr](https://github.c ## Supported target frameworks ## +- net461 - netcoreapp1.1 - netcoreapp2.0 - netcoreapp2.1 @@ -77,3 +78,21 @@ REM WARNING! The command below will run all the CoreClr benchmarks. cd PerformanceHarness/bin/x64/Release/netcoreapp2.0/publish dotnet PerformanceHarness.dll --perf:collect stopwatch+gcapi ``` + +### Example 4: Running with CoreClr's CoreRun.exe ### + +For this you will need to patch Core_Root with few assemblies: + +Managed (the assemblies used by the benchmarks are newer than the ones on Core_Root): + +- Microsoft.CodeAnalysis*.dll +- Newtonsoft.Json*.dll + +**Affected benchmarks: Serialization, and Roslyn. + +Native (assemblies used by TraceEvent not deployed with Core_Root): + +- \amd64\*.* +- \x86\*.* + +This [batch script](run-with-coreclr-corerun.cmd) is an example of the steps needed to run the CoreClr benchmarks with CoreRun. diff --git a/src/coreclr/common.props b/src/coreclr/common.props index 37d0c2da33d..0862e50af1b 100644 --- a/src/coreclr/common.props +++ b/src/coreclr/common.props @@ -23,6 +23,16 @@ $RepositoryUrl + + + true + + All diff --git a/src/coreclr/run-with-coreclr-corerun.cmd b/src/coreclr/run-with-coreclr-corerun.cmd new file mode 100644 index 00000000000..f8b6cc48a16 --- /dev/null +++ b/src/coreclr/run-with-coreclr-corerun.cmd @@ -0,0 +1,190 @@ +@REM This is a template script to run .NET Performance benchmarks with CoreRun.exe +@if not defined _echo echo off + + +setlocal + set "ERRORLEVEL=" + set "USAGE_DISPLAYED=" + set "PUBLISH_DIR=" + set "BENCHMARK_ASSEMBLY=" + + set "ARCHITECTURE=x64" + set "CONFIGURATION=Release" + set "TARGET_FRAMEWORK=netcoreapp2.0" + + call :parse_command_line_arguments %* || exit /b 1 + if defined USAGE_DISPLAYED exit /b %ERRORLEVEL% + cd "%PUBLISH_DIR%" + call :patch_core_root || exit /b 1 + call :common_env || exit /b 1 + + call :run_command %STABILITY_PREFIX% "%CORE_ROOT%\CoreRun.exe" PerformanceHarness.dll %BENCHMARK_ASSEMBLY% --perf:collect %COLLECTION_FLAGS% +endlocal& exit /b %ERRORLEVEL% + + +:common_env + echo/ Common .NET environment variables set. + set COMPlus_ + set DOTNET_ + set UseSharedCompilation + set XUNIT_ + exit /b 0 + + +:patch_core_root +rem **************************************************************************** +rem Copies latest managed binaries needed by the benchmarks, plus native +rem binaries used by TraceEvent. +rem **************************************************************************** + REM Managed binaries + for %%f in (Microsoft.CodeAnalysis Newtonsoft.Json) do ( + call :run_command xcopy.exe /VYRQKZ "%CD%\%%f*.dll" "%CORE_ROOT%\" || ( + call :print_error Failed to copy from: "%CD%\%%f*.dll", to: "%CORE_ROOT%\" + exit /b 1 + ) + ) + + REM Copy native libraries used by Microsoft.Diagnostics.Tracing.TraceEvent.dll + for %%a in (amd64 x86) do ( + for %%f in (KernelTraceControl msdia140) do ( + call :run_command xcopy.exe /VYRQKZ "%CD%\%%a\%%f.dll" "%CORE_ROOT%\%%a\" || ( + call :print_error Failed to copy from: "%CD%\%%a\%%f.dll", to: "%CORE_ROOT%\%%a\" + exit /b 1 + ) + ) + ) + exit /b %ERRORLEVEL% + + +:parse_command_line_arguments +rem **************************************************************************** +rem Parses the script's command line arguments. +rem **************************************************************************** + IF /I [%~1] == [--core-root] ( + set "CORE_ROOT=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--stability-prefix] ( + set "STABILITY_PREFIX=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--collection-flags] ( + set "COLLECTION_FLAGS=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--publish-dir] ( + set "PUBLISH_DIR=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--assembly] ( + set "BENCHMARK_ASSEMBLY=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + if /I [%~1] == [-?] ( + call :usage + exit /b 0 + ) + if /I [%~1] == [-h] ( + call :usage + exit /b 0 + ) + if /I [%~1] == [--help] ( + call :usage + exit /b 0 + ) + + if not defined CORE_ROOT ( + call :print_error CORE_ROOT was not defined. + exit /b 1 + ) + + if not defined PUBLISH_DIR ( + call :print_error --publish-dir was not specified. + exit /b 1 + ) + if not exist "%PUBLISH_DIR%" ( + call :print_error Specified published directory: %PUBLISH_DIR%, does not exist. + exit /b 1 + ) + + if not defined BENCHMARK_ASSEMBLY ( + call :print_to_console --assembly DOTNET_ASSEMBLY_NAME was not specified, all benchmarks will be run. + ) + + if not defined COLLECTION_FLAGS ( + call :print_to_console COLLECTION_FLAGS was not defined. Defaulting to stopwatch + set "COLLECTION_FLAGS=stopwatch" + ) + + exit /b %ERRORLEVEL% + + +:usage +rem **************************************************************************** +rem Script's usage. +rem **************************************************************************** + set "USAGE_DISPLAYED=1" + echo/ %~nx0 [OPTIONS] + echo/ + echo/ Options: + echo/ --collection-flags COLLECTION_FLAGS + echo/ xUnit-Performance Api valid performance collection flags: ^ + echo/ --core-root CORE_ROOT + echo/ CoreClr's CORE_ROOT directory (Binaries to be tested). + echo/ --publish-dir PUBLISH_DIR + echo/ Directory that contains the published .NET benchmarks. + echo/ --stability-prefix STABILITY_PREFIX + echo/ Command to prepend to the benchmark execution. + echo/ --assembly DOTNET_ASSEMBLY_NAME + echo/ .NET assembly to be tested. + exit /b %ERRORLEVEL% + + +:run_command +rem **************************************************************************** +rem Function wrapper used to send the command line being executed to the +rem console screen, before the command is executed. +rem **************************************************************************** + if "%~1" == "" ( + call :print_error No command was specified. + exit /b 1 + ) + + call :print_to_console $ %* + call %* + exit /b %ERRORLEVEL% + + +:print_error +rem **************************************************************************** +rem Function wrapper that unifies how errors are output by the script. +rem Functions output to the standard error. +rem **************************************************************************** + call :print_to_console [ERROR] %* 1>&2 + exit /b %ERRORLEVEL% + + +:print_to_console +rem **************************************************************************** +rem Sends text to the console screen. This can be useful to provide +rem information on where the script is executing. +rem **************************************************************************** + echo/ + echo/%USERNAME%@%COMPUTERNAME% "%CD%" + echo/[%DATE%][%TIME:~0,-3%] %* + exit /b %ERRORLEVEL% From 3a077b29ffc90c59298045b3c904e881878d3c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Tue, 22 May 2018 13:12:56 -0700 Subject: [PATCH 2/6] Marking default build to release. --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 337f6f3c853..2aca9710375 100755 --- a/build.py +++ b/build.py @@ -287,10 +287,10 @@ def process_arguments(): '-c', '--configuration', metavar='CONFIGURATION', required=False, - default='Debug', + default='release', choices=['debug', 'release'], type=str.casefold, - help='Configuration use for building the project (default "Debug").', + help='Configuration use for building the project (default "release").', ) parser.add_argument( '-f', '--frameworks', From 6f728570949857d933b7600907473c8e9d8eea3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Tue, 22 May 2018 14:12:23 -0700 Subject: [PATCH 3/6] Removed dead code. --- build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build.py b/build.py index 2aca9710375..cf276b69a62 100755 --- a/build.py +++ b/build.py @@ -37,7 +37,6 @@ def is_supported_version() -> bool: def get_script_path() -> str: '''Gets this script directory.''' return sys.path[0] - # return os.path.dirname(os.path.realpath(__file__)) def get_repo_root_path() -> str: From eb01ee1d8c5c02dfe6ea7d19606146271471c4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Tue, 22 May 2018 15:59:29 -0700 Subject: [PATCH 4/6] Refactor build.py and moved code to a module - Added missing annotations - Removed an extra coma on statement - Moved classes to its own file under the build module --- build.py | 329 ++------------------------ build/__init__.py | 0 build/common.py | 88 +++++++ build/exception/FatalError.py | 10 + build/exception/__init__.py | 0 build/parser/TargetFrameworkAction.py | 30 +++ build/parser/__init__.py | 0 build/process/DotNet.py | 98 ++++++++ build/process/__init__.py | 0 build/runner/RunCommand.py | 121 ++++++++++ build/runner/__init__.py | 0 11 files changed, 364 insertions(+), 312 deletions(-) create mode 100644 build/__init__.py create mode 100644 build/common.py create mode 100644 build/exception/FatalError.py create mode 100644 build/exception/__init__.py create mode 100644 build/parser/TargetFrameworkAction.py create mode 100644 build/parser/__init__.py create mode 100644 build/process/DotNet.py create mode 100644 build/process/__init__.py create mode 100644 build/runner/RunCommand.py create mode 100644 build/runner/__init__.py diff --git a/build.py b/build.py index cf276b69a62..74370dc335c 100755 --- a/build.py +++ b/build.py @@ -4,44 +4,27 @@ Builds the CoreClr Benchmarks ''' -from contextlib import contextmanager +from subprocess import CalledProcessError from traceback import format_exc +from typing import Tuple import argparse import datetime import logging import os -import subprocess import sys -import time - -LAUNCH_TIME = time.time() -LOGGING_FORMATTER = logging.Formatter( - fmt='[%(asctime)s][%(levelname)s] %(message)s', - datefmt="%Y-%m-%d %H:%M:%S") - - -class FatalError(Exception): - ''' - Raised for various script errors regarding environment and build - requirements. - ''' - - -def is_supported_version() -> bool: - '''Checks if the script is running on the supported version (>=3.5).''' - return sys.version_info.major > 2 and sys.version_info.minor > 4 - - -def get_script_path() -> str: - '''Gets this script directory.''' - return sys.path[0] - - -def get_repo_root_path() -> str: - '''Gets repository root directory.''' - return get_script_path() +from build.common import get_logging_console_handler +from build.common import get_logging_file_handler +from build.common import get_repo_root_path +from build.common import is_supported_version +from build.common import log_start_message +from build.common import LAUNCH_TIME +from build.common import LOGGING_FORMATTER +from build.exception.FatalError import FatalError +from build.parser.TargetFrameworkAction import TargetFrameworkAction +from build.process.DotNet import DotNet +from build.runner.RunCommand import RunCommand def generate_log_file_name() -> str: @@ -58,34 +41,6 @@ def generate_log_file_name() -> str: return os.path.join(log_dir, log_file_name) -def get_logging_console_handler( - fmt: logging.Formatter, - verbose: bool) -> logging.StreamHandler: - ''' - Gets a logging console handler (logging.StreamHandler) based on the - specified formatter (logging.Formatter) and verbosity. - ''' - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO if verbose else logging.WARNING) - console_handler.setFormatter(fmt) - return console_handler - - -def get_logging_file_handler( - file: str, - fmt: logging.Formatter, - set_formatter: bool = True) -> logging.FileHandler: - ''' - Gets a logging file handler (logging.FileHandler) based on the specified - formatter (logging.Formatter). - ''' - file_handler = logging.FileHandler(file) - file_handler.setLevel(logging.INFO) - if set_formatter: - file_handler.setFormatter(fmt) - return file_handler - - def init_logging(verbose: bool) -> str: '''Initializes the loggers used by the script.''' logging.getLogger().setLevel(logging.INFO) @@ -102,143 +57,7 @@ def init_logging(verbose: bool) -> str: return log_file_name -def log_start_message(name): - '''Used to log a start event message header.''' - start_msg = "Script started at {}".format( - str(datetime.datetime.fromtimestamp(LAUNCH_TIME))) - logging.getLogger(name).info('-' * len(start_msg)) - logging.getLogger(name).info(start_msg) - logging.getLogger(name).info('-' * len(start_msg)) - - -@contextmanager -def push_dir(path: str = None): - ''' - Adds the specified location to the top of a location stack, then changes to - the specified directory. - ''' - if path: - prev = os.getcwd() - try: - logging.getLogger('shell').info('pushd "%s"', path) - os.chdir(path) - yield - finally: - logging.getLogger('shell').info('popd') - os.chdir(prev) - else: - yield - - -class RunCommand(object): - ''' - This is a class wrapper around `subprocess.Popen` with an additional set - of logging features. - ''' - - def __init__( - self, - log_file, - cmdline: list, - success_exit_codes: list = None, - verbose: bool = False): - if not log_file: - raise TypeError('Unspecified log file.') - if cmdline is None: - raise TypeError('Unspecified command line to be executed.') - if not cmdline: - raise ValueError('Specified command line is empty.') - - self.__log_file = log_file - self.__cmdline = cmdline - self.__verbose = verbose - - if success_exit_codes is None: - self.__success_exit_codes = [0] - else: - self.__success_exit_codes = success_exit_codes - - @property - def log_file(self): - '''Log file name to write to.''' - return self.__log_file - - @property - def cmdline(self): - '''Command-line to use when starting the application.''' - return self.__cmdline - - @property - def success_exit_codes(self): - ''' - The successful exit codes that the associated process specifies when it - terminated. - ''' - return self.__success_exit_codes - - @property - def verbose(self): - '''Enables/Disables verbosity.''' - return self.__verbose - - def run(self, suffix: str = None, working_directory: str = None): - ''' - This is a function wrapper around `subprocess.Popen` with an additional - set of logging features. - ''' - should_pipe = self.verbose - with push_dir(working_directory): - quoted_cmdline = subprocess.list2cmdline(self.cmdline) - quoted_cmdline += ' > {}'.format( - os.devnull) if not should_pipe else '' - - logging.getLogger('shell').info(quoted_cmdline) - exe_name = os.path.basename(self.cmdline[0]).replace('.', '_') - - exe_log_file = self.log_file - if suffix is not None: - exe_log_file = exe_log_file.replace( - '.log', '.{}.log'.format(suffix)) - - exe_logger = logging.getLogger(exe_name) - exe_logger.handlers = [] - - file_handler = get_logging_file_handler( - exe_log_file, - LOGGING_FORMATTER, - set_formatter=(suffix is None)) - exe_logger.addHandler(file_handler) - - if suffix is not None: - log_start_message(exe_name) - - console_handler = get_logging_console_handler( - LOGGING_FORMATTER, self.verbose) - exe_logger.addHandler(console_handler) - - with open(os.devnull) as devnull: - proc = subprocess.Popen( - self.cmdline, - stdout=subprocess.PIPE if should_pipe else devnull, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - - if proc.stdout is not None: - for line in iter(proc.stdout.readline, ''): - line = line.rstrip() - exe_logger.info(line) - proc.stdout.close() - - proc.wait() - if proc.returncode not in self.success_exit_codes: - exe_logger.error( - "Process exited with status %s", proc.returncode) - raise subprocess.CalledProcessError( - proc.returncode, quoted_cmdline) - - -def check_requirements(log_file: str, verbose: bool): +def check_requirements(log_file: str, verbose: bool) -> None: ''' Checks that the requirements needs to build the CoreClr benchmarks are met. ''' @@ -250,31 +69,7 @@ def check_requirements(log_file: str, verbose: bool): raise FatalError("Cannot find dotnet.") -class TargetFrameworkAction(argparse.Action): - ''' - Used by the ArgumentParser to represent the information needed to parse the - supported .NET Core target frameworks argument from the command line. - ''' - - def __call__(self, parser, namespace, values, option_string=None): - if values: - wrong_choices = [] - for value in values: - if value not in self.supported_target_frameworks(): - wrong_choices.append(value) - if wrong_choices: - message = ', '.join(wrong_choices) - message = 'Invalid choice(s): {}'.format(message) - raise argparse.ArgumentError(self, message) - setattr(namespace, self.dest, values) - - @staticmethod - def supported_target_frameworks() -> list: - '''List of supported .NET Core target frameworks.''' - return ['netcoreapp1.1', 'netcoreapp2.0', 'netcoreapp2.1', 'net461'] - - -def process_arguments(): +def process_arguments() -> Tuple[str, list, bool]: ''' Function used to parse the command line arguments passed to this script through the cli. @@ -319,101 +114,11 @@ def process_arguments(): ) -class DotNet(object): - ''' - This is a class wrapper around the `dotnet` command line interface. - ''' - - def __init__( - self, - log_file: str, - working_directory: str, - csproj_file: str, - verbose: bool): - if not log_file: - raise TypeError('Unspecified log file.') - if not working_directory: - raise TypeError('Unspecified working directory.') - if not os.path.isdir(working_directory): - raise ValueError( - 'Specified working directory: {}, does not exist.'.format( - working_directory)) - - if os.path.isabs(csproj_file) and not os.path.exists(csproj_file): - raise ValueError( - 'Specified project file: {}, does not exist.'.format( - csproj_file)) - elif not os.path.exists(os.path.join(working_directory, csproj_file)): - raise ValueError( - 'Specified project file: {}, does not exist.'.format( - csproj_file)) - - self.__log_file = log_file - self.__working_directory = working_directory - self.__csproj_file = csproj_file - self.__verbose = verbose - - @property - def log_file(self): - '''Gets the log file name to write to.''' - return self.__log_file - - @property - def working_directory(self): - '''Gets the working directory for the dotnet process to be started.''' - return self.__working_directory - - @property - def csproj_file(self): - '''Gets the project file to run the dotnet cli against.''' - return self.__csproj_file - - @property - def verbose(self): - '''Gets a flag to whether verbosity if turned on or off.''' - return self.__verbose - - @property - def packages_path(self): - '''Gets the folder to restore packages to.''' - return os.path.join(get_repo_root_path(), 'packages') - - @property - def bin_path(self): - '''Gets the directory in which the built binaries will be placed.''' - return os.path.join(get_repo_root_path(), 'bin{}'.format(os.path.sep)) - - def restore(self): - ''' - Calls dotnet to restore the dependencies and tools of the specified - project. - ''' - cmdline = ['dotnet', 'restore', - '--packages', self.packages_path, - self.csproj_file] - RunCommand(self.log_file, cmdline, verbose=self.verbose).run( - 'dotnet-restore', self.working_directory) - - def publish(self, configuration: str, framework: str,): - ''' - Calls dotnet to pack the specified application and its dependencies - into the repo bin folder for deployment to a hosting system. - ''' - cmdline = ['dotnet', 'publish', - '--no-restore', - '--configuration', configuration, - '--framework', framework, - self.csproj_file, - '/p:BaseOutputPath={}'.format(self.bin_path)] - RunCommand(self.log_file, cmdline, verbose=self.verbose).run( - 'dotnet-publish', self.working_directory) - - def build_coreclr( log_file: str, configuration: str, frameworks: list, - verbose: bool): + verbose: bool) -> None: '''Builds the CoreClr set of benchmarks (Code Quality).''' working_directory = os.path.join( get_repo_root_path(), 'src', 'coreclr', 'PerformanceHarness') @@ -442,7 +147,7 @@ def main() -> int: return 0 except FatalError as ex: logging.getLogger('script').error(str(ex)) - except subprocess.CalledProcessError as ex: + except CalledProcessError as ex: logging.getLogger('script').error( 'Command: "%s", exited with status: %s', ex.cmd, ex.returncode) except IOError as ex: diff --git a/build/__init__.py b/build/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/common.py b/build/common.py new file mode 100644 index 00000000000..85f9cb25cd7 --- /dev/null +++ b/build/common.py @@ -0,0 +1,88 @@ +''' +Common functionality used by the .NET Performance Repository build scripts. +''' + +from contextlib import contextmanager + +import datetime +import logging +import os +import sys +import time + + +LAUNCH_TIME = time.time() +LOGGING_FORMATTER = logging.Formatter( + fmt='[%(asctime)s][%(levelname)s] %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + + +def is_supported_version() -> bool: + '''Checks if the script is running on the supported version (>=3.5).''' + return sys.version_info.major > 2 and sys.version_info.minor > 4 + + +def log_start_message(name) -> None: + '''Used to log a start event message header.''' + start_msg = "Script started at {}".format( + str(datetime.datetime.fromtimestamp(LAUNCH_TIME))) + logging.getLogger(name).info('-' * len(start_msg)) + logging.getLogger(name).info(start_msg) + logging.getLogger(name).info('-' * len(start_msg)) + + +def get_script_path() -> str: + '''Gets this script directory.''' + return sys.path[0] + + +def get_repo_root_path() -> str: + '''Gets repository root directory.''' + return get_script_path() + + +@contextmanager +def push_dir(path: str = None) -> None: + ''' + Adds the specified location to the top of a location stack, then changes to + the specified directory. + ''' + if path: + prev = os.getcwd() + try: + logging.getLogger('shell').info('pushd "%s"', path) + os.chdir(path) + yield + finally: + logging.getLogger('shell').info('popd') + os.chdir(prev) + else: + yield + + +def get_logging_console_handler( + fmt: logging.Formatter, + verbose: bool) -> logging.StreamHandler: + ''' + Gets a logging console handler (logging.StreamHandler) based on the + specified formatter (logging.Formatter) and verbosity. + ''' + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO if verbose else logging.WARNING) + console_handler.setFormatter(fmt) + return console_handler + + +def get_logging_file_handler( + file: str, + fmt: logging.Formatter, + set_formatter: bool = True) -> logging.FileHandler: + ''' + Gets a logging file handler (logging.FileHandler) based on the specified + formatter (logging.Formatter). + ''' + file_handler = logging.FileHandler(file) + file_handler.setLevel(logging.INFO) + if set_formatter: + file_handler.setFormatter(fmt) + return file_handler diff --git a/build/exception/FatalError.py b/build/exception/FatalError.py new file mode 100644 index 00000000000..85ea14bbc78 --- /dev/null +++ b/build/exception/FatalError.py @@ -0,0 +1,10 @@ +''' +Contains the definition of the FatalError exception. +''' + + +class FatalError(Exception): + ''' + Raised for various script errors regarding environment and build + requirements. + ''' diff --git a/build/exception/__init__.py b/build/exception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/parser/TargetFrameworkAction.py b/build/parser/TargetFrameworkAction.py new file mode 100644 index 00000000000..51b44101c89 --- /dev/null +++ b/build/parser/TargetFrameworkAction.py @@ -0,0 +1,30 @@ +''' +Contains the definition of the TargetFrameworkAction type used to parse +the .NET Cli the supported target frameworks. +''' + +import argparse + + +class TargetFrameworkAction(argparse.Action): + ''' + Used by the ArgumentParser to represent the information needed to parse the + supported .NET Core target frameworks argument from the command line. + ''' + + def __call__(self, parser, namespace, values, option_string=None): + if values: + wrong_choices = [] + for value in values: + if value not in self.supported_target_frameworks(): + wrong_choices.append(value) + if wrong_choices: + message = ', '.join(wrong_choices) + message = 'Invalid choice(s): {}'.format(message) + raise argparse.ArgumentError(self, message) + setattr(namespace, self.dest, values) + + @staticmethod + def supported_target_frameworks() -> list: + '''List of supported .NET Core target frameworks.''' + return ['netcoreapp1.1', 'netcoreapp2.0', 'netcoreapp2.1', 'net461'] diff --git a/build/parser/__init__.py b/build/parser/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/process/DotNet.py b/build/process/DotNet.py new file mode 100644 index 00000000000..2f32bf9f5c8 --- /dev/null +++ b/build/process/DotNet.py @@ -0,0 +1,98 @@ +''' +Contains the definition of DotNet process. +''' + +import os + +from ..common import get_repo_root_path +from ..runner.RunCommand import RunCommand + + +class DotNet(object): + ''' + This is a class wrapper around the `dotnet` command line interface. + ''' + + def __init__( + self, + log_file: str, + working_directory: str, + csproj_file: str, + verbose: bool): + if not log_file: + raise TypeError('Unspecified log file.') + if not working_directory: + raise TypeError('Unspecified working directory.') + if not os.path.isdir(working_directory): + raise ValueError( + 'Specified working directory: {}, does not exist.'.format( + working_directory)) + + if os.path.isabs(csproj_file) and not os.path.exists(csproj_file): + raise ValueError( + 'Specified project file: {}, does not exist.'.format( + csproj_file)) + elif not os.path.exists(os.path.join(working_directory, csproj_file)): + raise ValueError( + 'Specified project file: {}, does not exist.'.format( + csproj_file)) + + self.__log_file = log_file + self.__working_directory = working_directory + self.__csproj_file = csproj_file + self.__verbose = verbose + + @property + def log_file(self) -> str: + '''Gets the log file name to write to.''' + return self.__log_file + + @property + def working_directory(self) -> str: + '''Gets the working directory for the dotnet process to be started.''' + return self.__working_directory + + @property + def csproj_file(self) -> str: + '''Gets the project file to run the dotnet cli against.''' + return self.__csproj_file + + @property + def verbose(self) -> bool: + '''Gets a flag to whether verbosity if turned on or off.''' + return self.__verbose + + @property + def packages_path(self) -> str: + '''Gets the folder to restore packages to.''' + return os.path.join(get_repo_root_path(), 'packages') + + @property + def bin_path(self) -> str: + '''Gets the directory in which the built binaries will be placed.''' + return os.path.join(get_repo_root_path(), 'bin{}'.format(os.path.sep)) + + def restore(self) -> None: + ''' + Calls dotnet to restore the dependencies and tools of the specified + project. + ''' + cmdline = ['dotnet', 'restore', + '--packages', self.packages_path, + self.csproj_file] + RunCommand(self.log_file, cmdline, verbose=self.verbose).run( + 'dotnet-restore', self.working_directory) + + def publish(self, configuration: str, framework: str) -> None: + ''' + Calls dotnet to pack the specified application and its dependencies + into the repo bin folder for deployment to a hosting system. + ''' + cmdline = ['dotnet', 'publish', + '--no-restore', + '--configuration', configuration, + '--framework', framework, + self.csproj_file, + '/p:BaseOutputPath={}'.format(self.bin_path)] + RunCommand(self.log_file, cmdline, verbose=self.verbose).run( + 'dotnet-publish', self.working_directory) diff --git a/build/process/__init__.py b/build/process/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/runner/RunCommand.py b/build/runner/RunCommand.py new file mode 100644 index 00000000000..e339570f160 --- /dev/null +++ b/build/runner/RunCommand.py @@ -0,0 +1,121 @@ +''' +Contains the definition of RunCommand runner object. +''' + +import logging +import os +import subprocess + +from ..common import push_dir +from ..common import get_logging_console_handler +from ..common import get_logging_file_handler +from ..common import log_start_message +from ..common import LOGGING_FORMATTER + + +class RunCommand(object): + ''' + This is a class wrapper around `subprocess.Popen` with an additional set + of logging features. + ''' + + def __init__( + self, + log_file, + cmdline: list, + success_exit_codes: list = None, + verbose: bool = False): + if not log_file: + raise TypeError('Unspecified log file.') + if cmdline is None: + raise TypeError('Unspecified command line to be executed.') + if not cmdline: + raise ValueError('Specified command line is empty.') + + self.__log_file = log_file + self.__cmdline = cmdline + self.__verbose = verbose + + if success_exit_codes is None: + self.__success_exit_codes = [0] + else: + self.__success_exit_codes = success_exit_codes + + @property + def log_file(self) -> str: + '''Log file name to write to.''' + return self.__log_file + + @property + def cmdline(self) -> str: + '''Command-line to use when starting the application.''' + return self.__cmdline + + @property + def success_exit_codes(self) -> list: + ''' + The successful exit codes that the associated process specifies when it + terminated. + ''' + return self.__success_exit_codes + + @property + def verbose(self) -> bool: + '''Enables/Disables verbosity.''' + return self.__verbose + + def run(self, suffix: str = None, working_directory: str = None) -> None: + ''' + This is a function wrapper around `subprocess.Popen` with an additional + set of logging features. + ''' + should_pipe = self.verbose + with push_dir(working_directory): + quoted_cmdline = subprocess.list2cmdline(self.cmdline) + quoted_cmdline += ' > {}'.format( + os.devnull) if not should_pipe else '' + + logging.getLogger('shell').info(quoted_cmdline) + exe_name = os.path.basename(self.cmdline[0]).replace('.', '_') + + exe_log_file = self.log_file + if suffix is not None: + exe_log_file = exe_log_file.replace( + '.log', '.{}.log'.format(suffix)) + + exe_logger = logging.getLogger(exe_name) + exe_logger.handlers = [] + + file_handler = get_logging_file_handler( + exe_log_file, + LOGGING_FORMATTER, + set_formatter=(suffix is None)) + exe_logger.addHandler(file_handler) + + if suffix is not None: + log_start_message(exe_name) + + console_handler = get_logging_console_handler( + LOGGING_FORMATTER, self.verbose) + exe_logger.addHandler(console_handler) + + with open(os.devnull) as devnull: + proc = subprocess.Popen( + self.cmdline, + stdout=subprocess.PIPE if should_pipe else devnull, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + if proc.stdout is not None: + for line in iter(proc.stdout.readline, ''): + line = line.rstrip() + exe_logger.info(line) + proc.stdout.close() + + proc.wait() + if proc.returncode not in self.success_exit_codes: + exe_logger.error( + "Process exited with status %s", proc.returncode) + raise subprocess.CalledProcessError( + proc.returncode, quoted_cmdline) diff --git a/build/runner/__init__.py b/build/runner/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 21fe6cee6ea02b97e259dc071ba3aecfc721270d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Tue, 22 May 2018 16:15:13 -0700 Subject: [PATCH 5/6] Moving build.py into ./scripts/ --- build.py => scripts/build.py | 0 {build => scripts/build}/__init__.py | 0 {build => scripts/build}/common.py | 2 +- {build => scripts/build}/exception/FatalError.py | 0 {build => scripts/build}/exception/__init__.py | 0 {build => scripts/build}/parser/TargetFrameworkAction.py | 0 {build => scripts/build}/parser/__init__.py | 0 {build => scripts/build}/process/DotNet.py | 0 {build => scripts/build}/process/__init__.py | 0 {build => scripts/build}/runner/RunCommand.py | 0 {build => scripts/build}/runner/__init__.py | 0 11 files changed, 1 insertion(+), 1 deletion(-) rename build.py => scripts/build.py (100%) mode change 100755 => 100644 rename {build => scripts/build}/__init__.py (100%) rename {build => scripts/build}/common.py (97%) rename {build => scripts/build}/exception/FatalError.py (100%) rename {build => scripts/build}/exception/__init__.py (100%) rename {build => scripts/build}/parser/TargetFrameworkAction.py (100%) rename {build => scripts/build}/parser/__init__.py (100%) rename {build => scripts/build}/process/DotNet.py (100%) rename {build => scripts/build}/process/__init__.py (100%) rename {build => scripts/build}/runner/RunCommand.py (100%) rename {build => scripts/build}/runner/__init__.py (100%) diff --git a/build.py b/scripts/build.py old mode 100755 new mode 100644 similarity index 100% rename from build.py rename to scripts/build.py diff --git a/build/__init__.py b/scripts/build/__init__.py similarity index 100% rename from build/__init__.py rename to scripts/build/__init__.py diff --git a/build/common.py b/scripts/build/common.py similarity index 97% rename from build/common.py rename to scripts/build/common.py index 85f9cb25cd7..b811025dfc7 100644 --- a/build/common.py +++ b/scripts/build/common.py @@ -38,7 +38,7 @@ def get_script_path() -> str: def get_repo_root_path() -> str: '''Gets repository root directory.''' - return get_script_path() + return os.path.abspath(os.path.join(get_script_path(), '..')) @contextmanager diff --git a/build/exception/FatalError.py b/scripts/build/exception/FatalError.py similarity index 100% rename from build/exception/FatalError.py rename to scripts/build/exception/FatalError.py diff --git a/build/exception/__init__.py b/scripts/build/exception/__init__.py similarity index 100% rename from build/exception/__init__.py rename to scripts/build/exception/__init__.py diff --git a/build/parser/TargetFrameworkAction.py b/scripts/build/parser/TargetFrameworkAction.py similarity index 100% rename from build/parser/TargetFrameworkAction.py rename to scripts/build/parser/TargetFrameworkAction.py diff --git a/build/parser/__init__.py b/scripts/build/parser/__init__.py similarity index 100% rename from build/parser/__init__.py rename to scripts/build/parser/__init__.py diff --git a/build/process/DotNet.py b/scripts/build/process/DotNet.py similarity index 100% rename from build/process/DotNet.py rename to scripts/build/process/DotNet.py diff --git a/build/process/__init__.py b/scripts/build/process/__init__.py similarity index 100% rename from build/process/__init__.py rename to scripts/build/process/__init__.py diff --git a/build/runner/RunCommand.py b/scripts/build/runner/RunCommand.py similarity index 100% rename from build/runner/RunCommand.py rename to scripts/build/runner/RunCommand.py diff --git a/build/runner/__init__.py b/scripts/build/runner/__init__.py similarity index 100% rename from build/runner/__init__.py rename to scripts/build/runner/__init__.py From b8718b4865686b6eb150148136f061b39f2be752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Thu, 24 May 2018 10:10:34 -0700 Subject: [PATCH 6/6] Move the coreclr output to its own folder under bin --- scripts/build.py | 2 +- scripts/build/process/DotNet.py | 14 +++++++++--- scripts/build/runner/RunCommand.py | 35 +++++++++++++++--------------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/scripts/build.py b/scripts/build.py index 74370dc335c..d7769013100 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -127,7 +127,7 @@ def build_coreclr( dotnet = DotNet(log_file, working_directory, csproj_file, verbose) dotnet.restore() for framework in frameworks: - dotnet.publish(configuration, framework) + dotnet.publish(configuration, framework, 'CoreClr-Benchmarks') def main() -> int: diff --git a/scripts/build/process/DotNet.py b/scripts/build/process/DotNet.py index 2f32bf9f5c8..54144aa0c32 100644 --- a/scripts/build/process/DotNet.py +++ b/scripts/build/process/DotNet.py @@ -70,7 +70,7 @@ def packages_path(self) -> str: @property def bin_path(self) -> str: '''Gets the directory in which the built binaries will be placed.''' - return os.path.join(get_repo_root_path(), 'bin{}'.format(os.path.sep)) + return os.path.join(get_repo_root_path(), 'bin') def restore(self) -> None: ''' @@ -83,16 +83,24 @@ def restore(self) -> None: RunCommand(self.log_file, cmdline, verbose=self.verbose).run( 'dotnet-restore', self.working_directory) - def publish(self, configuration: str, framework: str) -> None: + def publish(self, + configuration: str, + framework: str, + product: str) -> None: ''' Calls dotnet to pack the specified application and its dependencies into the repo bin folder for deployment to a hosting system. ''' + if not product: + raise TypeError('Unspecified product name.') + base_output_path = '{}{}'.format( + os.path.join(self.bin_path, product), os.path.sep) + cmdline = ['dotnet', 'publish', '--no-restore', '--configuration', configuration, '--framework', framework, self.csproj_file, - '/p:BaseOutputPath={}'.format(self.bin_path)] + '/p:BaseOutputPath={}'.format(base_output_path)] RunCommand(self.log_file, cmdline, verbose=self.verbose).run( 'dotnet-publish', self.working_directory) diff --git a/scripts/build/runner/RunCommand.py b/scripts/build/runner/RunCommand.py index e339570f160..c935bc4df55 100644 --- a/scripts/build/runner/RunCommand.py +++ b/scripts/build/runner/RunCommand.py @@ -66,8 +66,7 @@ def verbose(self) -> bool: def run(self, suffix: str = None, working_directory: str = None) -> None: ''' - This is a function wrapper around `subprocess.Popen` with an additional - set of logging features. + Executes specified shell command. ''' should_pipe = self.verbose with push_dir(working_directory): @@ -100,22 +99,24 @@ def run(self, suffix: str = None, working_directory: str = None) -> None: exe_logger.addHandler(console_handler) with open(os.devnull) as devnull: - proc = subprocess.Popen( + with subprocess.Popen( self.cmdline, stdout=subprocess.PIPE if should_pipe else devnull, stderr=subprocess.STDOUT, universal_newlines=True, - ) - - if proc.stdout is not None: - for line in iter(proc.stdout.readline, ''): - line = line.rstrip() - exe_logger.info(line) - proc.stdout.close() - - proc.wait() - if proc.returncode not in self.success_exit_codes: - exe_logger.error( - "Process exited with status %s", proc.returncode) - raise subprocess.CalledProcessError( - proc.returncode, quoted_cmdline) + ) as proc: + + if proc.stdout is not None: + with proc.stdout: + for line in iter(proc.stdout.readline, ''): + line = line.rstrip() + exe_logger.info(line) + + proc.wait() + # FIXME: dotnet child processes are still running. + + if proc.returncode not in self.success_exit_codes: + exe_logger.error( + "Process exited with status %s", proc.returncode) + raise subprocess.CalledProcessError( + proc.returncode, quoted_cmdline)