diff --git a/extensions/commands/art/README.md b/extensions/commands/art/README.md index a7129e7..3affd26 100644 --- a/extensions/commands/art/README.md +++ b/extensions/commands/art/README.md @@ -77,7 +77,13 @@ optional arguments: ### Combining art:build-info and art:property to manage BuildInfo's in Artifactory -First, get a JSON output from a package build. This could come from a ``conan install`` or a ``conan create`` +Firstly, configure your Artifactory server with the url and credentials you want to use (this will come handy in the next commands): + +``` +conan art:server add my_artifactory --user= --password= +``` + +Now, get a JSON output from a package build. This could come from a ``conan install`` or a ``conan create``: ``` conan create . --format json -s build_type=Release > create_release.json @@ -90,25 +96,25 @@ Then upload the created package to your repository: conan upload ... -c -r ... ``` -Using the generated JSON files you can create a BuildInfo JSON. You have to pass the build -name and number and also the url and credentials for the Artifactory repository: +Using the generated JSON files you can create a BuildInfo JSON. To do this, you need to provide the build +name and number. You will also need to indicate the artifactory server to use: ``` -conan art:build-info create create_release.json mybuildname_release 1 --url= --user= --password= --with-dependencies > mybuildname_release.json -conan art:build-info create create_debug.json mybuildname_debug 1 --url= --user= --password= --with-dependencies > mybuildname_debug.json +conan art:build-info create create_release.json mybuildname_release 1 --server my_artifactory --with-dependencies > mybuildname_release.json +conan art:build-info create create_debug.json mybuildname_debug 1 --server my_artifactory --with-dependencies > mybuildname_debug.json ``` -Finally, you can upload the BuildInfo's +Finally, you can upload the BuildInfo's: ``` -conan art:build-info upload mybuildname_release.json --user= --password= -conan art:build-info upload mybuildname_debug.json --user= --password= +conan art:build-info upload mybuildname_release.json --server my_artifactory +conan art:build-info upload mybuildname_debug.json --server my_artifactory ``` -In this case we generated two BuilInfo's, for Release and Debug, we can merge those to +In this case we generated two BuildInfo's, for Release and Debug, we can merge those to create a new aggregated BuildInfo that we also will upload and set properties to: ``` -conan art:build-info append mybuildname_aggregated 1 --build-info=mybuildname_release,1 --build-info=mybuildname_debug,1 --user= --password= > mybuildname_aggregated.json -conan art:build-info upload mybuildname_aggregated.json --user= --password="" +conan art:build-info append mybuildname_aggregated 1 --build-info=mybuildname_release,1 --build-info=mybuildname_debug,1 --server my_artifactory > mybuildname_aggregated.json +conan art:build-info upload mybuildname_aggregated.json --server my_artifactory" ``` diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 0ab9f96..38b005b 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -1,3 +1,4 @@ +import base64 import datetime import json import os @@ -15,6 +16,8 @@ from conan import conan_version from conan.tools.scm import Version +from cmd_server import get_server + def response_to_str(response): content = response.content @@ -37,18 +40,15 @@ def response_to_str(response): content = "{}: {}".format(response.status_code, response.reason) return content - except Exception: return response.content -def api_request(method, request_url, user=None, password=None, apikey=None, json_data=None, +def api_request(method, request_url, user=None, password=None, json_data=None, sign_key_name=None): headers = {} if json_data: headers.update({"Content-Type": "application/json"}) - if apikey: - headers.update({"X-JFrog-Art-Api": apikey}) if sign_key_name: headers.update({"X-JFrog-Crypto-Key-Name": sign_key_name}) @@ -56,9 +56,6 @@ def api_request(method, request_url, user=None, password=None, apikey=None, json if user and password: response = requests_method(request_url, auth=( user, password), data=json_data, headers=headers) - elif apikey: - response = requests_method( - request_url, data=json_data, headers=headers) else: response = requests_method(request_url) @@ -154,7 +151,7 @@ def get_requested_by(nodes, node_id, artifact_type): class BuildInfo: def __init__(self, graph, name, number, repository, with_dependencies=False, - url=None, user=None, password=None, apikey=None): + url=None, user=None, password=None): self._graph = graph self._name = name self._number = number @@ -162,7 +159,6 @@ def __init__(self, graph, name, number, repository, with_dependencies=False, self._url = url self._user = user self._password = password - self._apikey = apikey self._cached_artifact_info = {} self._with_dependencies = with_dependencies @@ -220,7 +216,7 @@ def _get_remote_artifacts(): if not self._cached_artifact_info.get(request_url): checksums = None try: - response = api_request("get", request_url, self._user, self._password, self._apikey) + response = api_request("get", request_url, self._user, self._password) response_data = json.loads(response) checksums = response_data.get("checksums") self._cached_artifact_info[request_url] = checksums @@ -360,6 +356,42 @@ def manifest_from_build_info(build_info, repository, with_dependencies=True): return manifest +def assert_server_or_url_user_password(args): + if args.server and args.url: + raise ConanException("--server and --url (with --user & --password) flags cannot be used together.") + if not args.server and not args.url: + raise ConanException("Specify --server or --url (with --user & --password) flags to contact Artifactory.") + if args.url: + if not args.user or not args.password: + raise ConanException("Specify --user and --password to use with the --url flag to contact Artifactory.") + assert args.server or (args.url and args.user and args.password) + + +def get_url_user_password(args): + if args.server: + server_name = args.server.strip() + server = get_server(server_name) + url = server.get("url") + user = server.get("user") + password = server.get("password") + else: + url = args.url + user = args.user + password = args.password + return url, user, password + + +def _add_default_arguments(subparser, is_bi_create=False): + url_help = "Artifactory url, like: https://
/artifactory." + if is_bi_create: + url_help += " This may be not necessary if all the information for the Conan artifacts is present in the " \ + "local cache." + subparser.add_argument("--server", help="Server name of the Artifactory to get the build info from.") + subparser.add_argument("--url", help=url_help) + subparser.add_argument("--user", help="User name for the repository.") + subparser.add_argument("--password", help="Password for the user name.") + return subparser + @conan_command(group="Custom commands") def build_info(conan_api: ConanAPI, parser, *args): @@ -379,6 +411,7 @@ def build_info_create(conan_api: ConanAPI, parser, subparser, *args): """ Creates BuildInfo from a Conan graph json from a conan install or create. """ + _add_default_arguments(subparser, is_bi_create=True) check_min_required_conan_version("2.0.6") @@ -387,28 +420,20 @@ def build_info_create(conan_api: ConanAPI, parser, subparser, *args): subparser.add_argument("build_number", help="Build number property for BuildInfo.") subparser.add_argument("repository", help="Repository to look artifacts for.") - subparser.add_argument("--url", help="Artifactory url, like: https://
/artifactory. " - "This may be not necessary if all the information for the Conan " - "artifacts is present in the local cache.") - - - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") - subparser.add_argument("--with-dependencies", help="Whether to add dependencies information or not. Default: false.", action='store_true', default=False) args = parser.parse_args(*args) + url, user, password = get_url_user_password(args) + with open(args.json, 'r') as f: data = json.load(f) # remove the 'conanfile' node data["graph"]["nodes"].pop("0") bi = BuildInfo(data, args.build_name, args.build_number, args.repository, - with_dependencies=args.with_dependencies, url=args.url, user=args.user, password=args.password, - apikey=args.apikey) + with_dependencies=args.with_dependencies, url=url, user=user, password=password) cli_out_write(bi.create()) @@ -418,14 +443,14 @@ def build_info_upload(conan_api: ConanAPI, parser, subparser, *args): """ Uploads BuildInfo json to repository. """ + _add_default_arguments(subparser) subparser.add_argument("build_info", help="BuildInfo json file.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") args = parser.parse_args(*args) + assert_server_or_url_user_password(args) + + url, user, password = get_url_user_password(args) with open(args.build_info) as f: build_info_json = json.load(f) @@ -444,8 +469,8 @@ def build_info_upload(conan_api: ConanAPI, parser, subparser, *args): artifact_properties = {} artifact_path = artifact.get('path') try: - request_url = f"{args.url}/api/storage/{artifact_path}?properties" - props_response = api_request("get", request_url, args.user, args.password, args.apikey) + request_url = f"{url}/api/storage/{artifact_path}?properties" + props_response = api_request("get", request_url, user, password) artifact_properties = json.loads(props_response).get("properties") except: pass @@ -453,15 +478,12 @@ def build_info_upload(conan_api: ConanAPI, parser, subparser, *args): artifact_properties.setdefault("build.name", []).append(build_name) artifact_properties.setdefault("build.number", []).append(build_number) - request_url = f"{args.url}/api/metadata/{artifact_path}" - api_request("patch", request_url, args.user, args.password, - args.apikey, json_data=json.dumps({"props": artifact_properties})) - + request_url = f"{url}/api/metadata/{artifact_path}" + api_request("patch", request_url, user, password, json_data=json.dumps({"props": artifact_properties})) # now upload the BuildInfo - request_url = f"{args.url}/api/build" - response = api_request("put", request_url, args.user, args.password, - args.apikey, json_data=json.dumps(build_info_json)) + request_url = f"{url}/api/build" + response = api_request("put", request_url, user, password, json_data=json.dumps(build_info_json)) cli_out_write(response) @@ -470,10 +492,10 @@ def build_info_promote(conan_api: ConanAPI, parser, subparser, *args): """ Promote the BuildInfo from the source to the target repository. """ + _add_default_arguments(subparser) subparser.add_argument("build_name", help="BuildInfo name to promote.") subparser.add_argument("build_number", help="BuildInfo number to promote.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") subparser.add_argument("source_repo", help="Source repo for promotion.") subparser.add_argument("target_repo", help="Target repo for promotion.") @@ -481,26 +503,24 @@ def build_info_promote(conan_api: ConanAPI, parser, subparser, *args): action='store_true', default=False) subparser.add_argument("--comment", help="An optional comment describing the reason for promotion. Default: ''") - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") - args = parser.parse_args(*args) + assert_server_or_url_user_password(args) + + url, user, password = get_url_user_password(args) promotion_json = { "sourceRepo": args.source_repo, "targetRepo": args.target_repo, - # Conan promotions must always be copy, and the clean must be handled manually + # Conan's promotions must always be copy, and the clean must be handled manually # otherwise you can end up deleting recipe artifacts that other packages use "copy": "true", "dependencies": "true" if args.dependencies else "false", "comment": args.comment } - request_url = f"{args.url}/api/build/promote/{args.build_name}/{args.build_number}" + request_url = f"{url}/api/build/promote/{args.build_name}/{args.build_number}" - response = api_request("post", request_url, args.user, args.password, args.apikey, - json_data=json.dumps(promotion_json)) + response = api_request("post", request_url, user, password, json_data=json.dumps(promotion_json)) cli_out_write(response) @@ -510,20 +530,18 @@ def build_info_get(conan_api: ConanAPI, parser, subparser, *args): """ Get Build Info information. """ + _add_default_arguments(subparser) subparser.add_argument("build_name", help="BuildInfo name to get.") subparser.add_argument("build_number", help="BuildInfo number to get.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") - - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") args = parser.parse_args(*args) - request_url = f"{args.url}/api/build/{args.build_name}/{args.build_number}" + assert_server_or_url_user_password(args) + url, user, password = get_url_user_password(args) - response = api_request("get", request_url, args.user, args.password, args.apikey) + request_url = f"{url}/api/build/{args.build_name}/{args.build_number}" + response = api_request("get", request_url, user, password) cli_out_write(response) @@ -533,14 +551,13 @@ def build_info_delete(conan_api: ConanAPI, parser, subparser, *args): """ Removes builds stored in Artifactory. Useful for cleaning up old build info data. """ + _add_default_arguments(subparser) subparser.add_argument("build_name", help="BuildInfo name to delete.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") subparser.add_argument("--build-number", help="BuildInfo numbers to promote. You can add " \ "several build-numbers for the same build-name, like: --build-number=1 --build-number=2.", action='append') - subparser.add_argument("--delete-artifacts", help="Build artifacts are also removed " \ "provided they have the corresponding build.name and build.number properties attached to them. " \ "Default false.", @@ -548,11 +565,10 @@ def build_info_delete(conan_api: ConanAPI, parser, subparser, *args): subparser.add_argument("--delete-all", help="The whole build is removed. Default false.", action='store_true', default=False, ) - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") - args = parser.parse_args(*args) + assert_server_or_url_user_password(args) + + url, user, password = get_url_user_password(args) delete_json = { "buildName": args.build_name, @@ -561,10 +577,9 @@ def build_info_delete(conan_api: ConanAPI, parser, subparser, *args): "deleteAll": "true" if args.delete_all else "false", } - request_url = f"{args.url}/api/build/delete" + request_url = f"{url}/api/build/delete" - response = api_request("post", request_url, args.user, args.password, args.apikey, - json_data=json.dumps(delete_json)) + response = api_request("post", request_url, user, password, json_data=json.dumps(delete_json)) cli_out_write(response) @@ -574,38 +589,38 @@ def build_info_append(conan_api: ConanAPI, parser, subparser, *args): """ Append published build to the build info. """ + _add_default_arguments(subparser) subparser.add_argument("build_name", help="The current build name.") subparser.add_argument("build_number", help="The current build number.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") - - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") - - subparser.add_argument("--build-info", help="Name and number for the Build Info already published in Artifactory. You can add multiple Builds " \ - "like --build-info=build_name,build_number --build-info=build_name,build_number", + subparser.add_argument("--build-info", help="Name and number for the Build Info already published in Artifactory. " + "You can add multiple Builds like --build-info=build_name,build_number" + " --build-info=build_name,build_number", action="append") args = parser.parse_args(*args) + assert_server_or_url_user_password(args) + + url, user, password = get_url_user_password(args) for build_info in args.build_info: if not "," in build_info: - raise ConanException("Please, provide the build name and number to append in the format: --build-info=build_name,build_number") + raise ConanException("Please, provide the build name and number to append in the format: " + "--build-info=build_name,build_number") all_modules = [] for build_info in args.build_info: name, number = build_info.split(",") - request_url = f"{args.url}/api/build/{name}/{number}" - response = api_request("get", request_url, args.user, args.password, args.apikey) + request_url = f"{url}/api/build/{name}/{number}" + response = api_request("get", request_url, user, password) json_data = json.loads(response) build_info = json_data.get("buildInfo") for module in build_info.get("modules"): - # avoid repeating shared recipe modules between builds - if not any(d['id'] == module.get('id') for d in all_modules): - all_modules.append(module) + # avoid repeating shared recipe modules between builds + if not any(d['id'] == module.get('id') for d in all_modules): + all_modules.append(module) bi = BuildInfo(None, args.build_name, args.build_number, None) bi_json = bi.header() @@ -618,6 +633,7 @@ def build_info_create_bundle(conan_api: ConanAPI, parser, subparser, *args): """ Creates an Artifactory Release Bundle from the information of the Build Info """ + _add_default_arguments(subparser) subparser.add_argument("json", help="BuildInfo JSON.") @@ -626,17 +642,12 @@ def build_info_create_bundle(conan_api: ConanAPI, parser, subparser, *args): subparser.add_argument("bundle_name", help="The created bundle name.") subparser.add_argument("bundle_version", help="The created bundle version.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory. " - "This may be not necessary if all the information for the Conan " - "artifacts is present in the local cache.") - subparser.add_argument("sign_key_name", help="Signing Key name.") - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") - args = parser.parse_args(*args) + assert_server_or_url_user_password(args) + + url, user, password = get_url_user_password(args) with open(args.json, 'r') as f: data = json.load(f) @@ -647,9 +658,9 @@ def build_info_create_bundle(conan_api: ConanAPI, parser, subparser, *args): "payload": manifest } - request_url = f"{args.url}/api/release_bundles/from_files/{args.bundle_name}/{args.bundle_version}" + request_url = f"{url}/api/release_bundles/from_files/{args.bundle_name}/{args.bundle_version}" - response = api_request("post", request_url, args.user, args.password, args.apikey, - json_data=json.dumps(bundle_json), sign_key_name=args.sign_key_name) + response = api_request("post", request_url, user, password, json_data=json.dumps(bundle_json), + sign_key_name=args.sign_key_name) cli_out_write(response) diff --git a/extensions/commands/art/cmd_property.py b/extensions/commands/art/cmd_property.py index 9850175..9207fb0 100644 --- a/extensions/commands/art/cmd_property.py +++ b/extensions/commands/art/cmd_property.py @@ -1,5 +1,6 @@ +import base64 import json -from pathlib import Path +import os import requests @@ -9,34 +10,10 @@ from conans.model.package_ref import PkgReference from conan.errors import ConanException +from cmd_build_info import api_request, assert_server_or_url_user_password, get_url_user_password -def response_to_str(response): - content = response.content - try: - # A bytes message, decode it as str - if isinstance(content, bytes): - content = content.decode() - - content_type = response.headers.get("content-type") - - if content_type == "application/json": - # Errors from Artifactory looks like: - # {"errors" : [ {"status" : 400, "message" : "Bla bla bla"}]} - try: - data = json.loads(content)["errors"][0] - content = "{}: {}".format(data["status"], data["message"]) - except Exception: - pass - elif "text/html" in content_type: - content = "{}: {}".format(response.status_code, response.reason) - return content - - except Exception: - return response.content - - -def get_path_from_ref(ref): +def _get_path_from_ref(ref): try: package_ref = PkgReference.loads(ref) recipe_ref = package_ref.ref @@ -50,39 +27,13 @@ def get_path_from_ref(ref): return f"_/{recipe_ref.name}/{recipe_ref.version}{rrev_path}{pkgid_path}{prev_path}" -def api_request(type, request_url, user=None, password=None, apikey=None, json_data=None): - - headers = {} - if json_data: - headers.update({"Content-Type": "application/json"}) - if apikey: - headers.update({"X-JFrog-Art-Api": apikey}) - - requests_method = getattr(requests, type) - if user and password: - response = requests_method(request_url, auth=( - user, password), data=json_data, headers=headers) - elif apikey: - response = requests_method( - request_url, data=json_data, headers=headers) - else: - response = requests_method(request_url) - - if response.status_code == 401: - raise Exception(response_to_str(response)) - elif response.status_code not in [200, 204]: - raise Exception(response_to_str(response)) - - return response_to_str(response) - - -def add_default_arguments(subparser): - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") +def _add_default_arguments(subparser): subparser.add_argument("repository", help="Artifactory repository.") subparser.add_argument("reference", help="Conan reference.") + subparser.add_argument("--server", help="Server name of the Artifactory to get the build info from") + subparser.add_argument("--url", help="Artifactory url, like: https://
/artifactory") subparser.add_argument("--user", help="user name for the repository") subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") subparser.add_argument("--property", action='append', help='Property to add, like --property="build.name=buildname" --property="build.number=1"') @@ -102,10 +53,12 @@ def property_add(conan_api: ConanAPI, parser, subparser, *args): Append properties for artifacts under a Conan reference recursively. """ - add_default_arguments(subparser) + _add_default_arguments(subparser) args = parser.parse_args(*args) + assert_server_or_url_user_password(args) + if not args.property: raise ConanException("Please, add at least one property with the --property argument.") @@ -115,27 +68,27 @@ def property_add(conan_api: ConanAPI, parser, subparser, *args): list_folders = "1" # get artifacts recursively from a starting path - root_path = get_path_from_ref(args.reference) + root_path = _get_path_from_ref(args.reference) + + url, user, password = get_url_user_password(args) - request_url = f"{args.url}/api/storage/{args.repository}/{root_path}?list&deep=1&listFolders={list_folders}" + request_url = f"{url}/api/storage/{args.repository}/{root_path}?list&deep=1&listFolders={list_folders}" - data = json.loads(api_request("get", request_url, - args.user, args.password, args.apikey)) + data = json.loads(api_request("get", request_url, user, password)) # just consider those artifacts that have conan in the name - # conan_artifacts = [artifact for artifact in data.get("files") if "conan" in artifact.get('uri')] + # conan_artifacts = [artifact for artifact in data.get("files") if "conan" in artifact.get('uri')] # get properties for all artifacts for artifact in data.get("files"): uri = artifact.get('uri') - request_url = f"{args.url}/api/storage/{args.repository}/{root_path}{uri}?properties" + request_url = f"{url}/api/storage/{args.repository}/{root_path}{uri}?properties" artifact_properties = {} try: - props_response = api_request( - "get", request_url, args.user, args.password, args.apikey) + props_response = api_request("get", request_url, user, password) artifact_properties = json.loads(props_response).get("properties") except: pass @@ -145,9 +98,8 @@ def property_add(conan_api: ConanAPI, parser, subparser, *args): artifact_properties.setdefault(key, []).append(val) if artifact_properties: - request_url = f"{args.url}/api/metadata/{args.repository}/{root_path}{uri}?&recursiveProperties=0" - api_request("patch", request_url, args.user, args.password, - args.apikey, json_data=json.dumps({"props": artifact_properties})) + request_url = f"{url}/api/metadata/{args.repository}/{root_path}{uri}?&recursiveProperties=0" + api_request("patch", request_url, user, password, json_data=json.dumps({"props": artifact_properties})) @conan_subcommand() @@ -156,11 +108,12 @@ def property_set(conan_api: ConanAPI, parser, subparser, *args): Set properties for artifacts under a Conan reference recursively. """ - add_default_arguments(subparser) + _add_default_arguments(subparser) subparser.add_argument('--no-recursive', dest='recursive', action='store_false', help='Will not recursively set properties.') args = parser.parse_args(*args) + assert_server_or_url_user_password(args) if not args.property: raise ConanException("Please, add at least one property with the --property argument.") @@ -170,9 +123,10 @@ def property_set(conan_api: ConanAPI, parser, subparser, *args): json_data = json.dumps( {"props": {prop.split('=')[0]: prop.split('=')[1] for prop in args.property}}) - request_url = f"{args.url}/api/metadata/{args.repository}/{get_path_from_ref(args.reference)}?&recursiveProperties={recursive}" + url, user, password = get_url_user_password(args) + request_url = f"{url}/api/metadata/{args.repository}/{_get_path_from_ref(args.reference)}?&recursiveProperties={recursive}" - api_request("patch", request_url, args.user, args.password, args.apikey, json_data=json_data) + api_request("patch", request_url, user, password, json_data=json_data) @conan_subcommand() @@ -183,17 +137,18 @@ def property_build_info_add(conan_api: ConanAPI, parser, subparser, *args): """ subparser.add_argument("json", help="Build Info JSON.") - subparser.add_argument("url", help="Artifactory url, like: https://
/artifactory") subparser.add_argument("--property", action='append', help='Property to add, like --property="key1=value1" --property="key2=value2". \ If the property already exists, the values are appended.') - + + subparser.add_argument("--server", help="Server name of the Artifactory to get the build info from") + subparser.add_argument("--url", help="Artifactory url, like: https://
/artifactory") subparser.add_argument("--user", help="user name for the repository") subparser.add_argument("--password", help="password for the user name") - subparser.add_argument("--apikey", help="apikey for the repository") args = parser.parse_args(*args) + assert_server_or_url_user_password(args) with open(args.json, 'r') as f: data = json.load(f) @@ -201,13 +156,15 @@ def property_build_info_add(conan_api: ConanAPI, parser, subparser, *args): build_name = data.get("name") build_number = data.get("number") + url, user, password = get_url_user_password(args) + for module in data.get('modules'): for artifact in module.get('artifacts'): artifact_properties = {} artifact_path = artifact.get('path') try: - request_url = f"{args.url}/api/storage/{artifact_path}?properties" - props_response = api_request("get", request_url, args.user, args.password, args.apikey) + request_url = f"{url}/api/storage/{artifact_path}?properties" + props_response = api_request("get", request_url, user, password) artifact_properties = json.loads(props_response).get("properties") except: pass @@ -220,6 +177,5 @@ def property_build_info_add(conan_api: ConanAPI, parser, subparser, *args): key, val = property.split('=')[0], property.split('=')[1] artifact_properties.setdefault(key, []).append(val) - request_url = f"{args.url}/api/metadata/{artifact_path}" - api_request("patch", request_url, args.user, args.password, - args.apikey, json_data=json.dumps({"props": artifact_properties})) + request_url = f"{url}/api/metadata/{artifact_path}" + api_request("patch", request_url, user, password, json_data=json.dumps({"props": artifact_properties})) diff --git a/extensions/commands/art/cmd_server.py b/extensions/commands/art/cmd_server.py index 6512c1f..4adae54 100644 --- a/extensions/commands/art/cmd_server.py +++ b/extensions/commands/art/cmd_server.py @@ -37,20 +37,15 @@ def response_to_str(response): return response.content -def api_request(type, request_url, user=None, password=None, apikey=None, json_data=None): +def api_request(type, request_url, user=None, password=None, json_data=None): headers = {} if json_data: headers.update({"Content-Type": "application/json"}) - if apikey: - headers.update({"X-JFrog-Art-Api": apikey}) requests_method = getattr(requests, type) if user and password: response = requests_method(request_url, auth=( user, password), data=json_data, headers=headers) - elif apikey: - response = requests_method( - request_url, data=json_data, headers=headers) else: response = requests_method(request_url) @@ -62,7 +57,18 @@ def api_request(type, request_url, user=None, password=None, apikey=None, json_d return response_to_str(response) -def read_servers(): +def get_server(server_name): + servers = _read_servers() + server_names = [s["name"] for s in servers] + if server_name not in server_names: + raise ConanException(f"The server specified ({server_name}) is not configured. " + f"Use `conan art:server add {server_name}` to configure it.") + for s in servers: + if s["name"] == server_name: + return s + + +def _read_servers(): path = os.path.join(os.path.dirname(__file__), SERVERS_FILENAME) servers = [] if os.path.exists(path): @@ -74,27 +80,27 @@ def read_servers(): return servers -def write_servers(servers): +def _write_servers(servers): path = os.path.join(os.path.dirname(__file__), SERVERS_FILENAME) with open(path, "w") as servers_file: data = json.dumps({"servers": servers}) servers_file.write(base64.b64encode(data.encode('utf-8')).decode('utf-8')) -def assert_new_server(server_name, servers): +def _assert_new_server(server_name, servers): for s in servers: if server_name == s["name"]: raise ConanException(f"Server '{server_name}' ({s['url']}) already exist. " f"You can remove it using `conan art:server remove {server_name}`") -def assert_existing_server(server_name, servers): +def _assert_existing_server(server_name, servers): server_names = [s["name"] for s in servers] if server_name not in server_names: raise ConanException(f"Server '{server_name}' does not exist.") -def add_default_arguments(subparser): +def _add_default_arguments(subparser): subparser.add_argument("name", help="Name of the server") return subparser @@ -111,7 +117,7 @@ def server_add(conan_api: ConanAPI, parser, subparser, *args): """ Add Artifactory server and its credentials. """ - add_default_arguments(subparser) + _add_default_arguments(subparser) subparser.add_argument("url", help="URL of the artifactory server") subparser.add_argument("--user", help="user name for the repository") subparser.add_argument("--password", help="password for the user name") @@ -130,8 +136,8 @@ def server_add(conan_api: ConanAPI, parser, subparser, *args): name = args.name.strip() url = args.url.rstrip("/") - servers = read_servers() - assert_new_server(name, servers) + servers = _read_servers() + _assert_new_server(name, servers) token = api_request("get", f"{url}/api/security/encryptedPassword", user, password) # TODO: manage error with auth @@ -141,7 +147,7 @@ def server_add(conan_api: ConanAPI, parser, subparser, *args): "user": user, "password": token} servers.append(new_server) - write_servers(servers) + _write_servers(servers) ConanOutput().success(f"Server '{name}' ({url}) added successfully") @@ -150,12 +156,12 @@ def server_remove(conan_api: ConanAPI, parser, subparser, *args): """ Remove Artifactory servers. """ - add_default_arguments(subparser) + _add_default_arguments(subparser) args = parser.parse_args(*args) name = args.name.strip() - servers = read_servers() - assert_existing_server(name, servers) + servers = _read_servers() + _assert_existing_server(name, servers) url = None keep_servers = [] for s in servers: @@ -163,11 +169,11 @@ def server_remove(conan_api: ConanAPI, parser, subparser, *args): keep_servers.append(s) else: url = s["url"] - write_servers(keep_servers) + _write_servers(keep_servers) ConanOutput().success(f"Server '{name}' ({url}) removed successfully") -def output_server_list_text(servers): +def _output_server_list_text(servers): if servers: for s in servers: cli_out_write(f"{s['name']}:") @@ -178,15 +184,15 @@ def output_server_list_text(servers): cli_out_write("No servers configured. Use `conan art:server add` command to add one.") -def output_server_list_json(servers): +def _output_server_list_json(servers): [s.pop("password") for s in servers] cli_out_write(json.dumps({"servers": servers}, indent=4)) -@conan_subcommand(formatters={"text": output_server_list_text, - "json": output_server_list_json}) +@conan_subcommand(formatters={"text": _output_server_list_text, + "json": _output_server_list_json}) def server_list(conan_api: ConanAPI, parser, subparser, *args): """ List Artifactory servers. """ - return read_servers() + return _read_servers() diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 10a43dd..72a4b46 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -43,6 +43,7 @@ def test_build_info_create_no_deps(): build_number = "1" run(f"conan config install {repo}") + run(f'conan art:server add artifactory {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') run("conan new cmake_lib -d name=mypkg -d version=1.0 --force") run("conan create . --format json -tf='' -s build_type=Release > create_release.json") @@ -56,20 +57,20 @@ def test_build_info_create_no_deps(): run(f'conan art:build-info create create_release.json {build_name}_release {build_number} extensions-stg > {build_name}_release.json') run(f'conan art:build-info create create_debug.json {build_name}_debug {build_number} extensions-stg > {build_name}_debug.json') - run(f'conan art:build-info upload {build_name}_release.json {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info upload {build_name}_debug.json {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info upload {build_name}_release.json --server artifactory') + run(f'conan art:build-info upload {build_name}_debug.json --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') # aggregate the release and debug build infos into an aggregated one # we also have to set the properties so that the paths to the artifacts are linked # with the build info in Artifactory - run(f'conan art:build-info append {build_name}_aggregated {build_number} {os.getenv("ART_URL")} --build-info={build_name}_release,{build_number} --build-info={build_name}_debug,{build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" > {build_name}_aggregated.json') - run(f'conan art:build-info upload {build_name}_aggregated.json {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info append {build_name}_aggregated {build_number} --server artifactory --build-info={build_name}_release,{build_number} --build-info={build_name}_debug,{build_number} > {build_name}_aggregated.json') + run(f'conan art:build-info upload {build_name}_aggregated.json --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info get {build_name}_release {build_number} {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info get {build_name}_debug {build_number} {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info get {build_name}_aggregated {build_number} {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info get {build_name}_release {build_number} --server artifactory') + run(f'conan art:build-info get {build_name}_debug {build_number} --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info get {build_name}_aggregated {build_number} --server artifactory') - run(f'conan art:build-info promote {build_name}_aggregated {build_number} {os.getenv("ART_URL")} extensions-stg extensions-prod --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info promote {build_name}_aggregated {build_number} extensions-stg extensions-prod --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') # The local clean is because later I'm going to do a conan install from prod repo # and I want to make sure that the install succeeds because the package comes from @@ -85,9 +86,9 @@ def test_build_info_create_no_deps(): run('conan install --requires=mypkg/1.0 -r extensions-prod -s build_type=Release') run('conan install --requires=mypkg/1.0 -r extensions-prod -s build_type=Debug') - run(f'conan art:build-info delete {build_name}_release {os.getenv("ART_URL")} --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') - run(f'conan art:build-info delete {build_name}_debug {os.getenv("ART_URL")} --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') - run(f'conan art:build-info delete {build_name}_aggregated {os.getenv("ART_URL")} --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') + run(f'conan art:build-info delete {build_name}_release --server artifactory --build-number={build_number} --delete-all --delete-artifacts') + run(f'conan art:build-info delete {build_name}_debug --server artifactory --build-number={build_number} --delete-all --delete-artifacts') + run(f'conan art:build-info delete {build_name}_aggregated --url="{os.getenv("ART_URL")}" --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') # even deleting the builds, the folders will stay there, so manually cleaning run('conan remove mypkg* -c -r extensions-prod') @@ -117,6 +118,7 @@ def test_build_info_create_deps(): # +-----+ run(f"conan config install {repo}") + run(f'conan art:server add artifactory {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') run("conan remove '*' -c") @@ -143,18 +145,18 @@ def test_build_info_create_deps(): run("conan upload 'mypkg/1.0' -c -r extensions-stg") - run(f'conan art:build-info create create_release.json {build_name}_release {build_number} extensions-stg --url={os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --with-dependencies > {build_name}_release.json') + run(f'conan art:build-info create create_release.json {build_name}_release {build_number} extensions-stg --server artifactory --with-dependencies > {build_name}_release.json') run(f'conan art:build-info create create_debug.json {build_name}_debug {build_number} extensions-stg --url={os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --with-dependencies > {build_name}_debug.json') - run(f'conan art:build-info upload {build_name}_release.json {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info upload {build_name}_debug.json {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info upload {build_name}_release.json --server artifactory') + run(f'conan art:build-info upload {build_name}_debug.json --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info append {build_name}_aggregated {build_number} {os.getenv("ART_URL")} --build-info={build_name}_release,{build_number} --build-info={build_name}_debug,{build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" > {build_name}_aggregated.json') - run(f'conan art:build-info upload {build_name}_aggregated.json {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info append {build_name}_aggregated {build_number} --server artifactory --build-info={build_name}_release,{build_number} --build-info={build_name}_debug,{build_number} > {build_name}_aggregated.json') + run(f'conan art:build-info upload {build_name}_aggregated.json --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info get {build_name}_release {build_number} {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info get {build_name}_debug {build_number} {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') - run(f'conan art:build-info get {build_name}_aggregated {build_number} {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + run(f'conan art:build-info get {build_name}_release {build_number} --server artifactory') + run(f'conan art:build-info get {build_name}_debug {build_number} --server artifactory') + run(f'conan art:build-info get {build_name}_aggregated {build_number} --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') # FIXME: commenting this part, promote with --dependencies does not work # wait until it's fixed or the new BuildInfo promotion is released @@ -175,9 +177,9 @@ def test_build_info_create_deps(): # Promotions using Release Bundles do work with depdendencies, but they are not implemented in the testing Artifactory # conan art:build-info create-bundle ${build_name}_aggregated.json develop full_bundle 1.0 ${ART_URL} test_key_pair --user=${CONAN_LOGIN_USERNAME_DEVELOP} --password="${CONAN_PASSWORD_DEVELOP}" - run(f'conan art:build-info delete {build_name}_release {os.getenv("ART_URL")} --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') - run(f'conan art:build-info delete {build_name}_debug {os.getenv("ART_URL")} --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') - run(f'conan art:build-info delete {build_name}_aggregated {os.getenv("ART_URL")} --build-number={build_number} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') + run(f'conan art:build-info delete {build_name}_release --build-number={build_number} --server="artifactory" --delete-all --delete-artifacts') + run(f'conan art:build-info delete {build_name}_debug --build-number={build_number} --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') + run(f'conan art:build-info delete {build_name}_aggregated --build-number={build_number} --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" --delete-all --delete-artifacts') # even deleting the builds, the folders will stay there, so manually cleaning run('conan remove "*" -c -r extensions-prod') @@ -209,7 +211,7 @@ def test_fail_if_not_uploaded(): assert "Missing information in the Conan local cache, please provide " \ "the --url and --repository arguments to retrieve the information from" in out - out = run(f'conan art:build-info create create.json {build_name} {build_number} extensions-stg --url={os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" > {build_name}.json', error=True) + out = run(f'conan art:build-info create create.json {build_name} {build_number} extensions-stg --url="{os.getenv("ART_URL")}" --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}" > {build_name}.json', error=True) assert "There are no artifacts for the mypkg/1.0#" in out