diff --git a/sdk/template/azure-template/MANIFEST.in b/sdk/template/azure-template/MANIFEST.in index dd20dd67408b..9760aa1b05db 100644 --- a/sdk/template/azure-template/MANIFEST.in +++ b/sdk/template/azure-template/MANIFEST.in @@ -1,3 +1,4 @@ +include _meta.json include *.md include azure/__init__.py recursive-include tests *.py \ No newline at end of file diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index cbd97bf93ee9..7395e4787e69 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -1,14 +1,17 @@ import argparse import json import logging +import os from pathlib import Path import re from subprocess import check_call from .swaggertosdk.SwaggerToSdkCore import ( + read_config, CONFIG_FILE, ) from azure_devtools.ci_tools.git_tools import get_add_diff_file_list +from .swaggertosdk.autorest_tools import build_autorest_options from .generate_sdk import generate _LOGGER = logging.getLogger(__name__) @@ -37,6 +40,57 @@ def init_new_service(package_name, folder_name): with open(str(ci), 'w') as file_out: file_out.writelines(content) +def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme): + + readme_file = str(Path(spec_folder, input_readme)) + global_conf = config["meta"] + local_conf = config["projects"][readme_file] + + cmd = ["autorest", input_readme] + cmd += build_autorest_options(global_conf, local_conf) + + # metadata + metadata = { + "autorest": global_conf["autorest_options"]["version"], + "use": global_conf["autorest_options"]["use"], + "commit": data["headSha"], + "repository_url": data["repoHttpsUrl"], + "autorest_command": " ".join(cmd), + "readme": input_readme + } + + _LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2))) + + package_folder = Path(sdk_folder, folder_name, package_name).expanduser() + if not os.path.exists(package_folder): + _LOGGER.info(f"Package folder doesn't exist: {package_folder}") + _LOGGER.info("Failed to save metadata.") + return + + metadata_file_path = os.path.join(package_folder, "_meta.json") + with open(metadata_file_path, "w") as writer: + json.dump(metadata, writer, indent=2) + _LOGGER.info(f"Saved metadata to {metadata_file_path}") + + # Check whether MANIFEST.in includes _meta.json + require_meta = "include _meta.json\n" + manifest_file = os.path.join(package_folder, "MANIFEST.in") + if not os.path.exists(manifest_file): + _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") + return + + includes = [] + write_flag = False + with open(manifest_file, "r") as f: + includes = f.readlines() + if require_meta not in includes: + includes = [require_meta] + includes + write_flag = True + + if write_flag: + with open(manifest_file, "w") as f: + f.write("".join(includes)) + def main(generate_input, generate_output): with open(generate_input, "r") as reader: @@ -49,7 +103,7 @@ def main(generate_input, generate_output): for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f'[CODEGEN]({input_readme})codegen begin') - generate(CONFIG_FILE, + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, @@ -59,6 +113,7 @@ def main(generate_input, generate_output): package_names = get_package_names(sdk_folder) _LOGGER.info(f'[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]') + for folder_name, package_name in package_names: if package_name in package_total: continue @@ -77,6 +132,12 @@ def main(generate_input, generate_output): # Generate some necessary file for new service init_new_service(package_name, folder_name) + # Update metadata + try: + update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) + except Exception as e: + _LOGGER.info(str(e)) + # Setup package locally check_call(f'pip install --ignore-requires-python -e {str(Path(sdk_folder, folder_name, package_name))}', shell=True) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py index ceeb18c99a86..46e1a767790e 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py @@ -82,6 +82,7 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde local_conf, autorest_bin ) + return config def generate_main(): diff --git a/tools/azure-sdk-tools/tests/test_servicemetadata.py b/tools/azure-sdk-tools/tests/test_servicemetadata.py new file mode 100644 index 000000000000..e906671d5fb6 --- /dev/null +++ b/tools/azure-sdk-tools/tests/test_servicemetadata.py @@ -0,0 +1,154 @@ +import os +import json +import tempfile +import unittest +from pathlib import Path + +import pytest +from packaging_tools.auto_codegen import update_servicemetadata + + +""" +Create metadata file + +Update metadata file + +Update MANIFETS.IN + +No need to update MANIFETS.IN +""" + +MANIFEST_TEMP = """recursive-include tests *.py *.yaml +include *.md +include azure/__init__.py +include azure/mgmt/__init__.py +""" + +class TestServiceMetadata(unittest.TestCase): + + def setUp(self): + self.sdk_folder = "sdk" + self.data = { + "specFolder": "../azure-rest-api-specs", + "headSha": "e295fe97eee3709668d3e5f7f8b434026b814ef9", + "headRef": "master", + "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", + } + self.config = { + "meta": { + "autorest_options": { + "version": "3.0.6369", + "use": "@autorest/python@5.4.3", + "python": "", + "python-mode": "update", + "sdkrel:python-sdks-folder": "./sdk/.", + "multiapi": "", + "track2": "" + }, + "advanced_options": { + "create_sdk_pull_requests": True, + "sdk_generation_pull_request_base": "integration_branch" + }, + "repotag": "azure-sdk-for-python-track2", + "version": "0.2.0" + }, + "projects": { + "./azure-rest-api-specs/specification/operationalinsights/resource-manager/readme.md": {} + } + + } + self.folder_name = "monitor" + self.package_name = "azure-mgmt-monitor" + self.spec_folder = "./" + self.input_readme = "azure-rest-api-specs/specification/operationalinsights/resource-manager/readme.md" + + def test_metadata(self): + + with tempfile.TemporaryDirectory() as temp_dir: + # Init directories + self.sdk_folder = str(Path(temp_dir, "sdk")) + self.spec_folder = str(Path(temp_dir, "spec")) + os.makedirs(self.sdk_folder) + os.makedirs(self.spec_folder) + + readme_file = str(Path(self.spec_folder, self.input_readme)) + self.config["projects"][readme_file] = {} + + package_folder = Path(self.sdk_folder, self.folder_name, self.package_name).expanduser() + metadata_file_path = os.path.join(package_folder, "_meta.json") + manifest_file = os.path.join(package_folder, "MANIFEST.in") + os.makedirs(package_folder) + + # Test MANIFEST.in does not exist + update_servicemetadata( + sdk_folder=self.sdk_folder, + data=self.data, + config=self.config, + folder_name=self.folder_name, + package_name=self.package_name, + spec_folder=self.spec_folder, + input_readme=self.input_readme + ) + + assert os.path.isfile(metadata_file_path) == True + # Do not create MANIFEST.in, if it does not exist + assert os.path.isfile(manifest_file) == False + + # Test update metadata + with open(metadata_file_path, "w") as f: + json.dump({"autorest": "3.0.0"}, f, indent=2) + update_servicemetadata( + sdk_folder=self.sdk_folder, + data=self.data, + config=self.config, + folder_name=self.folder_name, + package_name=self.package_name, + spec_folder=self.spec_folder, + input_readme=self.input_readme + ) + + assert os.path.isfile(metadata_file_path) == True + assert os.path.isfile(manifest_file) == False + with open(metadata_file_path, "r") as f: + md = json.load(f) + assert md["autorest"] == "3.0.6369" + + # Test update MANIFEST.in + with open(manifest_file, "w") as f: + f.write(MANIFEST_TEMP) + update_servicemetadata( + sdk_folder=self.sdk_folder, + data=self.data, + config=self.config, + folder_name=self.folder_name, + package_name=self.package_name, + spec_folder=self.spec_folder, + input_readme=self.input_readme + ) + + assert os.path.isfile(metadata_file_path) == True + assert os.path.isfile(manifest_file) == True + with open(manifest_file, "r") as f: + meta_line = "include _meta.json\n" + line = f.readline() + assert meta_line == line + + # Test update MANIFEST.in again + update_servicemetadata( + sdk_folder=self.sdk_folder, + data=self.data, + config=self.config, + folder_name=self.folder_name, + package_name=self.package_name, + spec_folder=self.spec_folder, + input_readme=self.input_readme + ) + + assert os.path.isfile(metadata_file_path) == True + assert os.path.isfile(manifest_file) == True + with open(manifest_file, "r") as f: + meta_line = "include _meta.json\n" + line = f.readline() + second_line = f.readline() + assert meta_line == line + assert meta_line != second_line