From 17d1d96b7cb289bfd38179f349f554112958e4b0 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 3 Jul 2019 13:45:03 -0700 Subject: [PATCH 01/18] enabling samples for azure-storage code --- .gitignore | 1 + doc/sphinx/conf.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5d00286b1667..8c81b746955d 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ src/build *.iml /doc/_build /doc/sphinx/examples +/doc/sphinx/tests /.vs/config/applicationhost.config # Azure deployment credentials diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 7100227f1843..14f5b435c164 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -25,11 +25,16 @@ # FIX FOR EXAMPLE REFERENCES REPO_ROOT = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..', '..')) -glob_expansion = os.path.join(REPO_ROOT, 'sdk/*/*/examples/**/test*examples*.py') -example_files = glob.glob(glob_expansion, recursive=True) +examples_glob_expansion = os.path.join(REPO_ROOT, 'sdk/*/*/examples/**/test*example*.py') +samples_glob_expansion = os.path.join(REPO_ROOT, 'sdk/*/*/tests/**/test*sample*.py') + +example_files = glob.glob(examples_glob_expansion, recursive=True) +samples_files = glob.glob(samples_glob_expansion, recursive=True) + +all_files = [os.path.relpath(file, REPO_ROOT) for file in list(set(example_files + samples_files))] # now for each package, we need to copy it and write it to the relative path FROM THE CURRENT CWD -for example_file in [os.path.relpath(example_file, REPO_ROOT) for example_file in example_files]: +for example_file in all_files: relative_path_in_pkg = os.path.join(*(example_file.split(os.path.sep)[3:])) final_destination = os.path.abspath(os.path.join(os.path.dirname(__file__), relative_path_in_pkg)) From 18d9d8694d981dec27adba41815f401fc6d30d0d Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Sep 2019 18:50:40 -0700 Subject: [PATCH 02/18] update sdist to not install in develop --- eng/tox/tox.ini | 1 - scripts/dev_setup.py | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index eb020fb08aa9..422609c26749 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -59,7 +59,6 @@ commands = [testenv:sdist] skipsdist = false skip_install = false -usedevelop = true changedir = {toxinidir} deps = {[base]deps} diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py index 759d74bdb7da..0e976a825f78 100644 --- a/scripts/dev_setup.py +++ b/scripts/dev_setup.py @@ -42,6 +42,14 @@ def pip_command(command, additional_dir=".", error_ok=False): default="", help="Comma separated list of targeted packages. Used to limit the number of packages that dependencies will be installed for.", ) +parser.add_argument( + "--disabledevelop", + dest="develop_mode_disabled", + default=False, + help="Add this argument if you would prefer to install the package with a simple `pip install` versus `pip install -e`", +) + + args = parser.parse_args() packages = { @@ -122,9 +130,13 @@ def pip_command(command, additional_dir=".", error_ok=False): "install -r dev_requirements.txt", os.path.join(packages[package_name], package_name), ) + + mode_arg = "" if args.develop_mode_disabled else "-e" + pip_command( - "install --ignore-requires-python -e {}".format( - os.path.join(packages[package_name], package_name) + "install --ignore-requires-python {} {}".format( + os.path.join(packages[package_name], package_name), + mode_arg ) ) From 0583d10dc7e00b874b7e413a3cd9c8379bc8415b Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Sep 2019 18:52:05 -0700 Subject: [PATCH 03/18] allowing CI to run dev_setup without develop mode --- scripts/devops_tasks/setup_execute_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/setup_execute_tests.py b/scripts/devops_tasks/setup_execute_tests.py index d3eb6a7bdb07..53139504eeb0 100644 --- a/scripts/devops_tasks/setup_execute_tests.py +++ b/scripts/devops_tasks/setup_execute_tests.py @@ -65,8 +65,9 @@ def prep_tests(targeted_packages, python_version): [ python_version, dev_setup_script_location, + "--disabledevelop", "-p", - ",".join([os.path.basename(p) for p in targeted_packages]), + ",".join([os.path.basename(p) for p in targeted_packages]) ], root_dir, ) From afd339a96f6296d3b86da84cef07ae677cc16754 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Sep 2019 19:03:26 -0700 Subject: [PATCH 04/18] change order of format string arguments --- scripts/dev_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py index 0e976a825f78..2fa709abb2ef 100644 --- a/scripts/dev_setup.py +++ b/scripts/dev_setup.py @@ -135,8 +135,8 @@ def pip_command(command, additional_dir=".", error_ok=False): pip_command( "install --ignore-requires-python {} {}".format( - os.path.join(packages[package_name], package_name), - mode_arg + mode_arg, + os.path.join(packages[package_name], package_name) ) ) From 5dc1d9c7351bda9c2a7c35c8ae47d7f39cc57d42 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Sep 2019 19:04:06 -0700 Subject: [PATCH 05/18] default store_true properly --- scripts/dev_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py index 2fa709abb2ef..e5e974601a54 100644 --- a/scripts/dev_setup.py +++ b/scripts/dev_setup.py @@ -46,6 +46,7 @@ def pip_command(command, additional_dir=".", error_ok=False): "--disabledevelop", dest="develop_mode_disabled", default=False, + action="store_true" help="Add this argument if you would prefer to install the package with a simple `pip install` versus `pip install -e`", ) From 48adab949a4cf5ea23675fca5a5b79e19cb0b9a6 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Sep 2019 19:05:44 -0700 Subject: [PATCH 06/18] get some CI checks rolling --- sdk/redis/azure-mgmt-redis/dev_requirements.txt | 2 +- sdk/storage/azure-storage-queue/dev_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/redis/azure-mgmt-redis/dev_requirements.txt b/sdk/redis/azure-mgmt-redis/dev_requirements.txt index 6ccb7f031ddd..f6457a93d5e8 100644 --- a/sdk/redis/azure-mgmt-redis/dev_requirements.txt +++ b/sdk/redis/azure-mgmt-redis/dev_requirements.txt @@ -1 +1 @@ --e ../../../tools/azure-sdk-tools +-e ../../../tools/azure-sdk-tools \ No newline at end of file diff --git a/sdk/storage/azure-storage-queue/dev_requirements.txt b/sdk/storage/azure-storage-queue/dev_requirements.txt index c0c5aade709c..8657b2ccb464 100644 --- a/sdk/storage/azure-storage-queue/dev_requirements.txt +++ b/sdk/storage/azure-storage-queue/dev_requirements.txt @@ -2,4 +2,4 @@ -e ../../../tools/azure-sdk-tools -e ../../identity/azure-identity aiohttp>=3.0; python_version >= '3.5' -pytest \ No newline at end of file +pytest From 2e09d375b36f64ac934f4a421dc44016243c4f5d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 9 Sep 2019 20:06:37 -0700 Subject: [PATCH 07/18] updating appconfiguration manifest so that the sdist can properly install --- sdk/appconfiguration/azure-appconfiguration/MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/appconfiguration/azure-appconfiguration/MANIFEST.in b/sdk/appconfiguration/azure-appconfiguration/MANIFEST.in index fb0660bea2d2..9fd2bc9d0339 100644 --- a/sdk/appconfiguration/azure-appconfiguration/MANIFEST.in +++ b/sdk/appconfiguration/azure-appconfiguration/MANIFEST.in @@ -1,5 +1,5 @@ recursive-include tests *.py *.yaml -include *.rst +include *.md include azure/__init__.py include azure/data/__init__.py From f32f9bfd2206a5a2883f999b2e2ca276cccb222d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 9 Sep 2019 20:33:08 -0700 Subject: [PATCH 08/18] update applicationinsights manifest --- sdk/applicationinsights/azure-applicationinsights/MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/applicationinsights/azure-applicationinsights/MANIFEST.in b/sdk/applicationinsights/azure-applicationinsights/MANIFEST.in index bb37a2723dae..88830671bc4d 100644 --- a/sdk/applicationinsights/azure-applicationinsights/MANIFEST.in +++ b/sdk/applicationinsights/azure-applicationinsights/MANIFEST.in @@ -1 +1,3 @@ include *.rst + +include azure/__init__.py \ No newline at end of file From 9a3218a63217ae6c71dbb0f3cd081bcfa97849aa Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 9 Sep 2019 21:26:01 -0700 Subject: [PATCH 09/18] bugfix in dev_setup.py, bad syntax. adding 'develop' environment to the tox.ini --- eng/tox/tox.ini | 13 ++++++++++++- scripts/dev_setup.py | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 422609c26749..3e964ea47f0f 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -55,11 +55,22 @@ deps = commands = - mypy {toxinidir}/azure - [testenv:sdist] skipsdist = false skip_install = false changedir = {toxinidir} +deps = + {[base]deps} +commands = + pytest \ + {posargs} \ + {toxinidir} + +[testenv:develop] +skipsdist = false +skip_install = false +usedevelop = true +changedir = {toxinidir} deps = {[base]deps} commands = diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py index e5e974601a54..77b11d4f2e68 100644 --- a/scripts/dev_setup.py +++ b/scripts/dev_setup.py @@ -46,7 +46,7 @@ def pip_command(command, additional_dir=".", error_ok=False): "--disabledevelop", dest="develop_mode_disabled", default=False, - action="store_true" + action="store_true", help="Add this argument if you would prefer to install the package with a simple `pip install` versus `pip install -e`", ) From 62e94eec5e51f3228d0458ec0335bc15309d9627 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 9 Sep 2019 22:08:56 -0700 Subject: [PATCH 10/18] manifest broken. repairing --- sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in index bb37a2723dae..ac87972df424 100644 --- a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in +++ b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in @@ -1 +1,3 @@ include *.rst +include azure/__init__.py +include azure/mgmt/__init__.py From 4557b70021c4eacd81c3b8d5a8a4a16de1652271 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 9 Sep 2019 22:10:42 -0700 Subject: [PATCH 11/18] guessed that datalake-store had the same issue, so just fixing this before ci fails again --- sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in b/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in index bb37a2723dae..ac87972df424 100644 --- a/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in +++ b/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in @@ -1 +1,3 @@ include *.rst +include azure/__init__.py +include azure/mgmt/__init__.py From 276d7e1a8fb64a88cb7402f760948d4f9d9d46c2 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 10 Sep 2019 06:07:04 -0700 Subject: [PATCH 12/18] resolved problem with datalake analytics, needed to add the additional namespace to the __init__ --- sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in | 1 + sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in index ac87972df424..1c9ebaab0de4 100644 --- a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in +++ b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in @@ -1,3 +1,4 @@ include *.rst include azure/__init__.py include azure/mgmt/__init__.py +include azure/mgmt/datalake/__init__.py diff --git a/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in b/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in index ac87972df424..1c9ebaab0de4 100644 --- a/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in +++ b/sdk/datalake/azure-mgmt-datalake-store/MANIFEST.in @@ -1,3 +1,4 @@ include *.rst include azure/__init__.py include azure/mgmt/__init__.py +include azure/mgmt/datalake/__init__.py From 85e7bb03aadf7d0a74e40732ff72dff477eada55 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 10 Sep 2019 07:09:17 -0700 Subject: [PATCH 13/18] repair manifest for sdist azure-eventhub --- sdk/eventhub/azure-eventhubs/MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/eventhub/azure-eventhubs/MANIFEST.in b/sdk/eventhub/azure-eventhubs/MANIFEST.in index c9c2d50a3ec3..50c61fef797b 100644 --- a/sdk/eventhub/azure-eventhubs/MANIFEST.in +++ b/sdk/eventhub/azure-eventhubs/MANIFEST.in @@ -1,2 +1,2 @@ -include *.rst +include *.md include azure/__init__.py \ No newline at end of file From 7b88685bb3df35e7be5e64d0a4c39302246dc522 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 10 Sep 2019 11:37:11 -0700 Subject: [PATCH 14/18] adding manifest for identity --- sdk/identity/azure-identity/MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 sdk/identity/azure-identity/MANIFEST.in diff --git a/sdk/identity/azure-identity/MANIFEST.in b/sdk/identity/azure-identity/MANIFEST.in new file mode 100644 index 000000000000..50c61fef797b --- /dev/null +++ b/sdk/identity/azure-identity/MANIFEST.in @@ -0,0 +1,2 @@ +include *.md +include azure/__init__.py \ No newline at end of file From 08fc38340eee8cf4e6a7972787d41b89b22525d5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 10 Sep 2019 15:20:36 -0700 Subject: [PATCH 15/18] updating manifest for scheduler --- sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in b/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in index bb37a2723dae..ac87972df424 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in +++ b/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in @@ -1 +1,3 @@ include *.rst +include azure/__init__.py +include azure/mgmt/__init__.py From 0d8ef309e5377ac9cef3bfc6950a9005bc9ac4d4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 10 Sep 2019 15:29:34 -0700 Subject: [PATCH 16/18] repair manifest for servermanager --- sdk/servermanager/azure-mgmt-servermanager/MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/servermanager/azure-mgmt-servermanager/MANIFEST.in b/sdk/servermanager/azure-mgmt-servermanager/MANIFEST.in index bb37a2723dae..ac87972df424 100644 --- a/sdk/servermanager/azure-mgmt-servermanager/MANIFEST.in +++ b/sdk/servermanager/azure-mgmt-servermanager/MANIFEST.in @@ -1 +1,3 @@ include *.rst +include azure/__init__.py +include azure/mgmt/__init__.py From ad15a014c6729576e0f978ad5b06cc86b1c4bbd9 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 10 Sep 2019 18:45:28 -0700 Subject: [PATCH 17/18] update the doc pipeline to install in non-develop mode --- .azure-pipelines/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/docs.yml b/.azure-pipelines/docs.yml index b66bcd2a5072..83c1ff136d1f 100644 --- a/.azure-pipelines/docs.yml +++ b/.azure-pipelines/docs.yml @@ -22,6 +22,7 @@ jobs: displayName: 'Install All Packages' inputs: scriptPath: 'scripts/dev_setup.py' + arguments: '--disabledevelop' - powershell: | cd $(Build.SourcesDirectory)/doc/sphinx From e1a4f2216d37f022ef27a9afff734d7191f579ae Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 10 Sep 2019 19:46:40 -0700 Subject: [PATCH 18/18] Revert "update the doc pipeline to install in non-develop mode" This reverts commit ad15a014c6729576e0f978ad5b06cc86b1c4bbd9. --- .azure-pipelines/docs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/docs.yml b/.azure-pipelines/docs.yml index 83c1ff136d1f..b66bcd2a5072 100644 --- a/.azure-pipelines/docs.yml +++ b/.azure-pipelines/docs.yml @@ -22,7 +22,6 @@ jobs: displayName: 'Install All Packages' inputs: scriptPath: 'scripts/dev_setup.py' - arguments: '--disabledevelop' - powershell: | cd $(Build.SourcesDirectory)/doc/sphinx