Update for bannedDependency includes and beta_ tags (#13211)

* Update for bannedDependency includes and beta_ tags

* Update comment to remove sentence fragment

eng/jacoco-test-coverage/pom.xml

@@ -316,7 +316,8 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
+                <!--This is fine for in here but individual libraries need to list out everything. This mainly because
+                    com.microsoft.azure includes things that are not owned or built by the azure-sdk team. -->
                 <include>com.microsoft.azure:*</include>
               </includes>
             </bannedDependencies>

eng/versioning/set_versions.py

@@ -21,13 +21,13 @@
 # Use case: increment the version of a given artifact in the approprate version_[client|data|management].txt file
 #
 #    python eng/versioning/set_versions.py --bt [client|data|management] --increment-version --artifact-id <artifactId>
-# For example: To update increment the version of azure-core
+# For example: To increment the version of azure-core
 #    python eng/versioning/set_versions.py --bt client --iv --ar azure-core
 #
 # Use case: verify the version of a given artifact in the approprate version_[client|data|management].txt file
 #
 #    python eng/versioning/set_versions.py --bt [client|data|management] --verify-version --artifact-id <artifactId>
-# For example: To update increment the version of azure-core
+# For example: To verify the version of azure-core
 #    python eng/versioning/set_versions.py --bt client --vv --ar azure-core
 #
 # The script must be run at the root of azure-sdk-for-java.
@@ -210,6 +210,9 @@ def prep_version_file_for_source_testing(build_type):
     print('version_file=' + version_file)
     file_changed = False
 
+    # The version map is needed to get the 'current' version of any beta dependencies
+    # in order to update the beta_ version in the From Source runs
+    version_map = {}
     newlines = []
     with open(version_file, encoding='utf-8') as f:
         for raw_line in f:
@@ -221,6 +224,21 @@ def prep_version_file_for_source_testing(build_type):
                 if hasattr(module, 'current') and not module.current == module.dependency:
                     module.dependency = module.current
                     file_changed = True
+                # In order to ensure that the From Source runs are effectively testing everything
+                # together using the latest source built libraries, ensure that the beta_ dependency's
+                # version is set
+                elif module.name.startswith('beta_'):
+                    tempName = module.name[len('beta_'):]
+                    if tempName in version_map:
+                        # beta_ tags only have a dependency version, set that to
+                        # the current version of the non-beta dependency
+                        module.dependency = version_map[tempName].current
+                        file_changed = True
+                    else:
+                        # if the beta_ dependency doesn't have a non-beta entry in the version file then this is an error
+                        raise ValueError('prep_version_file_for_source_testing: beta library ({}) does not have a non-beta entry {} in version file {}'.format(module.name, tempName, version_file))
+
+                version_map[module.name] = module
                 newlines.append(module.string_for_version_file())
 
     with open(version_file, 'w', encoding='utf-8') as f:
@@ -280,8 +298,21 @@ def increment_library_version(build_type, artifact_id, group_id):
                     # The dependency version only needs to be updated it if is different from the current version.
                     # This would be the case where a library hasn't been released yet and has been released (either GA or preview)
                     if (module.dependency != module.current):
-                        print('library_to_update {}, previous dependency version={}, new dependency version={}'.format(library_to_update, module.dependency, module.current))
-                        module.dependency = module.current
+                        vDepMatch = version_regex_named.match(module.dependency)
+                        # If the dependency version is a beta then just set it to whatever the current
+                        # version is
+                        if (vDepMatch.group('prerelease') is not None):
+                            print('library_to_update {}, previous dependency version={}, new dependency version={}'.format(library_to_update, module.dependency, module.current))
+                            module.dependency = module.current
+                        # else, the dependency version isn't a pre-release version
+                        else:
+                            # if the dependency version isn't a beta and the current version is, don't
+                            # update the dependency version
+                            if (vmatch.group('prerelease') is not None):
+                                print('library_to_update {}, has a GA dependency version {} and a beta current version {}. The dependency version will be kept at the GA version. '.format(library_to_update, module.dependency, module.current))
+                            else:
+                                print('library_to_update {}, has both GA dependency {} and current {} versions. The dependency will be updated to {}. '.format(library_to_update, module.dependency, module.current, module.current))
+                                module.dependency = module.current
                     print('library_to_update {}, previous current version={}, new current version={}'.format(library_to_update, module.current, new_version))
                     module.current = new_version
                 newlines.append(module.string_for_version_file())

eng/versioning/update_versions.py

@@ -185,8 +185,20 @@ def load_version_map_from_file(the_file, version_map):
             if not stripped_line or stripped_line.startswith('#'):
                 continue
             module = CodeModule(stripped_line)
+            # verify no duplicate entries
             if (module.name in version_map):
                 raise ValueError('Version file: {0} contains a duplicate entry: {1}'.format(the_file, module.name))
+            # verify that if the module is beta_ or unreleased_ that there's a matching non-beta_ or non-unreleased_ entry
+            if (module.name.startswith('beta_') or module.name.startswith('unreleased_')):
+                tempName = module.name
+                if tempName.startswith('beta_'):
+                    tempName = module.name[len('beta_'):]
+                else:
+                    tempName = module.name[len('unreleased_'):]
+                # if there isn't a non beta or unreleased entry then raise an issue
+                if tempName not in version_map:
+                    raise ValueError('Version file: {0} does not contain a non-beta or non-unreleased entry for beta_/unreleased_ library: {1}'.format(the_file, module.name))
+
             version_map[module.name] = module
 
 def display_version_info(version_map):

sdk/core/azure-core-amqp/pom.xml

@@ -114,7 +114,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>com.microsoft.azure:qpid-proton-j-extensions:[1.2.3]</include> <!-- {x-include-update;com.microsoft.azure:qpid-proton-j-extensions;external_dependency} -->
                 <include>org.apache.qpid:proton-j:[0.33.4]</include> <!-- {x-include-update;org.apache.qpid:proton-j;external_dependency} -->
               </includes>

sdk/core/azure-core-http-netty/pom.xml

@@ -184,7 +184,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>io.netty:netty-buffer:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-buffer;external_dependency} -->
                 <include>io.netty:netty-codec-http:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http;external_dependency} -->
                 <include>io.netty:netty-codec-http2:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http2;external_dependency} -->
@@ -193,7 +192,7 @@
                 <include>io.netty:netty-transport-native-unix-common:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-unix-common;external_dependency} -->
                 <include>io.netty:netty-transport-native-epoll:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-epoll;external_dependency} -->
                 <include>io.projectreactor.netty:reactor-netty:[0.9.7.RELEASE]</include> <!-- {x-include-update;io.projectreactor.netty:reactor-netty;external_dependency} -->
-                <include>io.netty:netty-transport-native-kqueue:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-kqueue;external_dependency} -->                
+                <include>io.netty:netty-transport-native-kqueue:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-kqueue;external_dependency} -->
               </includes>
             </bannedDependencies>
           </rules>

sdk/core/azure-core-http-okhttp/pom.xml

@@ -140,7 +140,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>com.squareup.okhttp3:okhttp:[4.2.2]</include> <!-- {x-include-update;com.squareup.okhttp3:okhttp;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/core/azure-core-serializer-avro-apache/pom.xml

@@ -108,7 +108,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>org.apache.avro:avro:[1.9.2]</include> <!-- {x-include-update;org.apache.avro:avro;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/core/azure-core-serializer-avro-jackson/pom.xml

@@ -108,7 +108,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>com.fasterxml.jackson.dataformat:jackson-dataformat-avro:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.dataformat:jackson-dataformat-avro;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/core/azure-core-serializer-json-gson/pom.xml

@@ -114,7 +114,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>com.google.code.gson:gson:[2.8.5]</include> <!-- {x-include-update;com.google.code.gson:gson;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/core/azure-core-test/pom.xml

@@ -107,7 +107,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>io.projectreactor:reactor-test:[3.3.5.RELEASE]</include> <!-- {x-include-update;io.projectreactor:reactor-test;external_dependency} -->
                 <!-- special allowance for azure-core-test as it is not a shipping library: -->
                 <include>org.junit.jupiter:junit-jupiter-api:[5.6.2]</include> <!-- {x-include-update;org.junit.jupiter:junit-jupiter-api;external_dependency} -->

sdk/core/azure-core-tracing-opentelemetry/pom.xml

@@ -80,7 +80,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>io.opentelemetry:opentelemetry-api:[0.2.4]</include> <!-- {x-include-update;io.opentelemetry:opentelemetry-api;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/core/azure-core/pom.xml

@@ -180,7 +180,6 @@
                 </include> <!-- {x-include-update;com.fasterxml.jackson.dataformat:jackson-dataformat-xml;external_dependency} -->
                 <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1]
                 </include> <!-- {x-include-update;com.fasterxml.jackson.datatype:jackson-datatype-jsr310;external_dependency} -->
-                <include>com.google.code.findbugs:jsr305:[3.0.2]</include> <!-- {x-include-update;com.google.code.findbugs:jsr305;external_dependency} -->
                 <include>org.slf4j:slf4j-api:[1.7.28]</include> <!-- {x-include-update;org.slf4j:slf4j-api;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/cosmos/azure-cosmos-benchmark/pom.xml

@@ -223,7 +223,6 @@ Licensed under the MIT License.
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>com.beust:jcommander:[1.58]</include> <!-- {x-include-update;com.beust:jcommander;external_dependency} -->
                 <include>io.dropwizard.metrics:metrics-core:[4.1.0]</include> <!-- {x-include-update;cosmos_io.dropwizard.metrics:metrics-core;external_dependency} -->
                 <include>org.slf4j:slf4j-api:[1.7.28]</include> <!-- {x-include-update;org.slf4j:slf4j-api;external_dependency} -->

sdk/cosmos/azure-cosmos/pom.xml

@@ -277,13 +277,8 @@ Licensed under the MIT License.
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
-                <include>com.fasterxml.jackson.core:jackson-core:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.core:jackson-core;external_dependency} -->
-                <include>com.fasterxml.jackson.core:jackson-annotations:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.core:jackson-annotations;external_dependency} -->
-                <include>com.fasterxml.jackson.core:jackson-databind:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.core:jackson-databind;external_dependency} -->
                 <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.datatype:jackson-datatype-jsr310;external_dependency} -->
                 <include>com.fasterxml.jackson.module:jackson-module-afterburner:[2.10.1]</include> <!-- {x-include-update;cosmos_com.fasterxml.jackson.module:jackson-module-afterburner;external_dependency} -->
-                <include>com.google.code.findbugs:jsr305:[3.0.2]</include> <!-- {x-include-update;com.google.code.findbugs:jsr305;external_dependency} -->
                 <include>io.dropwizard.metrics:metrics-core:[4.1.0]</include> <!-- {x-include-update;cosmos_io.dropwizard.metrics:metrics-core;external_dependency} -->
                 <include>io.micrometer:micrometer-core:[1.2.0]</include> <!-- {x-include-update;io.micrometer:micrometer-core;external_dependency} -->
                 <include>io.netty:netty-codec-http:[4.1.49.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http;external_dependency} -->

sdk/cosmos/azure-spring-data-cosmos/pom.xml

@@ -191,7 +191,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>org.springframework:spring-beans:[5.2.6.RELEASE]</include> <!-- {x-include-update;org.springframework:spring-beans;external_dependency} -->
                 <include>org.springframework:spring-web:[5.2.6.RELEASE]</include> <!-- {x-include-update;org.springframework:spring-web;external_dependency} -->
                 <include>org.springframework:spring-tx:[5.2.6.RELEASE]</include> <!-- {x-include-update;org.springframework:spring-tx;external_dependency} -->
@@ -206,7 +205,6 @@
                 <include>org.json:json:[20140107]</include> <!-- {x-include-update;org.json:json;external_dependency} -->
                 <include>com.fasterxml.jackson.module:jackson-module-parameter-names:[2.10.0]</include> <!-- {x-include-update;com.fasterxml.jackson.module:jackson-module-parameter-names;external_dependency} -->
                 <include>javax.annotation:javax.annotation-api:[1.3.2]</include> <!-- {x-include-update;javax.annotation:javax.annotation-api;external_dependency} -->
-                <include>org.slf4j:slf4j-simple:[1.7.25]</include> <!-- {x-include-update;org.slf4j:slf4j-simple;external_dependency} -->
               </includes>
             </bannedDependencies>
           </rules>

sdk/e2e/pom.xml

@@ -86,7 +86,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>org.slf4j:slf4j-api:[1.7.28]</include> <!-- {x-include-update;org.slf4j:slf4j-api;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/identity/azure-identity/pom.xml

@@ -116,7 +116,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>com.microsoft.azure:msal4j:[1.3.0]</include> <!-- {x-include-update;com.microsoft.azure:msal4j;external_dependency} -->
                 <include>com.microsoft.azure:msal4j-persistence-extension:[0.1]</include> <!-- {x-include-update;com.microsoft.azure:msal4j-persistence-extension;external_dependency} -->
                 <include>com.nimbusds:oauth2-oidc-sdk:[6.14]</include> <!-- {x-include-update;com.nimbusds:oauth2-oidc-sdk;external_dependency} -->

sdk/parents/azure-client-sdk-parent/pom.xml

@@ -473,7 +473,7 @@
                 <exclude>*:*:*:*:compile</exclude>
                 <exclude>*:*:*:*:provided</exclude>
               </excludes>
-              <includes>
+              <includes combine.children="append">
                 <include>com.azure:*</include>
 
                 <!-- Used by many libraries to bring in annotations used by Reactor -->

sdk/schemaregistry/azure-data-schemaregistry-avro/pom.xml

@@ -85,7 +85,6 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
                 <include>org.apache.avro:avro:[1.9.2]</include> <!-- {x-include-update;org.apache.avro:avro;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/spring/azure-spring-boot-test-aad/pom.xml

@@ -47,8 +47,7 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
-                <include>com.microsoft.azure:*</include>
+                <include>com.microsoft.azure:azure-active-directory-spring-boot-starter:[2.3.3-beta.1]</include> <!-- {x-include-update;com.microsoft.azure:azure-active-directory-spring-boot-starter;current} -->
                 <include>org.springframework.boot:spring-boot-starter-web:[2.3.0.RELEASE]</include> <!-- {x-include-update;org.springframework.boot:spring-boot-starter-web;external_dependency} -->
               </includes>
             </bannedDependencies>

sdk/spring/azure-spring-boot-test-core/pom.xml

@@ -95,8 +95,8 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
-                <include>com.microsoft.azure:*</include>
+                <include>com.microsoft.azure:azure:[1.34.0]</include> <!-- {x-include-update;spring_com.microsoft.azure:azure;external_dependency} -->
+                <include>com.microsoft.azure:azure-spring-boot-starter:[2.3.3-beta.1]</include> <!-- {x-include-update;com.microsoft.azure:azure-spring-boot-starter;current} -->
                 <include>com.jcraft:jsch:[0.1.53]</include> <!-- {x-include-update;com.jcraft:jsch;external_dependency} -->
                 <include>com.github.cverges.expect4j:expect4j:[1.6]</include> <!-- {x-include-update;com.github.cverges.expect4j:expect4j;external_dependency} -->
                 <include>org.apache.maven:maven-embedder:[3.6.2]</include> <!-- {x-include-update;org.apache.maven:maven-embedder;external_dependency} -->

sdk/spring/azure-spring-boot-test-cosmosdb/pom.xml

@@ -47,8 +47,7 @@
           <rules>
             <bannedDependencies>
               <includes>
-                <include>com.azure:*</include>
-                <include>com.microsoft.azure:*</include>
+                <include>com.microsoft.azure:azure-cosmosdb-spring-boot-starter:[2.3.3-beta.1]</include> <!-- {x-include-update;com.microsoft.azure:azure-cosmosdb-spring-boot-starter;current} -->
                 <include>org.springframework.boot:spring-boot-starter-web:[2.3.0.RELEASE]</include> <!-- {x-include-update;org.springframework.boot:spring-boot-starter-web;external_dependency} -->
               </includes>
             </bannedDependencies>