Merge branch 'master' into make_some_bq_magic

.ci/magic-modules/generate-ansible.sh

@@ -13,10 +13,7 @@ pushd magic-modules-branched
 # Choose the author of the most recent commit as the downstream author
 COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 
-for i in $(find products/ -name 'ansible.yaml' -printf '%h\n');
-do
-  bundle exec compiler -p $i -e ansible -o "build/ansible/"
-done
+bundle exec compiler -a -e ansible -o "build/ansible/"
 
 ANSIBLE_COMMIT_MSG="$(cat .git/title)"
 

.ci/magic-modules/generate-inspec.sh

@@ -13,10 +13,7 @@ pushd magic-modules-branched
 # Choose the author of the most recent commit as the downstream author
 COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 
-for i in $(find products/ -name 'inspec.yaml' -printf '%h\n');
-do
-  bundle exec compiler -p $i -e inspec -o "build/inspec/"
-done
+bundle exec compiler -a -e inspec -o "build/inspec/"
 
 INSPEC_COMMIT_MSG="$(cat .git/title)"
 

api/async.rb

@@ -47,13 +47,20 @@ class Operation < Api::Object
       attr_reader :wait_ms
       attr_reader :timeouts
 
+      # Use this if the resource includes the full operation url.
+      attr_reader :full_url
+
       def validate
         super
 
         check :kind, type: String
         check :path, type: String, required: true
-        check :base_url, type: String, required: true
+        check :base_url, type: String
         check :wait_ms, type: Integer, required: true
+
+        check :full_url, type: String
+
+        conflicts %i[base_url full_url]
       end
     end
 

api/product/version.rb

@@ -25,7 +25,7 @@ class Version < Api::Object
       attr_reader :base_url
       attr_reader :name
 
-      ORDER = %w[ga beta alpha].freeze
+      ORDER = %w[ga beta alpha private].freeze
 
       def validate
         super

api/resource.rb

@@ -86,12 +86,13 @@ module Properties
       # within the collection (list) json. Will default to the
       # camelcase pluralize name of the resource.
       attr_reader :collection_url_key
-      # [Optional] This is an array with items that uniquely identify the
-      # resource.
-      # This is useful in case an API returns a list result and we need
-      # to fetch the particular resource we're interested in from that
-      # list.  Otherwise, it's safe to leave empty.
-      # If empty, we assume that `name` is the identifier.
+      # [Optional] An ordered list of names of parameters that uniquely identify
+      # the resource.
+      # Generally, it's safe to leave empty, in which case it defaults to `name`.
+      # Other values are normally useful in cases where an object has a parent
+      # and is identified by some non-name value, such as an ip+port pair.
+      # If you're writing a fine-grained resource (eg with nested_query) a value
+      # must be set.
       attr_reader :identity
       # [Optional] (Api::Resource::NestedQuery) This is useful in case you need
       # to change the query made for GET requests only. In particular, this is
@@ -191,7 +192,8 @@ def parameters
     end
 
     # Return the user-facing properties in client tools; this ends up meaning
-    # both properties and parameters.
+    # both properties and parameters but without any that are excluded due to
+    # version mismatches or manual exclusion
     def all_user_properties
       properties + parameters
     end
@@ -222,7 +224,7 @@ def gettable_properties
     # Returns the list of top-level properties once any nested objects with flatten_object
     # set to true have been collapsed
     def root_properties
-      properties.flat_map do |p|
+      all_user_properties.flat_map do |p|
         if p.flatten_object
           p.root_properties
         else
@@ -246,7 +248,7 @@ def identity
       if @identity.nil?
         props.select { |p| p.name == Api::Type::String::NAME.name }
       else
-        props.select { |p| @identity.include?(p.name) }
+        props.select { |p| @identity.include?(p.name) }.sort_by { |p| @identity.index p.name }
       end
     end
 

compiler.rb

@@ -34,7 +34,7 @@
 require 'provider/terraform_object_library'
 require 'pp' if ENV['COMPILER_DEBUG']
 
-product_names = nil
+products_to_compile = nil
 all_products = false
 yaml_dump = false
 output_path = nil
@@ -50,7 +50,7 @@
 # rubocop:disable Metrics/BlockLength
 OptionParser.new do |opt|
   opt.on('-p', '--product PRODUCT', Array, 'Folder[,Folder...] with product catalog') do |p|
-    product_names = p
+    products_to_compile = p
   end
   opt.on('-a', '--all', 'Build all products. Cannot be used with --product.') do
     all_products = true
@@ -86,32 +86,35 @@
 end.parse!
 # rubocop:enable Metrics/BlockLength
 
-raise 'Cannot use -p/--products and -a/--all simultaneously' if product_names && all_products
-raise 'Either -p/--products OR -a/--all must be present' if product_names.nil? && !all_products
+raise 'Cannot use -p/--products and -a/--all simultaneously' \
+  if products_to_compile && all_products
+raise 'Either -p/--products OR -a/--all must be present' \
+  if products_to_compile.nil? && !all_products
 raise 'Option -o/--output is a required parameter' if output_path.nil?
 raise 'Option -e/--engine is a required parameter' if provider_name.nil?
 
-if all_products
-  product_names = []
-  Dir['products/**/api.yaml'].each do |file_path|
-    product_names.push(File.dirname(file_path))
-  end
+all_product_files = []
+Dir['products/**/api.yaml'].each do |file_path|
+  all_product_files.push(File.dirname(file_path))
+end
 
-  if override_dir
-    Dir["#{override_dir}/products/**/api.yaml"].each do |file_path|
-      product = File.dirname(Pathname.new(file_path).relative_path_from(override_dir))
-      product_names.push(product) unless product_names.include? product
-    end
+if override_dir
+  Dir["#{override_dir}/products/**/api.yaml"].each do |file_path|
+    product = File.dirname(Pathname.new(file_path).relative_path_from(override_dir))
+    all_product_files.push(product) unless all_product_files.include? product
   end
-
-  raise 'No api.yaml files found. Check provider/engine name.' if product_names.empty?
 end
 
+products_to_compile = all_product_files if all_products
+
+raise 'No api.yaml files found. Check provider/engine name.' if products_to_compile.empty?
+
 start_time = Time.now
 
+products_for_version = []
 provider = nil
 # rubocop:disable Metrics/BlockLength
-product_names.each do |product_name|
+all_product_files.each do |product_name|
   product_override_path = ''
   provider_override_path = ''
   product_override_path = File.join(override_dir, product_name, 'api.yaml') if override_dir
@@ -143,9 +146,7 @@
   raise "Output path '#{output_path}' does not exist or is not a directory" \
     unless Dir.exist?(output_path)
 
-  Google::LOGGER.info "Compiling '#{product_name}' (at #{version}) output to '#{output_path}'"
-  Google::LOGGER.info \
-    "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
+  Google::LOGGER.info "Loading '#{product_name}' (at #{version})'"
 
   product_api = Api::Compiler.new(product_yaml).run
   product_api.validate
@@ -166,6 +167,16 @@
     product_api, provider_config, = \
       Provider::Config.parse(provider_override_path, product_api, version, override_dir)
   end
+  products_for_version.push(product_api.name)
+
+  unless products_to_compile.include?(product_name)
+    Google::LOGGER.info "Skipping product '#{product_name}' as it was not specified to be compiled"
+    next
+  end
+
+  Google::LOGGER.info "Compiling '#{product_name}' (at #{version}) output to '#{output_path}'"
+  Google::LOGGER.info \
+    "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
 
   pp provider_config if ENV['COMPILER_DEBUG']
 
@@ -179,8 +190,10 @@
     }
 
     provider_class = override_providers[force_provider]
-    raise "Invalid force provider option #{force_provider}" \
-      if provider_class.nil?
+    if provider_class.nil?
+      raise "Invalid force provider option #{force_provider}." \
+        + "\nPossible values #{override_providers} "
+    end
 
     provider = \
       override_providers[force_provider].new(provider_config, product_api, start_time)
@@ -193,6 +206,5 @@
 # of the products loop. This will get called with the provider from the final iteration
 # of the loop
 provider&.copy_common_files(output_path, version)
-provider&.compile_common_files(output_path, version)
-
+provider&.compile_common_files(output_path, version, products_for_version.sort)
 # rubocop:enable Metrics/BlockLength

google/yaml_validator.rb

@@ -82,6 +82,16 @@ def check(variable, **opts)
         unless opts[:allowed].include?(value)
     end
 
+    def conflicts(list)
+      value_checked = false
+      list.each do |item|
+        next if instance_variable_get("@#{item}").nil?
+        raise "#{list.join(',')} cannot be set at the same time" if value_checked
+
+        value_checked = true
+      end
+    end
+
     private
 
     def check_type(name, object, type)

products/binaryauthorization/api.yaml

@@ -84,18 +84,47 @@ objects:
                 - !ruby/object:Api::Type::String
                   name: id
                   description: |
-                    This field will be overwritten with key ID information, for
-                    example, an identifier extracted from a PGP public key. This
-                    field may not be updated.
-                  output: true
+                    The ID of this public key. Signatures verified by BinAuthz
+                    must include the ID of the public key that can be used to
+                    verify them, and that ID must match the contents of this
+                    field exactly. Additional restrictions on this field can
+                    be imposed based on which public key type is encapsulated.
+                    See the documentation on publicKey cases below for details.
                 - !ruby/object:Api::Type::String
                   name: asciiArmoredPgpPublicKey
                   description: |
                     ASCII-armored representation of a PGP public key, as the
                     entire output by the command
                     `gpg --export --armor foo@example.com` (either LF or CRLF
-                    line endings).
-                  required: true
+                    line endings). When using this field, id should be left
+                    blank. The BinAuthz API handlers will calculate the ID
+                    and fill it in automatically. BinAuthz computes this ID
+                    as the OpenPGP RFC4880 V4 fingerprint, represented as
+                    upper-case hex. If id is provided by the caller, it will
+                    be overwritten by the API-calculated ID.
+                - !ruby/object:Api::Type::NestedObject
+                  name: pkixPublicKey
+                  description: |
+                    A raw PKIX SubjectPublicKeyInfo format public key.
+                    
+                    NOTE: id may be explicitly provided by the caller when using this
+                    type of public key, but it MUST be a valid RFC3986 URI. If id is left
+                    blank, a default one will be computed based on the digest of the DER
+                    encoding of the public key.
+                  properties:
+                  - !ruby/object:Api::Type::String
+                    name: publicKeyPem
+                    description: |
+                      A PEM-encoded public key, as described in
+                      `https://tools.ietf.org/html/rfc7468#section-13`
+                  - !ruby/object:Api::Type::String
+                    name: signatureAlgorithm
+                    description: |
+                      The signature algorithm used to verify a message against
+                      a signature using this key. These signature algorithm must
+                      match the structure and any object identifiers encoded in
+                      publicKeyPem (i.e. this algorithm must match that of the
+                      public key).
           - !ruby/object:Api::Type::String
             name: delegationServiceAccountEmail
             description: |

products/binaryauthorization/terraform.yaml

@@ -23,6 +23,15 @@ overrides: !ruby/object:Overrides::ResourceOverrides
       vars:
         attestor_name: "test-attestor"
         note_name: "test-attestor-note"
+     - !ruby/object:Provider::Terraform::Examples
+      name: "binary_authorization_attestor_kms"
+      primary_resource_id: "attestor"
+      skip_test: true
+      vars:
+        attestor_name: "test-attestor"
+        note_name: "test-attestor-note"
+        key_name: "test-attestor-key"
+        keyring_name: "test-attestor-key-ring"
     custom_code: !ruby/object:Provider::Terraform::CustomCode
       encoder: templates/terraform/encoders/binauth_attestor_note_field_name.go.erb
       decoder: templates/terraform/decoders/binauth_attestor_note_field_name.go.erb