feat(bigquery): Add Routine

* Add Dataset#create_routine
* Add Argument
* Update StandardSql classes to expose public initializer
* Add Data#ddl_target_routine and QueryJob#ddl_target_routine

pr: #4707 
closes: #4430

google-cloud-bigquery/acceptance/bigquery/bigquery_test.rb

@@ -146,6 +146,7 @@
     # job.statement_type.must_equal "SELECT"
     job.ddl_operation_performed.must_be :nil?
     job.ddl_target_table.must_be :nil?
+    job.ddl_target_routine.must_be :nil?
   end
 
   it "should run a query job with dryrun flag" do

google-cloud-bigquery/acceptance/bigquery/model_test.rb

@@ -25,7 +25,7 @@
   end
   let(:model_id) { "model_#{SecureRandom.hex(4)}" }
   let :model_sql do
-    model_sql = <<-MODEL_SQL
+    model_sql = <<~MODEL_SQL
     CREATE MODEL #{dataset.dataset_id}.#{model_id}
     OPTIONS (
         model_type='linear_reg',

google-cloud-bigquery/acceptance/bigquery/routine_test.rb

@@ -0,0 +1,230 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "bigquery_helper"
+
+describe Google::Cloud::Bigquery, :bigquery do
+  let(:dataset_id) { "#{prefix}_dataset" }
+  let(:dataset) do
+    d = bigquery.dataset dataset_id
+    if d.nil?
+      d = bigquery.create_dataset dataset_id
+    end
+    d
+  end
+  let(:routine_id) { "routine_#{SecureRandom.hex(4)}" }
+  let :routine_sql do
+    routine_sql = <<~SQL
+    CREATE FUNCTION `#{routine_id}`(
+        arr ARRAY<STRUCT<name STRING, val INT64>>
+    ) AS (
+        (SELECT SUM(IF(elem.name = "foo",elem.val,null)) FROM UNNEST(arr) AS elem)
+    )
+    SQL
+  end
+
+  it "can create from SQL, list, read, update, and delete a routine" do
+    # create from sql
+    job = dataset.query_job routine_sql
+    job.wait_until_done!
+    job.wont_be :failed?
+    job.ddl_operation_performed.must_equal "CREATE"
+    routine = job.ddl_target_routine
+    routine.must_be_kind_of Google::Cloud::Bigquery::Routine
+    routine.reference?.must_equal true
+    routine.project_id.must_equal bigquery.project
+    routine.dataset_id.must_equal dataset.dataset_id
+    routine.routine_id.must_equal routine_id
+
+    # list
+    dataset.routines.all.map(&:routine_id).must_include routine_id
+
+    # list with filter
+    dataset.routines(filter: "routineType:SCALAR_FUNCTION").all.map(&:routine_id).must_include routine_id
+
+    # list with filter
+    dataset.routines(filter: "routineType:PROCEDURE").all.map(&:routine_id).wont_include routine_id
+
+    # get
+    routine = dataset.routine routine_id
+    routine.must_be_kind_of Google::Cloud::Bigquery::Routine
+    routine.project_id.must_equal bigquery.project
+    routine.dataset_id.must_equal dataset.dataset_id
+    routine.routine_id.must_equal routine_id
+
+    routine.description.must_be :nil?
+    routine.routine_type.must_equal "SCALAR_FUNCTION"
+    routine.language.must_equal "SQL"
+    routine.body.must_equal "(SELECT SUM(IF(elem.name = \"foo\",elem.val,null)) FROM UNNEST(arr) AS elem)"
+
+    arguments = routine.arguments
+    arguments.must_be_kind_of Array
+    arguments.size.must_equal 1
+
+    argument = arguments.first
+    argument.must_be_kind_of Google::Cloud::Bigquery::Argument
+    argument.argument_kind.must_be :nil?
+    argument.fixed_type?.must_equal true
+    argument.any_type?.must_equal false
+    argument.mode.must_be :nil?
+    argument.in?.must_equal false
+    argument.out?.must_equal false
+    argument.inout?.must_equal false
+    argument.name.must_equal "arr"
+
+    data_type = argument.data_type
+    data_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    data_type.type_kind.must_equal "ARRAY"
+    data_type.struct_type.must_be :nil?
+    data_type.array_element_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    data_type.array_element_type.type_kind.must_equal "STRUCT"
+    data_type.array_element_type.struct_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::StructType
+
+    struct_fields = data_type.array_element_type.struct_type.fields
+    struct_fields.must_be_kind_of Array
+    struct_fields.size.must_equal 2
+    struct_fields[0].must_be_kind_of Google::Cloud::Bigquery::StandardSql::Field
+    struct_fields[0].name.must_equal "name"
+    struct_fields[0].type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    struct_fields[0].type.type_kind.must_equal "STRING"
+    struct_fields[1].must_be_kind_of Google::Cloud::Bigquery::StandardSql::Field
+    struct_fields[1].name.must_equal "val"
+    struct_fields[1].type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    struct_fields[1].type.type_kind.must_equal "INT64"
+
+    # update
+    new_description = "Routine was updated #{Time.now}"
+    routine.description = new_description
+    routine.refresh!
+    routine.description.must_equal new_description
+
+    # delete
+    routine.delete.must_equal true
+
+    dataset.routine(routine_id).must_be_nil
+  end
+
+  it "can create, update and delete a routine" do
+    # create
+    routine = dataset.create_routine routine_id do |r|
+      r.routine_type = "SCALAR_FUNCTION"
+      r.language = :SQL
+      r.arguments = [
+        Google::Cloud::Bigquery::Argument.new(name: "x", data_type: "INT64")
+      ]
+      r.body = "x * 3"
+      r.description = "my description"
+    end
+
+    routine.must_be_kind_of Google::Cloud::Bigquery::Routine
+    routine.project_id.must_equal bigquery.project
+    routine.dataset_id.must_equal dataset.dataset_id
+    routine.routine_id.must_equal routine_id
+
+    routine.description.must_equal "my description"
+    routine.routine_type.must_equal "SCALAR_FUNCTION"
+    routine.language.must_equal "SQL"
+    routine.body.must_equal "x * 3"
+
+    arguments = routine.arguments
+    arguments.must_be_kind_of Array
+    arguments.size.must_equal 1
+
+    argument = arguments.first
+    argument.must_be_kind_of Google::Cloud::Bigquery::Argument
+    argument.argument_kind.must_be :nil?
+    argument.mode.must_be :nil?
+    argument.name.must_equal "x"
+
+    data_type = argument.data_type
+    data_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    data_type.type_kind.must_equal "INT64"
+    data_type.array_element_type.must_be :nil?
+    data_type.struct_type.must_be :nil?
+
+    # update 
+    new_body = "(SELECT SUM(IF(elem.name = \"foo\",elem.val,null)) FROM UNNEST(arr) AS elem)"
+    new_arguments = [
+      Google::Cloud::Bigquery::Argument.new(
+        name: "arr",
+        argument_kind: "FIXED_TYPE",
+        data_type: Google::Cloud::Bigquery::StandardSql::DataType.new(
+          type_kind: "ARRAY",
+          array_element_type: Google::Cloud::Bigquery::StandardSql::DataType.new(
+            type_kind: "STRUCT",
+            struct_type: Google::Cloud::Bigquery::StandardSql::StructType.new(
+              fields: [
+                Google::Cloud::Bigquery::StandardSql::Field.new(
+                  name: "name",
+                  type: Google::Cloud::Bigquery::StandardSql::DataType.new(type_kind: "STRING")
+                ),
+                Google::Cloud::Bigquery::StandardSql::Field.new(
+                  name: "val",
+                  type: Google::Cloud::Bigquery::StandardSql::DataType.new(type_kind: "INT64")
+                )
+              ]
+            )
+          )
+        )
+      )
+    ]
+
+    routine.update do |r|
+      r.body = new_body
+      r.arguments = new_arguments
+    end
+
+    routine.body.must_equal new_body
+
+    arguments = routine.arguments
+    arguments.must_be_kind_of Array
+    arguments.size.must_equal 1
+
+    argument = arguments.first
+    argument.must_be_kind_of Google::Cloud::Bigquery::Argument
+    argument.argument_kind.must_equal "FIXED_TYPE"
+    argument.mode.must_be :nil?
+    argument.name.must_equal "arr"
+
+    data_type = argument.data_type
+    data_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    data_type.type_kind.must_equal "ARRAY"
+    data_type.struct_type.must_be :nil?
+    data_type.array_element_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    data_type.array_element_type.type_kind.must_equal "STRUCT"
+    data_type.array_element_type.struct_type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::StructType
+
+    struct_fields = data_type.array_element_type.struct_type.fields
+    struct_fields.must_be_kind_of Array
+    struct_fields.size.must_equal 2
+    struct_fields[0].must_be_kind_of Google::Cloud::Bigquery::StandardSql::Field
+    struct_fields[0].name.must_equal "name"
+    struct_fields[0].type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    struct_fields[0].type.type_kind.must_equal "STRING"
+    struct_fields[1].must_be_kind_of Google::Cloud::Bigquery::StandardSql::Field
+    struct_fields[1].name.must_equal "val"
+    struct_fields[1].type.must_be_kind_of Google::Cloud::Bigquery::StandardSql::DataType
+    struct_fields[1].type.type_kind.must_equal "INT64"
+
+    # get
+    routine.reload!
+    routine.body.must_equal new_body
+    routine.arguments.first.data_type.array_element_type.struct_type.fields.last.type.type_kind.must_equal "INT64"
+
+    # delete
+    routine.delete.must_equal true
+
+    dataset.routine(routine_id).must_be_nil
+  end
+end

google-cloud-bigquery/acceptance/bigquery/standard_query_test.rb

@@ -112,4 +112,20 @@
     rows.count.must_equal 1
     rows.first[:value].must_equal ["foo", "bar", "baz"]
   end
+
+  it "queries a struct with no names" do
+    rows = bigquery.query "SELECT STRUCT(1, 'abc', 3.14) AS value", standard_sql: true
+
+    rows.class.must_equal Google::Cloud::Bigquery::Data
+    rows.count.must_equal 1
+    rows.first[:value].must_equal({ _field_1: 1, _field_2: "abc", _field_3: 3.14 })
+  end
+
+  it "queries a struct with duplicate names" do
+    rows = bigquery.query "SELECT STRUCT(1 AS x, 'abc' AS x, 3.14 AS x) AS value", standard_sql: true
+
+    rows.class.must_equal Google::Cloud::Bigquery::Data
+    rows.count.must_equal 1
+    rows.first[:value].must_equal({ x: 1, _field_2: "abc", _field_3: 3.14 })
+  end
 end