fix: Ensure that all currently supported GCP services allow querygcpTag

package.json

@@ -48,7 +48,7 @@
     "@google-cloud/logging": "^9.6.4",
     "@google-cloud/monitoring": "^2.3.5",
     "@google-cloud/notebooks": "^1.3.1",
-    "@google-cloud/resource-manager": "^3.0.0",
+    "@google-cloud/resource-manager": "^3.2.0",
     "@google-cloud/secret-manager": "^3.10.1",
     "@google-cloud/storage": "^5.16.1",
     "@google-cloud/vpc-access": "^1.1.2",

src/services/base/schema.graphql

@@ -22,4 +22,16 @@ type gcpKeyValue
     id: String! @id @search(by: [hash])
     key: String! @search(by: [hash, regexp])
     value: String @search(by: [hash, regexp])
+  }
+
+  type gcpKeyValues
+  @generate(
+    query: { get: true, query: true, aggregate: true }
+    mutation: { add: true, delete: false }
+    subscription: false
+  ) 
+  @key(fields: "id") {
+    id: String! @id @search(by: [hash])
+    key: String! @search(by: [hash, regexp])
+    values: [String] @search(by: [hash, regexp])
   }

src/services/organization/data.ts

@@ -1,9 +1,13 @@
-import { OrganizationsClient } from '@google-cloud/resource-manager'
+import {
+  OrganizationsClient,
+  TagKeysClient,
+  TagValuesClient,
+} from '@google-cloud/resource-manager'
 import { google } from '@google-cloud/resource-manager/build/protos/protos'
 import CloudGraph from '@cloudgraph/sdk'
 import groupBy from 'lodash/groupBy'
 import gcpLoggerText from '../../properties/logger'
-import { GcpServiceInput } from '../../types'
+import { GcpServiceInput, TagMap } from '../../types'
 import { initTestEndpoint, generateGcpErrorLog } from '../../utils'
 import { GLOBAL_REGION } from '../../config/constants'
 
@@ -17,6 +21,7 @@ export interface RawGcpOrganization
   id: string
   projectId: string
   region: string
+  tags?: TagMap
 }
 
 export const listOrganizationsData = async (
@@ -52,22 +57,78 @@ export const listOrganizationsData = async (
     resolve()
   })
 
+export const getTags = async ({
+  tagKeysClient,
+  tagValuesClient,
+  resourceId,
+}: {
+  tagKeysClient: TagKeysClient
+  tagValuesClient: TagValuesClient
+  resourceId: string
+}): Promise<TagMap> =>
+  new Promise(async resolve => {
+    const tags: TagMap = {}
+    try {
+      const iterable = tagKeysClient.listTagKeysAsync({ parent: resourceId })
+      for await (const response of iterable) {
+        if (response) {
+          const { name: parent,  shortName: tagKey } = response
+          const tagValuesIterable = await tagValuesClient.listTagValuesAsync({ parent })
+          const tagValues: string[] = []
+          for await (const tagValue of tagValuesIterable) {
+            if (tagValue) {
+              tagValues.push(tagValue.shortName)
+            }
+          }
+          tags[tagKey] = tagValues
+        }
+      }
+    } catch (error) {
+      generateGcpErrorLog(
+        serviceName,
+        'resourceManager:listTagKeysAsync',
+        error
+      )
+    }
+    resolve(tags)
+  })
+
 export default async ({
   config,
 }: GcpServiceInput): Promise<{
   [region: string]: RawGcpOrganization[]
 }> =>
   new Promise(async resolve => {
     const orgList: RawGcpOrganization[] = []
+    const tagKeysClient = new TagKeysClient({ ...config, apiEndpoint })
+    const tagValuesClient = new TagValuesClient({ ...config, apiEndpoint })
+    const tagsPromises = []
     const { projectId } = config
 
     const organizationsClient = new OrganizationsClient({
       ...config,
       apiEndpoint,
     })
 
+    // Get all Organizations
     await listOrganizationsData(organizationsClient, projectId, orgList)
 
+    // Add tags to each Organization
+    orgList.map(({ id }, idx) => {
+      const tagsPromise = new Promise<void>(async resolveTags => {
+        const tags = await getTags({
+          tagKeysClient,
+          tagValuesClient,
+          resourceId: id,
+        })
+
+        orgList[idx].tags = tags || {}
+        resolveTags()
+      })
+      tagsPromises.push(tagsPromise)
+    })
+    await Promise.all(tagsPromises)
+
     logger.debug(lt.foundResources(serviceName, orgList.length))
     resolve(groupBy(orgList, 'region'))
   })

src/services/organization/format.ts

@@ -2,7 +2,7 @@ import { google } from '@google-cloud/resource-manager/build/protos/protos'
 import { GcpOrganization } from '../../types/generated'
 import { RawGcpOrganization } from './data'
 import { toISOString } from '../../utils/dateutils'
-import { enumKeyToString } from '../../utils/format'
+import { enumKeyToString, formatKeyValuesFromMap } from '../../utils/format'
 
 export default ({
   service,
@@ -22,6 +22,7 @@ export default ({
     updateTime,
     deleteTime,
     etag,
+    tags = {},
   } = service
 
   return {
@@ -36,5 +37,6 @@ export default ({
     updateTime: toISOString(updateTime?.seconds?.toString()),
     deleteTime: toISOString(deleteTime?.seconds?.toString()),
     etag,
+    tags: formatKeyValuesFromMap(tags),
   }
 }

src/services/organization/schema.graphql

@@ -6,6 +6,7 @@ type gcpOrganization implements gcpBaseResource @key(fields: "id") {
   updateTime: String @search(by: [hash, regexp])
   deleteTime: String @search(by: [hash, regexp])
   etag: String @search(by: [hash, regexp])
+  tags: [gcpKeyValues]
   folders: [gcpFolder] @hasInverse(field: organization)
   projects: [gcpProject] @hasInverse(field: organization)
 }

src/services/tag/schema.graphql

@@ -2,9 +2,7 @@ type gcpTag implements gcpBaseResource @key(fields: "id") {
   id: String! @id @search(by: [hash])
   key: String! @search(by: [hash, regexp])
   value: String! @search(by: [hash, regexp])
-  folder: [gcpFolder]
   organization: [gcpOrganization]
-  project: [gcpProject]
   vmInstance: [gcpVmInstance]
 }
 

src/types/generated.ts

@@ -1933,6 +1933,12 @@ export type GcpKeyValue = {
   value?: Maybe<Scalars['String']>;
 };
 
+export type GcpKeyValues = {
+  id: Scalars['String'];
+  key: Scalars['String'];
+  values?: Maybe<Array<Maybe<Scalars['String']>>>;
+};
+
 export type GcpKmsCryptoKey = GcpBaseResource & {
   aiPlatformNotebooks?: Maybe<Array<Maybe<GcpAiPlatformNotebook>>>;
   createTime?: Maybe<Scalars['String']>;
@@ -2179,6 +2185,7 @@ export type GcpOrganization = GcpBaseResource & {
   folders?: Maybe<Array<Maybe<GcpFolder>>>;
   projects?: Maybe<Array<Maybe<GcpProject>>>;
   state?: Maybe<Scalars['String']>;
+  tags?: Maybe<Array<Maybe<GcpKeyValues>>>;
   updateTime?: Maybe<Scalars['String']>;
 };
 
@@ -2668,11 +2675,9 @@ export type GcpSubnetSecondaryRange = {
 };
 
 export type GcpTag = GcpBaseResource & {
-  folder?: Maybe<Array<Maybe<GcpFolder>>>;
   id: Scalars['String'];
   key: Scalars['String'];
   organization?: Maybe<Array<Maybe<GcpOrganization>>>;
-  project?: Maybe<Array<Maybe<GcpProject>>>;
   value: Scalars['String'];
   vmInstance?: Maybe<Array<Maybe<GcpVmInstance>>>;
 };

src/types/index.ts

@@ -4,21 +4,15 @@ export interface Tags {
 }
 
 export interface TagMap {
-  [property: string]: string
+  [property: string]: string | string []
 }
 
 export interface LabelMap {
   [property: string]: string
 }
 
 export interface KeyValueMapMap {
-  [k: string]: string | number | boolean | Long
-}
-
-export interface GcpServiceInput {
-  regions: string
-  config: GcpCredentials
-  rawData: rawDataInterface[]
+  [k: string]: string | number | boolean | Long | string[]
 }
 
 export interface GcpCredentials {
@@ -38,3 +32,9 @@ export interface rawDataInterface {
   subnet?: string[]
   vpcConnector?: string[]
 }
+
+export interface GcpServiceInput {
+  regions: string
+  config: GcpCredentials
+  rawData: rawDataInterface[]
+}

src/utils/format.ts

@@ -1,6 +1,6 @@
 import cuid from 'cuid'
 import { google } from '@google-cloud/bigquery-data-transfer/build/protos/protos'
-import { GcpKeyValue, GcpRawLabel, GcpRawTag, GcpBigQueryDataTransferParam } from '../types/generated'
+import { GcpKeyValue, GcpKeyValues, GcpRawLabel, GcpRawTag, GcpBigQueryDataTransferParam } from '../types/generated'
 import { TagMap, LabelMap, KeyValueMapMap } from '../types'
 
 export const formatKeyValueMap = (keyValueMap: KeyValueMapMap): GcpKeyValue[] => {
@@ -11,12 +11,23 @@ export const formatKeyValueMap = (keyValueMap: KeyValueMapMap): GcpKeyValue[] =>
   }))
 }
 
+export const formatKeyValuesMap = (keyValueMap: KeyValueMapMap): GcpKeyValues[] => {
+  return Object.keys(keyValueMap || {}).map(key => ({
+    id: cuid(),
+    key,
+    values: keyValueMap[key] as string[]
+  }))
+}
+
 // We need an id here to enfore uniqueness for Dgraph, otherwise we get duplicate tags
 export const formatLabelsFromMap = (labels: LabelMap): GcpRawLabel[] => formatKeyValueMap(labels)
 
 // We need an id here to enfore uniqueness for Dgraph, otherwise we get duplicate tags
 export const formatTagsFromMap = (tags: TagMap): GcpRawTag[] => formatKeyValueMap(tags)
 
+// We need an id here to enfore uniqueness for Dgraph, otherwise we get duplicate tags
+export const formatKeyValuesFromMap = (tags: TagMap): GcpKeyValues[] => formatKeyValuesMap(tags)
+
 export const obfuscateSensitiveString = (s: string): string => {
   const stars = '*'.repeat(Math.min(30, s.length - 6))
   return s.slice(0, 3) + stars + s.slice(stars.length + 3, s.length)

yarn.lock

@@ -897,10 +897,10 @@
   resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.4.tgz#9d8705ecb2baa41b6b2673f3a8e9b7b7e1abc52a"
   integrity sha512-j8yRSSqswWi1QqUGKVEKOG03Q7qOoZP6/h2zN2YO+F5h2+DHU0bSrHCK9Y7lo2DI9fBd8qGAw795sf+3Jva4yA==
 
-"@google-cloud/resource-manager@^3.0.0":
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/@google-cloud/resource-manager/-/resource-manager-3.0.0.tgz#4e81309e3523a410d61eda37d86884b53303bb5a"
-  integrity sha512-gumfF1JiglfnvhRPhpjjc9JbfcdNhcSKSbpYxAMUpFkjiPYd5z01NzEpiSwB/ucT8nUuFCqUrCHVel6gPLXk2g==
+"@google-cloud/resource-manager@^3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@google-cloud/resource-manager/-/resource-manager-3.2.0.tgz#dd0ad3e7efd591d56e06f4318fb1e4798a66fff2"
+  integrity sha512-5LDQ/l7QY7+TCHQzeK8+MKTwt79IxYb5hO5m3v0fd8b+VLBZhKp5bK9usluZZgdnyY7d0zIWsA1MiKtEHEzusA==
   dependencies:
     google-gax "^2.24.1"