Skip to content
This repository has been archived by the owner on Jan 26, 2023. It is now read-only.

Bulk paginated api for backfilling #29

Merged
merged 15 commits into from
Sep 19, 2019
Merged

Bulk paginated api for backfilling #29

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
0832827
first pass on bulk paginated api for backfilling
ensary Sep 9, 2019
dd1845c
typos and linter issues
ensary Sep 9, 2019
80d11e4
change naming
ensary Sep 9, 2019
219dc42
still can not get goimports to run properly from IDE (coneofshame)
ensary Sep 9, 2019
21d338e
gen.go and tests for handler
ensary Sep 11, 2019
deccf20
add page token and filtering resources by CF type
ensary Sep 12, 2019
c826c0d
remove enum value as it is not extensible, fix default integer value
ensary Sep 12, 2019
0a9641b
change the page api so that it does not clash with asset fetching by id
ensary Sep 16, 2019
3ab2aec
Add parameter handling for resource type, fix SQL issue, add paging code
ensary Sep 18, 2019
e25e779
linter/style
ensary Sep 18, 2019
e4e03a5
more tests
ensary Sep 18, 2019
f1ce3d6
linter
ensary Sep 18, 2019
87bbb8a
more tests
ensary Sep 19, 2019
d667a34
stop exposing internal structure for pageToken to caller, remove debu…
ensary Sep 19, 2019
1507f33
formatting
ensary Sep 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions api.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,75 @@ paths:
#! if eq .Response.Body.errorType "InvalidInput" !# 400
#! else !# 500
#! end !#, "bodyPassthrough": true}'
/v1/cloud/asset:
ensary marked this conversation as resolved.
Show resolved Hide resolved
get:
summary: "Retrieve a list of known cloud assets at point in time"
parameters:
- name: "time"
in: "query"
description: "The point in time details for asset list"
required: true
schema:
type: "string"
format: "date-time"
- name: "count"
in: "query"
description: "Number of cloud assets to return"
required: true
schema:
type: "integer"
minimum: 1
- name: "offset"
in: "query"
description: "Offset of the first cloud asset to return for paged API access. Zero-based."
required: false
schema:
type: "integer"
ensary marked this conversation as resolved.
Show resolved Hide resolved
minimum: 0
responses:
200:
description: "List of all assets found with the IP address at the given time"
content:
application/json:
schema:
$ref: "#/components/schemas/CloudAssets"
400:
description: "Invalid input"
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
404:
description: "The asset is not found"
x-transportd:
backend: app
enabled:
- "accesslog"
- "requestvalidation"
- "responsevalidation"
- "lambda"
lambda:
arn: "fetchAllAssetsByTime"
async: false
request: >
{
"time": "#!index .Request.Query.time 0!#",
"count": #!index .Request.Query.count 0!# #! if .Request.Query.offset !# ,
"offset": #!index .Request.Query.offset 0!#
#! end !#
}
success: '{"status": 200, "bodyPassthrough": true}'
error: >
{
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: After playing around with this syntax a bunch, I personally find something like this to be a little easier to read, since it separates the JSON response from the template syntax. There's more duplication here though, so I have no strong feelings either way.

            #! if eq .Response.Body.errorType "NotFound" !#
            { "status": 404 }
            #! else if eq .Response.Body.errorType "DependencyFailure" !#
            { "status": 502, "body": { "code": 502, "status": "Bad Gateway", "reason": "#!.Response.Body.errorMessage!#" }}
            #! else !#
            { "status": 500, "body": { "code": 500, "status": "Internal Server Error", "reason": "#!.Response.Body.errorMessage!#" }}
            #! end !#

"status":
#! if eq .Response.Body.errorType "InvalidInput" !# 400,
#! else !#
#! if eq .Response.Body.errorType "NotFound" !# 404,
#! else !# 500,
#! end !#
#! end !#
"bodyPassthrough": true
}
/v1/cloud/ip/{ipAddress}:
get:
summary: "Retrieve a cloud asset at a point in time by IP Address"
Expand Down
8 changes: 6 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.12

require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect
github.com/asecurityteam/logevent v0.0.0-20190225122144-b32737d8d51c // indirect
github.com/asecurityteam/runhttp v0.0.0-20190308211650-60620809c493
github.com/asecurityteam/serverfull v0.1.0
Expand All @@ -14,20 +15,23 @@ require (
github.com/go-yaml/yaml v2.1.0+incompatible // indirect
github.com/gobuffalo/genny v0.1.1 // indirect
github.com/gobuffalo/gogen v0.1.1 // indirect
github.com/gobuffalo/packr v0.0.0-20190416161152-cda4ac255773 // indirect
github.com/gobuffalo/packr/v2 v2.2.0
github.com/golang/mock v0.0.0-20190508161146-9fa652df1129
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da // indirect
github.com/karrick/godirwalk v1.10.0 // indirect
github.com/lib/pq v1.1.1
github.com/pkg/errors v0.8.1
github.com/rs/cors v1.7.0 // indirect
github.com/rs/xhandler v0.0.0-20151224012956-d9d9599b6aaf // indirect
github.com/rs/xlog v0.0.0-20171227185259-131980fab91b // indirect
github.com/rs/xstats v0.0.0-20170813190920-c67367528e16 // indirect
github.com/rs/zerolog v1.14.3 // indirect
github.com/sergi/go-diff v1.0.0 // indirect
github.com/spf13/cast v1.3.0 // indirect
github.com/stretchr/testify v1.3.0
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 // indirect
golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 // indirect
golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect
golang.org/x/tools v0.0.0-20190510151030-63859f3815cb // indirect
github.com/lib/pq v1.0.0
gopkg.in/yaml.v2 v2.2.2 // indirect
)
24 changes: 19 additions & 5 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/asecurityteam/logevent v0.0.0-20190225122144-b32737d8d51c h1:v630DsE4nvEfDCdi7OJhoHyNTPwuBplkTfGdv9WogbE=
github.com/asecurityteam/logevent v0.0.0-20190225122144-b32737d8d51c/go.mod h1:B8p/D2QVPuRA9tiDTByVrXegCsrfy8bWEgnAka23EO0=
github.com/asecurityteam/runhttp v0.0.0-20190308211650-60620809c493 h1:NlxDw3V5m6a7T/dY8LEzEW5DB+ByC602V8H26xT4xUg=
Expand Down Expand Up @@ -47,10 +49,7 @@ github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9h
github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU=
github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packr v0.0.0-20190416161152-cda4ac255773 h1:V40pHYQkyK/jF3gLB7g06umL066GasDSNmLc6r6HAgs=
github.com/gobuffalo/packr v0.0.0-20190416161152-cda4ac255773/go.mod h1:NqsGg8CSB2ZD+6RBIRs18G7aZqdYDlYNNvsSqP6T4/U=
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.1.0/go.mod h1:n90ZuXIc2KN2vFAOQascnPItp9A2g9QYSvYvS3AjQEM=
github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4=
Expand All @@ -60,16 +59,19 @@ github.com/golang/mock v0.0.0-20190508161146-9fa652df1129/go.mod h1:sBzyDLLjw3U8
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da h1:5y58+OCjoHCYB8182mpf/dEsq0vwTKPOo4zGfH0xW9A=
github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da/go.mod h1:oLH0CmIaxCGXD67VKGR5AacGXZSMznlmeqM8RzPrcY8=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.0 h1:fb2G3xs9hsG0CmH6fnx6sxTsvNeDQtcsIegljcXRQGU=
github.com/karrick/godirwalk v1.10.0/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k=
Expand All @@ -85,13 +87,20 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xhandler v0.0.0-20151224012956-d9d9599b6aaf h1:Df4QtDSQdFg5jenZonLrGr7iREOI/YAwYp18P7xjwPk=
github.com/rs/xhandler v0.0.0-20151224012956-d9d9599b6aaf/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xlog v0.0.0-20171227185259-131980fab91b h1:65vbRzwfvVUk63GnEiBy1lsY40FLZQev13NK+LnyHAE=
github.com/rs/xlog v0.0.0-20171227185259-131980fab91b/go.mod h1:PJ0wmxt3GdhZAbIT0S8HQXsHuLt11tPiF8bUKXUV77w=
github.com/rs/xstats v0.0.0-20170813190920-c67367528e16 h1:m0aigb++JZXs+tzTO60LOOKSOXWyr7scDxlaSvU6HN8=
github.com/rs/xstats v0.0.0-20170813190920-c67367528e16/go.mod h1:5Cg6M3g+Dp4RSFNYBtjJxxjksZc00LbESra5Sz6fGSU=
github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0=
github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
Expand All @@ -116,6 +125,7 @@ golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 h1:6M3SDHlHHDCx2PcQw3S4KsR17
golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -124,14 +134,18 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug=
golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190404132500-923d25813098/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190510151030-63859f3815cb h1:fuQCchFfDC5laXoMg4zQC0+xO4n1EJi6jYaXMdAu5q0=
golang.org/x/tools v0.0.0-20190510151030-63859f3815cb/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
18 changes: 12 additions & 6 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ func main() {
StatFn: domain.StatFromContext,
Fetcher: dbStorage,
}
fetchAllAssetsByTime := &v1.CloudFetchAllAssetsByTimeHandler{
LogFn: domain.LoggerFromContext,
StatFn: domain.StatFromContext,
Fetcher: dbStorage,
}
createPartition := &v1.CreatePartitionHandler{
LogFn: domain.LoggerFromContext,
Generator: dbStorage,
Expand All @@ -51,12 +56,13 @@ func main() {
Deleter: dbStorage,
}
handlers := map[string]serverfull.Function{
"insert": serverfull.NewFunction(insert.Handle),
"fetchByIP": serverfull.NewFunction(fetchByIP.Handle),
"fetchByHostname": serverfull.NewFunction(fetchByHostname.Handle),
"createPartition": serverfull.NewFunction(createPartition.Handle),
"getPartitions": serverfull.NewFunction(getPartitions.Handle),
"deletePartitions": serverfull.NewFunction(deletePartitions.Handle),
"insert": serverfull.NewFunction(insert.Handle),
"fetchByIP": serverfull.NewFunction(fetchByIP.Handle),
"fetchByHostname": serverfull.NewFunction(fetchByHostname.Handle),
"fetchAllAssetsByTime": serverfull.NewFunction(fetchAllAssetsByTime.Handle),
"createPartition": serverfull.NewFunction(createPartition.Handle),
"getPartitions": serverfull.NewFunction(getPartitions.Handle),
"deletePartitions": serverfull.NewFunction(deletePartitions.Handle),
}

fetcher := &serverfull.StaticFetcher{Functions: handlers}
Expand Down
5 changes: 5 additions & 0 deletions pkg/domain/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ type CloudAssetByHostnameFetcher interface {
FetchByHostname(ctx context.Context, when time.Time, hostname string) ([]CloudAssetDetails, error)
}

// CloudAllAssetsByTimeFetcher fetches details for all cloud assets based on limit and optional offset with a given point in time
type CloudAllAssetsByTimeFetcher interface {
FetchAll(ctx context.Context, when time.Time, count uint, offset uint) ([]CloudAssetDetails, error)
}

// Partition represents a database partition with the specified time range
type Partition struct {
Name string
Expand Down
49 changes: 49 additions & 0 deletions pkg/handlers/v1/cloud_fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package v1

import (
"context"
"errors"
"fmt"
"time"

Expand Down Expand Up @@ -38,6 +39,14 @@ type CloudAssetFetchByHostnameParameters struct {
Timestamp string `json:"time"`
}

// CloudAssetFetchAllByTimestampParameters represents the incoming payload for bulk fetching cloud assets for point in time with optional pagination
type CloudAssetFetchAllByTimestampParameters struct {
Timestamp string `json:"time"`
// we use the pointer type to detect if the value was not present in input as otherwise the int variable would be 0, which is a valid input
ensary marked this conversation as resolved.
Show resolved Hide resolved
Count *uint `json:"count"`
Offset *uint `json:"offset"`
}

// CloudFetchByIPHandler defines a lambda handler for fetching cloud assets with a given IP address
type CloudFetchByIPHandler struct {
LogFn domain.LogFn
Expand Down Expand Up @@ -108,6 +117,46 @@ func (h *CloudFetchByHostnameHandler) Handle(ctx context.Context, input CloudAss
return extractOutput(assets), nil
}

// CloudFetchAllAssetsByTimeHandler defines a lambda handler for bulk fetching cloud assets known at specific point in time
type CloudFetchAllAssetsByTimeHandler struct {
LogFn domain.LogFn
StatFn domain.StatFn
Fetcher domain.CloudAllAssetsByTimeFetcher
}

// Handle handles fetching cloud assets by hostname
func (h *CloudFetchAllAssetsByTimeHandler) Handle(ctx context.Context, input CloudAssetFetchAllByTimestampParameters) (CloudAssets, error) {
logger := h.LogFn(ctx)

ts, e := time.Parse(time.RFC3339Nano, input.Timestamp)
if e != nil {
logger.Info(logs.InvalidInput{Reason: e.Error()})
return CloudAssets{}, InvalidInput{Field: "time", Cause: e}
}

if input.Count == nil {
gcase555 marked this conversation as resolved.
Show resolved Hide resolved
ensary marked this conversation as resolved.
Show resolved Hide resolved
e = errors.New("missing or malformed required parameter count")
logger.Info(logs.InvalidInput{Reason: e.Error()})
return CloudAssets{}, InvalidInput{Field: "count", Cause: e}
}

var offset uint
if input.Offset != nil {
offset = *input.Offset
ensary marked this conversation as resolved.
Show resolved Hide resolved
}

assets, e := h.Fetcher.FetchAll(ctx, ts, *input.Count, offset)
if e != nil {
logger.Error(logs.StorageError{Reason: e.Error()})
return CloudAssets{}, e
}
if len(assets) == 0 {
return CloudAssets{}, NotFound{ID: "any"}
}

return extractOutput(assets), nil
}

func extractOutput(assets []domain.CloudAssetDetails) CloudAssets {
cloudAssets := CloudAssets{
Assets: make([]CloudAssetDetails, len(assets)),
Expand Down
59 changes: 59 additions & 0 deletions pkg/handlers/v1/cloud_fetch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/asecurityteam/asset-inventory-api/pkg/domain"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
Expand All @@ -27,6 +29,14 @@ func newFetchByHostnameHandler(fetcher domain.CloudAssetByHostnameFetcher) *Clou
}
}

func newCloudFetchAllAssetsByTimeHandler(fetcher domain.CloudAllAssetsByTimeFetcher) *CloudFetchAllAssetsByTimeHandler {
return &CloudFetchAllAssetsByTimeHandler{
LogFn: testLogFn,
StatFn: testStatFn,
Fetcher: fetcher,
}
}

func validFetchByIPInput() CloudAssetFetchByIPParameters {
return CloudAssetFetchByIPParameters{
IPAddress: "1.1.1.1",
Expand All @@ -41,6 +51,55 @@ func validFetchByHostnameInput() CloudAssetFetchByHostnameParameters {
}
}

func validFetchAllByTimestampInput() CloudAssetFetchAllByTimestampParameters {
var count uint = 100
var offset uint = 1
return CloudAssetFetchAllByTimestampParameters{
Timestamp: time.Now().Format(time.RFC3339Nano),
Count: &count,
Offset: &offset,
}
}

func TestCloudFetchAllAssetsByTimeInvalidDate(t *testing.T) {
input := validFetchAllByTimestampInput()
input.Timestamp = "not a valid date"
_, err := newCloudFetchAllAssetsByTimeHandler(nil).Handle(context.Background(), input)
require.NotNil(t, err)
}

func TestCloudFetchAllAssetsByTimeNoCount(t *testing.T) {
input := validFetchAllByTimestampInput()
input.Count = nil
_, err := newCloudFetchAllAssetsByTimeHandler(nil).Handle(context.Background(), input)
require.NotNil(t, err)
}

func TestCloudFetchAllAssetsByTimeStorageError(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

fetcher := NewMockCloudAllAssetsByTimeFetcher(ctrl)
input := validFetchAllByTimestampInput()
ts, _ := time.Parse(time.RFC3339Nano, input.Timestamp)
fetcher.EXPECT().FetchAll(gomock.Any(), ts, *input.Count, *input.Offset).Return([]domain.CloudAssetDetails{}, errors.New(""))

_, e := newCloudFetchAllAssetsByTimeHandler(fetcher).Handle(context.Background(), input)
require.NotNil(t, e)
}
func TestCloudFetchAllAssetsByTimeNoResults(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

fetcher := NewMockCloudAllAssetsByTimeFetcher(ctrl)
input := validFetchAllByTimestampInput()
ts, _ := time.Parse(time.RFC3339Nano, input.Timestamp)
fetcher.EXPECT().FetchAll(gomock.Any(), ts, *input.Count, *input.Offset).Return([]domain.CloudAssetDetails{}, nil)

_, e := newCloudFetchAllAssetsByTimeHandler(fetcher).Handle(context.Background(), input)
require.NotNil(t, e)
}

func TestFetchByIPInvalidInput(t *testing.T) {
tc := []struct {
name string
Expand Down
3 changes: 3 additions & 0 deletions pkg/handlers/v1/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package v1

//go:generate mockgen -destination mock_storage_test.go -package v1 github.com/asecurityteam/asset-inventory-api/pkg/domain PartitionGenerator,PartitionsGetter,PartitionsDeleter,CloudAssetStorer,CloudAssetByIPFetcher,CloudAssetByHostnameFetcher,CloudAllAssetsByTimeFetcher
Loading