From 87c13b0d4477f04caf54d32259fda18c433b47de Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Tue, 17 Sep 2024 15:04:03 +0300 Subject: [PATCH 1/2] introduce public_ip endpoint with filters and pagination support - implement the get publicIps database query and api handler - unify the default sorting function for twins/contracts/ips - write tests for the new endpoint - document the api filters/sorting params --- grid-proxy/docs/docs.go | 129 +++++++++++++++++- grid-proxy/docs/swagger.json | 129 +++++++++++++++++- grid-proxy/docs/swagger.yaml | 88 +++++++++++- grid-proxy/internal/explorer/db/postgres.go | 73 ++++++---- grid-proxy/internal/explorer/db/types.go | 1 + grid-proxy/internal/explorer/db/utils.go | 25 ++++ grid-proxy/internal/explorer/db_client.go | 9 ++ grid-proxy/internal/explorer/server.go | 42 ++++++ grid-proxy/pkg/client/grid_client.go | 36 +++++ grid-proxy/pkg/client/retrying_grid_client.go | 10 ++ .../pkg/client/retrying_grid_client_test.go | 5 + grid-proxy/pkg/types/farms.go | 9 -- grid-proxy/pkg/types/publicips.go | 17 +++ grid-proxy/tests/queries/mock_client/farms.go | 2 +- .../tests/queries/mock_client/publicips.go | 63 +++++++++ grid-proxy/tests/queries/mock_client/utils.go | 2 +- grid-proxy/tests/queries/publicip_test.go | 121 ++++++++++++++++ grid-proxy/tests/queries/utils.go | 2 +- 18 files changed, 725 insertions(+), 38 deletions(-) create mode 100644 grid-proxy/internal/explorer/db/utils.go create mode 100644 grid-proxy/pkg/types/publicips.go create mode 100644 grid-proxy/tests/queries/mock_client/publicips.go create mode 100644 grid-proxy/tests/queries/publicip_test.go diff --git a/grid-proxy/docs/docs.go b/grid-proxy/docs/docs.go index 8e677a898..7368d0697 100644 --- a/grid-proxy/docs/docs.go +++ b/grid-proxy/docs/docs.go @@ -785,6 +785,115 @@ const docTemplate = `{ } } }, + "/ips": { + "get": { + "description": "Get all public ips on the grid with pagination and filtration support.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "GridProxy" + ], + "summary": "Show public ips on the grid", + "parameters": [ + { + "type": "integer", + "description": "Page number", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Max result per page", + "name": "size", + "in": "query" + }, + { + "type": "boolean", + "description": "Set ips' count on headers based on filter", + "name": "ret_count", + "in": "query" + }, + { + "type": "boolean", + "description": "Get random patch of ips", + "name": "randomize", + "in": "query" + }, + { + "enum": [ + "ip", + "farm_id", + "contract_id" + ], + "type": "string", + "description": "Sort by specific ip field", + "name": "sort_by", + "in": "query" + }, + { + "enum": [ + "desc", + "asc" + ], + "type": "string", + "description": "The sorting order, default is 'asc'", + "name": "sort_order", + "in": "query" + }, + { + "type": "integer", + "description": "Get ips on specific farm", + "name": "farm_id", + "in": "query" + }, + { + "type": "string", + "description": "filter with the ip", + "name": "ip", + "in": "query" + }, + { + "type": "string", + "description": "filter with the gateway", + "name": "gateway", + "in": "query" + }, + { + "type": "boolean", + "description": "Get only the free ips, based on the ip have a contract id or not", + "name": "free", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/types.PublicIP" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, "/nodes": { "get": { "description": "Get all nodes on the grid, It has pagination", @@ -1618,6 +1727,9 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/types.Processor" } + }, + "updatedAt": { + "type": "integer" } } }, @@ -1714,6 +1826,12 @@ const docTemplate = `{ "farmingPolicyId": { "type": "integer" }, + "gpus": { + "type": "array", + "items": { + "$ref": "#/definitions/types.NodeGPU" + } + }, "gridVersion": { "type": "integer" }, @@ -1903,6 +2021,12 @@ const docTemplate = `{ "farmingPolicyId": { "type": "integer" }, + "gpus": { + "type": "array", + "items": { + "$ref": "#/definitions/types.NodeGPU" + } + }, "gridVersion": { "type": "integer" }, @@ -2004,7 +2128,7 @@ const docTemplate = `{ "type": "integer" }, "farm_id": { - "type": "string" + "type": "integer" }, "gateway": { "type": "string" @@ -2027,6 +2151,9 @@ const docTemplate = `{ "node_twin_id": { "type": "integer" }, + "updatedAt": { + "type": "integer" + }, "upload": { "description": "in bit/sec", "type": "number" diff --git a/grid-proxy/docs/swagger.json b/grid-proxy/docs/swagger.json index a11628492..d548ea13a 100644 --- a/grid-proxy/docs/swagger.json +++ b/grid-proxy/docs/swagger.json @@ -777,6 +777,115 @@ } } }, + "/ips": { + "get": { + "description": "Get all public ips on the grid with pagination and filtration support.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "GridProxy" + ], + "summary": "Show public ips on the grid", + "parameters": [ + { + "type": "integer", + "description": "Page number", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Max result per page", + "name": "size", + "in": "query" + }, + { + "type": "boolean", + "description": "Set ips' count on headers based on filter", + "name": "ret_count", + "in": "query" + }, + { + "type": "boolean", + "description": "Get random patch of ips", + "name": "randomize", + "in": "query" + }, + { + "enum": [ + "ip", + "farm_id", + "contract_id" + ], + "type": "string", + "description": "Sort by specific ip field", + "name": "sort_by", + "in": "query" + }, + { + "enum": [ + "desc", + "asc" + ], + "type": "string", + "description": "The sorting order, default is 'asc'", + "name": "sort_order", + "in": "query" + }, + { + "type": "integer", + "description": "Get ips on specific farm", + "name": "farm_id", + "in": "query" + }, + { + "type": "string", + "description": "filter with the ip", + "name": "ip", + "in": "query" + }, + { + "type": "string", + "description": "filter with the gateway", + "name": "gateway", + "in": "query" + }, + { + "type": "boolean", + "description": "Get only the free ips, based on the ip have a contract id or not", + "name": "free", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/types.PublicIP" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, "/nodes": { "get": { "description": "Get all nodes on the grid, It has pagination", @@ -1610,6 +1719,9 @@ "items": { "$ref": "#/definitions/types.Processor" } + }, + "updatedAt": { + "type": "integer" } } }, @@ -1706,6 +1818,12 @@ "farmingPolicyId": { "type": "integer" }, + "gpus": { + "type": "array", + "items": { + "$ref": "#/definitions/types.NodeGPU" + } + }, "gridVersion": { "type": "integer" }, @@ -1895,6 +2013,12 @@ "farmingPolicyId": { "type": "integer" }, + "gpus": { + "type": "array", + "items": { + "$ref": "#/definitions/types.NodeGPU" + } + }, "gridVersion": { "type": "integer" }, @@ -1996,7 +2120,7 @@ "type": "integer" }, "farm_id": { - "type": "string" + "type": "integer" }, "gateway": { "type": "string" @@ -2019,6 +2143,9 @@ "node_twin_id": { "type": "integer" }, + "updatedAt": { + "type": "integer" + }, "upload": { "description": "in bit/sec", "type": "number" diff --git a/grid-proxy/docs/swagger.yaml b/grid-proxy/docs/swagger.yaml index 59b5b3e93..9e257776f 100644 --- a/grid-proxy/docs/swagger.yaml +++ b/grid-proxy/docs/swagger.yaml @@ -91,6 +91,8 @@ definitions: items: $ref: '#/definitions/types.Processor' type: array + updatedAt: + type: integer type: object types.Farm: properties: @@ -153,6 +155,10 @@ definitions: type: string farmingPolicyId: type: integer + gpus: + items: + $ref: '#/definitions/types.NodeGPU' + type: array gridVersion: type: integer healthy: @@ -277,6 +283,10 @@ definitions: type: string farmingPolicyId: type: integer + gpus: + items: + $ref: '#/definitions/types.NodeGPU' + type: array gridVersion: type: integer healthy: @@ -344,7 +354,7 @@ definitions: contract_id: type: integer farm_id: - type: string + type: integer gateway: type: string id: @@ -359,6 +369,8 @@ definitions: type: number node_twin_id: type: integer + updatedAt: + type: integer upload: description: in bit/sec type: number @@ -952,6 +964,80 @@ paths: summary: Show the details for specific gateway tags: - GridProxy + /ips: + get: + consumes: + - application/json + description: Get all public ips on the grid with pagination and filtration support. + parameters: + - description: Page number + in: query + name: page + type: integer + - description: Max result per page + in: query + name: size + type: integer + - description: Set ips' count on headers based on filter + in: query + name: ret_count + type: boolean + - description: Get random patch of ips + in: query + name: randomize + type: boolean + - description: Sort by specific ip field + enum: + - ip + - farm_id + - contract_id + in: query + name: sort_by + type: string + - description: The sorting order, default is 'asc' + enum: + - desc + - asc + in: query + name: sort_order + type: string + - description: Get ips on specific farm + in: query + name: farm_id + type: integer + - description: filter with the ip + in: query + name: ip + type: string + - description: filter with the gateway + in: query + name: gateway + type: string + - description: Get only the free ips, based on the ip have a contract id or + not + in: query + name: free + type: boolean + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/types.PublicIP' + type: array + "400": + description: Bad Request + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: Show public ips on the grid + tags: + - GridProxy /nodes: get: consumes: diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index c01c04be6..2f26f5460 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -803,17 +803,7 @@ func (d *PostgresDatabase) GetTwins(ctx context.Context, filter types.TwinFilter } // Sorting - if limit.Randomize { - q = q.Order("random()") - } else if limit.SortBy != "" { - order := types.SortOrderAsc - if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { - order = types.SortOrderDesc - } - q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) - } else { - q = q.Order("twin.twin_id") - } + q = sort(q, "twin.twin_id", limit) var count int64 if limit.RetCount { @@ -911,20 +901,10 @@ func (d *PostgresDatabase) GetContracts(ctx context.Context, filter types.Contra } // Sorting - if limit.Randomize { - q = q.Order("random()") - } else if limit.SortBy != "" { - order := types.SortOrderAsc - if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { - order = types.SortOrderDesc - } - q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) - } else { - q = q.Order("contracts.contract_id") - } + q = sort(q, "contracts.contract_id", limit) var count int64 - if limit.Randomize || limit.RetCount { + if limit.RetCount { countQuery := q.Session(&gorm.Session{}) if res := countQuery.Count(&count); res.Error != nil { return nil, 0, errors.Wrap(res.Error, "couldn't get contract count") @@ -1035,3 +1015,50 @@ func (d *PostgresDatabase) GetContractsTotalBilledAmount(ctx context.Context, co return total, nil } + +// GetPublicIps return all ips based on filters and pagination +func (d *PostgresDatabase) GetPublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + q := d.gormDB.WithContext(ctx).Table("public_ip"). + Select( + "public_ip.id", + "public_ip.ip", + "public_ip.gateway", + "public_ip.contract_id", + "farm.farm_id", + ). + Joins("LEFT JOIN farm ON farm.id = public_ip.farm_id") + + if filter.FarmIDs != nil { + q = q.Where("farm.farm_id IN ?", filter.FarmIDs) + } + if filter.Free != nil { + q = q.Where("(contract_id = 0) = ?", *filter.Free) + } + if filter.Ip != nil { + q = q.Where("ip = ?", *filter.Ip) + } + if filter.Gateway != nil { + q = q.Where("gateway = ?", *filter.Gateway) + } + + // Sorting + q = sort(q, "public_ip.id", limit) + + // Counting + var count int64 + if limit.RetCount { + countQuery := q.Session(&gorm.Session{}) + if res := countQuery.Count(&count); res.Error != nil { + return nil, 0, errors.Wrap(res.Error, "couldn't get ips count") + } + } + + // Pagination + q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) + + ips := []types.PublicIP{} + if res := q.Scan(&ips); res.Error != nil { + return ips, uint(count), errors.Wrap(res.Error, "failed to scan returned ips from database") + } + return ips, uint(count), nil +} diff --git a/grid-proxy/internal/explorer/db/types.go b/grid-proxy/internal/explorer/db/types.go index 198fee807..023d8837d 100644 --- a/grid-proxy/internal/explorer/db/types.go +++ b/grid-proxy/internal/explorer/db/types.go @@ -26,6 +26,7 @@ type Database interface { GetContractBills(ctx context.Context, contractID uint32, limit types.Limit) ([]ContractBilling, uint, error) GetContractsLatestBillReports(ctx context.Context, contractsIds []uint32, limit uint) ([]ContractBilling, error) GetContractsTotalBilledAmount(ctx context.Context, contractIds []uint32) (uint64, error) + GetPublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) // indexer utils DeleteOldGpus(ctx context.Context, nodeTwinIds []uint32, expiration int64) error diff --git a/grid-proxy/internal/explorer/db/utils.go b/grid-proxy/internal/explorer/db/utils.go new file mode 100644 index 000000000..ae76c2cff --- /dev/null +++ b/grid-proxy/internal/explorer/db/utils.go @@ -0,0 +1,25 @@ +package db + +import ( + "fmt" + "strings" + + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" + "gorm.io/gorm" +) + +func sort(q *gorm.DB, defaultField string, limit types.Limit) *gorm.DB { + if limit.Randomize { + q = q.Order("random()") + } else if limit.SortBy != "" { + order := types.SortOrderAsc + if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { + order = types.SortOrderDesc + } + q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) + } else { + q = q.Order(defaultField) + } + + return q +} diff --git a/grid-proxy/internal/explorer/db_client.go b/grid-proxy/internal/explorer/db_client.go index 1434b7d93..62341e806 100644 --- a/grid-proxy/internal/explorer/db_client.go +++ b/grid-proxy/internal/explorer/db_client.go @@ -180,3 +180,12 @@ func (c *DBClient) GetTwinConsumption(ctx context.Context, twinId uint64) (types return consumption, err } + +func (c *DBClient) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + dbIps, count, err := c.DB.GetPublicIps(ctx, filter, limit) + if err != nil { + return nil, 0, err + } + + return dbIps, count, nil +} diff --git a/grid-proxy/internal/explorer/server.go b/grid-proxy/internal/explorer/server.go index 318eed114..911f99d24 100644 --- a/grid-proxy/internal/explorer/server.go +++ b/grid-proxy/internal/explorer/server.go @@ -400,6 +400,46 @@ func (a *App) listContracts(r *http.Request) (interface{}, mw.Response) { return dbContracts, resp } +// GetPublicIps godoc +// @Summary Show public ips on the grid +// @Description Get all public ips on the grid with pagination and filtration support. +// @Tags GridProxy +// @Accept json +// @Produce json +// @Param page query int false "Page number" +// @Param size query int false "Max result per page" +// @Param ret_count query bool false "Set ips' count on headers based on filter" +// @Param randomize query bool false "Get random patch of ips" +// @Param sort_by query string false "Sort by specific ip field" Enums(ip, farm_id, contract_id) +// @Param sort_order query string false "The sorting order, default is 'asc'" Enums(desc, asc) +// @Param farm_id query int false "Get ips on specific farm" +// @Param ip query string false "filter with the ip" +// @Param gateway query string false "filter with the gateway" +// @Param free query bool false "Get only the free ips, based on the ip have a contract id or not" +// @Success 200 {object} []types.PublicIP +// @Failure 400 {object} string +// @Failure 500 {object} string +// @Router /ips [get] +func (a *App) GetPublicIps(r *http.Request) (interface{}, mw.Response) { + filter := types.PublicIpFilter{} + limit := types.DefaultLimit() + if err := parseQueryParams(r, &filter, &limit); err != nil { + return nil, mw.BadRequest(err) + } + if err := limit.Valid(types.PublicIP{}); err != nil { + return nil, mw.BadRequest(err) + } + + ips, ipsCount, err := a.cl.PublicIps(r.Context(), filter, limit) + if err != nil { + log.Error().Err(err).Msg("failed to query public ips") + return nil, mw.Error(err) + } + + resp := createResponse(ipsCount, limit) + return ips, resp +} + // ping godoc // @Summary ping the server // @Description ping the server to check if it is running @@ -606,6 +646,8 @@ func Setup(router *mux.Router, gitCommit string, cl DBClient, relayClient rmb.Cl router.HandleFunc("/contracts/{contract_id:[0-9]+}", mw.AsHandlerFunc(a.getContract)) router.HandleFunc("/contracts/{contract_id:[0-9]+}/bills", mw.AsHandlerFunc(a.getContractBills)) + router.HandleFunc("/public_ips", mw.AsHandlerFunc(a.GetPublicIps)) + router.HandleFunc("/", mw.AsHandlerFunc(a.indexPage(router))) router.HandleFunc("/ping", mw.AsHandlerFunc(a.ping)) router.HandleFunc("/version", mw.AsHandlerFunc(a.version)) diff --git a/grid-proxy/pkg/client/grid_client.go b/grid-proxy/pkg/client/grid_client.go index 8dc2826fb..ad961ab66 100644 --- a/grid-proxy/pkg/client/grid_client.go +++ b/grid-proxy/pkg/client/grid_client.go @@ -44,6 +44,7 @@ type DBClient interface { Node(ctx context.Context, nodeID uint32) (res types.NodeWithNestedCapacity, err error) NodeStatus(ctx context.Context, nodeID uint32) (res types.NodeStatus, err error) Stats(ctx context.Context, filter types.StatsFilter) (res types.Stats, err error) + PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) } // Client a client to communicate with the grid proxy @@ -351,6 +352,41 @@ func (g *Clientimpl) ContractBills(ctx context.Context, contractID uint32, limit return contractBills, totalCount, nil } +// PublicIps returns all public ips on the chain based on filters and pagination params +func (g *Clientimpl) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + res, err := g.httpGet("public_ips", filter, limit) + if res != nil { + defer res.Body.Close() + } + if err != nil { + return nil, 0, err + } + + if res.StatusCode != http.StatusOK { + err = parseError(res.Body) + return nil, 0, err + } + + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, 0, err + } + + count, err := requestCounters(res) + if err != nil { + return nil, 0, err + } + + ips := []types.PublicIP{} + if err := json.Unmarshal(data, &ips); err != nil { + return nil, 0, err + } + + totalCount := uint(count) + + return ips, totalCount, nil +} + func (g *Clientimpl) newHTTPClient() *http.Client { return &http.Client{ Timeout: time.Second * 30, diff --git a/grid-proxy/pkg/client/retrying_grid_client.go b/grid-proxy/pkg/client/retrying_grid_client.go index a4e62ff09..849c2adba 100644 --- a/grid-proxy/pkg/client/retrying_grid_client.go +++ b/grid-proxy/pkg/client/retrying_grid_client.go @@ -137,3 +137,13 @@ func (g *RetryingClient) ContractBills(ctx context.Context, contractID uint32, l err = backoff.RetryNotify(f, bf(g.timeout), notify("contract_bills")) return } + +// PublicIps returns the public ips based on filters and pagination +func (g *RetryingClient) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) (res []types.PublicIP, totalCount uint, err error) { + f := func() error { + res, totalCount, err = g.cl.PublicIps(ctx, filter, limit) + return err + } + err = backoff.RetryNotify(f, bf(g.timeout), notify("public_ips")) + return +} diff --git a/grid-proxy/pkg/client/retrying_grid_client_test.go b/grid-proxy/pkg/client/retrying_grid_client_test.go index 622ea8f07..509c8c2a5 100644 --- a/grid-proxy/pkg/client/retrying_grid_client_test.go +++ b/grid-proxy/pkg/client/retrying_grid_client_test.go @@ -61,6 +61,11 @@ func (r *requestCounter) ContractBills(ctx context.Context, contractID uint32, l return nil, 0, errors.New("error") } +func (r *requestCounter) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + r.Counter++ + return nil, 0, errors.New("error") +} + func retryingConstructor(u ...string) Client { return NewRetryingClientWithTimeout(NewClient(u...), 1*time.Millisecond) } diff --git a/grid-proxy/pkg/types/farms.go b/grid-proxy/pkg/types/farms.go index 376f1af8e..3daeeb263 100644 --- a/grid-proxy/pkg/types/farms.go +++ b/grid-proxy/pkg/types/farms.go @@ -12,15 +12,6 @@ type Farm struct { PublicIps []PublicIP `json:"publicIps" sort:"public_ips"` } -// PublicIP info about public ip in the farm -type PublicIP struct { - ID string `json:"id"` - IP string `json:"ip"` - FarmID string `json:"farm_id"` - ContractID int `json:"contract_id"` - Gateway string `json:"gateway"` -} - // FarmFilter farm filters type FarmFilter struct { FreeIPs *uint64 `schema:"free_ips,omitempty"` diff --git a/grid-proxy/pkg/types/publicips.go b/grid-proxy/pkg/types/publicips.go new file mode 100644 index 000000000..0d911d1a5 --- /dev/null +++ b/grid-proxy/pkg/types/publicips.go @@ -0,0 +1,17 @@ +package types + +// PublicIP info about public ip in the farm +type PublicIP struct { + ID string `json:"id"` + IP string `json:"ip" sort:"ip"` + Gateway string `json:"gateway"` + ContractID uint64 `json:"contract_id" sort:"contract_id"` + FarmID uint64 `json:"farm_id,omitempty" sort:"farm_id"` +} + +type PublicIpFilter struct { + FarmIDs []uint64 `schema:"farm_ids,omitempty"` + Free *bool `schema:"free,omitempty"` + Ip *string `schema:"ip,omitempty"` + Gateway *string `schema:"gateway,omitempty"` +} diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index f71092363..1b7a13508 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -26,7 +26,7 @@ func (g *GridProxyMockClient) Farms(ctx context.Context, filter types.FarmFilter publicIPs[g.data.FarmIDMap[publicIP.FarmID]] = append(publicIPs[g.data.FarmIDMap[publicIP.FarmID]], types.PublicIP{ ID: publicIP.ID, IP: publicIP.IP, - ContractID: int(publicIP.ContractID), + ContractID: publicIP.ContractID, Gateway: publicIP.Gateway, }) } diff --git a/grid-proxy/tests/queries/mock_client/publicips.go b/grid-proxy/tests/queries/mock_client/publicips.go new file mode 100644 index 000000000..6fa927924 --- /dev/null +++ b/grid-proxy/tests/queries/mock_client/publicips.go @@ -0,0 +1,63 @@ +package mock + +import ( + "context" + "slices" + "sort" + + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" +) + +func (g *GridProxyMockClient) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + res := []types.PublicIP{} + if limit.Page == 0 { + limit.Page = 1 + } + if limit.Size == 0 { + limit.Size = 50 + } + + for _, ip := range g.data.PublicIPs { + if ip.satisfies(filter, &g.data) { + res = append(res, types.PublicIP{ + IP: ip.IP, + ID: ip.ID, + Gateway: ip.Gateway, + ContractID: ip.ContractID, + FarmID: g.data.FarmIDMap[ip.FarmID], + }) + } + } + + sort.Slice(res, func(i, j int) bool { + return res[i].ID < res[j].ID + }) + + res, count := getPage(res, limit) + + return res, uint(count), nil +} + +func (ip *PublicIp) satisfies(f types.PublicIpFilter, data *DBData) bool { + if f.Free != nil && + *f.Free != (ip.ContractID == 0) { + return false + } + + if len(f.FarmIDs) != 0 && + !slices.Contains(f.FarmIDs, data.FarmIDMap[ip.FarmID]) { + return false + } + + if f.Ip != nil && + *f.Ip != ip.IP { + return false + } + + if f.Gateway != nil && + *f.Gateway != ip.Gateway { + return false + } + + return true +} diff --git a/grid-proxy/tests/queries/mock_client/utils.go b/grid-proxy/tests/queries/mock_client/utils.go index 411f1a367..31ce4f660 100644 --- a/grid-proxy/tests/queries/mock_client/utils.go +++ b/grid-proxy/tests/queries/mock_client/utils.go @@ -7,7 +7,7 @@ import ( ) type Result interface { - types.Contract | types.Farm | types.Node | types.Twin + types.Contract | types.Farm | types.Node | types.Twin | types.PublicIP } func CalcFreeResources(total NodeResourcesTotal, used NodeResourcesTotal) NodeResourcesTotal { diff --git a/grid-proxy/tests/queries/publicip_test.go b/grid-proxy/tests/queries/publicip_test.go new file mode 100644 index 000000000..8da6606b5 --- /dev/null +++ b/grid-proxy/tests/queries/publicip_test.go @@ -0,0 +1,121 @@ +package test + +import ( + "context" + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" + mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" +) + +func TestPublicIP(t *testing.T) { + t.Parallel() + + t.Run("public_ip pagination test", func(t *testing.T) { + t.Parallel() + + f := types.PublicIpFilter{} + l := types.Limit{ + Size: 100, + Page: 1, + RetCount: true, + } + + for { + wantRes, wantCount, err := mockClient.PublicIps(context.Background(), f, l) + require.NoError(t, err) + + gotRes, gotCount, err := gridProxyClient.PublicIps(context.Background(), f, l) + require.NoError(t, err) + + require.Equal(t, wantCount, gotCount) + + require.True(t, reflect.DeepEqual(wantRes, gotRes), fmt.Sprintf("Used Filter:\n%s", SerializeFilter(f)), fmt.Sprintf("Difference:\n%s", cmp.Diff(wantRes, gotRes))) + + if l.Page*l.Size >= uint64(wantCount) { + break + } + l.Page++ + } + }) + + t.Run("public_ip filtration test", func(t *testing.T) { + f := types.PublicIpFilter{} + fv := reflect.ValueOf(&f).Elem() + l := types.Limit{ + Size: 9999999, + Page: 1, + RetCount: true, + } + agg := calcPublicIpsAggregates(&data) + + // for each field on the filters struct + for i := 0; i < fv.NumField(); i++ { + generator, ok := publicIpGen[fv.Type().Field(i).Name] + require.True(t, ok, "Filter field %s has no random value generator", fv.Type().Field(i).Name) + + // set the generated random values for the empty filter filed + randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } + if fv.Field(i).Type().Kind() != reflect.Slice { + fv.Field(i).Set(reflect.New(fv.Field(i).Type().Elem())) + } + fv.Field(i).Set(reflect.ValueOf(randomFieldValue)) + + // compare the clients responses + want, wantCount, err := mockClient.PublicIps(context.Background(), f, l) + require.NoError(t, err) + + got, gotCount, err := gridProxyClient.PublicIps(context.Background(), f, l) + require.NoError(t, err) + + require.Equal(t, wantCount, gotCount) + + require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Used Filter:\n%s", SerializeFilter(f)), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) + + fv.Field(i).Set(reflect.Zero(fv.Field(i).Type())) + } + }) +} + +type PublicIpsAggregate struct { + farmIds []uint64 + ips []string + gateway []string +} + +func calcPublicIpsAggregates(data *mock.DBData) (agg PublicIpsAggregate) { + for _, ip := range data.PublicIPs { + agg.farmIds = append(agg.farmIds, data.FarmIDMap[ip.FarmID]) + agg.ips = append(agg.ips, ip.IP) + agg.gateway = append(agg.gateway, ip.Gateway) + } + return +} + +var publicIpGen = map[string]func(agg PublicIpsAggregate) any{ + "Free": func(_ PublicIpsAggregate) any { + v := true + if flip(.5) { + v = false + } + return &v + }, + "FarmIDs": func(agg PublicIpsAggregate) any { + randomLen := rand.Intn(5) + return getRandomSliceFrom(agg.farmIds, randomLen) + }, + "Ip": func(agg PublicIpsAggregate) any { + return &agg.ips[rand.Intn(len(agg.ips))] + }, + "Gateway": func(agg PublicIpsAggregate) any { + return &agg.gateway[rand.Intn(len(agg.gateway))] + }, +} diff --git a/grid-proxy/tests/queries/utils.go b/grid-proxy/tests/queries/utils.go index 0f783c242..1f749fd99 100644 --- a/grid-proxy/tests/queries/utils.go +++ b/grid-proxy/tests/queries/utils.go @@ -10,7 +10,7 @@ import ( ) type Filter interface { - types.ContractFilter | types.NodeFilter | types.FarmFilter | types.TwinFilter | types.StatsFilter + types.ContractFilter | types.NodeFilter | types.FarmFilter | types.TwinFilter | types.StatsFilter | types.PublicIpFilter } func flip(success float32) bool { From 6d65b5b2d5ae10e2617887df80093cb977b28630 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Tue, 17 Sep 2024 15:15:09 +0300 Subject: [PATCH 2/2] fix the gridproxy mockclient in grid-client --- grid-client/mocks/grid_client_mock.go | 92 ++++++++++++++++++--------- 1 file changed, 62 insertions(+), 30 deletions(-) diff --git a/grid-client/mocks/grid_client_mock.go b/grid-client/mocks/grid_client_mock.go index 35c46e2e1..fdf02c523 100644 --- a/grid-client/mocks/grid_client_mock.go +++ b/grid-client/mocks/grid_client_mock.go @@ -82,21 +82,6 @@ func (mr *MockDBClientMockRecorder) Contracts(ctx, filter, pagination interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Contracts", reflect.TypeOf((*MockDBClient)(nil).Contracts), ctx, filter, pagination) } -// Stats mocks base method. -func (m *MockDBClient) Stats(ctx context.Context, filter types.StatsFilter) (types.Stats, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stats", ctx, filter) - ret0, _ := ret[0].(types.Stats) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Stats indicates an expected call of Stats. -func (mr *MockDBClientMockRecorder) Stats(ctx, filter interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockDBClient)(nil).Stats), ctx, filter) -} - // Farms mocks base method. func (m *MockDBClient) Farms(ctx context.Context, filter types.FarmFilter, pagination types.Limit) ([]types.Farm, int, error) { m.ctrl.T.Helper() @@ -159,6 +144,37 @@ func (mr *MockDBClientMockRecorder) Nodes(ctx, filter, pagination interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Nodes", reflect.TypeOf((*MockDBClient)(nil).Nodes), ctx, filter, pagination) } +// PublicIps mocks base method. +func (m *MockDBClient) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PublicIps", ctx, filter, limit) + ret0, _ := ret[0].([]types.PublicIP) + ret1, _ := ret[1].(uint) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// PublicIps indicates an expected call of PublicIps. +func (mr *MockDBClientMockRecorder) PublicIps(ctx, filter, limit interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublicIps", reflect.TypeOf((*MockDBClient)(nil).PublicIps), ctx, filter, limit) +} + +// Stats mocks base method. +func (m *MockDBClient) Stats(ctx context.Context, filter types.StatsFilter) (types.Stats, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stats", ctx, filter) + ret0, _ := ret[0].(types.Stats) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stats indicates an expected call of Stats. +func (mr *MockDBClientMockRecorder) Stats(ctx, filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockDBClient)(nil).Stats), ctx, filter) +} + // Twins mocks base method. func (m *MockDBClient) Twins(ctx context.Context, filter types.TwinFilter, pagination types.Limit) ([]types.Twin, int, error) { m.ctrl.T.Helper() @@ -245,21 +261,6 @@ func (mr *MockClientMockRecorder) Contracts(ctx, filter, pagination interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Contracts", reflect.TypeOf((*MockClient)(nil).Contracts), ctx, filter, pagination) } -// Stats mocks base method. -func (m *MockClient) Stats(ctx context.Context, filter types.StatsFilter) (types.Stats, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stats", ctx, filter) - ret0, _ := ret[0].(types.Stats) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Stats indicates an expected call of Stats. -func (mr *MockClientMockRecorder) Stats(ctx, filter interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockClient)(nil).Stats), ctx, filter) -} - // Farms mocks base method. func (m *MockClient) Farms(ctx context.Context, filter types.FarmFilter, pagination types.Limit) ([]types.Farm, int, error) { m.ctrl.T.Helper() @@ -336,6 +337,37 @@ func (mr *MockClientMockRecorder) Ping() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ping", reflect.TypeOf((*MockClient)(nil).Ping)) } +// PublicIps mocks base method. +func (m *MockClient) PublicIps(ctx context.Context, filter types.PublicIpFilter, limit types.Limit) ([]types.PublicIP, uint, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PublicIps", ctx, filter, limit) + ret0, _ := ret[0].([]types.PublicIP) + ret1, _ := ret[1].(uint) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// PublicIps indicates an expected call of PublicIps. +func (mr *MockClientMockRecorder) PublicIps(ctx, filter, limit interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublicIps", reflect.TypeOf((*MockClient)(nil).PublicIps), ctx, filter, limit) +} + +// Stats mocks base method. +func (m *MockClient) Stats(ctx context.Context, filter types.StatsFilter) (types.Stats, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stats", ctx, filter) + ret0, _ := ret[0].(types.Stats) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stats indicates an expected call of Stats. +func (mr *MockClientMockRecorder) Stats(ctx, filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockClient)(nil).Stats), ctx, filter) +} + // Twins mocks base method. func (m *MockClient) Twins(ctx context.Context, filter types.TwinFilter, pagination types.Limit) ([]types.Twin, int, error) { m.ctrl.T.Helper()