Skip to content

Commit

Permalink
Add api endpoint to render request payloads for querying the LIQUID api
Browse files Browse the repository at this point in the history
  • Loading branch information
Varsius committed Nov 5, 2024
1 parent d01aaf0 commit 1434463
Show file tree
Hide file tree
Showing 15 changed files with 371 additions and 21 deletions.
10 changes: 10 additions & 0 deletions docs/users/api-spec-resources.md
Original file line number Diff line number Diff line change
Expand Up @@ -798,3 +798,13 @@ The objects at `scrape_errors[]` may contain the following fields:
| `service_type` | string | Type name of the service where this resource scrape error was observed. |
| `checked_at` | integer | UNIX timestamp of the instant when this resource scrape error was observed in the specified project and service. |
| `message` | string | The exact error message that was observed. |

### GET /admin/liquid/service-capacity-request

Generates the request body payload for querying the LIQUID API endpoint /v1/report-capacity of a specific service.
Requires the `?service_type` query parameter.

### GET /admin/liquid/service-usage-request

Generates the request body payload for querying the LIQUID API endpoint /v1/projects/:uuid/report-usage of a specific service and project.
Requires the `?service_type` and `?project_id` query parameters.
3 changes: 3 additions & 0 deletions internal/api/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ func (p *v1Provider) AddTo(r *mux.Router) {
r.Methods("GET").Path("/v1/commitment-conversion/{service_type}/{resource_name}").HandlerFunc(p.GetCommitmentConversions)
r.Methods("POST").Path("/v1/domains/{domain_id}/projects/{project_id}/commitments/{commitment_id}/convert").HandlerFunc(p.ConvertCommitment)
r.Methods("POST").Path("/v1/domains/{domain_id}/projects/{project_id}/commitments/{commitment_id}/update-duration").HandlerFunc(p.UpdateCommitmentDuration)

r.Methods("GET").Path("/admin/liquid/service-capacity-request").HandlerFunc(p.GetServiceCapacityRequest)
r.Methods("GET").Path("/admin/liquid/service-usage-request").HandlerFunc(p.GetServiceUsageRequest)
}

// RequireJSON will parse the request body into the given data structure, or
Expand Down
121 changes: 121 additions & 0 deletions internal/api/liquid.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/******************************************************************************
*
* Copyright 2024 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package api

import (
"database/sql"
"errors"
"net/http"

"github.com/sapcc/go-bits/httpapi"
"github.com/sapcc/go-bits/respondwith"

"github.com/sapcc/limes/internal/core"
"github.com/sapcc/limes/internal/datamodel"
"github.com/sapcc/limes/internal/db"
)

// GetServiceCapacityRequest handles GET /admin/liquid/service-capacity-request?service_type=:type.
func (p *v1Provider) GetServiceCapacityRequest(w http.ResponseWriter, r *http.Request) {
httpapi.IdentifyEndpoint(r, "/admin/liquid/service-capacity-request")
token := p.CheckToken(r)
if !token.Require(w, "cluster:show") {
return
}

serviceType := r.URL.Query().Get("service_type")
if serviceType == "" {
http.Error(w, "missing required parameter: service_type", http.StatusBadRequest)
return
}

plugin, ok := p.Cluster.CapacityPlugins[serviceType]
if !ok {
http.Error(w, "invalid service type", http.StatusBadRequest)
return
}

backchannel := datamodel.NewCapacityPluginBackchannel(p.Cluster, p.DB)
serviceCapacityRequest, err := plugin.BuildServiceCapacityRequest(backchannel, p.Cluster.Config.AvailabilityZones)
if respondwith.ErrorText(w, err) {
return
}
if serviceCapacityRequest == nil {
http.Error(w, "capacity plugin does not support LIQUID requests", http.StatusNotImplemented)
return
}

respondwith.JSON(w, http.StatusOK, serviceCapacityRequest)
}

// p.GetServiceUsageRequest handles GET /admin/liquid/service-usage-request?service_type=:type&project_id=:id.
func (p *v1Provider) GetServiceUsageRequest(w http.ResponseWriter, r *http.Request) {
httpapi.IdentifyEndpoint(r, "/admin/liquid/service-usage-request")
token := p.CheckToken(r)
if !token.Require(w, "cluster:show") {
return
}

serviceType := r.URL.Query().Get("service_type")
if serviceType == "" {
http.Error(w, "missing required parameter: service_type", http.StatusBadRequest)
return
}

plugin, ok := p.Cluster.QuotaPlugins[db.ServiceType(serviceType)]
if !ok {
http.Error(w, "invalid service type", http.StatusBadRequest)
return
}

projectID := r.URL.Query().Get("project_id")
if projectID == "" {
http.Error(w, "missing required parameter: project_id", http.StatusBadRequest)
return
}

var dbProject db.Project
err := p.DB.SelectOne(&dbProject, `SELECT * FROM projects WHERE id = $1`, projectID)
if errors.Is(err, sql.ErrNoRows) {
http.Error(w, "project not found", http.StatusNotFound)
return
} else if respondwith.ErrorText(w, err) {
return
}

var dbDomain db.Domain
err = p.DB.SelectOne(&dbDomain, `SELECT * FROM domains WHERE id = $1`, dbProject.DomainID)
if respondwith.ErrorText(w, err) {
return
}

domain := core.KeystoneDomainFromDB(dbDomain)
project := core.KeystoneProjectFromDB(dbProject, domain)

serviceUsageRequest, err := plugin.BuildServiceUsageRequest(project, p.Cluster.Config.AvailabilityZones)
if respondwith.ErrorText(w, err) {
return
}
if serviceUsageRequest == nil {
http.Error(w, "quota plugin does not support LIQUID requests", http.StatusNotImplemented)
return
}

respondwith.JSON(w, http.StatusOK, serviceUsageRequest)
}
158 changes: 158 additions & 0 deletions internal/api/liquid_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
/*******************************************************************************
*
* Copyright 2024 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
"net/http"
"testing"

"github.com/sapcc/go-bits/assert"

"github.com/sapcc/limes/internal/test"
)

const (
liquidQuotaTestConfigYAML = `
availability_zones: [ az-one, az-two ]
discovery:
method: --test-static
params:
domains:
- { name: germany, id: uuid-for-germany }
projects:
uuid-for-germany:
- { name: berlin, id: uuid-for-berlin, parent_id: uuid-for-germany }
services:
- service_type: unittest
type: --test-generic
`
liquidCapacityTestConfigYAML = `
availability_zones: [ az-one, az-two ]
discovery:
method: --test-static
services:
- service_type: unittest
type: --test-generic
capacitors:
- id: unittest
type: --test-static
`
)

func TestGetServiceCapacityRequest(t *testing.T) {
t.Helper()
s := test.NewSetup(t,
test.WithConfig(liquidCapacityTestConfigYAML),
test.WithAPIHandler(NewV1API),
)

// endpoint requires cluster show permissions
s.TokenValidator.Enforcer.AllowView = false
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-capacity-request?service_type=unittest",
ExpectStatus: http.StatusForbidden,
}.Check(t, s.Handler)
s.TokenValidator.Enforcer.AllowView = true

// expect error when service type is missing
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-capacity-request",
ExpectStatus: http.StatusBadRequest,
ExpectBody: assert.StringData("missing required parameter: service_type\n"),
}.Check(t, s.Handler)

// expect error for invalid service type
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-capacity-request?service_type=invalid_service_type",
ExpectStatus: http.StatusBadRequest,
ExpectBody: assert.StringData("invalid service type\n"),
}.Check(t, s.Handler)

// TODO: Implement happy path test for liquid plugins
// Expect not implemented error for now
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-capacity-request?service_type=unittest",
ExpectStatus: 501,
ExpectBody: assert.StringData("capacity plugin does not support LIQUID requests\n"),
}.Check(t, s.Handler)
}

func TestServiceUsageRequest(t *testing.T) {
t.Helper()
s := test.NewSetup(t,
test.WithConfig(liquidQuotaTestConfigYAML),
test.WithAPIHandler(NewV1API),
test.WithDBFixtureFile("fixtures/start-data.sql"),
)

// endpoint requires cluster show permissions
s.TokenValidator.Enforcer.AllowView = false
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-usage-request?service_type=unittest&project_id=1",
ExpectStatus: http.StatusForbidden,
}.Check(t, s.Handler)
s.TokenValidator.Enforcer.AllowView = true

// expect error when service type is missing
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-usage-request?project_id=1",
ExpectStatus: http.StatusBadRequest,
ExpectBody: assert.StringData("missing required parameter: service_type\n"),
}.Check(t, s.Handler)

// expect error when project_id is missing
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-usage-request?service_type=unittest",
ExpectStatus: http.StatusBadRequest,
ExpectBody: assert.StringData("missing required parameter: project_id\n"),
}.Check(t, s.Handler)

// expect error for invalid service type
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-usage-request?service_type=invalid_service_type&project_id=1",
ExpectStatus: http.StatusBadRequest,
ExpectBody: assert.StringData("invalid service type\n"),
}.Check(t, s.Handler)

// expect error for invalid project_id
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-usage-request?service_type=unittest&project_id=-1",
ExpectStatus: http.StatusNotFound,
ExpectBody: assert.StringData("project not found\n"),
}.Check(t, s.Handler)

// TODO: Implement happy path test for liquid plugins
// Expect not implemented error for now
assert.HTTPRequest{
Method: "GET",
Path: "/admin/liquid/service-usage-request?service_type=unittest&project_id=1",
ExpectStatus: 501,
ExpectBody: assert.StringData("quota plugin does not support LIQUID requests\n"),
}.Check(t, s.Handler)
}
9 changes: 9 additions & 0 deletions internal/core/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,11 @@ type QuotaPlugin interface {
// The `serializedMetrics` return value is persisted in the Limes DB and
// supplied to all subsequent RenderMetrics calls.
Scrape(ctx context.Context, project KeystoneProject, allAZs []limes.AvailabilityZone) (result map[liquid.ResourceName]ResourceData, serializedMetrics []byte, err error)

// BuildServiceUsageRequest generates the request body payload for querying
// the LIQUID API endpoint /v1/projects/:uuid/report-usage
BuildServiceUsageRequest(project KeystoneProject, allAZs []limes.AvailabilityZone) (*liquid.ServiceUsageRequest, error)

// SetQuota updates the backend service's quotas for the given project in the
// given domain to the values specified here. The map is guaranteed to contain
// values for all resources defined by Resources().
Expand Down Expand Up @@ -233,6 +238,10 @@ type CapacityPlugin interface {
// supplied to all subsequent RenderMetrics calls.
Scrape(ctx context.Context, backchannel CapacityPluginBackchannel, allAZs []limes.AvailabilityZone) (result map[db.ServiceType]map[liquid.ResourceName]PerAZ[CapacityData], serializedMetrics []byte, err error)

// BuildServiceCapacityRequest generates the request body payload for querying
// the LIQUID API endpoint /v1/report-capacity
BuildServiceCapacityRequest(backchannel CapacityPluginBackchannel, allAZs []limes.AvailabilityZone) (*liquid.ServiceCapacityRequest, error)

// DescribeMetrics is called when Prometheus is scraping metrics from
// limes-collect, to provide an opportunity to the plugin to emit its own
// metrics.
Expand Down
43 changes: 26 additions & 17 deletions internal/plugins/capacity_liquid.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,25 +73,12 @@ func (p *liquidCapacityPlugin) Init(ctx context.Context, client *gophercloud.Pro

// Scrape implements the core.QuotaPlugin interface.
func (p *liquidCapacityPlugin) Scrape(ctx context.Context, backchannel core.CapacityPluginBackchannel, allAZs []limes.AvailabilityZone) (result map[db.ServiceType]map[liquid.ResourceName]core.PerAZ[core.CapacityData], serializedMetrics []byte, err error) {
req := liquid.ServiceCapacityRequest{
AllAZs: allAZs,
DemandByResource: make(map[liquid.ResourceName]liquid.ResourceDemand, len(p.LiquidServiceInfo.Resources)),
}

for resName, resInfo := range p.LiquidServiceInfo.Resources {
if !resInfo.HasCapacity {
continue
}
if !resInfo.NeedsResourceDemand {
continue
}
req.DemandByResource[resName], err = backchannel.GetResourceDemand(p.ServiceType, resName)
if err != nil {
return nil, nil, fmt.Errorf("while getting resource demand for %s/%s: %w", p.ServiceType, resName, err)
}
req, err := p.BuildServiceCapacityRequest(backchannel, allAZs)
if err != nil {
return nil, nil, err
}

resp, err := p.LiquidClient.GetCapacityReport(ctx, req)
resp, err := p.LiquidClient.GetCapacityReport(ctx, *req)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -131,6 +118,28 @@ func (p *liquidCapacityPlugin) Scrape(ctx context.Context, backchannel core.Capa
return result, serializedMetrics, nil
}

func (p *liquidCapacityPlugin) BuildServiceCapacityRequest(backchannel core.CapacityPluginBackchannel, allAZs []limes.AvailabilityZone) (*liquid.ServiceCapacityRequest, error) {
req := &liquid.ServiceCapacityRequest{
AllAZs: allAZs,
DemandByResource: make(map[liquid.ResourceName]liquid.ResourceDemand, len(p.LiquidServiceInfo.Resources)),
}

var err error
for resName, resInfo := range p.LiquidServiceInfo.Resources {
if !resInfo.HasCapacity {
continue
}
if !resInfo.NeedsResourceDemand {
continue
}
req.DemandByResource[resName], err = backchannel.GetResourceDemand(p.ServiceType, resName)
if err != nil {
return nil, fmt.Errorf("while getting resource demand for %s/%s: %w", p.ServiceType, resName, err)
}
}
return req, nil
}

// DescribeMetrics implements the core.QuotaPlugin interface.
func (p *liquidCapacityPlugin) DescribeMetrics(ch chan<- *prometheus.Desc) {
liquidDescribeMetrics(ch, p.LiquidServiceInfo.CapacityMetricFamilies, nil)
Expand Down
4 changes: 4 additions & 0 deletions internal/plugins/capacity_manual.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ func (p *capacityManualPlugin) Scrape(ctx context.Context, _ core.CapacityPlugin
return result, nil, nil
}

func (p *capacityManualPlugin) BuildServiceCapacityRequest(backchannel core.CapacityPluginBackchannel, allAZs []limes.AvailabilityZone) (*liquid.ServiceCapacityRequest, error) {
return nil, nil
}

// DescribeMetrics implements the core.CapacityPlugin interface.
func (p *capacityManualPlugin) DescribeMetrics(ch chan<- *prometheus.Desc) {
// not used by this plugin
Expand Down
Loading

0 comments on commit 1434463

Please sign in to comment.