Skip to content

Commit

Permalink
Support distributed tracing (#79)
Browse files Browse the repository at this point in the history 
* Add trace ID to executor client and bbs calls

Signed-off-by: Nick Rohn <nrohn@vmware.com>

* Use garden client factory to send requests with trace ID

* Rename GardenClientFactory

Signed-off-by: Renee Chu <reneec@vmware.com>

---------

Signed-off-by: Nick Rohn <nrohn@vmware.com>
Signed-off-by: Renee Chu <reneec@vmware.com>
  • Loading branch information
@mariash
mariash authored May 23, 2023
1 parent b452e62 commit 1c64887
Show file tree
Hide file tree
Showing 22 changed files with 587 additions and 402 deletions.
8 changes: 4 additions & 4 deletions client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ import (

type Client interface {
Ping(logger lager.Logger) error
AllocateContainers(logger lager.Logger, requests []AllocationRequest) []AllocationFailure
AllocateContainers(logger lager.Logger, traceID string, requests []AllocationRequest) []AllocationFailure
GetContainer(logger lager.Logger, guid string) (Container, error)
RunContainer(lager.Logger, *RunRequest) error
RunContainer(lager.Logger, string, *RunRequest) error
UpdateContainer(lager.Logger, *UpdateRequest) error
StopContainer(logger lager.Logger, guid string) error
DeleteContainer(logger lager.Logger, guid string) error
StopContainer(logger lager.Logger, traceID string, guid string) error
DeleteContainer(logger lager.Logger, traceID string, guid string) error
ListContainers(lager.Logger) ([]Container, error)
GetBulkMetrics(lager.Logger) (map[string]Metrics, error)
RemainingResources(lager.Logger) (ExecutorResources, error)
Expand Down
7 changes: 4 additions & 3 deletions depot/containerstore/containerreaper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func newContainerReaper(logger lager.Logger, config *ContainerConfig, clock cloc
func (r *containerReaper) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
logger := r.logger.Session("container-reaper")
timer := r.clock.NewTimer(r.config.ReapInterval)
traceID := "" // requests to reap are not originated through API

close(ready)

Expand All @@ -41,7 +42,7 @@ func (r *containerReaper) Run(signals <-chan os.Signal, ready chan<- struct{}) e
logger.Error("failed-to-reap-extra-containers", err)
}

err = r.reapMissingGardenContainers(logger.Session("reap-missing-garden-containers"))
err = r.reapMissingGardenContainers(logger.Session("reap-missing-garden-containers"), traceID)
if err != nil {
logger.Error("failed-to-reap-missing-containers", err)
}
Expand Down Expand Up @@ -76,7 +77,7 @@ func (r *containerReaper) reapExtraGardenContainers(logger lager.Logger) error {
return nil
}

func (r *containerReaper) reapMissingGardenContainers(logger lager.Logger) error {
func (r *containerReaper) reapMissingGardenContainers(logger lager.Logger, traceID string) error {
logger.Info("starting")
defer logger.Info("complete")

Expand All @@ -86,7 +87,7 @@ func (r *containerReaper) reapMissingGardenContainers(logger lager.Logger) error
return err
}

r.containers.CompleteMissing(logger, snapshotGuids, handles)
r.containers.CompleteMissing(logger, traceID, snapshotGuids, handles)

return nil
}
Expand Down
65 changes: 32 additions & 33 deletions depot/containerstore/containerstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"code.cloudfoundry.org/executor/depot/event"
"code.cloudfoundry.org/executor/depot/transformer"
"code.cloudfoundry.org/executor/initializer/configuration"
"code.cloudfoundry.org/garden"
"code.cloudfoundry.org/lager/v3"
"code.cloudfoundry.org/volman"
"github.com/tedsuo/ifrit"
Expand All @@ -25,15 +24,15 @@ var (

type ContainerStore interface {
// Setters
Reserve(logger lager.Logger, req *executor.AllocationRequest) (executor.Container, error)
Destroy(logger lager.Logger, guid string) error
Reserve(logger lager.Logger, traceID string, req *executor.AllocationRequest) (executor.Container, error)
Destroy(logger lager.Logger, traceID string, guid string) error

// Container Operations
Initialize(logger lager.Logger, req *executor.RunRequest) error
Create(logger lager.Logger, guid string) (executor.Container, error)
Run(logger lager.Logger, guid string) error
Create(logger lager.Logger, traceID string, guid string) (executor.Container, error)
Run(logger lager.Logger, traceID string, guid string) error
Update(logger lager.Logger, req *executor.UpdateRequest) error
Stop(logger lager.Logger, guid string) error
Stop(logger lager.Logger, traceID string, guid string) error

// Getters
Get(logger lager.Logger, guid string) (executor.Container, error)
Expand Down Expand Up @@ -63,18 +62,18 @@ type ContainerConfig struct {
}

type containerStore struct {
containerConfig ContainerConfig
gardenClient garden.Client
dependencyManager DependencyManager
volumeManager volman.Manager
credManager CredManager
transformer transformer.Transformer
containers *nodeMap
eventEmitter event.Hub
clock clock.Clock
metronClient loggingclient.IngressClient
rootFSSizer configuration.RootFSSizer
logManager LogManager
containerConfig ContainerConfig
gardenClientFactory GardenClientFactory
dependencyManager DependencyManager
volumeManager volman.Manager
credManager CredManager
transformer transformer.Transformer
containers *nodeMap
eventEmitter event.Hub
clock clock.Clock
metronClient loggingclient.IngressClient
rootFSSizer configuration.RootFSSizer
logManager LogManager

useDeclarativeHealthCheck bool
declarativeHealthcheckPath string
Expand All @@ -95,7 +94,7 @@ type containerStore struct {
func New(
containerConfig ContainerConfig,
totalCapacity *executor.ExecutorResources,
gardenClient garden.Client,
gardenClientFactory GardenClientFactory,
dependencyManager DependencyManager,
volumeManager volman.Manager,
credManager CredManager,
Expand All @@ -116,7 +115,7 @@ func New(
) ContainerStore {
return &containerStore{
containerConfig: containerConfig,
gardenClient: gardenClient,
gardenClientFactory: gardenClientFactory,
dependencyManager: dependencyManager,
volumeManager: volumeManager,
credManager: credManager,
Expand Down Expand Up @@ -144,7 +143,7 @@ func (cs *containerStore) Cleanup(logger lager.Logger) {
cs.dependencyManager.Stop(logger)
}

func (cs *containerStore) Reserve(logger lager.Logger, req *executor.AllocationRequest) (executor.Container, error) {
func (cs *containerStore) Reserve(logger lager.Logger, traceID string, req *executor.AllocationRequest) (executor.Container, error) {
logger = logger.Session("containerstore-reserve", lager.Data{"guid": req.Guid})
logger.Debug("starting")
defer logger.Debug("complete")
Expand All @@ -156,7 +155,7 @@ func (cs *containerStore) Reserve(logger lager.Logger, req *executor.AllocationR
cs.useDeclarativeHealthCheck,
cs.declarativeHealthcheckPath,
container,
cs.gardenClient,
cs.gardenClientFactory,
cs.clock,
cs.dependencyManager,
cs.volumeManager,
Expand All @@ -179,7 +178,7 @@ func (cs *containerStore) Reserve(logger lager.Logger, req *executor.AllocationR
return executor.Container{}, err
}

cs.eventEmitter.Emit(executor.NewContainerReservedEvent(container))
cs.eventEmitter.Emit(executor.NewContainerReservedEvent(container, traceID))
return container, nil
}

Expand All @@ -202,7 +201,7 @@ func (cs *containerStore) Initialize(logger lager.Logger, req *executor.RunReque
return nil
}

func (cs *containerStore) Create(logger lager.Logger, guid string) (executor.Container, error) {
func (cs *containerStore) Create(logger lager.Logger, traceID string, guid string) (executor.Container, error) {
logger = logger.Session("containerstore-create", lager.Data{"guid": guid})
logger.Info("starting")
defer logger.Info("complete")
Expand All @@ -213,7 +212,7 @@ func (cs *containerStore) Create(logger lager.Logger, guid string) (executor.Con
return executor.Container{}, err
}

err = node.Create(logger)
err = node.Create(logger, traceID)
if err != nil {
logger.Error("failed-to-create-container", err)
return executor.Container{}, err
Expand All @@ -222,7 +221,7 @@ func (cs *containerStore) Create(logger lager.Logger, guid string) (executor.Con
return node.Info(), nil
}

func (cs *containerStore) Run(logger lager.Logger, guid string) error {
func (cs *containerStore) Run(logger lager.Logger, traceID string, guid string) error {
logger = logger.Session("containerstore-run")

logger.Info("starting")
Expand All @@ -235,7 +234,7 @@ func (cs *containerStore) Run(logger lager.Logger, guid string) error {
return err
}

err = node.Run(logger)
err = node.Run(logger, traceID)
if err != nil {
logger.Error("failed-to-run-container", err)
return err
Expand All @@ -259,7 +258,7 @@ func (cs *containerStore) Update(logger lager.Logger, req *executor.UpdateReques
return node.Update(logger, req)
}

func (cs *containerStore) Stop(logger lager.Logger, guid string) error {
func (cs *containerStore) Stop(logger lager.Logger, traceID string, guid string) error {
logger = logger.Session("containerstore-stop", lager.Data{"Guid": guid})

logger.Info("starting")
Expand All @@ -271,12 +270,12 @@ func (cs *containerStore) Stop(logger lager.Logger, guid string) error {
return err
}

node.Stop(logger)
node.Stop(logger, traceID)

return nil
}

func (cs *containerStore) Destroy(logger lager.Logger, guid string) error {
func (cs *containerStore) Destroy(logger lager.Logger, traceID string, guid string) error {
logger = logger.Session("containerstore.destroy", lager.Data{"Guid": guid})

logger.Info("starting")
Expand All @@ -288,7 +287,7 @@ func (cs *containerStore) Destroy(logger lager.Logger, guid string) error {
return err
}

err = node.Destroy(logger)
err = node.Destroy(logger, traceID)
if err != nil {
logger.Error("failed-to-destroy-container", err)
return err
Expand Down Expand Up @@ -343,7 +342,7 @@ func (cs *containerStore) Metrics(logger lager.Logger) (map[string]executor.Cont
}

logger.Debug("getting-metrics-in-garden")
gardenMetrics, err := cs.gardenClient.BulkMetrics(containerGuids)
gardenMetrics, err := cs.gardenClientFactory.NewGardenClient(logger, "").BulkMetrics(containerGuids)
if err != nil {
logger.Error("getting-metrics-in-garden-failed", err)
return nil, err
Expand Down Expand Up @@ -397,5 +396,5 @@ func (cs *containerStore) NewRegistryPruner(logger lager.Logger) ifrit.Runner {
}

func (cs *containerStore) NewContainerReaper(logger lager.Logger) ifrit.Runner {
return newContainerReaper(logger, &cs.containerConfig, cs.clock, cs.containers, cs.gardenClient)
return newContainerReaper(logger, &cs.containerConfig, cs.clock, cs.containers, cs.gardenClientFactory.NewGardenClient(logger, ""))
}
Loading

0 comments on commit 1c64887

Please sign in to comment.