Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dependency Updates and Performance Improvements #58

Merged
merged 13 commits into from
Jan 4, 2025
Merged

Dependency Updates and Performance Improvements #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
0943c9e
move deprecated docker types
QuintenQVD0 Dec 8, 2024
2579f05
push
QuintenQVD0 Dec 11, 2024
4ef5326
Merge branch 'pelican-dev:main' into archives_and_dockertypes
QuintenQVD0 Dec 21, 2024
a8d5bf5
Update packages + go from `github.com/mholt/archiver/v4` -> `github.c…
QuintenQVD0 Dec 21, 2024
6da3a29
add back the check for ignored files when compressing
QuintenQVD0 Dec 21, 2024
269c6ef
Make unpacking .tar.gz, .tar.bz2, .7z, .tar, .rar work
QuintenQVD0 Dec 24, 2024
7a8d2d9
Also try to handle other formats
QuintenQVD0 Dec 24, 2024
7d149c4
Import the log library
QuintenQVD0 Dec 24, 2024
47ff07e
The error: "invalid argument" was fixt in mholt/archives
QuintenQVD0 Dec 27, 2024
f7361e8
Update imports
QuintenQVD0 Dec 27, 2024
863f37d
Allow for rar Extraction
QuintenQVD0 Dec 27, 2024
0082f58
Remove the duplicate conversion
QuintenQVD0 Dec 27, 2024
82d2d12
Merge branch 'pelican-dev:main' into archives_and_dockertypes
QuintenQVD0 Dec 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
log.WithField("error", err).Fatal("failed to initialize cron system")
} else {
log.WithField("subsystem", "cron").Info("starting cron processes")
s.StartAsync()
s.Start()
}

go func() {
Expand Down
3 changes: 1 addition & 2 deletions environment/docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (

"emperror.dev/errors"
"github.com/apex/log"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"

Expand Down Expand Up @@ -39,7 +38,7 @@ func ConfigureDocker(ctx context.Context) error {
}

nw := config.Get().Docker.Network
resource, err := cli.NetworkInspect(ctx, nw.Name, types.NetworkInspectOptions{})
resource, err := cli.NetworkInspect(ctx, nw.Name, network.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return err
Expand Down
21 changes: 10 additions & 11 deletions environment/docker/container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (
"emperror.dev/errors"
"github.com/apex/log"
"github.com/buger/jsonparser"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
dockerImage "github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
dockerImage "github.com/docker/docker/api/types/image" // Alias the correct images package

"github.com/pelican-dev/wings/config"
"github.com/pelican-dev/wings/environment"
Expand Down Expand Up @@ -196,17 +196,17 @@ func (e *Environment) Create() error {

networkMode := container.NetworkMode(cfg.Docker.Network.Mode)
if a.ForceOutgoingIP {
enableIPv6 := false // define a bool variable
enableIPv6 := false
e.log().Debug("environment/docker: forcing outgoing IP address")
networkName := "ip-" + strings.ReplaceAll(strings.ReplaceAll(a.DefaultMapping.Ip, ".", "-"), ":", "-")
networkMode = container.NetworkMode(networkName)

if _, err := e.client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{}); err != nil {
if _, err := e.client.NetworkInspect(ctx, networkName, network.InspectOptions{}); err != nil {
if !client.IsErrNotFound(err) {
return err
}

if _, err := e.client.NetworkCreate(ctx, networkName, types.NetworkCreate{
if _, err := e.client.NetworkCreate(ctx, networkName, network.CreateOptions{
Driver: "bridge",
EnableIPv6: &enableIPv6,
Internal: false,
Expand Down Expand Up @@ -440,16 +440,15 @@ func (e *Environment) ensureImageExists(image string) error {
}

func (e *Environment) convertMounts() []mount.Mount {
var out []mount.Mount

for _, m := range e.Configuration.Mounts() {
out = append(out, mount.Mount{
mounts := e.Configuration.Mounts()
out := make([]mount.Mount, len(mounts))
for i, m := range mounts {
out[i] = mount.Mount{
Type: mount.TypeBind,
Source: m.Source,
Target: m.Target,
ReadOnly: m.ReadOnly,
})
}
}

return out
}
8 changes: 4 additions & 4 deletions environment/docker/stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"time"

"emperror.dev/errors"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/goccy/go-json"

"github.com/pelican-dev/wings/environment"
Expand Down Expand Up @@ -57,7 +57,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
case <-ctx.Done():
return ctx.Err()
default:
var v types.StatsJSON
var v container.StatsResponse
if err := dec.Decode(&v); err != nil {
if err != io.EOF && !errors.Is(err, context.Canceled) {
e.log().WithField("error", err).Warn("error while processing Docker stats output for container")
Expand Down Expand Up @@ -103,7 +103,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
// bothering me about it. It should also reflect a slightly more correct memory value anyways.
//
// @see https://github.com/docker/cli/blob/96e1d1d6/cli/command/container/stats_helpers.go#L227-L249
func calculateDockerMemory(stats types.MemoryStats) uint64 {
func calculateDockerMemory(stats container.MemoryStats) uint64 {
if v, ok := stats.Stats["total_inactive_file"]; ok && v < stats.Usage {
return stats.Usage - v
}
Expand All @@ -119,7 +119,7 @@ func calculateDockerMemory(stats types.MemoryStats) uint64 {
// by the defined CPU limits on the container.
//
// @see https://github.com/docker/cli/blob/aa097cf1aa19099da70930460250797c8920b709/cli/command/container/stats_helpers.go#L166
func calculateDockerAbsoluteCpu(pStats types.CPUStats, stats types.CPUStats) float64 {
func calculateDockerAbsoluteCpu(pStats container.CPUStats, stats container.CPUStats) float64 {
// Calculate the change in CPU usage between the current and previous reading.
cpuDelta := float64(stats.CPUUsage.TotalUsage) - float64(pStats.CPUUsage.TotalUsage)

Expand Down
53 changes: 28 additions & 25 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,50 @@ require (
github.com/buger/jsonparser v1.1.1
github.com/cenkalti/backoff/v4 v4.3.0
github.com/creasty/defaults v1.8.0
github.com/docker/docker v27.2.0+incompatible
github.com/docker/docker v27.4.1+incompatible
github.com/docker/go-connections v0.5.0
github.com/fatih/color v1.17.0
github.com/fatih/color v1.18.0
github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf
github.com/gabriel-vasile/mimetype v1.4.5
github.com/gabriel-vasile/mimetype v1.4.7
github.com/gammazero/workerpool v1.1.3
github.com/gbrlsnchs/jwt/v3 v3.0.1
github.com/gin-gonic/gin v1.10.0
github.com/glebarez/sqlite v1.11.0
github.com/go-co-op/gocron v1.37.0
github.com/goccy/go-json v0.10.3
github.com/go-co-op/gocron/v2 v2.14.0
github.com/goccy/go-json v0.10.4
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/iancoleman/strcase v0.3.0
github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0
github.com/juju/ratelimit v1.0.2
github.com/klauspost/compress v1.17.9
github.com/klauspost/compress v1.17.11
github.com/klauspost/pgzip v1.2.6
github.com/magiconair/properties v1.8.7
github.com/magiconair/properties v1.8.9
github.com/mattn/go-colorable v0.1.13
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/mholt/archives v0.0.0-20241226194006-fc8400ac3529
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/sftp v1.13.6
github.com/pkg/sftp v1.13.7
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/shirou/gopsutil/v3 v3.24.5
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
golang.org/x/crypto v0.27.0
golang.org/x/sync v0.8.0
golang.org/x/sys v0.25.0
github.com/stretchr/testify v1.10.0
golang.org/x/crypto v0.31.0
golang.org/x/sync v0.10.0
golang.org/x/sys v0.28.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/gorm v1.25.11
gorm.io/gorm v1.25.12
)

require (
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Microsoft/hcsshim v0.12.2 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/STARRY-S/zip v0.2.1 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.5.1 // indirect
github.com/bodgit/sevenzip v1.6.0 // indirect
github.com/bodgit/windows v1.0.1 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
Expand All @@ -66,7 +67,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gammazero/deque v0.2.1 // indirect
Expand All @@ -79,17 +80,18 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.15.0 // indirect
Expand All @@ -101,7 +103,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
Expand All @@ -113,6 +115,7 @@ require (
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sorairolake/lzip-go v0.3.5 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
Expand All @@ -127,16 +130,16 @@ require (
go.opentelemetry.io/otel/metric v1.25.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.25.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/term v0.24.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.31.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/tools v0.22.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gotest.tools/v3 v3.0.2 // indirect
Expand Down
Loading
Loading