Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize db transactions #429

Merged
merged 10 commits into from
Jan 24, 2024
Merged

Optimize db transactions #429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0b41041
api_cluster: Optimize db transactions
monstermunchkin Jan 22, 2024
086030e
daemon_images: Optimize db transactions
monstermunchkin Jan 22, 2024
d74fe2d
daemon_storage: Optimize db transactions
monstermunchkin Jan 22, 2024
c5dae22
images: Optimize db transactions
monstermunchkin Jan 22, 2024
b7d9fc5
storage_volumes_snapshot: Optimize db transactions
monstermunchkin Jan 22, 2024
1a1a635
instance/drivers: Optimize db transactions
monstermunchkin Jan 24, 2024
82ca586
driver_ovn: Optimize db transactions
monstermunchkin Jan 24, 2024
275cbf5
network/acl: Optimize db transactions
monstermunchkin Jan 24, 2024
b0bc76f
network/zone: Optimize db transactions
monstermunchkin Jan 24, 2024
78faceb
storage_volumes: Optimize db transactions
monstermunchkin Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 38 additions & 67 deletions cmd/incusd/api_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,7 @@ func clusterPutJoin(d *Daemon, r *http.Request, req api.ClusterPut) response.Res

// Get all defined storage pools and networks, so they can be compared to the ones in the cluster.
pools := []api.StoragePool{}
networks := []api.InitNetworksProjectPost{}

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
poolNames, err := tx.GetStoragePoolNames(ctx)
Expand All @@ -645,17 +646,9 @@ func clusterPutJoin(d *Daemon, r *http.Request, req api.ClusterPut) response.Res
pools = append(pools, *pool)
}

return nil
})
if err != nil {
return err
}
// Get a list of projects for networks.
var projects []dbCluster.Project

// Get a list of projects for networks.
var projects []dbCluster.Project
networks := []api.InitNetworksProjectPost{}

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
projects, err = dbCluster.GetProjects(ctx, tx.Tx())
if err != nil {
return fmt.Errorf("Failed to load projects for networks: %w", err)
Expand Down Expand Up @@ -2791,66 +2784,54 @@ type internalClusterPostHandoverRequest struct {
}

func clusterCheckStoragePoolsMatch(cluster *db.Cluster, reqPools []api.StoragePool) error {
var err error
var poolNames []string

err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
poolNames, err = tx.GetCreatedStoragePoolNames(ctx)

return err
})
if err != nil && !response.IsNotFoundError(err) {
return err
}
return cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
poolNames, err := tx.GetCreatedStoragePoolNames(ctx)
if err != nil && !response.IsNotFoundError(err) {
return err
}

for _, name := range poolNames {
found := false
for _, reqPool := range reqPools {
if reqPool.Name != name {
continue
}
for _, name := range poolNames {
found := false
for _, reqPool := range reqPools {
if reqPool.Name != name {
continue
}

found = true
found = true

var pool *api.StoragePool
var pool *api.StoragePool

err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
_, pool, _, err = tx.GetStoragePoolInAnyState(ctx, name)
if err != nil {
return err
}

return err
})
if err != nil {
return err
}
if pool.Driver != reqPool.Driver {
return fmt.Errorf("Mismatching driver for storage pool %s", name)
}
// Exclude the keys which are node-specific.
exclude := db.NodeSpecificStorageConfig
err = localUtil.CompareConfigs(pool.Config, reqPool.Config, exclude)
if err != nil {
return fmt.Errorf("Mismatching config for storage pool %s: %w", name, err)
}

if pool.Driver != reqPool.Driver {
return fmt.Errorf("Mismatching driver for storage pool %s", name)
}
// Exclude the keys which are node-specific.
exclude := db.NodeSpecificStorageConfig
err = localUtil.CompareConfigs(pool.Config, reqPool.Config, exclude)
if err != nil {
return fmt.Errorf("Mismatching config for storage pool %s: %w", name, err)
break
}

break
if !found {
return fmt.Errorf("Missing storage pool %s", name)
}
}

if !found {
return fmt.Errorf("Missing storage pool %s", name)
}
}
return nil
return nil
})
}

func clusterCheckNetworksMatch(cluster *db.Cluster, reqNetworks []api.InitNetworksProjectPost) error {
var err error

// Get a list of projects for networks.
var networkProjectNames []string

err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
networkProjectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx())
return cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Get a list of projects for networks.
networkProjectNames, err := dbCluster.GetProjectNames(ctx, tx.Tx())
if err != nil {
return fmt.Errorf("Failed to load projects for networks: %w", err)
}
Expand Down Expand Up @@ -2898,11 +2879,6 @@ func clusterCheckNetworksMatch(cluster *db.Cluster, reqNetworks []api.InitNetwor

return nil
})
if err != nil {
return err
}

return nil
}

// Used as low-level recovering helper.
Expand Down Expand Up @@ -3181,7 +3157,7 @@ func internalClusterHeal(d *Daemon, r *http.Request) response.Response {
}

func evacuateClusterSetState(s *state.State, name string, state int) error {
err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Get the node.
node, err := tx.GetNodeByName(ctx, name)
if err != nil {
Expand Down Expand Up @@ -3211,11 +3187,6 @@ func evacuateClusterSetState(s *state.State, name string, state int) error {

return nil
})
if err != nil {
return err
}

return nil
}

// evacuateHostShutdownDefaultTimeout default timeout (in seconds) for waiting for clean shutdown to complete.
Expand Down
50 changes: 19 additions & 31 deletions cmd/incusd/daemon_images.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,34 +207,26 @@ func ImageDownload(r *http.Request, s *state.State, op *operations.Operation, ar
// Check if the image is available locally or it's on another node. Do this before creating
// the missing DB record so we don't include ourself in the search results.
nodeAddress, err = tx.LocateImage(ctx, imgInfo.Fingerprint)
if err != nil {
return fmt.Errorf("Locate image %q in the cluster: %w", imgInfo.Fingerprint, err)
}

return err
})
if err != nil {
return nil, fmt.Errorf("Locate image %q in the cluster: %w", imgInfo.Fingerprint, err)
}

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// We need to insert the database entry for this project, including the node ID entry.
return tx.CreateImage(ctx, args.ProjectName, imgInfo.Fingerprint, imgInfo.Filename, imgInfo.Size, args.Public, imgInfo.AutoUpdate, imgInfo.Architecture, imgInfo.CreatedAt, imgInfo.ExpiresAt, imgInfo.Properties, imgInfo.Type, nil)
})
if err != nil {
return nil, fmt.Errorf("Failed creating image record for project: %w", err)
}

// Mark the image as "cached" if downloading for an instance.
if args.SetCached {
err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return tx.SetImageCachedAndLastUseDate(ctx, args.ProjectName, imgInfo.Fingerprint, time.Now().UTC())
})
err = tx.CreateImage(ctx, args.ProjectName, imgInfo.Fingerprint, imgInfo.Filename, imgInfo.Size, args.Public, imgInfo.AutoUpdate, imgInfo.Architecture, imgInfo.CreatedAt, imgInfo.ExpiresAt, imgInfo.Properties, imgInfo.Type, nil)
if err != nil {
return nil, fmt.Errorf("Failed setting cached flag and last use date: %w", err)
return fmt.Errorf("Failed creating image record for project: %w", err)
}
}

var id int
// Mark the image as "cached" if downloading for an instance.
if args.SetCached {
err = tx.SetImageCachedAndLastUseDate(ctx, args.ProjectName, imgInfo.Fingerprint, time.Now().UTC())
if err != nil {
return fmt.Errorf("Failed setting cached flag and last use date: %w", err)
}
}

var id int

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
id, imgInfo, err = tx.GetImage(ctx, fp, cluster.ImageFilter{Project: &args.ProjectName})
if err != nil {
return err
Expand All @@ -245,6 +237,7 @@ func ImageDownload(r *http.Request, s *state.State, op *operations.Operation, ar
if err != nil {
return nil, err
}

// Transfer image if needed (after database record has been created above).
if nodeAddress != "" {
// The image is available from another node, let's try to import it.
Expand All @@ -269,20 +262,15 @@ func ImageDownload(r *http.Request, s *state.State, op *operations.Operation, ar
ctxMap["pool"] = args.StoragePool

var poolID int64
var poolIDs []int64

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Get the ID of the target storage pool.
poolID, err = tx.GetStoragePoolID(ctx, args.StoragePool)
if err != nil {
return err
}

return err
})
if err != nil {
return nil, err
}

var poolIDs []int64

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Check if the image is already in the pool.
poolIDs, err = tx.GetPoolsWithImage(ctx, info.Fingerprint)

Expand Down
29 changes: 10 additions & 19 deletions cmd/incusd/daemon_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,19 +155,16 @@ func daemonStorageValidate(s *state.State, target string) error {
}

var poolID int64
var snapshots []db.StorageVolumeArgs

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Validate pool exists.
poolID, _, _, err = tx.GetStoragePool(ctx, poolName)
if err != nil {
return fmt.Errorf("Unable to load storage pool %q: %w", poolName, err)
}

return err
})
if err != nil {
return fmt.Errorf("Unable to load storage pool %q: %w", poolName, err)
}

// Confirm volume exists.
err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Confirm volume exists.
dbVol, err := tx.GetStoragePoolVolume(ctx, poolID, api.ProjectDefaultName, db.StoragePoolVolumeTypeCustom, volumeName, true)
if err != nil {
return fmt.Errorf("Failed loading storage volume %q in %q project: %w", target, api.ProjectDefaultName, err)
Expand All @@ -177,21 +174,15 @@ func daemonStorageValidate(s *state.State, target string) error {
return fmt.Errorf("Storage volume %q in %q project is not filesystem content type", target, api.ProjectDefaultName)
}

return nil
})
if err != nil {
return err
}

var snapshots []db.StorageVolumeArgs

err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
snapshots, err = tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, api.ProjectDefaultName, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
if err != nil {
return fmt.Errorf("Unable to load storage volume snapshots %q in %q project: %w", target, api.ProjectDefaultName, err)
}

return err
return nil
})
if err != nil {
return fmt.Errorf("Unable to load storage volume snapshots %q in %q project: %w", target, api.ProjectDefaultName, err)
return err
}

if len(snapshots) != 0 {
Expand Down
Loading
Loading