Profiles & instance listing efficency improvements (from Incus)

lxd/api_internal_recover.go

@@ -104,14 +104,24 @@ func internalRecoverScan(s *state.State, userPools []api.StoragePoolsPost, valid
 			return err
 		}
 
+		profileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
+		if err != nil {
+			return err
+		}
+
+		profileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
+		if err != nil {
+			return err
+		}
+
 		// Convert to map for lookups by project name later.
 		projectProfiles = make(map[string][]*api.Profile)
 		for _, profile := range profiles {
 			if projectProfiles[profile.Project] == nil {
 				projectProfiles[profile.Project] = []*api.Profile{}
 			}
 
-			apiProfile, err := profile.ToAPI(ctx, tx.Tx())
+			apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
 			if err != nil {
 				return err
 			}

lxd/backup/backup_config_utils.go

@@ -50,8 +50,20 @@ func ConfigToInstanceDBArgs(state *state.State, c *config.Config, projectName st
 				return err
 			}
 
+			// Get all the profile configs.
+			profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
+			// Get all the profile devices.
+			profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
 			for _, profile := range profiles {
-				apiProfile, err := profile.ToAPI(ctx, tx.Tx())
+				apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
 				if err != nil {
 					return err
 				}

lxd/db/cluster/instances.go

@@ -77,16 +77,30 @@ type InstanceFilter struct {
 }
 
 // ToAPI converts the database Instance to API type.
-func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, globalConfig map[string]any) (*api.Instance, error) {
+func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, globalConfig map[string]any, instanceDevices map[int][]Device, profileConfigs map[int]map[string]string, profileDevices map[int][]Device) (*api.Instance, error) {
 	profiles, err := GetInstanceProfiles(ctx, tx, i.ID)
 	if err != nil {
 		return nil, err
 	}
 
+	if profileConfigs == nil {
+		profileConfigs, err = GetConfig(ctx, tx, "profile")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if profileDevices == nil {
+		profileDevices, err = GetDevices(ctx, tx, "profile")
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	apiProfiles := make([]api.Profile, 0, len(profiles))
 	profileNames := make([]string, 0, len(profiles))
 	for _, p := range profiles {
-		apiProfile, err := p.ToAPI(ctx, tx)
+		apiProfile, err := p.ToAPI(ctx, tx, profileConfigs, profileDevices)
 		if err != nil {
 			return nil, err
 		}
@@ -95,9 +109,18 @@ func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, globalConfig map[strin
 		profileNames = append(profileNames, p.Name)
 	}
 
-	devices, err := GetInstanceDevices(ctx, tx, i.ID)
-	if err != nil {
-		return nil, err
+	var devices map[string]Device
+	if instanceDevices != nil {
+		devices = map[string]Device{}
+
+		for _, dev := range instanceDevices[i.ID] {
+			devices[dev.Name] = dev
+		}
+	} else {
+		devices, err = GetInstanceDevices(ctx, tx, i.ID)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	apiDevices := DevicesToAPI(devices)

lxd/db/cluster/profiles.go

@@ -51,22 +51,41 @@ type ProfileFilter struct {
 }
 
 // ToAPI returns a cluster Profile as an API struct.
-func (p *Profile) ToAPI(ctx context.Context, tx *sql.Tx) (*api.Profile, error) {
-	config, err := GetProfileConfig(ctx, tx, p.ID)
-	if err != nil {
-		return nil, err
+func (p *Profile) ToAPI(ctx context.Context, tx *sql.Tx, profileConfigs map[int]map[string]string, profileDevices map[int][]Device) (*api.Profile, error) {
+	var err error
+
+	var dbConfig map[string]string
+	if profileConfigs != nil {
+		dbConfig = profileConfigs[p.ID]
+		if dbConfig == nil {
+			dbConfig = map[string]string{}
+		}
+	} else {
+		dbConfig, err = GetProfileConfig(ctx, tx, p.ID)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	devices, err := GetProfileDevices(ctx, tx, p.ID)
-	if err != nil {
-		return nil, err
+	var dbDevices map[string]Device
+	if profileDevices != nil {
+		dbDevices = map[string]Device{}
+
+		for _, dev := range profileDevices[p.ID] {
+			dbDevices[dev.Name] = dev
+		}
+	} else {
+		dbDevices, err = GetProfileDevices(ctx, tx, p.ID)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	profile := &api.Profile{
 		Name:        p.Name,
 		Description: p.Description,
-		Config:      config,
-		Devices:     DevicesToAPI(devices),
+		Config:      dbConfig,
+		Devices:     DevicesToAPI(dbDevices),
 	}
 
 	return profile, nil

lxd/db/instances.go

@@ -534,6 +534,18 @@ func (c *ClusterTx) instanceProfilesFill(ctx context.Context, snapshotsMode bool
 		return fmt.Errorf("Failed loading profiles: %w", err)
 	}
 
+	// Get all the profile configs.
+	profileConfigs, err := cluster.GetConfig(context.TODO(), c.Tx(), "profile")
+	if err != nil {
+		return fmt.Errorf("Failed loading profile configs: %w", err)
+	}
+
+	// Get all the profile devices.
+	profileDevices, err := cluster.GetDevices(context.TODO(), c.Tx(), "profile")
+	if err != nil {
+		return fmt.Errorf("Failed loading profile devices: %w", err)
+	}
+
 	// Populate profilesByID map entry for referenced profiles.
 	// This way we only call ToAPI() on the profiles actually referenced by the instances in
 	// the list, which can reduce the number of queries run.
@@ -543,7 +555,7 @@ func (c *ClusterTx) instanceProfilesFill(ctx context.Context, snapshotsMode bool
 			continue
 		}
 
-		profilesByID[profile.ID], err = profile.ToAPI(context.TODO(), c.tx)
+		profilesByID[profile.ID], err = profile.ToAPI(context.TODO(), c.tx, profileConfigs, profileDevices)
 		if err != nil {
 			return err
 		}

lxd/db/profiles.go

@@ -60,7 +60,7 @@ func (c *ClusterTx) GetProfile(ctx context.Context, project, name string) (int64
 	profile := profiles[0]
 	id := int64(profile.ID)
 
-	result, err := profile.ToAPI(ctx, c.tx)
+	result, err := profile.ToAPI(ctx, c.tx, nil, nil)
 	if err != nil {
 		return -1, nil, err
 	}
@@ -77,8 +77,20 @@ func (c *ClusterTx) GetProfiles(ctx context.Context, projectName string, profile
 		return nil, err
 	}
 
+	// Get all the profile configs.
+	profileConfigs, err := cluster.GetConfig(ctx, c.Tx(), "profile")
+	if err != nil {
+		return nil, err
+	}
+
+	// Get all the profile devices.
+	profileDevices, err := cluster.GetDevices(ctx, c.Tx(), "profile")
+	if err != nil {
+		return nil, err
+	}
+
 	for i, profile := range dbProfiles {
-		apiProfile, err := profile.ToAPI(ctx, c.tx)
+		apiProfile, err := profile.ToAPI(ctx, c.tx, profileConfigs, profileDevices)
 		if err != nil {
 			return nil, err
 		}

lxd/instance_patch.go

@@ -191,8 +191,18 @@ func instancePatch(d *Daemon, r *http.Request) response.Response {
 			return err
 		}
 
+		profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
+		if err != nil {
+			return err
+		}
+
+		profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
+		if err != nil {
+			return err
+		}
+
 		for _, profile := range profiles {
-			apiProfile, err := profile.ToAPI(ctx, tx.Tx())
+			apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
 			if err != nil {
 				return err
 			}

lxd/instance_post.go

@@ -509,9 +509,19 @@ func instancePostMigration(s *state.State, inst instance.Instance, newName strin
 				return err
 			}
 
+			profileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
+			profileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
 			apiProfiles = make([]api.Profile, 0, len(profiles))
 			for _, profile := range profiles {
-				apiProfile, err := profile.ToAPI(ctx, tx.Tx())
+				apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
 				if err != nil {
 					return err
 				}

lxd/instance_put.go

@@ -139,8 +139,18 @@ func instancePut(d *Daemon, r *http.Request) response.Response {
 				return err
 			}
 
+			profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
+			profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
 			for _, profile := range profiles {
-				apiProfile, err := profile.ToAPI(ctx, tx.Tx())
+				apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
 				if err != nil {
 					return err
 				}

lxd/instances_post.go

@@ -1171,6 +1171,16 @@ func instancesPost(d *Daemon, r *http.Request) response.Response {
 				return err
 			}
 
+			dbProfileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
+			dbProfileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
+			if err != nil {
+				return err
+			}
+
 			profilesByName := make(map[string]dbCluster.Profile, len(dbProfiles))
 			for _, dbProfile := range dbProfiles {
 				profilesByName[dbProfile.Name] = dbProfile
@@ -1182,7 +1192,7 @@ func instancesPost(d *Daemon, r *http.Request) response.Response {
 					return fmt.Errorf("Requested profile %q doesn't exist", profileName)
 				}
 
-				apiProfile, err := profile.ToAPI(ctx, tx.Tx())
+				apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileConfigs, dbProfileDevices)
 				if err != nil {
 					return err
 				}

lxd/network/acl/acl_load.go

@@ -145,11 +145,19 @@ func UsedBy(s *state.State, aclProjectName string, usageFunc func(ctx context.Co
 			return err
 		}
 
+		// Get all the profile devices.
+		profileDevicesByID, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
+		if err != nil {
+			return err
+		}
+
 		for _, profile := range profiles {
-			profileDevices[profile.Name], err = cluster.GetProfileDevices(ctx, tx.Tx(), profile.ID)
-			if err != nil {
-				return err
+			devices := map[string]cluster.Device{}
+			for _, dev := range profileDevicesByID[profile.ID] {
+				devices[dev.Name] = dev
 			}
+
+			profileDevices[profile.Name] = devices
 		}
 
 		return nil

lxd/network/network_utils.go

@@ -188,17 +188,19 @@ func UsedBy(s *state.State, networkProjectName string, networkID int64, networkN
 
 	// Look for profiles. Next cheapest to do.
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
+		// Get all profiles
 		profiles, err := cluster.GetProfiles(ctx, tx.Tx())
 		if err != nil {
 			return err
 		}
 
-		for _, profile := range profiles {
-			profileDevices, err := cluster.GetProfileDevices(ctx, tx.Tx(), profile.ID)
-			if err != nil {
-				return err
-			}
+		// Get all the profile devices.
+		profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
+		if err != nil {
+			return err
+		}
 
+		for _, profile := range profiles {
 			profileProject, err := cluster.GetProject(ctx, tx.Tx(), profile.Project)
 			if err != nil {
 				return err
@@ -209,7 +211,12 @@ func UsedBy(s *state.State, networkProjectName string, networkID int64, networkN
 				return err
 			}
 
-			inUse, err := usedByProfileDevices(profileDevices, apiProfileProject, networkProjectName, networkName, networkType)
+			devices := map[string]cluster.Device{}
+			for _, dev := range profileDevices[profile.ID] {
+				devices[dev.Name] = dev
+			}
+
+			inUse, err := usedByProfileDevices(devices, apiProfileProject, networkProjectName, networkName, networkType)
 			if err != nil {
 				return err
 			}