Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix memory leak during distribution of a table with a lot of partitions #6722

Merged
merged 1 commit into from
Feb 17, 2023
Merged

fix memory leak during distribution of a table with a lot of partitions #6722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/backend/distributed/commands/create_distributed_table.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1116,12 +1116,27 @@ CreateDistributedTable(Oid relationId, char *distributionColumnName,
char *relationName = get_rel_name(relationId);
char *parentRelationName = quote_qualified_identifier(schemaName, relationName);

/*
* when there are many partitions, each call to CreateDistributedTable
* accumulates used memory. Create and free context for each call.
*/
MemoryContext citusPartitionContext =
AllocSetContextCreate(CurrentMemoryContext,
"citus_per_partition_context",
ALLOCSET_DEFAULT_SIZES);
MemoryContext oldContext = MemoryContextSwitchTo(citusPartitionContext);

foreach_oid(partitionRelationId, partitionList)
{
MemoryContextReset(citusPartitionContext);

CreateDistributedTable(partitionRelationId, distributionColumnName,
distributionMethod, shardCount, false,
parentRelationName);
}

MemoryContextSwitchTo(oldContext);
MemoryContextDelete(citusPartitionContext);
}

/* copy over data for hash distributed and reference tables */
Expand Down
36 changes: 22 additions & 14 deletions src/backend/distributed/operations/create_shards.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool
{
bool colocatedShard = true;
List *insertedShardPlacements = NIL;
List *insertedShardIds = NIL;

/* make sure that tables are hash partitioned */
CheckHashPartitionedTable(targetRelationId);
Expand Down Expand Up @@ -254,7 +255,9 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool
foreach_ptr(sourceShardInterval, sourceShardIntervalList)
{
uint64 sourceShardId = sourceShardInterval->shardId;
uint64 newShardId = GetNextShardId();
uint64 *newShardIdPtr = (uint64 *) palloc0(sizeof(uint64));
*newShardIdPtr = GetNextShardId();
insertedShardIds = lappend(insertedShardIds, newShardIdPtr);

int32 shardMinValue = DatumGetInt32(sourceShardInterval->minValue);
int32 shardMaxValue = DatumGetInt32(sourceShardInterval->maxValue);
Expand All @@ -263,7 +266,7 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool
List *sourceShardPlacementList = ShardPlacementListSortedByWorker(
sourceShardId);

InsertShardRow(targetRelationId, newShardId, targetShardStorageType,
InsertShardRow(targetRelationId, *newShardIdPtr, targetShardStorageType,
shardMinValueText, shardMaxValueText);

ShardPlacement *sourcePlacement = NULL;
Expand All @@ -272,21 +275,26 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool
int32 groupId = sourcePlacement->groupId;
const uint64 shardSize = 0;

/*
* Optimistically add shard placement row the pg_dist_shard_placement, in case
* of any error it will be roll-backed.
*/
uint64 shardPlacementId = InsertShardPlacementRow(newShardId,
INVALID_PLACEMENT_ID,
shardSize,
groupId);

ShardPlacement *shardPlacement = LoadShardPlacement(newShardId,
shardPlacementId);
insertedShardPlacements = lappend(insertedShardPlacements, shardPlacement);
InsertShardPlacementRow(*newShardIdPtr,
INVALID_PLACEMENT_ID,
shardSize,
groupId);
}
}

/*
* load shard placements for the shard at once after all placement insertions
* finished. That prevents MetadataCache from rebuilding unnecessarily after
* each placement insertion.
*/
uint64 *shardIdPtr;
foreach_ptr(shardIdPtr, insertedShardIds)
{
List *placementsForShard = ShardPlacementList(*shardIdPtr);
insertedShardPlacements = list_concat(insertedShardPlacements,
placementsForShard);
}

CreateShardsOnWorkers(targetRelationId, insertedShardPlacements,
useExclusiveConnections, colocatedShard);
}
Expand Down