Skip to content

Commit

Permalink
Add fullstack test case
Browse files Browse the repository at this point in the history
  • Loading branch information
@JaySon-Huang
JaySon-Huang committed Nov 21, 2023
1 parent 6865a01 commit db0784c
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 162 deletions.
225 changes: 90 additions & 135 deletions dbms/src/TiDB/Schema/SchemaBuilder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,68 +493,63 @@ void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(DatabaseID databa
}

applyDropTable(db_info->id, table_id);
return;
}
else

assert(table_info->replica_info.count != 0);
// if set not 0, we first check whether the storage exists, and then check the replica_count and available
auto & tmt_context = context.getTMTContext();
auto storage = tmt_context.getStorages().get(keyspace_id, table_info->id);
if (storage == nullptr)
{
// if set not 0, we first check whether the storage exists, and then check the replica_count and available
auto & tmt_context = context.getTMTContext();
auto storage = tmt_context.getStorages().get(keyspace_id, table_info->id);
if (storage != nullptr)
if (!table_id_map.tableIDInDatabaseIdMap(table_id))
{
if (storage->getTombstone() == 0)
{
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
auto storage_replica_info = managed_storage->getTableInfo().replica_info;
if (storage_replica_info.count == table_info->replica_info.count
&& storage_replica_info.available == table_info->replica_info.available)
{
return;
}
else
{
if (table_info->isLogicalPartitionTable())
{
for (const auto & part_def : table_info->partition.definitions)
{
auto new_part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper);
auto part_storage = tmt_context.getStorages().get(keyspace_id, new_part_table_info->id);
if (part_storage != nullptr)
{
auto alter_lock = part_storage->lockForAlter(getThreadNameAndID());
part_storage->alterSchemaChange(
alter_lock,
*new_part_table_info,
name_mapper.mapDatabaseName(db_info->id, keyspace_id),
name_mapper.mapTableName(*new_part_table_info),
context);
}
else
table_id_map.emplacePartitionTableID(part_def.id, table_id);
}
}
auto alter_lock = storage->lockForAlter(getThreadNameAndID());
storage->alterSchemaChange(
alter_lock,
*table_info,
name_mapper.mapDatabaseName(db_info->id, keyspace_id),
name_mapper.mapTableName(*table_info),
context);
}
return;
}
else
{
applyRecoverTable(db_info->id, table_id);
}
applyCreateTable(db_info->id, table_id);
}
else
return;
}

if (storage->getTombstone() != 0)
{
applyRecoverTable(db_info->id, table_id);
return;
}

auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
auto storage_replica_info = managed_storage->getTableInfo().replica_info;
if (storage_replica_info.count == table_info->replica_info.count
&& storage_replica_info.available == table_info->replica_info.available)
{
return;
}

if (table_info->isLogicalPartitionTable())
{
for (const auto & part_def : table_info->partition.definitions)
{
if (!table_id_map.tableIDInDatabaseIdMap(table_id))
auto new_part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper);
auto part_storage = tmt_context.getStorages().get(keyspace_id, new_part_table_info->id);
if (part_storage != nullptr)
{
applyCreateTable(db_info->id, table_id);
auto alter_lock = part_storage->lockForAlter(getThreadNameAndID());
part_storage->alterSchemaChange(
alter_lock,
*new_part_table_info,
name_mapper.mapDatabaseName(db_info->id, keyspace_id),
name_mapper.mapTableName(*new_part_table_info),
context);
}
else
table_id_map.emplacePartitionTableID(part_def.id, table_id);
}
}
auto alter_lock = storage->lockForAlter(getThreadNameAndID());
storage->alterSchemaChange(
alter_lock,
*table_info,
name_mapper.mapDatabaseName(db_info->id, keyspace_id),
name_mapper.mapTableName(*table_info),
context);
}

template <typename Getter, typename NameMapper>
Expand Down Expand Up @@ -807,15 +802,17 @@ void SchemaBuilder<Getter, NameMapper>::applyRecoverTable(DatabaseID database_id
for (const auto & part_def : table_info->partition.definitions)
{
auto new_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper);
applyRecoverPhysicalTable(db_info, new_table_info);
tryRecoverPhysicalTable(db_info, new_table_info);
}
}

applyRecoverPhysicalTable(db_info, table_info);
tryRecoverPhysicalTable(db_info, table_info);
}

// Return true - the Storage instance exists and is recovered (or not tombstone)
// false - the Storage instance does not exist
template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRecoverPhysicalTable(
bool SchemaBuilder<Getter, NameMapper>::tryRecoverPhysicalTable(
const TiDB::DBInfoPtr & db_info,
const TiDB::TableInfoPtr & table_info)
{
Expand All @@ -827,48 +824,47 @@ void SchemaBuilder<Getter, NameMapper>::applyRecoverPhysicalTable(
log,
"Storage instance does not exist, tryRecoverPhysicalTable is ignored, table_id={}",
table_info->id);
return false;
}
else
{
if (!storage->isTombstone())
{
LOG_INFO(
log,
"Trying to recover table {} but it is not marked as tombstone, skip, database_id={} table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
return;
}

if (!storage->isTombstone())
{
LOG_INFO(
log,
"Create table {} by recover begin, database_id={} table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
AlterCommands commands;
{
AlterCommand command;
command.type = AlterCommand::RECOVER;
commands.emplace_back(std::move(command));
}
auto alter_lock = storage->lockForAlter(getThreadNameAndID());
storage->updateTombstone(
alter_lock,
commands,
name_mapper.mapDatabaseName(*db_info),
*table_info,
name_mapper,
context);
LOG_INFO(
log,
"Create table {} by recover end, database_id={} table_id={}",
"Trying to recover table {} but it is not marked as tombstone, skip, database_id={} table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
return;
return true;
}

LOG_INFO(
log,
"Create table {} by recover begin, database_id={} table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
AlterCommands commands;
{
AlterCommand command;
command.type = AlterCommand::RECOVER;
commands.emplace_back(std::move(command));
}
auto alter_lock = storage->lockForAlter(getThreadNameAndID());
storage->updateTombstone(
alter_lock,
commands,
name_mapper.mapDatabaseName(*db_info),
*table_info,
name_mapper,
context);
LOG_INFO(
log,
"Create table {} by recover end, database_id={} table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
return true;
}

static ASTPtr parseCreateStatement(const String & statement)
Expand Down Expand Up @@ -1098,53 +1094,12 @@ void SchemaBuilder<Getter, NameMapper>::applyCreateStorageInstance(
db_info->id,
table_info->id);

/// Check if this is a RECOVER table.
/// Try to recover the existing storage instance
if (tryRecoverPhysicalTable(db_info, table_info))
{
auto & tmt_context = context.getTMTContext();
if (auto * storage = tmt_context.getStorages().get(keyspace_id, table_info->id).get(); storage)
{
if (!storage->isTombstone())
{
LOG_DEBUG(
log,
"Trying to create table {}, but it already exists and is not marked as tombstone, database_id={} "
"table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
return;
}

LOG_DEBUG(
log,
"Recovering table {} with database_id={}, table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
AlterCommands commands;
{
AlterCommand command;
command.type = AlterCommand::RECOVER;
commands.emplace_back(std::move(command));
}
auto alter_lock = storage->lockForAlter(getThreadNameAndID());
storage->updateTombstone(
alter_lock,
commands,
name_mapper.mapDatabaseName(*db_info),
*table_info,
name_mapper,
context);
LOG_INFO(
log,
"Created table {}, database_id={} table_id={}",
name_mapper.debugCanonicalName(*db_info, *table_info),
db_info->id,
table_info->id);
return;
}
return;
}

// Else the storage instance does not exist, create it.
/// Normal CREATE table.
if (table_info->engine_type == StorageEngine::UNSPECIFIED)
{
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/TiDB/Schema/SchemaBuilder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ struct SchemaBuilder

void applyRecoverTable(DatabaseID database_id, TiDB::TableID table_id);

void applyRecoverPhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info);
bool tryRecoverPhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info);

/// Parameter schema_name should be mapped.
void applyDropPhysicalTable(const String & db_name, TableID table_id);
Expand Down
25 changes: 1 addition & 24 deletions dbms/src/TiDB/Schema/TiDBSchemaSyncer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,29 +152,6 @@ Int64 TiDBSchemaSyncer<mock_getter, mock_mapper>::syncAllSchemas(Context & conte
return version;
}

template <bool mock_getter, bool mock_mapper>
std::tuple<bool, DatabaseID, TableID> TiDBSchemaSyncer<mock_getter, mock_mapper>::findDatabaseIDAndTableID(
TableID physical_table_id)
{
auto database_id = table_id_map.findTableIDInDatabaseMap(physical_table_id);
TableID logical_table_id = physical_table_id;
if (database_id == -1)
{
/// if we can't find physical_table_id in table_id_to_database_id,
/// we should first try to find it in partition_id_to_logical_id because it could be the pysical_table_id of partition tables
logical_table_id = table_id_map.findTableIDInPartitionMap(physical_table_id);
if (logical_table_id != -1)
database_id = table_id_map.findTableIDInDatabaseMap(logical_table_id);
}

if (database_id != -1 && logical_table_id != -1)
{
return std::make_tuple(true, database_id, logical_table_id);
}

return std::make_tuple(false, 0, 0);
}

template <bool mock_getter, bool mock_mapper>
std::tuple<bool, String> TiDBSchemaSyncer<mock_getter, mock_mapper>::trySyncTableSchema(
Context & context,
Expand All @@ -184,7 +161,7 @@ std::tuple<bool, String> TiDBSchemaSyncer<mock_getter, mock_mapper>::trySyncTabl
{
// Get logical_table_id and database_id by physical_table_id.
// If the table is a partition table, logical_table_id != physical_table_id, otherwise, logical_table_id == physical_table_id;
auto [found, database_id, logical_table_id] = findDatabaseIDAndTableID(physical_table_id);
auto [found, database_id, logical_table_id] = table_id_map.findDatabaseIDAndLogicalTableID(physical_table_id);
if (!found)
{
String message = fmt::format(
Expand Down
2 changes: 0 additions & 2 deletions dbms/src/TiDB/Schema/TiDBSchemaSyncer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,6 @@ class TiDBSchemaSyncer : public SchemaSyncer
}
}

std::tuple<bool, DatabaseID, TableID> findDatabaseIDAndTableID(TableID physical_table_id);

public:
TiDBSchemaSyncer(KVClusterPtr cluster_, KeyspaceID keyspace_id_)
: cluster(std::move(cluster_))
Expand Down

0 comments on commit db0784c

Please sign in to comment.