Skip to content

Commit

Permalink
chore: remove drop protection before cleanup (#2863)
Browse files Browse the repository at this point in the history 
The cleanup function used to remove old test databases did not remove the drop protection flag before trying to drop a database.
  • Loading branch information
@olavloite
olavloite authored Feb 6, 2024
1 parent 968877e commit 1765e08
Show file tree
Hide file tree
Showing 4 changed files with 126 additions and 147 deletions.
67 changes: 51 additions & 16 deletions google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,15 @@

package com.google.cloud.spanner.testing;

import com.google.api.client.util.BackOff;
import com.google.api.client.util.ExponentialBackOff;
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.spanner.BatchClient;
import com.google.cloud.spanner.Database;
import com.google.cloud.spanner.DatabaseClient;
import com.google.cloud.spanner.DatabaseId;
import com.google.cloud.spanner.Dialect;
import com.google.cloud.spanner.ErrorCode;
import com.google.cloud.spanner.InstanceId;
import com.google.cloud.spanner.Spanner;
import com.google.cloud.spanner.SpannerException;
Expand All @@ -34,6 +37,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand All @@ -54,7 +58,7 @@ public class RemoteSpannerHelper {
private static int dbRolePrefix = new Random().nextInt(Integer.MAX_VALUE);
private static final AtomicInteger backupSeq = new AtomicInteger();
private static final int backupPrefix = new Random().nextInt(Integer.MAX_VALUE);
private final List<Database> dbs = new ArrayList<>();
private final List<DatabaseId> databaseIds = new ArrayList<>();

protected RemoteSpannerHelper(SpannerOptions options, InstanceId instanceId, Spanner client) {
this.options = options;
Expand Down Expand Up @@ -132,30 +136,59 @@ public String getUniqueBackupId() {
public Database createTestDatabase(Dialect dialect, Iterable<String> statements)
throws SpannerException {
String dbId = getUniqueDatabaseId();
DatabaseId databaseId = DatabaseId.of(instanceId.getProject(), instanceId.getInstance(), dbId);
Database databaseToCreate =
client
.getDatabaseAdminClient()
.newDatabaseBuilder(
DatabaseId.of(instanceId.getProject(), instanceId.getInstance(), dbId))
.setDialect(dialect)
.build();
client.getDatabaseAdminClient().newDatabaseBuilder(databaseId).setDialect(dialect).build();
try {
Iterable<String> ddlStatements =
dialect == Dialect.POSTGRESQL ? Collections.emptyList() : statements;
OperationFuture<Database, CreateDatabaseMetadata> op =
client.getDatabaseAdminClient().createDatabase(databaseToCreate, ddlStatements);
Database db = op.get();
Database db = null;
final int maxAttempts = 20;
BackOff backOff =
new ExponentialBackOff.Builder()
.setInitialIntervalMillis(10_000)
.setMaxIntervalMillis(60_000)
.setMaxElapsedTimeMillis(120_000)
.build();
for (int attempts = 0; attempts < maxAttempts; attempts++) {
try {
OperationFuture<Database, CreateDatabaseMetadata> op =
client.getDatabaseAdminClient().createDatabase(databaseToCreate, ddlStatements);
db = op.get();
break;
} catch (ExecutionException executionException) {
SpannerException spannerException =
SpannerExceptionFactory.asSpannerException(executionException.getCause());
if (spannerException.getErrorCode() != ErrorCode.RESOURCE_EXHAUSTED) {
throw executionException;
}
} catch (SpannerException spannerException) {
if (spannerException.getErrorCode() != ErrorCode.RESOURCE_EXHAUSTED) {
throw spannerException;
}
}
long sleep = backOff.nextBackOffMillis();
if (sleep > 0L) {
Thread.sleep(sleep);
}
}
if (db == null) {
throw SpannerExceptionFactory.newSpannerException(
ErrorCode.RESOURCE_EXHAUSTED,
String.format("Failed to create test database after %d attempts", maxAttempts));
}
if (dialect == Dialect.POSTGRESQL && Iterables.size(statements) > 0) {
client
.getDatabaseAdminClient()
.updateDatabaseDdl(instanceId.getInstance(), dbId, statements, null)
.get();
}
logger.log(Level.FINE, "Created test database {0}", db.getId());
dbs.add(db);
return db;
} catch (Exception e) {
throw SpannerExceptionFactory.newSpannerException(e);
} finally {
databaseIds.add(databaseId);
}
}

Expand All @@ -167,13 +200,15 @@ public Database createTestDatabase(Iterable<String> statements) throws SpannerEx
public void cleanUp() {
// Drop all the databases we created explicitly.
int numDropped = 0;
for (Database db : dbs) {
for (DatabaseId databaseId : databaseIds) {
try {
logger.log(Level.INFO, "Dropping test database {0}", db.getId());
db.drop();
logger.log(Level.INFO, "Dropping test database {0}", databaseId);
client
.getDatabaseAdminClient()
.dropDatabase(databaseId.getInstanceId().getInstance(), databaseId.getDatabase());
++numDropped;
} catch (SpannerException e) {
logger.log(Level.SEVERE, "Failed to drop test database " + db.getId(), e);
} catch (Throwable e) {
logger.log(Level.SEVERE, "Failed to drop test database " + databaseId, e);
}
}
logger.log(Level.INFO, "Dropped {0} test database(s)", numDropped);
Expand Down
34 changes: 18 additions & 16 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/IntegrationTestEnv.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@

import com.google.api.client.util.ExponentialBackOff;
import com.google.api.gax.longrunning.OperationFuture;
import com.google.api.gax.paging.Page;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.DatabaseInfo.DatabaseField;
import com.google.cloud.spanner.testing.EmulatorSpannerHelper;
import com.google.cloud.spanner.testing.RemoteSpannerHelper;
import com.google.common.collect.Iterators;
Expand Down Expand Up @@ -213,27 +213,29 @@ private void cleanUpOldDatabases(InstanceId instanceId) {
long OLD_DB_THRESHOLD_SECS = TimeUnit.SECONDS.convert(6L, TimeUnit.HOURS);
Timestamp currentTimestamp = Timestamp.now();
int numDropped = 0;
Page<Database> page = databaseAdminClient.listDatabases(instanceId.getInstance());
String TEST_DB_REGEX = "(testdb_(.*)_(.*))|(mysample-(.*))";

logger.log(Level.INFO, "Dropping old test databases from {0}", instanceId.getName());
while (page != null) {
for (Database db : page.iterateAll()) {
try {
long timeDiff = currentTimestamp.getSeconds() - db.getCreateTime().getSeconds();
// Delete all databases which are more than OLD_DB_THRESHOLD_SECS seconds
// old.
if ((db.getId().getDatabase().matches(TEST_DB_REGEX))
&& (timeDiff > OLD_DB_THRESHOLD_SECS)) {
logger.log(Level.INFO, "Dropping test database {0}", db.getId());
db.drop();
++numDropped;
for (Database db : databaseAdminClient.listDatabases(instanceId.getInstance()).iterateAll()) {
try {
long timeDiff = currentTimestamp.getSeconds() - db.getCreateTime().getSeconds();
// Delete all databases which are more than OLD_DB_THRESHOLD_SECS seconds old.
if ((db.getId().getDatabase().matches(TEST_DB_REGEX))
&& (timeDiff > OLD_DB_THRESHOLD_SECS)) {
logger.log(Level.INFO, "Dropping test database {0}", db.getId());
if (db.isDropProtectionEnabled()) {
Database updatedDatabase =
databaseAdminClient.newDatabaseBuilder(db.getId()).disableDropProtection().build();
databaseAdminClient
.updateDatabase(updatedDatabase, DatabaseField.DROP_PROTECTION)
.get();
}
} catch (SpannerException e) {
logger.log(Level.SEVERE, "Failed to drop test database " + db.getId(), e);
db.drop();
++numDropped;
}
} catch (SpannerException | ExecutionException | InterruptedException e) {
logger.log(Level.SEVERE, "Failed to drop test database " + db.getId(), e);
}
page = page.getNextPage();
}
logger.log(Level.INFO, "Dropped {0} test database(s)", numDropped);
}
Expand Down
68 changes: 15 additions & 53 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITDatabaseAdminTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import com.google.cloud.spanner.testing.RemoteSpannerHelper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata;
import com.google.spanner.admin.database.v1.UpdateDatabaseMetadata;
import java.util.ArrayList;
Expand All @@ -50,7 +49,6 @@
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
Expand All @@ -68,33 +66,20 @@ public class ITDatabaseAdminTest {
private static final Logger logger = Logger.getLogger(ITDatabaseAdminTest.class.getName());
private DatabaseAdminClient dbAdminClient;
private RemoteSpannerHelper testHelper;
private List<Database> dbs = new ArrayList<>();

@Before
public void setUp() {
testHelper = env.getTestHelper();
dbAdminClient = testHelper.getClient().getDatabaseAdminClient();
}

@After
public void tearDown() {
for (Database db : dbs) {
db.drop();
}
dbs.clear();
}

@Test
public void testDatabaseOperations() throws Exception {
final String databaseId = testHelper.getUniqueDatabaseId();
final String instanceId = testHelper.getInstanceId().getInstance();
final String createTableT = "CREATE TABLE T (\n" + " K STRING(MAX),\n" + ") PRIMARY KEY(K)";

final Database createdDatabase =
dbAdminClient
.createDatabase(instanceId, databaseId, ImmutableList.of(createTableT))
.get(5, TimeUnit.MINUTES);
dbs.add(createdDatabase);
final Database createdDatabase = testHelper.createTestDatabase(createTableT);
final String databaseId = createdDatabase.getId().getDatabase();

assertEquals(databaseId, createdDatabase.getId().getDatabase());
assertEquals(Dialect.GOOGLE_STANDARD_SQL, createdDatabase.getDialect());
Expand Down Expand Up @@ -124,7 +109,6 @@ public void testDatabaseOperations() throws Exception {
assertEquals(databaseDdl, ImmutableList.of(createTableT, createTableT2));

dbAdminClient.dropDatabase(instanceId, databaseId);
dbs.clear();

try {
dbAdminClient.getDatabase(instanceId, databaseId);
Expand All @@ -136,13 +120,11 @@ public void testDatabaseOperations() throws Exception {

@Test
public void updateDdlRetry() throws Exception {
String dbId = testHelper.getUniqueDatabaseId();
String instanceId = testHelper.getInstanceId().getInstance();
String statement1 = "CREATE TABLE T (\n" + " K STRING(MAX),\n" + ") PRIMARY KEY(K)";
OperationFuture<Database, CreateDatabaseMetadata> op =
dbAdminClient.createDatabase(instanceId, dbId, ImmutableList.of(statement1));
Database db = op.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
dbs.add(db);
Database db = testHelper.createTestDatabase(statement1);
String dbId = db.getId().getDatabase();

String statement2 = "CREATE TABLE T2 (\n" + " K2 STRING(MAX),\n" + ") PRIMARY KEY(K2)";
OperationFuture<Void, UpdateDatabaseDdlMetadata> op1 =
dbAdminClient.updateDatabaseDdl(instanceId, dbId, ImmutableList.of(statement2), "myop");
Expand All @@ -163,13 +145,9 @@ public void updateDdlRetry() throws Exception {

@Test
public void databaseOperationsViaEntity() throws Exception {
String dbId = testHelper.getUniqueDatabaseId();
String instanceId = testHelper.getInstanceId().getInstance();
String statement1 = "CREATE TABLE T (\n" + " K STRING(MAX),\n" + ") PRIMARY KEY(K)";
OperationFuture<Database, CreateDatabaseMetadata> op =
dbAdminClient.createDatabase(instanceId, dbId, ImmutableList.of(statement1));
Database db = op.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
dbs.add(db);
Database db = testHelper.createTestDatabase(statement1);
String dbId = db.getId().getDatabase();
assertThat(db.getId().getDatabase()).isEqualTo(dbId);

db = db.reload();
Expand All @@ -181,7 +159,6 @@ public void databaseOperationsViaEntity() throws Exception {
Iterable<String> statementsInDb = db.getDdl();
assertThat(statementsInDb).containsExactly(statement1, statement2);
db.drop();
dbs.clear();
try {
db.reload();
fail("Expected exception");
Expand All @@ -191,16 +168,11 @@ public void databaseOperationsViaEntity() throws Exception {
}

@Test
public void listPagination() throws Exception {
List<String> dbIds =
ImmutableList.of(
testHelper.getUniqueDatabaseId(),
testHelper.getUniqueDatabaseId(),
testHelper.getUniqueDatabaseId());

public void listPagination() {
String instanceId = testHelper.getInstanceId().getInstance();
for (String dbId : dbIds) {
dbs.add(dbAdminClient.createDatabase(instanceId, dbId, ImmutableList.of()).get());
List<String> dbIds = new ArrayList<>(3);
for (int n = 0; n < 3; n++) {
dbIds.add(testHelper.createTestDatabase().getId().getDatabase());
}
Page<Database> page = dbAdminClient.listDatabases(instanceId, Options.pageSize(1));
List<String> dbIdsGot = new ArrayList<>();
Expand Down Expand Up @@ -228,10 +200,7 @@ public void createAndListDatabaseRoles() throws Exception {
testHelper.getUniqueDatabaseRole());

String instanceId = testHelper.getInstanceId().getInstance();
Database database =
dbAdminClient
.createDatabase(instanceId, testHelper.getUniqueDatabaseId(), ImmutableList.of())
.get();
Database database = testHelper.createTestDatabase();

// Create the roles in Db.
List<String> dbRolesCreateStatements = new ArrayList<>();
Expand Down Expand Up @@ -280,13 +249,9 @@ public void createAndListDatabaseRoles() throws Exception {
}

@Test
public void updateDatabaseInvalidFieldsToUpdate() throws Exception {
public void updateDatabaseInvalidFieldsToUpdate() {
assumeFalse("Emulator does not drop database protection", isUsingEmulator());
String instanceId = testHelper.getInstanceId().getInstance();
Database database =
dbAdminClient
.createDatabase(instanceId, testHelper.getUniqueDatabaseId(), ImmutableList.of())
.get();
Database database = testHelper.createTestDatabase();
logger.log(Level.INFO, "Created database: {0}", database.getId().getName());

Database databaseToUpdate =
Expand All @@ -304,10 +269,7 @@ public void updateDatabaseInvalidFieldsToUpdate() throws Exception {
public void dropDatabaseWithProtectionEnabled() throws Exception {
assumeFalse("Emulator does not drop database protection", isUsingEmulator());
String instanceId = testHelper.getInstanceId().getInstance();
Database database =
dbAdminClient
.createDatabase(instanceId, testHelper.getUniqueDatabaseId(), ImmutableList.of())
.get();
Database database = testHelper.createTestDatabase();
logger.log(Level.INFO, "Created database: {0}", database.getId().getName());

// Enable drop protection for the database.
Expand Down
Loading

0 comments on commit 1765e08

Please sign in to comment.