Skip to content

Commit

Permalink
Address comments from @jrhee17
Browse files Browse the repository at this point in the history
  • Loading branch information
@minwoox
minwoox committed Sep 10, 2024
1 parent 9a5b3e8 commit d0eba6b
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 51 deletions.
11 changes: 11 additions & 0 deletions ...com/linecorp/centraldogma/server/internal/mirror/RemovingHostnamePatternsServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@

package com.linecorp.centraldogma.server.internal.mirror;

import static com.linecorp.centraldogma.server.internal.mirror.RemovingHostnamePatternsService.REMOVING_HOSTNAME_JOB_LOG;
import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson;
import static org.assertj.core.api.Assertions.assertThat;

import java.time.Instant;
import java.util.Map;

import org.junit.jupiter.api.extension.RegisterExtension;
Expand All @@ -35,8 +37,10 @@
import com.linecorp.centraldogma.common.Revision;
import com.linecorp.centraldogma.internal.Jackson;
import com.linecorp.centraldogma.server.mirror.MirrorCredential;
import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer;
import com.linecorp.centraldogma.server.storage.project.Project;
import com.linecorp.centraldogma.server.storage.project.ProjectManager;
import com.linecorp.centraldogma.server.storage.repository.Repository;
import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension;

class RemovingHostnamePatternsServiceTest {
Expand Down Expand Up @@ -168,6 +172,13 @@ void removeHostnamePatterns(String hostnamePatterns) throws Exception {
mirrorCredential = Jackson.treeToValue(entry3.contentAsJson(), MirrorCredential.class);
assertThat(mirrorCredential.hostnamePatterns()).isEmpty();
assertThat(mirrorCredential.id()).isEqualTo("credential-3");

// Make sure that the log is written.
final Repository dogmaRepo = projectManager.get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA)
.repos().get(Project.REPO_DOGMA);
final Map<String, Entry<?>> log = dogmaRepo.find(Revision.HEAD, REMOVING_HOSTNAME_JOB_LOG).join();
final JsonNode data = log.get(REMOVING_HOSTNAME_JOB_LOG).contentAsJson();
assertThat(Jackson.readValue(data.get("timestamp").asText(), Instant.class)).isBefore(Instant.now());
}

private static void assertCredential(Entry<?> entry, String credential) throws JsonParseException {
Expand Down
126 changes: 75 additions & 51 deletions ...ava/com/linecorp/centraldogma/server/internal/mirror/RemovingHostnamePatternsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,36 +18,46 @@

import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.PATH_CREDENTIALS;

import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;

import com.linecorp.centraldogma.common.Author;
import com.linecorp.centraldogma.common.Change;
import com.linecorp.centraldogma.common.Entry;
import com.linecorp.centraldogma.common.EntryType;
import com.linecorp.centraldogma.common.Markup;
import com.linecorp.centraldogma.common.Revision;
import com.linecorp.centraldogma.internal.Jackson;
import com.linecorp.centraldogma.server.command.Command;
import com.linecorp.centraldogma.server.command.CommandExecutor;
import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor;
import com.linecorp.centraldogma.server.management.ServerStatus;
import com.linecorp.centraldogma.server.command.CommitResult;
import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer;
import com.linecorp.centraldogma.server.storage.project.Project;
import com.linecorp.centraldogma.server.storage.project.ProjectManager;
import com.linecorp.centraldogma.server.storage.repository.MetaRepository;
import com.linecorp.centraldogma.server.storage.repository.Repository;

final class RemovingHostnamePatternsService {

private static final Logger logger = LoggerFactory.getLogger(RemovingHostnamePatternsService.class);

@VisibleForTesting
static final String REMOVING_HOSTNAME_JOB_LOG = "/removing-hostname-job.json";

private final ProjectManager projectManager;
private final CommandExecutor commandExecutor;

Expand All @@ -57,65 +67,79 @@ final class RemovingHostnamePatternsService {
}

void start() throws Exception {
// Enter read-only mode.
commandExecutor.execute(Command.updateServerStatus(ServerStatus.REPLICATION_ONLY))
.get(1, TimeUnit.MINUTES);
logger.info("Start removing hostnamePatterns in credential ...");
if (commandExecutor instanceof ZooKeeperCommandExecutor) {
logger.debug("Waiting for 30 seconds to make sure that all cluster have been notified of the " +
"read-only mode ...");
Thread.sleep(30000);
if (hasJobLog()) {
return;
}
logger.info("Start removing hostnamePatterns in credential ...");

final Stopwatch stopwatch = Stopwatch.createStarted();
int numProjects = 0;
try {
for (Project project : projectManager.list().values()) {
if (InternalProjectInitializer.INTERNAL_PROJECT_DOGMA.equals(project.name())) {
continue;
}
try {
logger.info("Removing hostnamePatterns in credential files in the project: {} ...",
project.name());
final List<Change<?>> changes = new ArrayList<>();
final MetaRepository repository = project.metaRepo();
for (Entry<?> entry : repository.find(Revision.HEAD, PATH_CREDENTIALS + "**")
.get().values()) {
if (entry.type() != EntryType.JSON) {
continue;
}
final JsonNode content = (JsonNode) entry.content();
if (content.get("hostnamePatterns") == null) {
continue;
}
changes.add(Change.ofJsonUpsert(entry.path(),
((ObjectNode) content).without("hostnamePatterns")));
for (Project project : projectManager.list().values()) {
if (InternalProjectInitializer.INTERNAL_PROJECT_DOGMA.equals(project.name())) {
continue;
}
try {
logger.info("Removing hostnamePatterns in credential files in the project: {} ...",
project.name());
final List<Change<?>> changes = new ArrayList<>();
final MetaRepository repository = project.metaRepo();
for (Entry<?> entry : repository.find(Revision.HEAD, PATH_CREDENTIALS + "**")
.get().values()) {
if (entry.type() != EntryType.JSON) {
continue;
}

if (changes.isEmpty()) {
final JsonNode content = (JsonNode) entry.content();
if (content.get("hostnamePatterns") == null) {
continue;
}
changes.add(Change.ofJsonUpsert(entry.path(),
((ObjectNode) content).without("hostnamePatterns")));
}

numProjects++;
logger.info("hostnamePatterns in credentials are removed in the project: {}",
project.name());

commandExecutor.execute(Command.forcePush(Command.push(
Author.SYSTEM, project.name(), Project.REPO_META, Revision.HEAD,
"Remove hostnamePatterns in credentials.", "", Markup.PLAINTEXT,
changes)))
.get(1, TimeUnit.MINUTES);
} catch (Throwable t) {
logger.warn("Failed to remove hostnamePatterns in credential files in the project: {}",
project.name(), t);
if (changes.isEmpty()) {
continue;
}

numProjects++;
logger.info("hostnamePatterns in credentials are removed in the project: {}",
project.name());

executeCommand(Command.push(
Author.SYSTEM, project.name(), Project.REPO_META, Revision.HEAD,
"Remove hostnamePatterns in credentials.", "", Markup.PLAINTEXT,
changes));
} catch (Throwable t) {
logger.warn("Failed to remove hostnamePatterns in credential files in the project: {}",
project.name(), t);
}
logger.info("hostnamePatterns are removed in {} projects. (took: {} ms.)",
numProjects, stopwatch.elapsed().toMillis());
} finally {
// Exit read-only mode.
commandExecutor.execute(Command.updateServerStatus(ServerStatus.WRITABLE))
.get(1, TimeUnit.MINUTES);
}
logger.info("hostnamePatterns are removed in {} projects. (took: {} ms.)",
numProjects, stopwatch.elapsed().toMillis());
logRemovingJob(numProjects);
}

private boolean hasJobLog() throws Exception {
final Project internalProj = projectManager.get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA);
final Repository repository = internalProj.repos().get(Project.REPO_DOGMA);
final Map<String, Entry<?>> entries = repository.find(Revision.HEAD, REMOVING_HOSTNAME_JOB_LOG).get();
final Entry<?> entry = entries.get(REMOVING_HOSTNAME_JOB_LOG);
return entry != null;
}

private CommitResult executeCommand(Command<CommitResult> command)
throws InterruptedException, ExecutionException, TimeoutException {
return commandExecutor.execute(command).get(1, TimeUnit.MINUTES);
}

private void logRemovingJob(int numProjects) throws Exception {
final ImmutableMap<String, Object> data =
ImmutableMap.of("timestamp", Instant.now(),
"projects", numProjects);
final Change<JsonNode> change = Change.ofJsonUpsert(REMOVING_HOSTNAME_JOB_LOG,
Jackson.writeValueAsString(data));
executeCommand(Command.push(Author.SYSTEM, InternalProjectInitializer.INTERNAL_PROJECT_DOGMA,
Project.REPO_DOGMA, Revision.HEAD,
"Removing hostnamePatterns from " + numProjects+ " projects has been done.", "",
Markup.PLAINTEXT, change));
}
}

0 comments on commit d0eba6b

Please sign in to comment.