Skip to content

Commit

Permalink
test: Convert replicaton tests to the v1 API (#116)
Browse files Browse the repository at this point in the history 
These tests now only use the v1 API and the associated test cluster
machinery.
  • Loading branch information
@freeekanayaka
freeekanayaka authored Dec 28, 2023
2 parents 72b4804 + 1bad892 commit 6577440
Show file tree
Hide file tree
Showing 27 changed files with 2,052 additions and 1,028 deletions.
32 changes: 14 additions & 18 deletions src/client.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ static int clientSubmitConfiguration(struct raft *r, struct raft_entry *entry)

rv = configurationDecode(&entry->buf, &configuration);
if (rv != 0) {
assert(rv == RAFT_NOMEM || rv == RAFT_MALFORMED);
goto err;
}

Expand All @@ -34,6 +35,7 @@ static int clientSubmitConfiguration(struct raft *r, struct raft_entry *entry)
if (configuration.n != r->configuration.n) {
rv = progressRebuildArray(r, &configuration);
if (rv != 0) {
assert(rv == RAFT_NOMEM);
goto err_after_decode;
}
}
Expand All @@ -48,7 +50,7 @@ static int clientSubmitConfiguration(struct raft *r, struct raft_entry *entry)
err_after_decode:
configurationClose(&configuration);
err:
assert(rv != 0);
assert(rv == RAFT_NOMEM || rv == RAFT_MALFORMED);
return rv;
}

Expand All @@ -65,7 +67,6 @@ int ClientSubmit(struct raft *r, struct raft_entry *entries, unsigned n)
if (r->state != RAFT_LEADER || r->transfer != NULL) {
rv = RAFT_NOTLEADER;
ErrMsgFromCode(r->errmsg, rv);
tracef("raft_apply not leader");
goto err;
}

Expand All @@ -84,6 +85,9 @@ int ClientSubmit(struct raft *r, struct raft_entry *entries, unsigned n)

rv = logAppend(r->log, entry->term, entry->type, &entry->buf, NULL);
if (rv != 0) {
/* This logAppend call can't fail with RAFT_BUSY, because these are
* brand new entries. */
assert(rv == RAFT_NOMEM);
goto err_after_log_append;
}

Expand All @@ -97,6 +101,7 @@ int ClientSubmit(struct raft *r, struct raft_entry *entries, unsigned n)

rv = replicationTrigger(r, index);
if (rv != 0) {
/* TODO: assert the possible error values */
goto err_after_log_append;
}

Expand All @@ -105,7 +110,7 @@ int ClientSubmit(struct raft *r, struct raft_entry *entries, unsigned n)
err_after_log_append:
logDiscard(r->log, index);
err:
assert(rv != 0);
assert(rv == RAFT_NOTLEADER || rv == RAFT_MALFORMED || rv == RAFT_NOMEM);
return rv;
}

Expand All @@ -120,8 +125,6 @@ int raft_apply(struct raft *r,
struct raft_entry entry;
int rv;

tracef("raft_apply n %d", n);

assert(r != NULL);
assert(bufs != NULL);
assert(n == 1);
Expand Down Expand Up @@ -196,8 +199,9 @@ int raft_barrier(struct raft *r, struct raft_barrier *req, raft_barrier_cb cb)
return rv;
}

int ClientChangeConfiguration(struct raft *r,
const struct raft_configuration *configuration)
static int clientChangeConfiguration(
struct raft *r,
const struct raft_configuration *configuration)
{
struct raft_entry entry;
struct raft_event event;
Expand Down Expand Up @@ -241,8 +245,6 @@ int raft_add(struct raft *r,
return rv;
}

tracef("add server: id %llu, address %s", id, address);

/* Make a copy of the current configuration, and add the new server to
* it. */
rv = configurationCopy(&r->configuration, &configuration);
Expand All @@ -258,7 +260,7 @@ int raft_add(struct raft *r,
req->cb = cb;
req->catch_up_id = 0;

rv = ClientChangeConfiguration(r, &configuration);
rv = clientChangeConfiguration(r, &configuration);
if (rv != 0) {
goto err_after_configuration_copy;
}
Expand Down Expand Up @@ -323,7 +325,6 @@ int raft_assign(struct raft *r,

r->now = r->io->time(r->io);

tracef("raft_assign to id:%llu the role:%d", id, role);
if (role != RAFT_STANDBY && role != RAFT_VOTER && role != RAFT_SPARE) {
rv = RAFT_BADROLE;
ErrMsgFromCode(r->errmsg, rv);
Expand Down Expand Up @@ -384,9 +385,8 @@ int raft_assign(struct raft *r,
int old_role = r->configuration.servers[server_index].role;
r->configuration.servers[server_index].role = role;

rv = ClientChangeConfiguration(r, &r->configuration);
rv = clientChangeConfiguration(r, &r->configuration);
if (rv != 0) {
tracef("ClientChangeConfiguration failed %d", rv);
r->configuration.servers[server_index].role = old_role;
return rv;
}
Expand Down Expand Up @@ -432,8 +432,6 @@ int raft_remove(struct raft *r,
goto err;
}

tracef("remove server: id %llu", id);

/* Make a copy of the current configuration, and remove the given server
* from it. */
rv = configurationCopy(&r->configuration, &configuration);
Expand All @@ -449,7 +447,7 @@ int raft_remove(struct raft *r,
req->cb = cb;
req->catch_up_id = 0;

rv = ClientChangeConfiguration(r, &configuration);
rv = clientChangeConfiguration(r, &configuration);
if (rv != 0) {
goto err_after_configuration_copy;
}
Expand Down Expand Up @@ -545,9 +543,7 @@ int raft_transfer(struct raft *r,
unsigned i;
int rv;

tracef("transfer to %llu", id);
if (r->state != RAFT_LEADER || r->transfer != NULL) {
tracef("transfer error - state:%d", r->state);
rv = RAFT_NOTLEADER;
ErrMsgFromCode(r->errmsg, rv);
goto err;
Expand Down
19 changes: 14 additions & 5 deletions src/client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,20 @@

#include "../include/raft.h"

/* Submit the given entries and start replicating them. */
/* Submit the given entries and start replicating them.
*
* Errors:
*
* RAFT_NOTLEADER
* The server is not leader, or a leadership transfer is in progress.
*
* RAFT_MALFORMED
* The submitted entry is of type RAFT_CHANGE, but the encoded configuration
* is invalid.
*
* RAFT_NOMEM
* Memory could not be allocated to store the new entry.
*/
int ClientSubmit(struct raft *r, struct raft_entry *entries, unsigned n);

/* Start catching-up the given server. */
Expand All @@ -12,8 +25,4 @@ void ClientCatchUp(struct raft *r, raft_id server_id);
/* Start transferring leadership to the given server. */
int ClientTransfer(struct raft *r, raft_id server_id);

/* Submit a new RAFT_CHANGE entry with the given new configuration. */
int ClientChangeConfiguration(struct raft *r,
const struct raft_configuration *configuration);

#endif /* CLIENT_H_ */
70 changes: 41 additions & 29 deletions src/convert.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
#include "request.h"
#include "tracing.h"

#define tracef(...) Tracef(r->tracer, __VA_ARGS__)
#define infof(...) Infof(r->tracer, " " __VA_ARGS__)

/* Convenience for setting a new state value and asserting that the transition
* is valid. */
Expand Down Expand Up @@ -92,13 +92,18 @@ void convertToFollower(struct raft *r)
r->follower_state.current_leader.address = NULL;
}

int convertToCandidate(struct raft *r, bool disrupt_leader)
int convertToCandidate(struct raft *r, const bool disrupt_leader)
{
const struct raft_server *server;
size_t n_voters = configurationVoterCount(&r->configuration);

(void)server; /* Only used for assertions. */

/* Check that we're a voter in the current configuration. */
server = configurationGet(&r->configuration, r->id);
assert(server != NULL);
assert(server->role == RAFT_VOTER);

convertClear(r);
convertSetState(r, RAFT_CANDIDATE);

Expand All @@ -112,12 +117,8 @@ int convertToCandidate(struct raft *r, bool disrupt_leader)

/* Fast-forward to leader if we're the only voting server in the
* configuration. */
server = configurationGet(&r->configuration, r->id);
assert(server != NULL);
assert(server->role == RAFT_VOTER);

if (n_voters == 1) {
tracef("self elect and convert to leader");
infof("self elect and convert to leader");
return convertToLeader(r);
}

Expand All @@ -129,10 +130,9 @@ int convertToCandidate(struct raft *r, bool disrupt_leader)

int convertToLeader(struct raft *r)
{
size_t n_voters;
int rv;

tracef("become leader for term %llu", r->current_term);

convertClear(r);
convertSetState(r, RAFT_LEADER);

Expand All @@ -143,7 +143,8 @@ int convertToLeader(struct raft *r)
/* Allocate and initialize the progress array. */
rv = progressBuildArray(r);
if (rv != 0) {
return rv;
assert(rv == RAFT_NOMEM);
goto err;
}

/* Reset promotion state. */
Expand All @@ -154,19 +155,23 @@ int convertToLeader(struct raft *r)

/* By definition, all entries until the last_stored entry will be committed
* if we are the only voter around. */
size_t n_voters = configurationVoterCount(&r->configuration);
if (n_voters == 1 && (r->last_stored > r->commit_index)) {
tracef("apply log entries after self election %llu %llu",
r->last_stored, r->commit_index);
r->commit_index = r->last_stored;
r->update->flags |= RAFT_UPDATE_COMMIT_INDEX;
} else if (n_voters > 1) {
n_voters = configurationVoterCount(&r->configuration);
assert(n_voters > 0);
if (n_voters == 1) {
if (r->last_stored > r->commit_index) {
infof("apply log entries after self election %llu %llu",
r->last_stored, r->commit_index);
r->commit_index = r->last_stored;
r->update->flags |= RAFT_UPDATE_COMMIT_INDEX;
}
} else {
/* Raft Dissertation, paragraph 6.4:
* The Leader Completeness Property guarantees that a leader has all
* committed entries, but at the start of its term, it may not know
* which those are. To find out, it needs to commit an entry from its
* term. Raft handles this by having each leader commit a blank no-op
* entry into the log at the start of its term. */
*
* The Leader Completeness Property guarantees that a leader has all
* committed entries, but at the start of its term, it may not know
* which those are. To find out, it needs to commit an entry from its
* term. Raft handles this by having each leader commit a blank no-op
* entry into the log at the start of its term. */
struct raft_entry entry;

entry.type = RAFT_BARRIER;
Expand All @@ -175,21 +180,28 @@ int convertToLeader(struct raft *r)
entry.buf.base = raft_malloc(entry.buf.len);

if (entry.buf.base == NULL) {
return RAFT_NOMEM;
rv = RAFT_NOMEM;
goto err;
}

rv = ClientSubmit(r, &entry, 1);
if (rv != 0) {
tracef(
"failed to send no-op barrier entry after leader conversion: "
"%d",
rv);
/* This call to ClientSubmit can only fail with RAFT_NOMEM, because
* it's not a RAFT_CHANGE entry (RAFT_MALFORMED can't be returned)
* and we're leader (RAFT_NOTLEADER can't be returned) */
assert(rv == RAFT_NOMEM);
infof("can't submit no-op after converting to leader: %s",
raft_strerror(rv));
raft_free(entry.buf.base);
return rv;
goto err;
}
}

return 0;

err:
assert(rv == RAFT_NOMEM);
return rv;
}

void convertToUnavailable(struct raft *r)
Expand All @@ -202,4 +214,4 @@ void convertToUnavailable(struct raft *r)
convertSetState(r, RAFT_UNAVAILABLE);
}

#undef tracef
#undef infof
16 changes: 14 additions & 2 deletions src/convert.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,13 @@ void convertToFollower(struct raft *r);
* On conversion to candidate, start election
*
* If the disrupt_leader flag is true, the server will set the disrupt leader
* flag of the RequestVote messages it sends. */
* flag of the RequestVote messages it sends.
*
* Errors:
*
* RAFT_NOMEM
* Memory for the votes array could not be allocated.
*/
int convertToCandidate(struct raft *r, bool disrupt_leader);

/* Convert from candidate to leader.
Expand All @@ -44,7 +50,13 @@ int convertToCandidate(struct raft *r, bool disrupt_leader);
* The leader maintains a nextIndex for each follower, which is the index
* of the next log entry the leader will send to that follower. When a
* leader first comes to power, it initializes all nextIndex values to the
* index just after the last one in its log. */
* index just after the last one in its log.
* Errors:
*
* RAFT_NOMEM
* Memory for the progress array or for the initial no-op entry could
* not be allocated.
*/
int convertToLeader(struct raft *r);

void convertToUnavailable(struct raft *r);
Expand Down
Loading

0 comments on commit 6577440

Please sign in to comment.