Skip to content
This repository has been archived by the owner on May 6, 2020. It is now read-only.

Commit

Permalink
connect: Try to reconnect to proxy
Browse files Browse the repository at this point in the history 
Try to reconnect to proxy if connection is lost (e.g. proxy died).
Timeout is defined either during configure phase (--with-timeout) or
by passing --reconnect-timeout command line parameter.

This patch is needed to implement high availability of cc-proxy.

Fixes #53.

Signed-off-by: Dmitry Voytik <dmitry.voytik@huawei.com>
  • Loading branch information
Dmitry Voytik committed Oct 2, 2017
1 parent 69b6246 commit b8325e8
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 29 deletions.
6 changes: 6 additions & 0 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,10 @@ AS_IF([test $CPPCHECK = 1],
AC_SUBST(CPPCHECK_PATH)]
)

AC_ARG_WITH([timeout], [AS_HELP_STRING([--with-timeout],
[reconnection timeout to proxy in seconds])])
AS_IF([test x"$with_timeout" != "x"],
[AC_DEFINE_UNQUOTED([RECONNECT_TIMEOUT_S], [$with_timeout],
[reconnection timeout to proxy in seconds])])

AC_OUTPUT
140 changes: 111 additions & 29 deletions src/shim.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,23 @@
#include <sys/socket.h>
#include <unistd.h>
#include <netdb.h>
#include <time.h>

#include "config.h"
#include "utils.h"
#include "log.h"
#include "shim.h"

// How many seconds shim tries to reconnect to proxy before it exits
#ifndef RECONNECT_TIMEOUT_S
#define RECONNECT_TIMEOUT_S 10
#endif

// Period between attempts to reconnect to proxy in milliseconds
#ifndef RECONNECT_POLL_MS
#define RECONNECT_POLL_MS 100
#endif

/* globals */

struct pollfd poll_fds[MAX_POLL_FDS] = {{-1}};
Expand Down Expand Up @@ -312,6 +323,7 @@ send_connect_command(struct cc_shim *shim)
return false;
}

shim_debug("Sending connect command\n");
if ( verify_base64url_format(shim->token) != 0) {
shim_error("Invalid token: %s, base64 encoded token expected\n",
shim->token);
Expand All @@ -337,31 +349,34 @@ send_connect_command(struct cc_shim *shim)
return ret;
}

bool reconnect_to_proxy(struct cc_shim *shim);

/*!
* Read message received from proxy.
*
* \param fd File descriptor to read the data from.
* \param shim \ref cc_shim
* \param buf Buffer to store the data in.
* \param size Size in bytes to be read.
*
* \return true on success, false otherwise
*/
bool read_wire_data(int fd, uint8_t *buf, size_t size)
bool read_wire_data(struct cc_shim *shim, uint8_t *buf, size_t size)
{
ssize_t ret;
size_t offset = 0;

if ( fd < 0 || ! buf ) {
if ( shim->proxy_sock_fd < 0 || ! buf ) {
return false;
}

while(offset < size) {
ret = recv(fd, buf+offset, size-offset, 0);
ret = recv(shim->proxy_sock_fd, buf+offset, size-offset, 0);
if (ret == 0) {
shim_debug("Received EOF on file descriptor\n");
// TODO: Exit for now, add logic to try to reconnect
// to proxy.
exit(EXIT_FAILURE);
if (! reconnect_to_proxy(shim)) {
exit(EXIT_FAILURE);
}
return false;
} else if (ret < 0) {
shim_error("Failed to read from fd: %s\n",
strerror(errno));
Expand Down Expand Up @@ -403,7 +418,7 @@ read_frame(struct cc_shim *shim)
abort();
}

if (! read_wire_data(shim->proxy_sock_fd, buf, size)) {
if (! read_wire_data(shim, buf, size)) {
goto error;
}

Expand Down Expand Up @@ -434,8 +449,7 @@ read_frame(struct cc_shim *shim)
abort();
}

if (! read_wire_data(shim->proxy_sock_fd, buf + size,
header_size_in_bytes - size)) {
if (! read_wire_data(shim, buf + size, header_size_in_bytes - size)) {
shim_error("Error while reading frame from proxy at %s\n",
shim->proxy_address);
goto error;
Expand All @@ -453,8 +467,7 @@ read_frame(struct cc_shim *shim)
}

memset(buf, 0, fr->header.payload_len + 1);
if (! read_wire_data(shim->proxy_sock_fd, buf,
fr->header.payload_len)) {
if (! read_wire_data(shim, buf, fr->header.payload_len)) {
goto error;
}
fr->payload = buf;
Expand Down Expand Up @@ -845,7 +858,7 @@ parse_connection_uri(struct cc_shim *shim, char *uri)
* \return true on success, false on failure
*/
bool
connect_to_proxy(struct cc_shim *shim)
establish_connection_to_proxy(struct cc_shim *shim)
{
int sockfd = -1;
char *port_str = NULL;
Expand Down Expand Up @@ -876,7 +889,7 @@ connect_to_proxy(struct cc_shim *shim)

if (connect(sockfd, (struct sockaddr *)&remote,
sizeof(struct sockaddr_un)) == -1) {
shim_error("Error while connecting to proxy "
shim_warning("Error while connecting to proxy "
"with address %s: %s\n", shim->proxy_address,
strerror(errno));
goto out;
Expand Down Expand Up @@ -954,6 +967,71 @@ connect_to_proxy(struct cc_shim *shim)
return false;
}

/*
* Establish tcp/unix connection with proxy and sends a Connect command to it
*
* \param shim \ref cc_shim
*
* \return true on success, false on failure
*/
bool connect_to_proxy(struct cc_shim *shim)
{
if (! establish_connection_to_proxy(shim)) {
return false;
}

/* Send a Connect command to the proxy and wait for the response
*/
if (! send_connect_command(shim)) {
shim_error("Could not send connect command to "
PROXY "\n");
return false;
}
return true;
}

inline void sleep_ms(int ms)
{
struct timespec ts = { 0, ms * 1000000L };
nanosleep(&ts, NULL);
}

/*
* Try to re-establish tcp/unix connection with proxy in shim->timeout seconds.
*
* \param shim \ref cc_shim
*
* \return true on success, false on failure
*/
bool reconnect_to_proxy(struct cc_shim *shim)
{
shim_warning("Reconnecting to " PROXY " (timeout %d s)\n",
shim->timeout);
int time = 0;
while (1) {
sleep_ms(RECONNECT_POLL_MS);
time += RECONNECT_POLL_MS;
if (time >= shim->timeout * 1000) {
shim_error("Failed to reconnect to " PROXY
" (timeout %d s)\n", shim->timeout);
return false;
}

close(shim->proxy_sock_fd);
if (! connect_to_proxy(shim)) {
continue;
}

break;
}

// Update poll_fds because connect_to_proxy(shim) might have updated
// shim->proxy_sock_fd
add_pollfd(poll_fds, PROXY_SOCK_INDEX, shim->proxy_sock_fd,
POLLIN | POLLPRI);
return true;
}

/*
* Print version information.
*/
Expand All @@ -968,12 +1046,16 @@ show_version(void) {
void
print_usage(void) {
printf("%s: Usage\n", program_name);
printf(" -c, --container-id Container id\n");
printf(" -d, --debug Enable debug output\n");
printf(" -t, --token Connection token passed by cc-proxy\n");
printf(" -u, --uri Connection uri. Supported schemes are tcp: and unix:\n");
printf(" -v, --version Show version\n");
printf(" -h, --help Display this help message\n");
printf(" -c, --container-id Container id\n");
printf(" -d, --debug Enable debug output\n");
printf(" -r, --reconnect-timeout Reconnection timeout to " PROXY
" in seconds\n");
printf(" -t, --token Connection token passed by " PROXY
"\n");
printf(" -u, --uri Connection uri. Supported schemes "
"are tcp: and unix:\n");
printf(" -v, --version Show version\n");
printf(" -h, --help Display this help message\n");
}

int
Expand All @@ -983,6 +1065,7 @@ main(int argc, char **argv)
.container_id = NULL,
.proxy_sock_fd = -1,
.token = NULL,
.timeout = RECONNECT_TIMEOUT_S,
.proxy_address = NULL,
.proxy_port = -1,
};
Expand All @@ -999,20 +1082,27 @@ main(int argc, char **argv)
{"container-id", required_argument, 0, 'c'},
{"debug", no_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{"reconnect-timeout", required_argument, 0, 'r'},
{"token", required_argument, 0, 't'},
{"uri", required_argument, 0, 'u'},
{"version", no_argument, 0, 'v'},
{ 0, 0, 0, 0},
};

while ((c = getopt_long(argc, argv, "c:dht:u:v", prog_opts, NULL))!= -1) {
while ((c = getopt_long(argc, argv, "c:dhr:t:u:v", prog_opts, NULL))!= -1) {
switch (c) {
case 'c':
shim.container_id = strdup(optarg);
break;
case 't':
shim.token = strdup(optarg);
break;
case 'r':
shim.timeout = atoi(optarg);
if (shim.timeout <= 0) {
shim.timeout = RECONNECT_TIMEOUT_S;
}
break;
case 'd':
debug = true;
break;
Expand Down Expand Up @@ -1077,14 +1167,6 @@ main(int argc, char **argv)
goto out;
}

/* Send a Connect command to the proxy and wait for the response
*/
shim_debug("Sending initial connect command\n");
if (! send_connect_command(&shim)) {
shim_error("Could not send connect command to proxy\n");
goto out;
}

add_pollfd(poll_fds, PROXY_SOCK_INDEX, shim.proxy_sock_fd, POLLIN | POLLPRI);

/* Add stdin only if it is attached to a terminal.
Expand Down
3 changes: 3 additions & 0 deletions src/shim.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

#include <stdio.h>

#define PROXY "cc-proxy"

/* The shim would be handling fixed number of predefined fds.
* This would be signal fd, stdin fd and a proxy socket connection fd.
*/
Expand All @@ -23,6 +25,7 @@ struct cc_shim {
char *container_id;
int proxy_sock_fd;
char *token;
int timeout; /* reconnection timeout to proxy */
char *proxy_address;
int proxy_port;
};
Expand Down

0 comments on commit b8325e8

Please sign in to comment.