Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tmp delete support #41

Merged
merged 1 commit into from
Oct 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions include/pg.h
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ extern "C" {

#include "libpq/pqformat.h"

#include "miscadmin.h"

#include "libpq/libpq-be.h"

#include "postmaster/postmaster.h"

#ifdef __cplusplus
}
#endif
Expand Down
2 changes: 1 addition & 1 deletion include/xvacuum.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@
#define EXTERNC
#endif

EXTERNC int yezzey_delete_chunk_internal(const char *external_chunk_path);
EXTERNC int yezzey_delete_chunk_internal(const char *external_chunk_path, int segindx);

#endif /* YEZZEY_XVACUUM_H */
18 changes: 18 additions & 0 deletions include/ydeleter.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#pragma once

#include <string>

class YDeleter {
public:
virtual ~YDeleter(){

};

// write() attempts to write up to count bytes from the buffer.
// Always return 0 if EOF, no matter how many times it's invoked. Throw
// exception if encounters errors.
virtual bool deleteChunk(const std::string &fn) = 0;

// This should be reentrant, has no side effects when called multiple times.
virtual bool close() = 0;
};
33 changes: 28 additions & 5 deletions include/yproxy.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include "ylister.h"
#include "yreader.h"
#include "ywriter.h"
#include "ydeleter.h"
#include <memory>
#include <string>
#include <vector>
Expand Down Expand Up @@ -44,8 +45,7 @@ class YProxyReader : public YReader {
int retry_limit{1};
};

// write to external storage, using gpwriter.
// encrypt all data with gpg
// Write into external storage using yproxy
class YProxyWriter : public YWriter {
public:
explicit YProxyWriter(std::shared_ptr<IOadv> adv, ssize_t segindx,
Expand All @@ -62,12 +62,9 @@ class YProxyWriter : public YWriter {
int prepareYproxyConnection();
std::vector<char> ConstructPutRequest(std::string fileName);
std::vector<char> ConstructCopyDataRequest(const char *buffer, size_t amount);
std::vector<char> CostructCommandCompleteRequest();
int readRFQResponce();

private:
std::string createXPath();
int write_full(const std::vector<char> &msg);

std::shared_ptr<IOadv> adv_;
ssize_t segindx_;
Expand All @@ -83,6 +80,32 @@ class YProxyWriter : public YWriter {
XLogRecPtr getInsertionStorageLsn() { return insertion_rec_ptr_; }
};



/* Delete specified file from external storage, bypassing all sanity checks */
class YProxyDeleter : public YDeleter {
public:
explicit YProxyDeleter(std::shared_ptr<IOadv> adv, ssize_t segindx);

virtual ~YProxyDeleter();

virtual bool close();

virtual bool deleteChunk(const std::string &chunkName);

protected:
/* prepare connection for chunk reading */
int prepareYproxyConnection();
std::vector<char> ConstructDeleteRequest(std::string fileName);

private:
std::shared_ptr<IOadv> adv_;
ssize_t segindx_;

int client_fd_{-1};
};


// list external storage using yproxy
class YProxyLister : public YLister {
public:
Expand Down
20 changes: 10 additions & 10 deletions src/xvacuum.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#include "xvacuum.h"
#include "gpcleaner.h"
#include "yproxy.h"
#include "gucs.h"
#include "pg.h"
#include <string>
Expand All @@ -12,7 +12,7 @@
* TBD: check, that chunk status is obsolete and other sanity checks
* to avoid deleting chunk, which can we needed to read relation data
*/
int yezzey_delete_chunk_internal(const char *external_chunk_path) {
int yezzey_delete_chunk_internal(const char *external_chunk_path, int segindx) {
try {
auto ioadv = std::make_shared<IOadv>(
std::string(gpg_engine_path), std::string(gpg_key_id),
Expand All @@ -23,16 +23,16 @@ int yezzey_delete_chunk_internal(const char *external_chunk_path) {
"" /* coords */, InvalidOid /* reloid */, std::string(walg_bin_path),
std::string(walg_config_path), use_gpg_crypto, yproxy_socket);

auto x_path = std::string(external_chunk_path);
auto init_url = getYezzeyExtrenalStorageBucket(ioadv->host.c_str(),
ioadv->bucket.c_str()) +
storage_url_add_options(x_path, ioadv->config_path.c_str());
std::string storage_path(external_chunk_path);

elog(LOG, "removing external chunk with url %s", init_url.c_str());
auto deleter =
std::make_shared<YProxyDeleter>(ioadv, ssize_t(segindx));

auto cleaner = cleaner_init(init_url.c_str());
int rc = cleaner->clean();
return rc;
if (deleter->deleteChunk(storage_path)) {
return 0;
}

return -1;
} catch (...) {
elog(ERROR, "failed to prepare x-storage reader for chunk");
return 0;
Expand Down
Loading
Loading