Skip to content

Commit

Permalink
Issue #1295: switch to Mojo::AWS::S3
Browse files Browse the repository at this point in the history 
Tentatively implement missing methods
  • Loading branch information
@bschmalhofer
bschmalhofer committed Oct 12, 2021
1 parent e318bad commit 227bc1a
Showing 1 changed file with 99 additions and 42 deletions.
141 changes: 99 additions & 42 deletions Kernel/System/Ticket/Article/Backend/MIMEBase/ArticleStorageS3.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ use Data::Dumper;
use File::Basename qw(basename);

# CPAN modules
use Amazon::S3::Thin;
use Sub::Override;
use Mojo::UserAgent;
use Mojo::Date;
use Mojo::URL;
Expand Down Expand Up @@ -67,24 +65,21 @@ sub new {
# Call new() on Base.pm to execute the common code.
my $Self = $Type->SUPER::new(%Param);

# get values from SysConfig
my $ConfigObject = $Kernel::OM->Get('Kernel::Config');
my $Region = $ConfigObject->Get('Ticket::Article::Backend::MIMEBase::ArticleStorageS3::Region')
|| die 'Got no AWS Region!';
$Self->{S3Client} = Amazon::S3::Thin->new({
aws_access_key_id => 'test',
aws_secret_access_key => 'test',
region => $Region,
});

# Bucket created on Docker host with:
# docker_admin> export AWS_ACCESS_KEY_ID=test
# docker_admin> export AWS_SECRET_ACCESS_KEY=test
# docker_admin> aws --endpoint-url=http://localhost:4566 s3 mb s3://otobo-20211010a
# make_bucket: otobo-20211010a

$Self->{Bucket} = 'otobo-20211010a';
# TODO: eliminate hardcoded values
$Self->{Bucket} = 'otobo-20211010a';
$Self->{UserAgent} = Mojo::UserAgent->new();
$Self->{AWSObject} = Mojo::AWS::S3->new(
$Self->{S3Object} = Mojo::AWS::S3->new(
transactor => $Self->{UserAgent}->transactor,
service => 's3',
region => $Region,
Expand Down Expand Up @@ -142,10 +137,23 @@ sub ArticleDeletePlain {
}
}

# delete attachments
# TODO_S3
# delete plain
my $FilePath = $Self->_FilePath($Param{ArticleID}, 'plain.txt');
my $Now = Mojo::Date->new(time)->to_datetime;
my $URL = Mojo::URL->new->scheme('https')->host('localstack:4566')->path($FilePath); # run within container
my $Transaction = $Self->{S3Object}->signed_request(
method => 'DELETE',
datetime => $Now,
url => $URL,
);

# run blocking request
$Self->{UserAgent}->start($Transaction);

warn Dumper(['XXX ArticleDeletePlain', $Transaction] );

# the S3 backend does not support storing articles in mixed backends
# TODO: check success
return 1;
}

Expand All @@ -164,10 +172,51 @@ sub ArticleDeleteAttachment {
}
}

# delete attachments
# TODO_S3
# query all attachments so that they can be deleted
my $XML4Delete;
{
my %AttachmentIndex = $Self->ArticleAttachmentIndexRaw(%Param);

my $ArticlePrefix = $Self->_ArticlePrefix($Param{ArticleID});
my @Keys;
for my $FileID (sort { $a <=> $b } keys %AttachmentIndex ) {
push @Keys, $ArticlePrefix . $AttachmentIndex{$FileID}->{File};
}

# TODO: proper XML quoting
my @ObjectNodes = map { sprintf <<'END_OBJECT_NODE', $_ } @Keys;
<Object>
<Key>%s</Key>
</Object>
END_OBJECT_NODE
$XML4Delete = sprintf <<'END_XML4DELETE', join "\n", @ObjectNodes;
<Delete xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
%s
<Quiet>boolean</Quiet>
</Delete>
END_XML4DELETE
}

# See https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
# delete plain
my $FilePath = $Self->_FilePath($Param{ArticleID}, 'plain.txt');
my $Now = Mojo::Date->new(time)->to_datetime;
my $URL = Mojo::URL->new->scheme('https')->host('localstack:4566')->path($FilePath); # run within container
$URL->query('delete=');
my $Transaction = $Self->{S3Object}->signed_request(
method => 'POST',
datetime => $Now,
url => $URL,
payload => [$XML4Delete],
);

# run blocking request
$Self->{UserAgent}->start($Transaction);

warn Dumper(['XXX ArticleDeleteAttachment', $Transaction] );

# the S3 backend does not support storing articles in mixed backends
# TODO: check success
return 1;
}

Expand All @@ -191,7 +240,7 @@ sub ArticleWritePlain {
my $FilePath = $Self->_FilePath($Param{ArticleID}, 'plain.txt');
my $Now = Mojo::Date->new(time)->to_datetime;
my $URL = Mojo::URL->new->scheme('https')->host('localstack:4566')->path($FilePath); # run within container
my $Transaction = $Self->{AWSObject}->signed_request(
my $Transaction = $Self->{S3Object}->signed_request(
method => 'PUT',
datetime => $Now,
url => $URL,
Expand Down Expand Up @@ -270,7 +319,7 @@ sub ArticleWriteAttachment {
my $FilePath = $Self->_FilePath($Param{ArticleID}, $NewFilename);
my $Now = Mojo::Date->new(time)->to_datetime;
my $URL = Mojo::URL->new->scheme('https')->host('localstack:4566')->path($FilePath); # run within container
my $Transaction = $Self->{AWSObject}->signed_request(
my $Transaction = $Self->{S3Object}->signed_request(
method => 'PUT',
datetime => $Now,
url => $URL,
Expand Down Expand Up @@ -305,21 +354,23 @@ sub ArticlePlain {
$Param{ArticleID} =~ s/\0//g;

# retrieve plain from S3
my $Prefix = $Self->_ArticlePrefix($Param{ArticleID});
my $Key = $Prefix . 'plain.txt';
my $Response;
{
my $Override = Sub::Override->new( 'Amazon::S3::Thin::Resource::_region_specific_host' => sub { return 'localstack:4566' } );
$Response = $Self->{S3Client}->get_object(
$Self->{Bucket},
$Key,
);
}
my $FilePath = $Self->_FilePath($Param{ArticleID}, 'plain.txt');
my $Now = Mojo::Date->new(time)->to_datetime;
my $URL = Mojo::URL->new->scheme('https')->host('localstack:4566')->path($FilePath); # run within container
my $Transaction = $Self->{S3Object}->signed_request(
method => 'GET',
datetime => $Now,
url => $URL,
);

# run blocking request
$Self->{UserAgent}->start($Transaction);

warn Dumper( ['XXX ArticlePlain', $Transaction ] );

# TODO: check success
# the S3 backend does not support storing articles in mixed backends
warn Dumper( ['XXX ArticlePlain', $Response ] );
return $Response->content;
# TODO: check success
return $Transaction->res->body;
}


Expand All @@ -338,21 +389,27 @@ sub ArticleAttachmentIndexRaw {

# TODO: caching is essential here, because often ArticleAttachment() after ArticleAttachmentIndexRaw()

my $Prefix = $Self->_ArticlePrefix($Param{ArticleID});
my $Response;
{
my $Override = Sub::Override->new( 'Amazon::S3::Thin::Resource::_region_specific_host' => sub { return 'localstack:4566' } );
$Response = $Self->{S3Client}->list_objects(
$Self->{Bucket},
{
prefix => $Prefix,
delimiter => '/',
}
);
}
# retrieve plain from S3
my $Now = Mojo::Date->new(time)->to_datetime;
my $URL = Mojo::URL->new->scheme('https')->host('localstack:4566')->path($Self->{Bucket}); # run within container

# the parameters are passed in the Query-URL
my $ArticlePrefix = $Self->_ArticlePrefix($Param{ArticleID});
$URL->query([prefix => $ArticlePrefix, delimiter => '/']);

my $Transaction = $Self->{S3Object}->signed_request(
method => 'GET',
datetime => $Now,
url => $URL,
);

# run blocking request
$Self->{UserAgent}->start($Transaction);

# the S3 backend does not support storing articles in mixed backends
# TODO: check success
# parse the returned XML
my $Content = $Response->content;
my $Content = $Transaction->res->body;
my $XMLSimpleObject = $Kernel::OM->Get('Kernel::System::XML::Simple');
my $ParsedXML = $XMLSimpleObject->XMLIn(
XMLInput => $Content,
Expand All @@ -363,7 +420,7 @@ sub ArticleAttachmentIndexRaw {
},
);

warn Dumper( ['YYY', $Response, $Content, $ParsedXML ] );
warn Dumper( ['Debug ArticleAttachmentIndexRaw', $Transaction, $ParsedXML ] );

my %Index;
if ( ref $ParsedXML eq 'HASH' && $ParsedXML->{Contents} && ref $ParsedXML->{Contents} eq 'ARRAY' ) {
Expand Down

0 comments on commit 227bc1a

Please sign in to comment.