Issue #1295: first version that acutually stores a file

RotherOSS · Oct 8, 2021 · 5146f97 · 5146f97
1 parent a9f33a8
commit 5146f97
Show file tree

Hide file tree

Showing 6 changed files with 110 additions and 12 deletions.
diff --git a/Kernel/Config/Files/XML/Ticket.xml b/Kernel/Config/Files/XML/Ticket.xml
@@ -209,7 +209,7 @@
         </Value>
     </Setting>
     <Setting Name="Ticket::Article::Backend::MIMEBase::ArticleStorage" Required="1" Valid="1" ConfigLevel="200">
-        <Description Translatable="1">Saves the attachments of articles. "DB" stores all data in the database (not recommended for storing big attachments). "FS" stores the data on the filesystem; this is faster but the webserver should run under the OTOBO user. You can switch between the modules even on a system that is already in production without any loss of data. Note: Searching for attachment names is not supported when "FS" is used.</Description>
+        <Description Translatable="1">Saves the attachments of articles. "DB" stores all data in the database (not recommended for storing big attachments). "FS" stores the data on the filesystem; this is faster but the webserver should run under the OTOBO user. You can switch between the modules even on a system that is already in production without any loss of data. Note: Searching for attachment names is not supported when "FS" is used. "S3" is experimental</Description>
         <Navigation>Core::CommunicationChannel::MIMEBase</Navigation>
         <Value>
             <Item ValueType="PerlModule" ValueFilter="Kernel/System/Ticket/Article/Backend/MIMEBase/ArticleStorage*.pm">Kernel::System::Ticket::Article::Backend::MIMEBase::ArticleStorageDB</Item>
@@ -236,6 +236,13 @@
             <Item ValueType="Checkbox">1</Item>
         </Value>
     </Setting>
+    <Setting Name="Ticket::Article::Backend::MIMEBase::ArticleStorageS3::Region" Required="1" Valid="1" ConfigLevel="200">
+        <Description Translatable="1">Specifies the AWS region</Description>
+        <Navigation>Core::CommunicationChannel::MIMEBase</Navigation>
+        <Value>
+            <Item ValueType="String" ValueRegex="">eu-central-1</Item>
+        </Value>
+    </Setting>
     <Setting Name="OTOBOEscalationEvents::DecayTime" Required="0" Valid="1">
         <Description Translatable="1">The duration in minutes after emitting an event, in which the new escalation notify and start events are suppressed.</Description>
         <Navigation>Core::Ticket</Navigation>

diff --git a/Kernel/System/Console/Command/Admin/Article/StorageSwitch.pm b/Kernel/System/Console/Command/Admin/Article/StorageSwitch.pm
@@ -36,10 +36,10 @@ sub Configure {
     $Self->Description('Migrate article files from one storage backend to another on the fly.');
     $Self->AddOption(
         Name        => 'target',
-        Description => "Specify the target backend to migrate to (ArticleStorageDB|ArticleStorageFS).",
+        Description => 'Specify the target backend to migrate to. One of ArticleStorageDB, ArticleStorageFS, or ArticleStorageS3.',
         Required    => 1,
         HasValue    => 1,
-        ValueRegex  => qr/^(?:ArticleStorageDB|ArticleStorageFS)$/smx,
+        ValueRegex  => qr/^ArticleStorage(?:DB|FS|S3)$/smx,
     );
     $Self->AddOption(
         Name        => 'tickets-closed-before-date',
@@ -88,6 +88,7 @@ To reduce load on the database for a running system, you can use the <yellow>--m
 
  <green>otobo.Console.pl $Self->{Name} --target ArticleStorageFS --micro-sleep 1000</green>
 EOF
+
     return;
 }
 
@@ -159,6 +160,7 @@ sub Run {
     my %Target2Source = (
         ArticleStorageFS => 'ArticleStorageDB',
         ArticleStorageDB => 'ArticleStorageFS',
+        ArticleStorageS3 => 'ArticleStorageDB',
     );
 
     my $MicroSleep = $Self->GetOption('micro-sleep');

diff --git a/Kernel/System/Ticket/Article/Backend/MIMEBase/ArticleStorageS3.pm b/Kernel/System/Ticket/Article/Backend/MIMEBase/ArticleStorageS3.pm
@@ -24,14 +24,17 @@ use utf8;
 use parent qw(Kernel::System::Ticket::Article::Backend::MIMEBase::Base);
 
 # core modules
+use Data::Dumper;
 
 # CPAN modules
 use Amazon::S3::Thin;
+use Sub::Override;
 
 # OTOBO modules
 use Kernel::System::VariableCheck qw(IsStringWithData);
 
 our @ObjectDependencies = (
+    'Kernel::Config',
     'Kernel::System::Log',
     'Kernel::System::Main',
 );
@@ -52,6 +55,28 @@ L<Kernel::System::Ticket::Article::Backend::MIMEBase::ArticleStorageDB>.
 
 =cut
 
+sub new {
+    my ( $Type, %Param ) = @_;
+
+    # Call new() on Base.pm to execute the common code.
+    my $Self = $Type->SUPER::new(%Param);
+
+    my $ConfigObject = $Kernel::OM->Get('Kernel::Config');
+    my $Region = $ConfigObject->Get('Ticket::Article::Backend::MIMEBase::ArticleStorageS3::Region')
+        || die 'Got no AWS Region!';
+    $Self->{S3Client} = Amazon::S3::Thin->new({
+        aws_access_key_id     => 'test',
+        aws_secret_access_key => 'test',
+        region                => $Region,
+    });
+
+    $Self->{Bucket} = 'otobo-20211008a';
+
+    # for now the S3 backend does not cache
+
+    return $Self;
+}
+
 sub ArticleDelete {
     my ( $Self, %Param ) = @_;
 
@@ -142,8 +167,22 @@ sub ArticleWritePlain {
     }
 
     # write article to S3
-    # TODO_S3
+    # TODO_S3 third
+    my $Prefix = $Self->_ArticlePrefix($Param{ArticleID});
+    my $Key = join '/', $Prefix, 'plain.txt';
+    my $Response;
+    {
+        my $Override = Sub::Override->new( 'Amazon::S3::Thin::Resource::_region_specific_host' => sub { return 'localstack:4566' } );
+        $Response = $Self->{S3Client}->put_object(
+            $Self->{Bucket},
+            $Key,
+            $Param{Email},
+        );
+    }
+
+    warn Dumper( ['AAA', $Response, $Response->content ] );
 
+    # TODO: check success
     return 1;
 }
 
@@ -188,9 +227,6 @@ sub ArticleWriteAttachment {
         }
     }
 
-    # get file name
-    $Param{Filename} = $NewFileName;
-
     # get attachment size
     $Param{Filesize} = bytes::length( $Param{Content} );
 
@@ -206,9 +242,23 @@ sub ArticleWriteAttachment {
     }
     $Disposition //= '';
 
-    # write attachment to db
-    # TODO_S3
+    # write attachment to S3
+    # TODO_S3 second
+    my $Prefix = $Self->_ArticlePrefix($Param{ArticleID});
+    my $Key = join '/', $Prefix, $NewFileName;
+    my $Response;
+    {
+        my $Override = Sub::Override->new( 'Amazon::S3::Thin::Resource::_region_specific_host' => sub { return 'localstack:4566' } );
+        $Response = $Self->{S3Client}->put_object(
+            $Self->{Bucket},
+            $Key,
+            $Param{Content},
+        );
+    }
+
+    warn Dumper( ['ZZZ', $Response, $Response->content ] );
 
+    # TODO: check success
     return 1;
 }
 
@@ -248,16 +298,27 @@ sub ArticleAttachmentIndexRaw {
         return;
     }
 
+    # TODO_S3 first
+    my $Prefix = $Self->_ArticlePrefix($Param{ArticleID});
+    my $Response;
+    {
+        my $Override = Sub::Override->new( 'Amazon::S3::Thin::Resource::_region_specific_host' => sub { return 'localstack:4566' } );
+        $Response = $Self->{S3Client}->list_objects(
+            $Self->{Bucket},
+            { prefix => "$Prefix/", delimiter => '/' }
+        );
+    }
+
+    warn Dumper( ['YYY', $Response, $Response->content ] );
+
     my %Index;
     my $Counter = 0;
 
-    # TODO_S3
-
     # return existing index
     return %Index if %Index;
 
     # the S3 backend does not support storing articles in mixed backends
-    return 1;
+    return;
 }
 
 sub ArticleAttachment {
@@ -296,4 +357,11 @@ sub ArticleAttachment {
     return;
 }
 
+sub _ArticlePrefix {
+    my ($Self, $ArticleID) = @_;
+
+    # TODO: what about the content path set up in the table article_data_mime 
+    return join '/', 'OTOBO', 'articles', $ArticleID;
+}
+
 1;
diff --git a/bin/otobo.CheckModules.pl b/bin/otobo.CheckModules.pl
@@ -590,6 +590,18 @@ =head1 DESCRIPTION
             ports  => undef,
         },
     },
+    {
+        Module    => 'Sub::Override',
+        Features  => ['aws'],
+        Comment   => 'for hacking Amazon::S3::Thin',
+        InstTypes => {
+            aptget => undef,
+            emerge => undef,
+            yum    => undef,
+            zypper => undef,
+            ports  => undef,
+        },
+    },
 
     # Feature db
     {

diff --git a/cpanfile b/cpanfile
@@ -112,6 +112,9 @@ feature 'aws', 'Amazon Web Services, currently only S3' => sub {
     # Required for article storage in S3
     requires 'Amazon::S3::Thin';
 
+    # for hacking Amazon::S3::Thin
+    requires 'Sub::Override';
+
 };
 
 feature 'db:mysql', 'Support for database MySQL' => sub {
@@ -251,6 +254,9 @@ feature 'optional', 'Suppport for optional' => sub {
     # Required for article storage in S3
     requires 'Amazon::S3::Thin';
 
+    # for hacking Amazon::S3::Thin
+    requires 'Sub::Override';
+
     # Required to connect to a MySQL database.
     requires 'DBD::mysql';
 

diff --git a/cpanfile.docker b/cpanfile.docker
@@ -109,6 +109,9 @@ requires 'Const::Fast';
     # Required for article storage in S3
     requires 'Amazon::S3::Thin';
 
+    # for hacking Amazon::S3::Thin
+    requires 'Sub::Override';
+
 # };
 
 # feature 'db:mysql', 'Support for database MySQL' => sub {