[Rt-commit] rt branch, 4.4/external-storage, created. rt-4.2.11-5-g428c9c4
Shawn Moore
shawn at bestpractical.com
Thu May 21 15:58:24 EDT 2015
The branch, 4.4/external-storage has been created
at 428c9c45c4c552c2c469a3d14acef766882742fb (commit)
- Log -----------------------------------------------------------------
commit 428c9c45c4c552c2c469a3d14acef766882742fb
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Thu May 21 19:56:46 2015 +0000
Verbatim copy of RT::Extension::ExternalStorage
at a9df4596ba43cd64552a2c9f3173e82bd0690b78
diff --git a/lib/RT/Extension/ExternalStorage.pm b/lib/RT/Extension/ExternalStorage.pm
new file mode 100644
index 0000000..cfac2d8
--- /dev/null
+++ b/lib/RT/Extension/ExternalStorage.pm
@@ -0,0 +1,283 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+use 5.008003;
+use warnings;
+use strict;
+
+package RT::Extension::ExternalStorage;
+
+our $VERSION = '0.61';
+
+use Digest::SHA qw//;
+
+require RT::Extension::ExternalStorage::Backend;
+
+=head1 NAME
+
+RT::Extension::ExternalStorage - Store attachments outside the database
+
+=head1 SYNOPSIS
+
+ Set( @Plugins, 'RT::Extension::ExternalStorage' );
+
+ Set(%ExternalStorage,
+ Type => 'Disk',
+ Path => '/opt/rt4/var/attachments',
+ );
+
+=head1 DESCRIPTION
+
+By default, RT stores attachments in the database. This extension moves
+all attachments that RT does not need efficient access to (which include
+textual content and images) to outside of the database. This may either
+be on local disk, or to a cloud storage solution. This decreases the
+size of RT's database, in turn decreasing the burden of backing up RT's
+database, at the cost of adding additional locations which must be
+configured or backed up. Attachment storage paths are calculated based
+on file contents; this provides de-duplication.
+
+The files are initially stored in the database when RT receives them;
+this guarantees that the user does not need to wait for the file to be
+transferred to disk or to the cloud, and makes it durable to transient
+failures of cloud connectivity. The provided C<bin/extract-attachments>
+script, to be run regularly via cron, takes care of moving attachments
+out of the database at a later time.
+
+=head1 INSTALLATION
+
+=over
+
+=item C<perl Makefile.PL>
+
+=item C<make>
+
+=item C<make install>
+
+May need root permissions
+
+=item Edit your F</opt/rt4/etc/RT_SiteConfig.pm>
+
+If you are using RT 4.2 or greater, add this line:
+
+ Plugin('RT::Extension::ExternalStorage');
+
+For RT 4.0, add this line:
+
+ Set(@Plugins, qw(RT::Extension::ExternalStorage));
+
+or add C<RT::Extension::ExternalStorage> to your existing C<@Plugins> line.
+
+You will also need to configure the C<%ExternalStorage> option,
+depending on how and where you want your data stored; see
+L</CONFIGURATION>.
+
+=item Restart your webserver
+
+Restarting the webserver before the next step (extracting existing
+attachments) is important to ensure that files remain available as they
+are extracted.
+
+=item Extract existing attachments
+
+Run C<bin/extract-attachments>; this may take some time, depending on
+the existing size of the database. This task may be safely cancelled
+and re-run to resume.
+
+=item Schedule attachments extraction
+
+Schedule C<bin/extract-attachments> to run at regular intervals via
+cron. For instance, the following F</etc/cron.d/rt> entry will run it
+daily, which may be good to concentrate network or disk usage to times
+when RT is less in use:
+
+ 0 0 * * * root /opt/rt4/local/plugins/RT-Extension-ExternalStorage/bin/extract-attachments
+
+=back
+
+=head1 CONFIGURATION
+
+This module comes with a number of possible backends; see the
+documentation in each for necessary configuration details:
+
+=over
+
+=item L<RT::Extension::ExternalStorage::Disk>
+
+=item L<RT::Extension::ExternalStorage::Dropbox>
+
+=item L<RT::Extension::ExternalStorage::AmazonS3>
+
+=back
+
+=head1 CAVEATS
+
+This extension is not currently compatibile with RT's C<shredder> tool;
+attachments which are shredded will not be removed from external
+storage.
+
+=cut
+
+our $BACKEND;
+our $WRITE;
+$RT::Config::META{ExternalStorage} = {
+ Type => 'HASH',
+ PostLoadCheck => sub {
+ my $self = shift;
+ my %hash = $self->Get('ExternalStorage');
+ return unless keys %hash;
+ $hash{Write} = $WRITE;
+ $BACKEND = RT::Extension::ExternalStorage::Backend->new( %hash );
+ },
+};
+
+sub Store {
+ my $class = shift;
+ my $content = shift;
+
+ my $key = Digest::SHA::sha256_hex( $content );
+ my ($ok, $msg) = $BACKEND->Store( $key => $content );
+ return ($ok, $msg) unless defined $ok;
+
+ return ($key);
+}
+
+
+package RT::Record;
+
+no warnings 'redefine';
+my $__DecodeLOB = __PACKAGE__->can('_DecodeLOB');
+*_DecodeLOB = sub {
+ my $self = shift;
+ my $ContentType = shift || '';
+ my $ContentEncoding = shift || 'none';
+ my $Content = shift;
+ my $Filename = @_;
+
+ return $__DecodeLOB->($self, $ContentType, $ContentEncoding, $Content, $Filename)
+ unless $ContentEncoding eq "external";
+
+ unless ($BACKEND) {
+ RT->Logger->error( "Failed to load $Content; external storage not configured" );
+ return ("");
+ };
+
+ my ($ok, $msg) = $BACKEND->Get( $Content );
+ unless (defined $ok) {
+ RT->Logger->error( "Failed to load $Content from external storage: $msg" );
+ return ("");
+ }
+
+ return $__DecodeLOB->($self, $ContentType, 'none', $ok, $Filename);
+};
+
+package RT::ObjectCustomFieldValue;
+
+sub StoreExternally {
+ my $self = shift;
+ my $type = $self->CustomFieldObj->Type;
+ my $length = length($self->LargeContent || '');
+
+ return 0 if $length == 0;
+
+ return 1 if $type eq "Binary";
+
+ return 1 if $type eq "Image" and $length > 10 * 1024 * 1024;
+
+ return 0;
+}
+
+package RT::Attachment;
+
+sub StoreExternally {
+ my $self = shift;
+ my $type = $self->ContentType;
+ my $length = $self->ContentLength;
+
+ return 0 if $length == 0;
+
+ if ($type =~ m{^multipart/}) {
+ return 0;
+ } elsif ($type =~ m{^(text|message)/}) {
+ # If textual, we only store externally if it's _large_ (> 10M)
+ return 1 if $length > 10 * 1024 * 1024;
+ return 0;
+ } elsif ($type =~ m{^image/}) {
+ # Ditto images, which may be displayed inline
+ return 1 if $length > 10 * 1024 * 1024;
+ return 0;
+ } else {
+ return 1;
+ }
+}
+
+=head1 AUTHOR
+
+Best Practical Solutions, LLC E<lt>modules at bestpractical.comE<gt>
+
+=head1 BUGS
+
+All bugs should be reported via email to
+
+ L<bug-RT-Extension-ExternalStorage at rt.cpan.org|mailto:bug-RT-Extension-ExternalStorage at rt.cpan.org>
+
+or via the web at
+
+ L<rt.cpan.org|http://rt.cpan.org/Public/Dist/Display.html?Name=RT-Extension-ExternalStorage>.
+
+=head1 COPYRIGHT
+
+This extension is Copyright (C) 2009-2015 Best Practical Solutions, LLC.
+
+This is free software, licensed under:
+
+ The GNU General Public License, Version 2, June 1991
+
+=cut
+
+1;
diff --git a/lib/RT/Extension/ExternalStorage/AmazonS3.pm b/lib/RT/Extension/ExternalStorage/AmazonS3.pm
new file mode 100644
index 0000000..773ab4d
--- /dev/null
+++ b/lib/RT/Extension/ExternalStorage/AmazonS3.pm
@@ -0,0 +1,197 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+use 5.008003;
+use warnings;
+use strict;
+
+package RT::Extension::ExternalStorage::AmazonS3;
+
+use Role::Basic qw/with/;
+with 'RT::Extension::ExternalStorage::Backend';
+
+our( $S3, $BUCKET);
+sub Init {
+ my $self = shift;
+ my %self = %{$self};
+
+ if (not Amazon::S3->require) {
+ RT->Logger->error("Required module Amazon::S3 is not installed");
+ return;
+ } elsif (not $self{AccessKeyId}) {
+ RT->Logger->error("AccessKeyId not provided for AmazonS3");
+ return;
+ } elsif (not $self{SecretAccessKey}) {
+ RT->Logger->error("SecretAccessKey not provided for AmazonS3");
+ return;
+ } elsif (not $self{Bucket}) {
+ RT->Logger->error("Bucket not provided for AmazonS3");
+ return;
+ }
+
+
+ $S3 = Amazon::S3->new( {
+ aws_access_key_id => $self{AccessKeyId},
+ aws_secret_access_key => $self{SecretAccessKey},
+ retry => 1,
+ } );
+
+ my $buckets = $S3->bucket( $self{Bucket} );
+ unless ( $buckets ) {
+ RT->Logger->error("Can't list buckets of AmazonS3: ".$S3->errstr);
+ return;
+ }
+ unless ( grep {$_->bucket eq $self{Bucket}} @{$buckets->{buckets}} ) {
+ my $ok = $S3->add_bucket( {
+ bucket => $self{Bucket},
+ acl_short => 'private',
+ } );
+ unless ($ok) {
+ RT->Logger->error("Can't create new bucket '$self{Bucket}' on AmazonS3: ".$S3->errstr);
+ return;
+ }
+ }
+
+ return $self;
+}
+
+sub Get {
+ my $self = shift;
+ my ($sha) = @_;
+
+ my $ok = $S3->bucket($self->{Bucket})->get_key( $sha );
+ return (undef, "Could not retrieve from AmazonS3:" . $S3->errstr)
+ unless $ok;
+ return ($ok->{value});
+}
+
+sub Store {
+ my $self = shift;
+ my ($sha, $content) = @_;
+
+ # No-op if the path exists already
+ return (1) if $S3->bucket($self->{Bucket})->head_key( $sha );
+
+ $S3->bucket($self->{Bucket})->add_key(
+ $sha => $content
+ ) or return (undef, "Failed to write to AmazonS3: " . $S3->errstr);
+
+ return (1);
+}
+
+=head1 NAME
+
+RT::Extension::ExternalStorage::Dropbox - Store files in the Dropbox cloud
+
+=head1 SYNOPSIS
+
+ Set(%ExternalStorage,
+ Type => 'Dropbox',
+ AccessToken => '...',
+ );
+
+=head1 DESCRIPTION
+
+This storage option places attachments in the Dropbox shared file
+service. The files are de-duplicated when they are saved; as such, if
+the same file appears in multiple transactions, only one copy will be
+stored on in Dropbox.
+
+Files in Dropbox C<must not be modified or removed>; doing so may cause
+internal inconsistency.
+
+=head1 SETUP
+
+In order to use this stoage type, a new application must be registered
+with Dropbox:
+
+=over
+
+=item 1.
+
+Log into Dropbox as the user you wish to store files as.
+
+=item 2.
+
+Click C<Create app> on L<https://www.dropbox.com/developers/apps>
+
+=item 3.
+
+Choose B<Dropbox API app> as the type of app.
+
+=item 4.
+
+Choose the B<Files and datastores> as the type of data to store.
+
+=item 5.
+
+Choose B<Yes>, your application only needs access to files it creates.
+
+=item 6.
+
+Enter a descriptive name -- C<Request Tracker files> is fine.
+
+=item 7.
+
+Under C<Generated access token>, click the C<Generate> button.
+
+=item 8.
+
+Copy the provided value into your F<RT_SiteConfig.pm> file as the
+C<AccessToken>:
+
+ Set(%ExternalStorage,
+ Type => 'Dropbox',
+ AccessToken => '...', # Replace the value here, between the quotes
+ );
+
+=back
+
+=cut
+
+1;
diff --git a/lib/RT/Extension/ExternalStorage/Backend.pm b/lib/RT/Extension/ExternalStorage/Backend.pm
new file mode 100644
index 0000000..03b0898
--- /dev/null
+++ b/lib/RT/Extension/ExternalStorage/Backend.pm
@@ -0,0 +1,89 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+use 5.008003;
+use warnings;
+use strict;
+
+package RT::Extension::ExternalStorage::Backend;
+
+use Role::Basic;
+
+requires 'Init';
+requires 'Get';
+requires 'Store';
+
+sub new {
+ my $class = shift;
+ my %args = @_;
+
+ $class = delete $args{Type};
+ if (not $class) {
+ RT->Logger->error("No storage engine type provided");
+ return undef;
+ } elsif ($class->require) {
+ } else {
+ my $long = "RT::Extension::ExternalStorage::$class";
+ if ($long->require) {
+ $class = $long;
+ } else {
+ RT->Logger->error("Can't load external storage engine $class: $@");
+ return undef;
+ }
+ }
+
+ unless ($class->DOES("RT::Extension::ExternalStorage::Backend")) {
+ RT->Logger->error("External storage engine $class doesn't implement RT::Extension::ExternalStorage::Backend");
+ return undef;
+ }
+
+ my $self = bless \%args, $class;
+ $self->Init;
+}
+
+1;
diff --git a/lib/RT/Extension/ExternalStorage/Disk.pm b/lib/RT/Extension/ExternalStorage/Disk.pm
new file mode 100644
index 0000000..e93fb84
--- /dev/null
+++ b/lib/RT/Extension/ExternalStorage/Disk.pm
@@ -0,0 +1,144 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+use 5.008003;
+use warnings;
+use strict;
+
+package RT::Extension::ExternalStorage::Disk;
+
+use File::Path qw//;
+
+use Role::Basic qw/with/;
+with 'RT::Extension::ExternalStorage::Backend';
+
+sub Init {
+ my $self = shift;
+
+ my %self = %{$self};
+ if (not $self{Path}) {
+ RT->Logger->error("No path provided for local storage");
+ return;
+ } elsif (not -e $self{Path}) {
+ RT->Logger->error("Path provided for local storage ($self{Path}) does not exist");
+ return;
+ } elsif ($self{Write} and not -w $self{Path}) {
+ RT->Logger->error("Path provided for local storage ($self{Path}) is not writable");
+ return;
+ }
+
+ return $self;
+}
+
+sub Get {
+ my $self = shift;
+ my ($sha) = @_;
+
+ $sha =~ m{^(...)(...)(.*)};
+ my $path = $self->{Path} . "/$1/$2/$3";
+
+ return (undef, "File does not exist") unless -e $path;
+
+ open(my $fh, "<", $path) or return (undef, "Cannot read file on disk: $!");
+ my $content = do {local $/; <$fh>};
+ $content = "" unless defined $content;
+ close $fh;
+
+ return ($content);
+}
+
+sub Store {
+ my $self = shift;
+ my ($sha, $content) = @_;
+
+ $sha =~ m{^(...)(...)(.*)};
+ my $dir = $self->{Path} . "/$1/$2";
+ my $path = "$dir/$3";
+
+ return (1) if -f $path;
+
+ File::Path::make_path($dir, {error => \my $err});
+ return (undef, "Making directory failed") if @{$err};
+
+ open( my $fh, ">:raw", $path ) or return (undef, "Cannot write file on disk: $!");
+ print $fh $content or return (undef, "Cannot write file to disk: $!");
+ close $fh or return (undef, "Cannot write file to disk: $!");
+
+ return (1);
+}
+
+=head1 NAME
+
+RT::Extension::ExternalStorage::Disk - On-disk storage of attachments
+
+=head1 SYNOPSIS
+
+ Set(%ExternalStorage,
+ Type => 'Disk',
+ Path => '/opt/rt4/var/attachments',
+ );
+
+=head1 DESCRIPTION
+
+This storage option places attachments on disk under the given C<Path>,
+uncompressed. The files are de-duplicated when they are saved; as such,
+if the same file appears in multiple transactions, only one copy will be
+stored on disk.
+
+The C<Path> must be readable by the webserver, and writable by the
+C<bin/extract-attachments> script. Because the majority of the
+attachments are in the filesystem, a simple database backup is thus
+incomplete. It is B<extremely important> that I<backups include the
+on-disk attachments directory>.
+
+Files also C<must not be modified or removed>; doing so may cause
+internal inconsistency.
+
+=cut
+
+1;
diff --git a/lib/RT/Extension/ExternalStorage/Dropbox.pm b/lib/RT/Extension/ExternalStorage/Dropbox.pm
new file mode 100644
index 0000000..d57bf6b
--- /dev/null
+++ b/lib/RT/Extension/ExternalStorage/Dropbox.pm
@@ -0,0 +1,184 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+use 5.008003;
+use warnings;
+use strict;
+
+package RT::Extension::ExternalStorage::Dropbox;
+
+use Role::Basic qw/with/;
+with 'RT::Extension::ExternalStorage::Backend';
+
+our $DROPBOX;
+sub Init {
+ my $self = shift;
+ my %self = %{$self};
+
+ if (not File::Dropbox->require) {
+ RT->Logger->error("Required module File::Dropbox is not installed");
+ return;
+ } elsif (not $self{AccessToken}) {
+ RT->Logger->error("AccessToken not provided for Dropbox. Register a new application"
+ . " at https://www.dropbox.com/developers/apps and generate an access token.");
+ return;
+ }
+
+
+ $DROPBOX = File::Dropbox->new(
+ oauth2 => 1,
+ access_token => $self{AccessToken},
+ root => 'sandbox',
+ furlopts => { timeout => 60 },
+ );
+
+ return $self;
+}
+
+sub Get {
+ my $self = shift;
+ my ($sha) = @_;
+
+ open( $DROPBOX, "<", $sha)
+ or return (undef, "Failed to retrieve file from dropbox: $!");
+ my $content = do {local $/; <$DROPBOX>};
+ close $DROPBOX;
+
+ return ($content);
+}
+
+sub Store {
+ my $self = shift;
+ my ($sha, $content) = @_;
+
+ # No-op if the path exists already. This forces a metadata read.
+ return (1) if open( $DROPBOX, "<", $sha);
+
+ open( $DROPBOX, ">", $sha )
+ or return (undef, "Open for write on dropbox failed: $!");
+ print $DROPBOX $content
+ or return (undef, "Write to dropbox failed: $!");
+ close $DROPBOX
+ or return (undef, "Flush to dropbox failed: $!");
+
+ return (1);
+}
+
+=head1 NAME
+
+RT::Extension::ExternalStorage::Dropbox - Store files in the Dropbox cloud
+
+=head1 SYNOPSIS
+
+ Set(%ExternalStorage,
+ Type => 'Dropbox',
+ AccessToken => '...',
+ );
+
+=head1 DESCRIPTION
+
+This storage option places attachments in the Dropbox shared file
+service. The files are de-duplicated when they are saved; as such, if
+the same file appears in multiple transactions, only one copy will be
+stored on in Dropbox.
+
+Files in Dropbox C<must not be modified or removed>; doing so may cause
+internal inconsistency. It is also important to ensure that the Dropbox
+account used has sufficient space for the attachments, and to monitor
+its space usage.
+
+=head1 SETUP
+
+In order to use this stoage type, a new application must be registered
+with Dropbox:
+
+=over
+
+=item 1.
+
+Log into Dropbox as the user you wish to store files as.
+
+=item 2.
+
+Click C<Create app> on L<https://www.dropbox.com/developers/apps>
+
+=item 3.
+
+Choose B<Dropbox API app> as the type of app.
+
+=item 4.
+
+Choose the B<Files and datastores> as the type of data to store.
+
+=item 5.
+
+Choose B<Yes>, your application only needs access to files it creates.
+
+=item 6.
+
+Enter a descriptive name -- C<Request Tracker files> is fine.
+
+=item 7.
+
+Under C<Generated access token>, click the C<Generate> button.
+
+=item 8.
+
+Copy the provided value into your F<RT_SiteConfig.pm> file as the
+C<AccessToken>:
+
+ Set(%ExternalStorage,
+ Type => 'Dropbox',
+ AccessToken => '...', # Replace the value here, between the quotes
+ );
+
+=back
+
+=cut
+
+1;
diff --git a/lib/RT/Extension/ExternalStorage/Test.pm.in b/lib/RT/Extension/ExternalStorage/Test.pm.in
new file mode 100644
index 0000000..9032217
--- /dev/null
+++ b/lib/RT/Extension/ExternalStorage/Test.pm.in
@@ -0,0 +1,98 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+use strict;
+use warnings;
+
+### after: use lib qw(@RT_LIB_PATH@);
+use lib qw(/opt/rt4/local/lib /opt/rt4/lib);
+
+package RT::Extension::ExternalStorage::Test;
+
+=head2 RT::Extension::ExternalStorage::Test
+
+Initialization for testing.
+
+=cut
+
+use base qw(RT::Test);
+use File::Spec;
+use File::Path 'mkpath';
+
+sub import {
+ my $class = shift;
+ my %args = @_;
+
+ $args{'requires'} ||= [];
+ if ( $args{'testing'} ) {
+ unshift @{ $args{'requires'} }, 'RT::Extension::ExternalStorage';
+ } else {
+ $args{'testing'} = 'RT::Extension::ExternalStorage';
+ }
+
+ $class->SUPER::import( %args );
+ $class->export_to_level(1);
+
+ require RT::Extension::ExternalStorage;
+}
+
+sub attachments_dir {
+ my $dir = File::Spec->catdir( RT::Test->temp_directory, qw(attachments) );
+ mkpath($dir);
+ return $dir;
+}
+
+sub bootstrap_more_config {
+ my $self = shift;
+ my ($config) = @_;
+
+ my $dir = $self->attachments_dir;
+ print $config qq|Set( %ExternalStorage, Type => 'Disk', Path => '$dir' );\n|;
+}
+
+1;
diff --git a/sbin/extract-attachments.in b/sbin/extract-attachments.in
new file mode 100755
index 0000000..ce5dedb
--- /dev/null
+++ b/sbin/extract-attachments.in
@@ -0,0 +1,167 @@
+#!/usr/bin/env perl
+### before: #!@PERL@
+
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+use strict;
+use warnings;
+
+### after: use lib qw(@RT_LIB_PATH@);
+use lib qw(/opt/rt4/local/lib /opt/rt4/lib);
+
+BEGIN { $RT::Extension::ExternalStorage::WRITE = 1 };
+use RT -init;
+
+# Ensure we only run one of these processes at once
+use Fcntl ':flock';
+exit unless flock main::DATA, LOCK_EX | LOCK_NB;
+
+die "\%ExternalStorage is not configured\n"
+ unless RT->Config->Get("ExternalStorage");
+
+exit unless $RT::Extension::ExternalStorage::BACKEND;
+
+my $last = RT->System->FirstAttribute("ExternalStorage");
+$last = $last ? $last->Content : {};
+
+for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+ my $column = $class eq 'RT::Attachments' ? "Content" : "LargeContent";
+ my $id = $last->{$class} || 0;
+
+ while (1) {
+ my $attach = $class->new($RT::SystemUser);
+ $attach->Limit(
+ FIELD => 'id',
+ OPERATOR => '>',
+ VALUE => $id,
+ );
+ $attach->Limit(
+ FIELD => 'ContentEncoding',
+ OPERATOR => '!=',
+ VALUE => 'external',
+ ENTRYAGGREGATOR => 'AND',
+ );
+ if ($class eq "RT::Attachments") {
+ $attach->_OpenParen('applies');
+ $attach->Limit(
+ FIELD => 'ContentType',
+ OPERATOR => 'NOT STARTSWITH',
+ VALUE => $_,
+ SUBCLAUSE => 'applies',
+ ENTRYAGGREGATOR => "AND",
+ ) for "text/", "message/", "image/", "multipart/";
+ $attach->_CloseParen('applies');
+ $attach->Limit(
+ FUNCTION => 'LENGTH(main.Content)',
+ OPERATOR => '>',
+ VALUE => 10*1024*1024,
+ SUBCLAUSE => 'applies',
+ ENTRYAGGREGATOR => 'OR',
+ );
+ } else {
+ my $cfs = $attach->Join(
+ ALIAS1 => 'main',
+ FIELD1 => 'CustomField',
+ TABLE2 => 'CustomFields',
+ FIELD2 => 'id',
+ );
+ # TODO: use IN operator once we increase required RT version to 4.2
+ $attach->Limit(
+ ALIAS => $cfs,
+ FIELD => "Type",
+ VALUE => $_,
+ ) for qw(Binary Image);
+ $attach->{'find_expired_rows'} = 1;
+ }
+
+ $attach->RowsPerPage(100);
+ $RT::Handle->dbh->begin_work;
+ while ( my $a = $attach->Next ) {
+ $id = $a->id;
+ next unless $a->StoreExternally;
+
+ # Explicitly get bytes (not characters, which ->$column would do)
+ my $content = $a->_DecodeLOB(
+ "application/octet-stream",
+ $a->ContentEncoding,
+ $a->_Value( $column, decode_utf8 => 0),
+ );
+
+ # Attempt to write that out
+ my ($key, $msg) = RT::Extension::ExternalStorage->Store( $content );
+ unless ($key) {
+ RT->Logger->error("Failed to store $class $id: $msg");
+ exit 1;
+ }
+
+ (my $status, $msg ) = $a->__Set(
+ Field => $column, Value => $key
+ );
+ unless ($status) {
+ RT->Logger->error("Failed to update $column of $class $id: $msg");
+ exit 2;
+ }
+
+ ( $status, $msg ) = $a->__Set(
+ Field => 'ContentEncoding', Value => 'external',
+ );
+ unless ($status) {
+ RT->Logger->error("Failed to update ContentEncoding of $class $id: $msg");
+ exit 2;
+ }
+ }
+ $RT::Handle->dbh->commit;
+
+ last unless $attach->Count;
+ }
+ $last->{$class} = $id;
+}
+
+RT->System->SetAttribute( Name => "ExternalStorage", Content => $last );
+
+__DATA__
diff --git a/t/externalstorage/basic.t b/t/externalstorage/basic.t
new file mode 100644
index 0000000..281d2e7
--- /dev/null
+++ b/t/externalstorage/basic.t
@@ -0,0 +1,76 @@
+use strict;
+use warnings;
+
+use RT::Extension::ExternalStorage::Test tests => undef;
+
+my $queue = RT::Test->load_or_create_queue(Name => 'General');
+ok $queue && $queue->id;
+
+my $message = MIME::Entity->build(
+ From => 'root at localhost',
+ Subject => 'test',
+ Data => 'test',
+);
+$message->attach(
+ Type => 'image/special',
+ Filename => 'afile.special',
+ Data => 'boo',
+);
+$message->attach(
+ Type => 'application/octet-stream',
+ Filename => 'otherfile.special',
+ Data => 'thing',
+);
+my $ticket = RT::Ticket->new( RT->SystemUser );
+my ($id) = $ticket->Create(
+ Queue => $queue,
+ Subject => 'test',
+ MIMEObj => $message,
+);
+
+ok $id, 'created a ticket';
+
+my @attachs = @{ $ticket->Transactions->First->Attachments->ItemsArrayRef };
+is scalar @attachs, 4, "Contains a multipart and two sub-parts";
+
+is $attachs[0]->ContentType, "multipart/mixed", "Found the top multipart";
+ok !$attachs[0]->StoreExternally, "Shouldn't store multipart part on disk";
+
+is $attachs[1]->ContentType, "text/plain", "Found the text part";
+is $attachs[1]->Content, 'test', "Can get the text part content";
+is $attachs[1]->ContentEncoding, "none", "Content is not encoded";
+ok !$attachs[1]->StoreExternally, "Won't store text part on disk";
+
+is $attachs[2]->ContentType, "image/special", "Found the image part";
+is $attachs[2]->Content, 'boo', "Can get the image content";
+is $attachs[2]->ContentEncoding, "none", "Content is not encoded";
+ok !$attachs[2]->StoreExternally, "Won't store images on disk";
+
+is $attachs[3]->ContentType, "application/octet-stream", "Found the binary part";
+is $attachs[3]->Content, 'thing', "Can get the binary content";
+is $attachs[3]->ContentEncoding, "none", "Content is not encoded";
+ok $attachs[3]->StoreExternally, "Will store binary data on disk";
+
+my $dir = RT::Extension::ExternalStorage::Test->attachments_dir;
+ok !<$dir/*>, "Attachments directory is empty";
+
+
+ok -e 'sbin/extract-attachments', "Found extract-attachments script";
+ok -x 'sbin/extract-attachments', "extract-attachments is executable";
+ok !system('sbin/extract-attachments'), "extract-attachments ran successfully";
+
+ at attachs = @{ $ticket->Transactions->First->Attachments->ItemsArrayRef };
+is $attachs[1]->Content, 'test', "Can still get the text part content";
+is $attachs[1]->ContentEncoding, "none", "Content is not encoded";
+
+is $attachs[2]->Content, 'boo', "Can still get the image content";
+is $attachs[2]->ContentEncoding, "none", "Content is not encoded";
+
+is $attachs[3]->ContentType, "application/octet-stream", "Found the binary part";
+is $attachs[3]->Content, 'thing', "Can still get the binary content";
+isnt $attachs[3]->__Value('Content'), "thing", "Content in database is not the raw content";
+is $attachs[3]->ContentEncoding, "external", "Content encoding is 'external'";
+
+ok <$dir/*>, "Attachments directory contains files";
+
+done_testing();
-----------------------------------------------------------------------
More information about the rt-commit
mailing list