[Rt-commit] rt branch, 4.4/external-storage, repushed
Shawn Moore
shawn at bestpractical.com
Fri Jun 5 14:52:13 EDT 2015
The branch 4.4/external-storage was deleted and repushed:
was 62c2211ebd13eea0b8b5c6a40b3b93c746ee8709
now 08b1636199ad8b107bc45a9a5adc1771d906aacf
1: 428c9c4 = 1: 428c9c4 Verbatim copy of RT::Extension::ExternalStorage
2: d4b8dd6 ! 2: b4bebf8 Move ExternalStorage files into proper namespaces
@@ -1,8 +1,45 @@
Author: Shawn M Moore <shawn at bestpractical.com>
- RT::Extension::ExternalStorage -> RT::ExternalStorage
+ Move ExternalStorage files into proper namespaces
- Simple namespace adjustment, no more.
+ This includes only the bare minimum of code changes in order to make
+ this look like it was part of core RT all along.
+
+diff --git a/.gitignore b/.gitignore
+--- a/.gitignore
++++ b/.gitignore
+@@
+ /sbin/rt-email-dashboards
+ /sbin/rt-email-digest
+ /sbin/rt-email-group-admin
++/sbin/rt-externalize-attachments
+ /sbin/rt-fulltext-indexer
+ /sbin/rt-preferences-viewer
+ /sbin/rt-server
+
+diff --git a/Makefile.in b/Makefile.in
+--- a/Makefile.in
++++ b/Makefile.in
+@@
+ rt-email-dashboards \
+ rt-email-digest \
+ rt-email-group-admin \
++ rt-externalize-attachments \
+ rt-fulltext-indexer \
+ rt-importer \
+ rt-preferences-viewer \
+
+diff --git a/configure.ac b/configure.ac
+--- a/configure.ac
++++ b/configure.ac
+@@
+ sbin/rt-test-dependencies
+ sbin/rt-email-digest
+ sbin/rt-email-dashboards
++ sbin/rt-externalize-attachments
+ sbin/rt-clean-sessions
+ sbin/rt-shredder
+ sbin/rt-validator
diff --git a/lib/RT/Extension/ExternalStorage.pm b/lib/RT/Extension/ExternalStorage.pm
deleted file mode 100644
@@ -1124,12 +1161,12 @@
+configured or backed up. Attachment storage paths are calculated based
+on file contents; this provides de-duplication.
+
-+The files are initially stored in the database when RT receives them;
-+this guarantees that the user does not need to wait for the file to be
-+transferred to disk or to the cloud, and makes it durable to transient
-+failures of cloud connectivity. The provided C<bin/extract-attachments>
-+script, to be run regularly via cron, takes care of moving attachments
-+out of the database at a later time.
++The files are initially stored in the database when RT receives
++them; this guarantees that the user does not need to wait for
++the file to be transferred to disk or to the cloud, and makes it
++durable to transient failures of cloud connectivity. The provided
++C<sbin/rt-externalize-attachments> script, to be run regularly via cron,
++takes care of moving attachments out of the database at a later time.
+
+=head1 INSTALLATION
+
@@ -1167,18 +1204,18 @@
+
+=item Extract existing attachments
+
-+Run C<bin/extract-attachments>; this may take some time, depending on
-+the existing size of the database. This task may be safely cancelled
++Run C<sbin/rt-externalize-attachments>; this may take some time, depending
++on the existing size of the database. This task may be safely cancelled
+and re-run to resume.
+
+=item Schedule attachments extraction
+
-+Schedule C<bin/extract-attachments> to run at regular intervals via
++Schedule C<sbin/rt-externalize-attachments> to run at regular intervals via
+cron. For instance, the following F</etc/cron.d/rt> entry will run it
+daily, which may be good to concentrate network or disk usage to times
+when RT is less in use:
+
-+ 0 0 * * * root /opt/rt4/local/plugins/RT-Extension-ExternalStorage/bin/extract-attachments
++ 0 0 * * * root /opt/rt4/sbin/rt-externalize-attachments
+
+=back
+
@@ -1760,7 +1797,7 @@
+stored on disk.
+
+The C<Path> must be readable by the webserver, and writable by the
-+C<bin/extract-attachments> script. Because the majority of the
++C<sbin/rt-externalize-attachments> script. Because the majority of the
+attachments are in the filesystem, a simple database backup is thus
+incomplete. It is B<extremely important> that I<backups include the
+on-disk attachments directory>.
@@ -1962,10 +1999,10 @@
+
+1;
-diff --git a/lib/RT/ExternalStorage/Test.pm.in b/lib/RT/ExternalStorage/Test.pm.in
+diff --git a/lib/RT/Test/ExternalStorage.pm b/lib/RT/Test/ExternalStorage.pm
new file mode 100644
--- /dev/null
-+++ b/lib/RT/ExternalStorage/Test.pm.in
++++ b/lib/RT/Test/ExternalStorage.pm
@@
+# BEGIN BPS TAGGED BLOCK {{{
+#
@@ -2015,40 +2052,12 @@
+#
+# END BPS TAGGED BLOCK }}}
+
++package RT::Test::ExternalStorage;
+use strict;
+use warnings;
-+
-+### after: use lib qw(@RT_LIB_PATH@);
-+use lib qw(/opt/rt4/local/lib /opt/rt4/lib);
-+
-+package RT::ExternalStorage::Test;
-+
-+=head2 RT::ExternalStorage::Test
-+
-+Initialization for testing.
-+
-+=cut
-+
+use base qw(RT::Test);
+use File::Spec;
+use File::Path 'mkpath';
-+
-+sub import {
-+ my $class = shift;
-+ my %args = @_;
-+
-+ $args{'requires'} ||= [];
-+ if ( $args{'testing'} ) {
-+ unshift @{ $args{'requires'} }, 'RT::ExternalStorage';
-+ } else {
-+ $args{'testing'} = 'RT::ExternalStorage';
-+ }
-+
-+ $class->SUPER::import( %args );
-+ $class->export_to_level(1);
-+
-+ require RT::ExternalStorage;
-+}
+
+sub attachments_dir {
+ my $dir = File::Spec->catdir( RT::Test->temp_directory, qw(attachments) );
@@ -2058,44 +2067,374 @@
+
+sub bootstrap_more_config {
+ my $self = shift;
-+ my ($config) = @_;
++ my $handle = shift;
++ my $args = shift;
++
++ $self->SUPER::bootstrap_more_config($handle, $args, @_);
+
+ my $dir = $self->attachments_dir;
-+ print $config qq|Set( %ExternalStorage, Type => 'Disk', Path => '$dir' );\n|;
++ print $handle qq|Set( %ExternalStorage, Type => 'Disk', Path => '$dir' );\n|;
+}
+
+1;
diff --git a/sbin/extract-attachments.in b/sbin/extract-attachments.in
+deleted file mode 100755
--- a/sbin/extract-attachments.in
-+++ b/sbin/extract-attachments.in
++++ /dev/null
@@
- ### after: use lib qw(@RT_LIB_PATH@);
- use lib qw(/opt/rt4/local/lib /opt/rt4/lib);
-
+-#!/usr/bin/env perl
+-### before: #!@PERL@
+-
+-# BEGIN BPS TAGGED BLOCK {{{
+-#
+-# COPYRIGHT:
+-#
+-# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
+-# <sales at bestpractical.com>
+-#
+-# (Except where explicitly superseded by other copyright notices)
+-#
+-#
+-# LICENSE:
+-#
+-# This work is made available to you under the terms of Version 2 of
+-# the GNU General Public License. A copy of that license should have
+-# been provided with this software, but in any event can be snarfed
+-# from www.gnu.org.
+-#
+-# This work is distributed in the hope that it will be useful, but
+-# WITHOUT ANY WARRANTY; without even the implied warranty of
+-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+-# General Public License for more details.
+-#
+-# You should have received a copy of the GNU General Public License
+-# along with this program; if not, write to the Free Software
+-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+-# 02110-1301 or visit their web page on the internet at
+-# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+-#
+-#
+-# CONTRIBUTION SUBMISSION POLICY:
+-#
+-# (The following paragraph is not intended to limit the rights granted
+-# to you to modify and distribute this software under the terms of
+-# the GNU General Public License and is only of importance to you if
+-# you choose to contribute your changes and enhancements to the
+-# community by submitting them to Best Practical Solutions, LLC.)
+-#
+-# By intentionally submitting any modifications, corrections or
+-# derivatives to this work, or any other work intended for use with
+-# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+-# you are the copyright holder for those contributions and you grant
+-# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+-# royalty-free, perpetual, license to use, copy, create derivative
+-# works based on those contributions, and sublicense and distribute
+-# those contributions and any derivatives thereof.
+-#
+-# END BPS TAGGED BLOCK }}}
+-use strict;
+-use warnings;
+-
+-### after: use lib qw(@RT_LIB_PATH@);
+-use lib qw(/opt/rt4/local/lib /opt/rt4/lib);
+-
-BEGIN { $RT::Extension::ExternalStorage::WRITE = 1 };
+-use RT -init;
+-
+-# Ensure we only run one of these processes at once
+-use Fcntl ':flock';
+-exit unless flock main::DATA, LOCK_EX | LOCK_NB;
+-
+-die "\%ExternalStorage is not configured\n"
+- unless RT->Config->Get("ExternalStorage");
+-
+-exit unless $RT::Extension::ExternalStorage::BACKEND;
+-
+-my $last = RT->System->FirstAttribute("ExternalStorage");
+-$last = $last ? $last->Content : {};
+-
+-for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+- my $column = $class eq 'RT::Attachments' ? "Content" : "LargeContent";
+- my $id = $last->{$class} || 0;
+-
+- while (1) {
+- my $attach = $class->new($RT::SystemUser);
+- $attach->Limit(
+- FIELD => 'id',
+- OPERATOR => '>',
+- VALUE => $id,
+- );
+- $attach->Limit(
+- FIELD => 'ContentEncoding',
+- OPERATOR => '!=',
+- VALUE => 'external',
+- ENTRYAGGREGATOR => 'AND',
+- );
+- if ($class eq "RT::Attachments") {
+- $attach->_OpenParen('applies');
+- $attach->Limit(
+- FIELD => 'ContentType',
+- OPERATOR => 'NOT STARTSWITH',
+- VALUE => $_,
+- SUBCLAUSE => 'applies',
+- ENTRYAGGREGATOR => "AND",
+- ) for "text/", "message/", "image/", "multipart/";
+- $attach->_CloseParen('applies');
+- $attach->Limit(
+- FUNCTION => 'LENGTH(main.Content)',
+- OPERATOR => '>',
+- VALUE => 10*1024*1024,
+- SUBCLAUSE => 'applies',
+- ENTRYAGGREGATOR => 'OR',
+- );
+- } else {
+- my $cfs = $attach->Join(
+- ALIAS1 => 'main',
+- FIELD1 => 'CustomField',
+- TABLE2 => 'CustomFields',
+- FIELD2 => 'id',
+- );
+- # TODO: use IN operator once we increase required RT version to 4.2
+- $attach->Limit(
+- ALIAS => $cfs,
+- FIELD => "Type",
+- VALUE => $_,
+- ) for qw(Binary Image);
+- $attach->{'find_expired_rows'} = 1;
+- }
+-
+- $attach->RowsPerPage(100);
+- $RT::Handle->dbh->begin_work;
+- while ( my $a = $attach->Next ) {
+- $id = $a->id;
+- next unless $a->StoreExternally;
+-
+- # Explicitly get bytes (not characters, which ->$column would do)
+- my $content = $a->_DecodeLOB(
+- "application/octet-stream",
+- $a->ContentEncoding,
+- $a->_Value( $column, decode_utf8 => 0),
+- );
+-
+- # Attempt to write that out
+- my ($key, $msg) = RT::Extension::ExternalStorage->Store( $content );
+- unless ($key) {
+- RT->Logger->error("Failed to store $class $id: $msg");
+- exit 1;
+- }
+-
+- (my $status, $msg ) = $a->__Set(
+- Field => $column, Value => $key
+- );
+- unless ($status) {
+- RT->Logger->error("Failed to update $column of $class $id: $msg");
+- exit 2;
+- }
+-
+- ( $status, $msg ) = $a->__Set(
+- Field => 'ContentEncoding', Value => 'external',
+- );
+- unless ($status) {
+- RT->Logger->error("Failed to update ContentEncoding of $class $id: $msg");
+- exit 2;
+- }
+- }
+- $RT::Handle->dbh->commit;
+-
+- last unless $attach->Count;
+- }
+- $last->{$class} = $id;
+-}
+-
+-RT->System->SetAttribute( Name => "ExternalStorage", Content => $last );
+-
+-__DATA__
+
+diff --git a/sbin/rt-externalize-attachments.in b/sbin/rt-externalize-attachments.in
+new file mode 100755
+--- /dev/null
++++ b/sbin/rt-externalize-attachments.in
+@@
++#!@PERL@
++# BEGIN BPS TAGGED BLOCK {{{
++#
++# COPYRIGHT:
++#
++# This software is Copyright (c) 1996-2015 Best Practical Solutions, LLC
++# <sales at bestpractical.com>
++#
++# (Except where explicitly superseded by other copyright notices)
++#
++#
++# LICENSE:
++#
++# This work is made available to you under the terms of Version 2 of
++# the GNU General Public License. A copy of that license should have
++# been provided with this software, but in any event can be snarfed
++# from www.gnu.org.
++#
++# This work is distributed in the hope that it will be useful, but
++# WITHOUT ANY WARRANTY; without even the implied warranty of
++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++# General Public License for more details.
++#
++# You should have received a copy of the GNU General Public License
++# along with this program; if not, write to the Free Software
++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
++# 02110-1301 or visit their web page on the internet at
++# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
++#
++#
++# CONTRIBUTION SUBMISSION POLICY:
++#
++# (The following paragraph is not intended to limit the rights granted
++# to you to modify and distribute this software under the terms of
++# the GNU General Public License and is only of importance to you if
++# you choose to contribute your changes and enhancements to the
++# community by submitting them to Best Practical Solutions, LLC.)
++#
++# By intentionally submitting any modifications, corrections or
++# derivatives to this work, or any other work intended for use with
++# Request Tracker, to Best Practical Solutions, LLC, you confirm that
++# you are the copyright holder for those contributions and you grant
++# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
++# royalty-free, perpetual, license to use, copy, create derivative
++# works based on those contributions, and sublicense and distribute
++# those contributions and any derivatives thereof.
++#
++# END BPS TAGGED BLOCK }}}
++use strict;
++use warnings;
++
++# fix lib paths, some may be relative
++BEGIN { # BEGIN RT CMD BOILERPLATE
++ require File::Spec;
++ require Cwd;
++ my @libs = ("@RT_LIB_PATH@", "@LOCAL_LIB_PATH@");
++ my $bin_path;
++
++ for my $lib (@libs) {
++ unless ( File::Spec->file_name_is_absolute($lib) ) {
++ $bin_path ||= ( File::Spec->splitpath(Cwd::abs_path(__FILE__)) )[1];
++ $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
++ }
++ unshift @INC, $lib;
++ }
++
++}
++
+BEGIN { $RT::ExternalStorage::WRITE = 1 };
- use RT -init;
-
- # Ensure we only run one of these processes at once
-@@
- die "\%ExternalStorage is not configured\n"
- unless RT->Config->Get("ExternalStorage");
-
--exit unless $RT::Extension::ExternalStorage::BACKEND;
++use RT -init;
++
++# Ensure we only run one of these processes at once
++use Fcntl ':flock';
++exit unless flock main::DATA, LOCK_EX | LOCK_NB;
++
++die "\%ExternalStorage is not configured\n"
++ unless RT->Config->Get("ExternalStorage");
++
+exit unless $RT::ExternalStorage::BACKEND;
-
- my $last = RT->System->FirstAttribute("ExternalStorage");
- $last = $last ? $last->Content : {};
-@@
- );
-
- # Attempt to write that out
-- my ($key, $msg) = RT::Extension::ExternalStorage->Store( $content );
++
++my $last = RT->System->FirstAttribute("ExternalStorage");
++$last = $last ? $last->Content : {};
++
++for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
++ my $column = $class eq 'RT::Attachments' ? "Content" : "LargeContent";
++ my $id = $last->{$class} || 0;
++
++ while (1) {
++ my $attach = $class->new($RT::SystemUser);
++ $attach->Limit(
++ FIELD => 'id',
++ OPERATOR => '>',
++ VALUE => $id,
++ );
++ $attach->Limit(
++ FIELD => 'ContentEncoding',
++ OPERATOR => '!=',
++ VALUE => 'external',
++ ENTRYAGGREGATOR => 'AND',
++ );
++ if ($class eq "RT::Attachments") {
++ $attach->_OpenParen('applies');
++ $attach->Limit(
++ FIELD => 'ContentType',
++ OPERATOR => 'NOT STARTSWITH',
++ VALUE => $_,
++ SUBCLAUSE => 'applies',
++ ENTRYAGGREGATOR => "AND",
++ ) for "text/", "message/", "image/", "multipart/";
++ $attach->_CloseParen('applies');
++ $attach->Limit(
++ FUNCTION => 'LENGTH(main.Content)',
++ OPERATOR => '>',
++ VALUE => 10*1024*1024,
++ SUBCLAUSE => 'applies',
++ ENTRYAGGREGATOR => 'OR',
++ );
++ } else {
++ my $cfs = $attach->Join(
++ ALIAS1 => 'main',
++ FIELD1 => 'CustomField',
++ TABLE2 => 'CustomFields',
++ FIELD2 => 'id',
++ );
++ # TODO: use IN operator once we increase required RT version to 4.2
++ $attach->Limit(
++ ALIAS => $cfs,
++ FIELD => "Type",
++ VALUE => $_,
++ ) for qw(Binary Image);
++ $attach->{'find_expired_rows'} = 1;
++ }
++
++ $attach->RowsPerPage(100);
++ $RT::Handle->dbh->begin_work;
++ while ( my $a = $attach->Next ) {
++ $id = $a->id;
++ next unless $a->StoreExternally;
++
++ # Explicitly get bytes (not characters, which ->$column would do)
++ my $content = $a->_DecodeLOB(
++ "application/octet-stream",
++ $a->ContentEncoding,
++ $a->_Value( $column, decode_utf8 => 0),
++ );
++
++ # Attempt to write that out
+ my ($key, $msg) = RT::ExternalStorage->Store( $content );
- unless ($key) {
- RT->Logger->error("Failed to store $class $id: $msg");
- exit 1;
++ unless ($key) {
++ RT->Logger->error("Failed to store $class $id: $msg");
++ exit 1;
++ }
++
++ (my $status, $msg ) = $a->__Set(
++ Field => $column, Value => $key
++ );
++ unless ($status) {
++ RT->Logger->error("Failed to update $column of $class $id: $msg");
++ exit 2;
++ }
++
++ ( $status, $msg ) = $a->__Set(
++ Field => 'ContentEncoding', Value => 'external',
++ );
++ unless ($status) {
++ RT->Logger->error("Failed to update ContentEncoding of $class $id: $msg");
++ exit 2;
++ }
++ }
++ $RT::Handle->dbh->commit;
++
++ last unless $attach->Count;
++ }
++ $last->{$class} = $id;
++}
++
++RT->System->SetAttribute( Name => "ExternalStorage", Content => $last );
++
++__DATA__
diff --git a/t/externalstorage/basic.t b/t/externalstorage/basic.t
--- a/t/externalstorage/basic.t
@@ -2105,7 +2444,7 @@
use warnings;
-use RT::Extension::ExternalStorage::Test tests => undef;
-+use RT::ExternalStorage::Test tests => undef;
++use RT::Test::ExternalStorage tests => undef;
my $queue = RT::Test->load_or_create_queue(Name => 'General');
ok $queue && $queue->id;
@@ -2114,8 +2453,17 @@
ok $attachs[3]->StoreExternally, "Will store binary data on disk";
-my $dir = RT::Extension::ExternalStorage::Test->attachments_dir;
-+my $dir = RT::ExternalStorage::Test->attachments_dir;
++my $dir = RT::Test::ExternalStorage->attachments_dir;
ok !<$dir/*>, "Attachments directory is empty";
+-ok -e 'sbin/extract-attachments', "Found extract-attachments script";
+-ok -x 'sbin/extract-attachments', "extract-attachments is executable";
+-ok !system('sbin/extract-attachments'), "extract-attachments ran successfully";
++ok -e 'sbin/rt-externalize-attachments', "Found rt-externalize-attachments script";
++ok -x 'sbin/rt-externalize-attachments', "rt-externalize-attachments is executable";
++ok !system('sbin/rt-externalize-attachments'), "rt-externalize-attachments ran successfully";
+
+ @attachs = @{ $ticket->Transactions->First->Attachments->ItemsArrayRef };
+ is $attachs[1]->Content, 'test', "Can still get the text part content";
3: fb84a1f < --: ------- bin/extract-attachments -> sbin/rt-externalize-attachments
4: fe97b43 < --: ------- Remove VERSION from RT::ExternalStorage
5: 68e81bc < --: ------- Remove boilerplate CPAN doc from RT::ExternalStorage
6: a12cd1a < --: ------- Remove 5.8.3 requirement from RT::ExternalStorage modules
7: a7a2b74 < --: ------- Install sbin/rt-externalize-attachments
8: 1e4a67b < --: ------- Ignore generated sbin/rt-externalize-attachments
9: 2817a0e < --: ------- Use the standard script setup stanza in externalize-attachments
10: 111e4da < --: ------- Explain that ExternalStorage and $last is high-water mark
11: d485335 < --: ------- Explain seemingly-spurious __DATA__
12: c3dbdce < --: ------- compatibile -> compatible
13: 7caac89 < --: ------- Remove straggling references to ExternalStorage "extension"
14: dd72a1d < --: ------- Remove now-unnecessary install and @Plugins instructions
15: ce9a822 < --: ------- Remove AmazonS3.pm's copy of Dropbox.pm's documentation
16: a647740 < --: ------- Unify installation and configuration
17: f73eb38 < --: ------- Explain empty if block
18: 36f47ff < --: ------- Include rt-externalize-attachments cronjob in README
19: 4f76abf < --: ------- Describe %ExternalStorage in RT_Config.pm
20: 3b2b6b5 < --: ------- Describe backups with ExternalStorage
21: 0d74b22 < --: ------- Use an accessor for Disk's Path attribute
22: 43b49b2 < --: ------- Move StoreExternally monkeypatches right into their classes
23: 2ae713c < --: ------- Rename StoreExternally to ShouldStoreExternally
24: 45d8658 < --: ------- Die with a message if backend is missing
25: 3c63421 < --: ------- Fix reference to now-removed %self
26: 0b02f39 < --: ------- Move additional META entry into RT::Config
27: 55b3d17 < --: ------- Use OPERATOR => 'IN'
--: ------- > 3: 713c3f2 Update ExternalStorage documentation
--: ------- > 4: 686daae Cleanup pass of ExternalStorage code
37: 7f3ffb0 ! 5: ae65f3a Add --with-attachment-store to ./configure
@@ -2,7 +2,8 @@
Add --with-attachment-store to ./configure
- This sets up defaults for rt-test-dependencies
+ This sets up defaults for rt-test-dependencies to install Amazon::S3
+ or File::Dropbox as appropriate
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
@@ -29,6 +30,15 @@
--- a/sbin/rt-test-dependencies.in
+++ b/sbin/rt-test-dependencies.in
@@
+ 'with-USERLOGO',
+ 'with-HTML-DOC',
+
++ 'with-S3', 'with-DROPBOX',
++
+ 'list-deps',
+ 'siteinstall!',
+ 'help|h',
+@@
# Set up defaults
my %default = (
@@ -43,7 +53,7 @@
- 'with-GD' => @RT_GD@,
+ 'with-CORE' => 1,
+ 'with-CLI' => 1,
-+ 'with-MAILGATE' => 1,
++ 'with-MAILGATE' => 1,
+ 'with-DEVELOPER' => @RT_DEVELOPER@,
+ 'with-GPG' => @RT_GPG_DEPS@,
+ 'with-SMIME' => @RT_SMIME_DEPS@,
@@ -53,8 +63,6 @@
'with-DASHBOARDS' => 1,
- 'with-USERLOGO' => 1,
- 'with-HTML-DOC' => @RT_DEVELOPER@,
-- 'with-S3' => 0,
-- 'with-DROPBOX' => 0,
+ 'with-USERLOGO' => 1,
+ 'with-HTML-DOC' => @RT_DEVELOPER@,
+ 'with-S3' => (uc(q{@ATTACHMENT_STORE@}) eq 'S3'),
@@ -62,4 +70,19 @@
);
$args{$_} = $default{$_} foreach grep !exists $args{$_}, keys %default;
+@@
+ Pod::Simple 3.24
+ .
+
++$deps{'S3'} = [ text_to_hash( <<'.') ];
++Amazon::S3
++.
++
++$deps{'DROPBOX'} = [ text_to_hash( <<'.') ];
++File::Dropbox
++.
++
+ my %AVOID = (
+ 'DBD::Oracle' => [qw(1.23)],
+ 'Devel::StackTrace' => [qw(1.28 1.29)],
28: 0672f48 ! 6: ead55c8 New config for ExternalStorageCutoffSize
@@ -2,7 +2,9 @@
New config for ExternalStorageCutoffSize
- (Honestly, mostly because I want to test with much smaller files)
+ This lets you adjust how large attachments must be to go into
+ external storage. Its value is changed from 10 MiB to 10 MB for
+ parity with MaxAttachmentSize.
diff --git a/etc/RT_Config.pm.in b/etc/RT_Config.pm.in
--- a/etc/RT_Config.pm.in
@@ -13,19 +15,19 @@
+=item C<$ExternalStorageCutoffSize>
+
-+Certain object types, like values for Binary custom fields, are always
-+put into external storage. However, for other object types, like images
-+and text, there is a line in the sand where you want small objects in the
-+database but large objects in external storage. By default, objects larger
-+than 10 MiB (10*1024*1024 bytes) will be put into external storage.
-+C<$ExternalStorageCutoffSize> adjusts that line in the sand.
++Certain object types, like values for Binary (aka file upload) custom
++fields, are always put into external storage. However, for other
++object types, like images and text, there is a line in the sand where
++you want small objects in the database but large objects in external
++storage. By default, objects larger than 10 MB will be put into external
++storage. C<$ExternalStorageCutoffSize> adjusts that line in the sand.
+
+Note that changing this setting does not affect existing attachments, only
+the new ones that C<sbin/rt-externalize-attachments> hasn't seen yet.
+
+=cut
+
-+Set($ExternalStorageCutoffSize, 10*1024*1024);
++Set($ExternalStorageCutoffSize, 10_000_000);
+
=back
29: 701dcbd < --: ------- Add optional deps on Amazon::S3 or File::Dropbox
30: 4cfc6b5 < --: ------- Inline RT::Record::_DecodeLOB monkeypatch
31: cf750bf < --: ------- Hang $RT::ExternalStorage::BACKEND off RT->System
32: 187de01 < --: ------- Move Store routine from RT::ExternalStorage to script
33: f2a9727 < --: ------- Explain what a Binary CF is
34: fe32673 < --: ------- Use a method call to confirm writeability rather than a global
35: 59f2652 < --: ------- Fix externalstorage tests to be more like core RT
36: c8b6bd0 < --: ------- WIP
38: 20c99d6 < --: ------- Import overlays for each ExternalStorage PM
39: db2889e < --: ------- Move Digest::SHA load to externalize-attachments
40: 4cd2063 < --: ------- Comment explaining the fan-out for ExternalStorage on disk
41: 94b7ee4 < --: ------- Use an accessor for Dropbox instead of global $DROPBOX
42: de60f1c < --: ------- Remove unused $BUCKET
43: 61a9dc1 < --: ------- Use S3 accessor instead of global $S3
44: 7262974 < --: ------- Factor out AmazonS3->BucketObj
45: c89ddab < --: ------- Switch from %self to $self
46: aabdb18 < --: ------- Make an accessor for Bucket
47: ebfef40 < --: ------- Use accessors for AccessKeyId and SecretAccessKey too
48: c244b30 < --: ------- Add ->ExternalStoreDigest for Attachment and OCFV
49: 3838e23 < --: ------- Add ExternalStorage->DirectLinkForObject API
50: 84fb165 < --: ------- Extract Digest directly to avoid loading the file from external storage
51: 4131abf < --: ------- DirectLinkForObject -> DownloadURLFor
52: 56979bd < --: ------- RT->System->ExternalStorageURLFor($Object)
53: 25ee9a3 < --: ------- Add config to disable ExternalStorageDirectLink
54: 2053b60 < --: ------- Use direct external-storage URL for OCFVs where applicable
55: ced12a6 < --: ------- Use direct external-storage URL for txn display
56: d9c6088 < --: ------- Use direct external-storage URL for attachment list
57: b4a8c18 < --: ------- Use direct external-storage URL for mobile UI attachments
58: e8431bc < --: ------- Use direct external-storage URL for show email record
59: ba31033 < --: ------- Document the ExternalStorage-related RT::System methods
60: e2c2f85 < --: ------- Point users to ExternalStorage::Dropbox doc when no access token
61: f8bad3f < --: ------- Dropbox has deprecated the datastore API
62: 4fbcb97 < --: ------- Explain that S3 is the only engine that can do direct links
63: 1a90b2c < --: ------- Improve _DecodeLOB's doc and mention external storage
64: 2d814a2 < --: ------- Make encoding=external a sibling branch
65: 473b1d1 < --: ------- Improve the use of variables in _DecodeLOB/external
66: da32d14 < --: ------- Better error messages for missing attributes
67: f01f43e < --: ------- Copyediting for ExternalStorage docs
68: 2ec02ca < --: ------- Log when we create a new bucket
69: d90eb37 < --: ------- S3 setup instructions
70: 9288f35 < --: ------- This should be bold not comment
71: 9f3ce5f < --: ------- Add --verbose to rt-externalize-attachments
--: ------- > 7: 2d16571 New config for ExternalStorageDirectLink
72: d5d2770 ! 8: 7100684 Add --verbose to rt-externalize-attachments
@@ -1,6 +1,6 @@
Author: Shawn M Moore <shawn at bestpractical.com>
- Explain _why_ we're not uploading to external storage
+ Add --verbose to rt-externalize-attachments
diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
--- a/lib/RT/Attachment.pm
@@ -68,19 +68,100 @@
--- a/sbin/rt-externalize-attachments.in
+++ b/sbin/rt-externalize-attachments.in
@@
- $RT::Handle->dbh->begin_work;
+
+ }
+
++# Read in the options
++my %opts;
++use Getopt::Long;
++GetOptions( \%opts,
++ "help|h", "verbose|v",
++);
++
++if ($opts{'help'}) {
++ require Pod::Usage;
++ print Pod::Usage::pod2usage(-verbose => 2);
++ exit;
++}
++
+ use RT -init;
+
+ # Ensure we only run one of these processes at once
+@@
+ $attach->RowsPerPage(100);
while ( my $a = $attach->Next ) {
$id = $a->id;
- next unless $a->ShouldStoreExternally;
+
+ my ($ok, $why) = $a->ShouldStoreExternally;
+ if (!$ok) {
-+ RT->Logger->info("Skipping $class $id because: $why");
++ if ($opts{verbose}) {
++ RT->Logger->info("Skipping $class $id because: $why");
++ }
+ next;
+ }
# Explicitly get bytes (not characters, which ->$column would do)
my $content = $a->_DecodeLOB(
+@@
+ );
+
+ # Attempt to write that out
++ if ($opts{verbose}) {
++ RT->Logger->info("Storing $class $id");
++ }
++
+ my ($key, $msg) = Store( $content );
+ unless ($key) {
+ RT->Logger->error("Failed to store $class $id: $msg");
+@@
+ }
+
+ $RT::Handle->dbh->commit;
++
++ if ($opts{verbose}) {
++ RT->Logger->info("Stored $class $id as $key");
++ }
+ }
+
+ last unless $attach->Count;
+@@
+ return ($key);
+ }
+
++=head1 NAME
++
++rt-externalize-attachments - Move attachments from database to external storage
++
++=head1 SYNOPSIS
++
++ rt-externalize-attachments [options]
++
++=head1 OPTIONS
++
++This tool supports a few options. Most are for debugging.
++
++=over 8
++
++=item -h
++
++=item --help
++
++Display this documentation
++
++=item -v
++
++=item --verbose
++
++Log a message (at level "info") for each attachment, both before its
++upload begins and after it completes.
++
++=back
++
++=cut
++
+ # don't remove; for locking (see call to flock above)
+ __DATA__
diff --git a/t/externalstorage/basic.t b/t/externalstorage/basic.t
--- a/t/externalstorage/basic.t
73: ef18fcb ! 9: 08b1636 Filter should-store-externally? in Perl, not SQL
@@ -1,6 +1,6 @@
Author: Shawn M Moore <shawn at bestpractical.com>
- Filter should store externally in Perl, not SQL
+ Filter should-store-externally? in Perl, not SQL
This reduces performance, but it improves visibility and flexibility.
The code for answering "should store externally?" is no longer
@@ -55,4 +55,3 @@
$attach->{'find_expired_rows'} = 1;
}
-
74: fccaa3f < --: ------- Limit the transaction to each upload
75: fa121bf < --: ------- Turn off direct linking by default
76: f57e298 < --: ------- Implement and document direct linking for S3
77: 4fbaa8a < --: ------- Handle a single bucket in a region
78: 017246b < --: ------- Fix failing POD test
79: 62a8193 < --: ------- Use 10_000_000 for ExternalStorageCutoffSize
80: 62c2211 < --: ------- Only log skipped attachments under --verbose
More information about the rt-commit
mailing list