[Rt-commit] rt branch, 4.4/improve-rt-externalize-attachments, created. rt-4.4.2-59-g8d52a1a
Craig Kaiser
craig at bestpractical.com
Tue Dec 26 14:16:25 EST 2017
The branch, 4.4/improve-rt-externalize-attachments has been created
at 8d52a1a45989f21c149dcce2054c1b03b6ea053b (commit)
- Log -----------------------------------------------------------------
commit 8d52a1a45989f21c149dcce2054c1b03b6ea053b
Author: Craig Kaiser <craig at bestpractical.com>
Date: Tue Dec 26 14:09:44 2017 -0500
Improve rt-externalize-attachments
Support for 'attachments' flag that allows the number of attachments that will be
processed to be specified. Add 'attachment_id' flag to submit a start ID, where
attachments prior to the specified ID will be limited.
diff --git a/sbin/rt-externalize-attachments b/sbin/rt-externalize-attachments
new file mode 100755
index 0000000..5c58275
--- /dev/null
+++ b/sbin/rt-externalize-attachments
@@ -0,0 +1,304 @@
+#!/Users/craigkaiser/perl5/perlbrew/perls/perl-5.24.3/bin/perl
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2017 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+use strict;
+use warnings;
+use POSIX qw(strftime);
+
+# fix lib paths, some may be relative
+BEGIN { # BEGIN RT CMD BOILERPLATE
+ require File::Spec;
+ require Cwd;
+ my @libs = ("./lib", "./local/lib");
+ my $bin_path;
+
+ for my $lib (@libs) {
+ unless ( File::Spec->file_name_is_absolute($lib) ) {
+ $bin_path ||= ( File::Spec->splitpath(Cwd::abs_path(__FILE__)) )[1];
+ $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
+ }
+ unshift @INC, $lib;
+ }
+
+}
+
+# Read in the options
+my %opts;
+use Getopt::Long;
+GetOptions( \%opts,
+ "help|h",
+ "verbose|v",
+ "age=s",
+ "batchsize=s",
+ "dry-run",
+ "attachments=s",
+ "attachment_id=s",
+);
+
+if ($opts{'help'}) {
+ require Pod::Usage;
+ print Pod::Usage::pod2usage(-verbose => 2);
+ exit;
+}
+
+if ($opts{'dry-run'}) {
+ RT->Logger->info("(DRY-RUN, no changes made)");
+ # dry-run implies verbose output
+ $opts{verbose} = 1;
+}
+
+use RT -init;
+
+# Ensure we only run one of these processes at once
+use Fcntl ':flock';
+exit unless flock main::DATA, LOCK_EX | LOCK_NB;
+
+use Digest::SHA qw//;
+
+my $ExternalStorage = RT->System->ExternalStorage;
+
+die "\%ExternalStorage is not configured\n"
+ unless $ExternalStorage;
+
+if ($ExternalStorage->can('IsWriteable')) {
+ my ($ok, $msg) = $ExternalStorage->IsWriteable;
+ die $msg if !$ok;
+}
+
+# pull out the previous high-water mark for each object type
+my $last = RT->System->FirstAttribute("ExternalStorage");
+$last = $last ? $last->Content : {};
+
+for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+ my $column = $class eq 'RT::Attachments' ? "Content" : "LargeContent";
+ my $id = $last->{$class} || 0;
+ my $batchsize = $opts{'batchsize'} || 1;
+ my $attachments = $opts{'attachments'} || 9999;
+ my $attachment_id = $opts{'attachment_id'};
+
+ while (1) {
+ my $attach = $class->new($RT::SystemUser);
+ $attach->Limit(
+ FIELD => 'id',
+ OPERATOR => '>',
+ VALUE => $id,
+ );
+ $attach->Limit(
+ FIELD => 'ContentEncoding',
+ OPERATOR => '!=',
+ VALUE => 'external',
+ ENTRYAGGREGATOR => 'AND',
+ );
+ if ($opts{'age'}) {
+ my $agelimit = strftime "%F", localtime(time()-$opts{'age'}*24*60*60);
+ $attach->Limit(
+ FIELD => 'Created',
+ OPERATOR => '<',
+ VALUE => $agelimit,
+ ENTRYAGGREGATOR => 'AND',
+ );
+ }
+ if ($opts{'attachment_id'}) {
+ $attach->Limit(
+ FIELD => 'ID',
+ OPERATOR => '>=',
+ VALUE => $attachment_id,
+ ENTRYAGGREGATOR => 'AND',
+ );
+ }
+
+ if ($class eq "RT::ObjectCustomFieldValues") {
+ $attach->{'find_disabled_rows'} = 1;
+ }
+
+ $attach->RowsPerPage(100);
+ while ( my $a = $attach->Next and $batchsize > 0) {
+ $id = $a->id;
+
+ if ($opts{'batchsize'}) {
+ $batchsize -= 1;
+ }
+
+ my ($ok, $why) = $a->ShouldStoreExternally;
+ if (!$ok) {
+ if ($opts{verbose}) {
+ RT->Logger->info("Skipping $class $id because: $why");
+ }
+ next;
+ }
+
+ # Explicitly get bytes (not characters, which ->$column would do)
+ my $content = $a->_DecodeLOB(
+ "application/octet-stream",
+ $a->ContentEncoding,
+ $a->_Value( $column, decode_utf8 => 0),
+ );
+
+ # Attempt to write that out
+ if ($opts{verbose}) {
+ RT->Logger->info("Storing $class $id");
+ }
+
+ my ($key, $msg) = ('DRY-RUN', '');
+ if (!$opts{'dry-run'}) {
+ ($key, $msg) = Store( $content, $a );
+ unless ($key) {
+ RT->Logger->error("Failed to store $class $id: $msg");
+ exit 1;
+ }
+
+ $RT::Handle->dbh->begin_work;
+
+ (my $status, $msg ) = $a->__Set(
+ Field => $column, Value => $key
+ );
+ unless ($status) {
+ RT->Logger->error("Failed to update $column of $class $id: $msg");
+ exit 2;
+ }
+
+ ( $status, $msg ) = $a->__Set(
+ Field => 'ContentEncoding', Value => 'external',
+ );
+ unless ($status) {
+ RT->Logger->error("Failed to update ContentEncoding of $class $id: $msg");
+ exit 2;
+ }
+
+ $RT::Handle->dbh->commit;
+ }
+
+ if ($opts{verbose}) {
+ RT->Logger->info("Stored $class $id as $key");
+ }
+ if ( $attach->Count >= $attachments ) {
+ RT->Logger->info("Reached attachment limit.");
+ exit 2;
+ }
+ }
+ last unless ( $attach->Count and $batchsize > 0 );
+ }
+ $last->{$class} = $id;
+}
+
+if (!$opts{'dry-run'}) {
+ # update high-water mark for each object type
+ RT->System->SetAttribute( Name => "ExternalStorage", Content => $last );
+}
+
+sub Store {
+ my $content = shift;
+ my $attachment = shift;
+
+ my $sha = Digest::SHA::sha256_hex( $content );
+ return $ExternalStorage->Store( $sha => $content, $attachment );
+}
+
+=head1 NAME
+
+rt-externalize-attachments - Move attachments from database to external storage
+
+=head1 SYNOPSIS
+
+ rt-externalize-attachments [options]
+
+=head1 DESCRIPTION
+
+rt-externalize-attachments is used to move attachments out of the database to
+some external storage.
+
+=head1 OPTIONS
+
+=over 8
+
+=item --age=AGE
+
+If age is given, then only attachments older than C<AGE> days will be moved.
+By default everything is moved.
+
+=item --batchsize=NUM
+
+If batchsize is given, then only C<NUM> number of attachments will be moved.
+By default everything is moved.
+
+=item -h
+
+=item --help
+
+Display this documentation
+
+=item -v
+
+=item --verbose
+
+Log a message (at level "info") for each attachment, both before its
+upload begins and after it completes.
+
+=item --dry-run
+
+Make a trial run and do no changes (mostly the same output as a real run is
+produced). C<dry-run> implies verbose output.
+
+=item --attachments
+
+Profide the number of attachments you would like to process.
+
+=item --attachment_id
+
+Provide a attachment ID to have as the starting point of the externalize.
+
+=back
+
+=head1 SEE ALSO
+
+L<RT_Config.html/External-storage>, L<RT::ExternalStorage>
+
+=cut
+
+# don't remove; for locking (see call to flock above)
+__DATA__
-----------------------------------------------------------------------
More information about the rt-commit
mailing list