[Rt-commit] rt branch 4.4/shred-external-storage created. rt-4.4.6-74-g02f2eaed9b
BPS Git Server
git at git.bestpractical.com
Fri Jul 21 16:53:50 UTC 2023
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "rt".
The branch, 4.4/shred-external-storage has been created
at 02f2eaed9b001041221861cdaedb12c2f5bdd4ad (commit)
- Log -----------------------------------------------------------------
commit 02f2eaed9b001041221861cdaedb12c2f5bdd4ad
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Jul 21 12:49:17 2023 -0400
Implement Delete for Dropbox external storage
The Delete method is used by shredder when running with
--include-external-storage.
diff --git a/lib/RT/ExternalStorage/Dropbox.pm b/lib/RT/ExternalStorage/Dropbox.pm
index 2b4e790b6f..3a5639e61f 100644
--- a/lib/RT/ExternalStorage/Dropbox.pm
+++ b/lib/RT/ExternalStorage/Dropbox.pm
@@ -161,6 +161,17 @@ sub DownloadURLFor {
return;
}
+sub Delete {
+ my $self = shift;
+ my $sha = shift;
+ my $path = $self->_FilePath($sha);
+
+ if ( $self->_PathExists($path) && !$self->Dropbox->delete($path) ) {
+ return ( undef, "Delete $sha from dropbox failed: " . $self->Dropbox->error );
+ }
+ return ($sha);
+}
+
=head1 NAME
RT::ExternalStorage::Dropbox - Store files in the Dropbox cloud
commit b19e22178b743abf235adeada22c5c080264e5a4
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Jul 21 12:39:15 2023 -0400
Switch to WebService::Dropbox to use Dropbox API v2
Dropbox API v1 has been turned off for years, sadly that File::Dropbox
doesn't support v2.
As there are no persistent access tokens any more(tokens are only valid
in 4 hours), we need to configure RefreshToken instead to automatically
generate new access tokens.
diff --git a/lib/RT/ExternalStorage/Dropbox.pm b/lib/RT/ExternalStorage/Dropbox.pm
index 8f8a965e27..2b4e790b6f 100644
--- a/lib/RT/ExternalStorage/Dropbox.pm
+++ b/lib/RT/ExternalStorage/Dropbox.pm
@@ -62,78 +62,99 @@ sub Dropbox {
return $self->{Dropbox};
}
-sub AccessToken {
+sub AppKey {
my $self = shift;
- return $self->{AccessToken};
+ return $self->{AppKey};
}
-sub Init {
+sub AppSecret {
my $self = shift;
+ return $self->{AppSecret};
+}
- {
- # suppress given/warn is experimental warnings from File::Dropbox 0.6
- # https://rt.cpan.org/Ticket/Display.html?id=108107
-
- my $original_warn_handler = $SIG{__WARN__};
- local $SIG{__WARN__} = sub {
- return if $_[0] =~ /(given|when) is experimental/;
+sub RefreshToken {
+ my $self = shift;
+ return $self->{RefreshToken};
+}
- # Avoid reporting this anonymous call frame as the source of the warning.
- goto &$original_warn_handler;
- };
+sub Init {
+ my $self = shift;
- if (not File::Dropbox->require) {
- RT->Logger->error("Required module File::Dropbox is not installed");
- return;
- } elsif (not $self->AccessToken) {
- RT->Logger->error("Required option 'AccessToken' not provided for Dropbox external storage. See the documentation for " . __PACKAGE__ . " for setting up this integration.");
- return;
- }
+ if ( not WebService::Dropbox->require ) {
+ RT->Logger->error("Required module WebService::Dropbox is not installed");
+ return;
+ }
+ WebService::Dropbox->import;
+ for my $item (qw/AppKey AppSecret RefreshToken/) {
+ next if $self->$item;
+ RT->Logger->error(
+ "Required option '$item' not provided for Dropbox external storage. See the documentation for "
+ . __PACKAGE__
+ . " for setting up this integration." );
+ return;
}
-
- my $dropbox = File::Dropbox->new(
- oauth2 => 1,
- access_token => $self->AccessToken,
- root => 'sandbox',
- furlopts => { timeout => 60 },
+ my $dropbox = WebService::Dropbox->new(
+ {
+ key => $self->AppKey,
+ secret => $self->AppSecret,
+ }
);
+
+ $dropbox->refresh_access_token( $self->RefreshToken );
$self->Dropbox($dropbox);
return $self;
}
-sub Get {
+# Dropbox requires the "/" prefix
+sub _FilePath {
my $self = shift;
- my ($sha) = @_;
+ my $sha = shift;
+ return "/$sha";
+}
- my $dropbox = $self->Dropbox;
+sub _PathExists {
+ my $self = shift;
+ my $path = shift;
- open( $dropbox, "<", $sha)
- or return (undef, "Failed to retrieve file from dropbox: $!");
- my $content = do {local $/; <$dropbox>};
- close $dropbox;
+ # Get rid of expected warnings when path doesn't exist
+ local $SIG{__WARN__} = sub {};
+ return $self->Dropbox->get_metadata($path);
+}
- return ($content);
+sub Get {
+ my $self = shift;
+ my ($sha) = @_;
+ my $path = $self->_FilePath($sha);
+
+ my $content;
+ open my $fh, '>', \$content;
+ $self->Dropbox->download($path, $fh);
+ close $fh;
+ if ( $content ) {
+ return ($content);
+ }
+ else {
+ return ( undef, "Read $sha from dropbox failed: " . $self->Dropbox->error );
+ }
}
sub Store {
my $self = shift;
my ($sha, $content, $attachment) = @_;
- my $dropbox = $self->Dropbox;
+ my $path = $self->_FilePath($sha);
# No-op if the path exists already. This forces a metadata read.
- return ($sha) if open( $dropbox, "<", $sha);
-
- open( $dropbox, ">", $sha )
- or return (undef, "Open for write on dropbox failed: $!");
- print $dropbox $content
- or return (undef, "Write to dropbox failed: $!");
- close $dropbox
- or return (undef, "Flush to dropbox failed: $!");
+ return ($sha) if $self->_PathExists($path);
- return ($sha);
+ if ( $self->Dropbox->upload( $path, $content ) ) {
+ return ($sha);
+ }
+ else {
+ return ( undef, "Write $sha to dropbox failed: " . $self->Dropbox->error );
+ }
}
sub DownloadURLFor {
@@ -147,8 +168,10 @@ RT::ExternalStorage::Dropbox - Store files in the Dropbox cloud
=head1 SYNOPSIS
Set(%ExternalStorage,
- Type => 'Dropbox',
- AccessToken => '...',
+ Type => 'Dropbox',
+ AccessKey => '...',
+ AccessSecret => '...',
+ RefreshToken => '...',
);
=head1 DESCRIPTION
@@ -180,11 +203,11 @@ Click C<Create app> on L<https://www.dropbox.com/developers/apps>
=item 3.
-Choose B<Dropbox API app> as the type of app.
+Choose B<Scoped access> as the API.
=item 4.
-Choose B<Yes>, your application only needs access to files it creates.
+Choose B<App folder> as the type of access.
=item 5.
@@ -192,16 +215,38 @@ Enter a descriptive name -- C<Request Tracker files> is fine.
=item 6.
-Under C<Generated access token>, click the C<Generate> button.
+After creation, grant the following permissions on Permissions tab:
+
+ files.metadata.write
+ files.metadata.read
+ files.content.write
+ files.content.read
=item 7.
-Copy the provided value into your F<RT_SiteConfig.pm> file as the
-C<AccessToken>:
+On Settings tab, get C<App key>/C<App secret> and then access the following
+URL:
+
+ https://www.dropbox.com/oauth2/authorize?token_access_type=offline&response_type=code&client_id=<App key>
+
+Where <App key> is the one you got earlier.
+
+After a confirmation page, you will receive a code, use it along with C<App
+key> and C<App secret> in the following command and run it:
+
+ curl https://api.dropbox.com/oauth2/token -d code=<received code> -d grant_type=authorization_code -u <App key>:<App secret>
+
+The response shall contain C<refresh_token> value.
+
+=item 8.
+
+Copy the provided values into your F<RT_SiteConfig.pm>:
Set(%ExternalStorage,
- Type => 'Dropbox',
- AccessToken => '...', # Replace the value here, between the quotes
+ Type => 'Dropbox',
+ AccessKey => '...',
+ AccessSecret => '...',
+ RefreshToken => '...', # Replace the value here, between the quotes
);
=back
diff --git a/sbin/rt-test-dependencies.in b/sbin/rt-test-dependencies.in
index 14dd5a8e26..2365f599bb 100644
--- a/sbin/rt-test-dependencies.in
+++ b/sbin/rt-test-dependencies.in
@@ -304,7 +304,7 @@ Amazon::S3
.
$deps{'DROPBOX'} = [ text_to_hash( <<'.') ];
-File::Dropbox
+WebService::Dropbox
.
my %AVOID = (
commit 900af9c2105075284c95580de701aba1c7f1395c
Author: sunnavy <sunnavy at bestpractical.com>
Date: Thu Jul 20 03:48:02 2023 -0400
Update tests as we added ExternalStorageDump plugin
diff --git a/t/shredder/03plugin.t b/t/shredder/03plugin.t
index de5d44fa7d..19633fb92a 100644
--- a/t/shredder/03plugin.t
+++ b/t/shredder/03plugin.t
@@ -3,10 +3,10 @@ use strict;
use warnings;
use Test::Deep;
-use RT::Test::Shredder nodb => 1, tests => 28;
+use RT::Test::Shredder nodb => 1, tests => undef;
my $test = "RT::Test::Shredder";
-my @PLUGINS = sort qw(Attachments Base Objects SQLDump Summary Tickets Users);
+my @PLUGINS = sort qw(Attachments Base ExternalStorageDump Objects SQLDump Summary Tickets Users);
use_ok('RT::Shredder::Plugin');
{
@@ -37,3 +37,4 @@ use_ok('RT::Shredder::Plugin');
ok(!$status, "not loaded plugin - not exist");
}
+done_testing;
commit 798639d826e72f1000b9c2cfcb9c5317ed547731
Author: sunnavy <sunnavy at bestpractical.com>
Date: Thu Jul 20 02:00:47 2023 -0400
Support to shred external contents of attachments/objectcustomfieldvalues
To undo changes, here we insert external contents back to RT database
before wiping them out, so the SQL dump contains all the data. Then
users can re-externalize them using commands saved in a separate file
*.external-storage.sh that is also generated by shredder.
Note that this commit doesn't touch the broken Dropbox storage, because
File::Dropbox(last released in 2016) doesn't support Dropbox's current
API v2. We will migrate it in a separate commit later.
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index a72aa1d0dd..3ba250ab0f 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -172,6 +172,18 @@ sub DownloadURLFor {
return "https://" . $self->Bucket . ".s3.amazonaws.com/" . $digest;
}
+sub Delete {
+ my $self = shift;
+ my $sha = shift;
+
+ if ( $self->BucketObj->head_key($sha) ) {
+ $self->BucketObj->delete_key($sha)
+ or return ( undef, "Failed to delete $sha from AmazonS3: " . $self->S3->errstr );
+ }
+
+ return ($sha);
+}
+
=head1 NAME
RT::ExternalStorage::AmazonS3 - Store files in Amazon's S3 cloud
diff --git a/lib/RT/ExternalStorage/Backend.pm b/lib/RT/ExternalStorage/Backend.pm
index 6ec8cf38b5..e3fd0c9bf9 100644
--- a/lib/RT/ExternalStorage/Backend.pm
+++ b/lib/RT/ExternalStorage/Backend.pm
@@ -57,6 +57,7 @@ requires 'Init';
requires 'Get';
requires 'Store';
requires 'DownloadURLFor';
+requires 'Delete';
sub new {
my $class = shift;
diff --git a/lib/RT/ExternalStorage/Disk.pm b/lib/RT/ExternalStorage/Disk.pm
index bd505906cf..b51fe81797 100644
--- a/lib/RT/ExternalStorage/Disk.pm
+++ b/lib/RT/ExternalStorage/Disk.pm
@@ -52,7 +52,7 @@ use strict;
package RT::ExternalStorage::Disk;
use File::Path qw//;
-
+use File::Basename 'dirname';
use Role::Basic qw/with/;
with 'RT::ExternalStorage::Backend';
@@ -85,12 +85,21 @@ sub IsWriteable {
return (1);
}
+sub _FilePath {
+ my $self = shift;
+ my $sha = shift;
+
+ # fan out to avoid one gigantic directory which slows down all file access
+ $sha =~ m{^(...)(...)(.*)};
+ return $self->Path . "/$1/$2/$3";
+}
+
sub Get {
my $self = shift;
my ($sha) = @_;
$sha =~ m{^(...)(...)(.*)};
- my $path = $self->Path . "/$1/$2/$3";
+ my $path = $self->_FilePath($sha);
return (undef, "File does not exist") unless -e $path;
@@ -105,15 +114,11 @@ sub Get {
sub Store {
my $self = shift;
my ($sha, $content, $attachment) = @_;
-
- # fan out to avoid one gigantic directory which slows down all file access
- $sha =~ m{^(...)(...)(.*)};
- my $dir = $self->Path . "/$1/$2";
- my $path = "$dir/$3";
+ my $path = $self->_FilePath($sha);
return ($sha) if -f $path;
- File::Path::make_path($dir, {error => \my $err});
+ File::Path::make_path(dirname($path), {error => \my $err});
return (undef, "Making directory failed") if @{$err};
open( my $fh, ">:raw", $path ) or return (undef, "Cannot write file on disk: $!");
@@ -127,6 +132,17 @@ sub DownloadURLFor {
return;
}
+sub Delete {
+ my $self = shift;
+ my $sha = shift;
+ my $path = $self->_FilePath($sha);
+
+ if ( -f $path ) {
+ unlink $path or return ( undef, "Cannot delete file: $!" );
+ }
+ return ($sha);
+}
+
=head1 NAME
RT::ExternalStorage::Disk - On-disk storage of attachments
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 6dc972542a..e565258631 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -2615,7 +2615,62 @@ sub _AsInsertQuery
return $res;
}
-sub BeforeWipeout { return 1 }
+sub BeforeWipeout {
+ my $self = shift;
+ if ( $RT::Shredder::IncludeExternalStorage
+ && ( $self->isa('RT::Attachment') || $self->isa('RT::ObjectCustomFieldValue') )
+ && ( $self->ContentEncoding // '' ) eq 'external' )
+ {
+ my $digest = $self->ExternalStoreDigest;
+
+ # Delete external resource only if there are no objects referenced to it
+ for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+ my $objects = $class->new( $self->CurrentUser );
+ $objects->Limit( FIELD => 'ContentEncoding', VALUE => 'external' );
+ $objects->Limit( FIELD => 'Content', VALUE => $digest );
+ $objects->Limit( FIELD => 'id', VALUE => $self->Id, OPERATOR => '!=' );
+ if ( $objects->First ) {
+ RT->Logger->info("$digest is referenced by other objects, skipping");
+ return 1;
+ }
+ }
+
+ my $storage = RT->System->ExternalStorage;
+ unless ($storage) {
+ RT->Logger->error("External storage not configured");
+ RT::Shredder::Exception::Info->throw('InvalidExternalStorage');
+ }
+
+ # Internalize content so we can re-create objects easily from generated SQL
+ if ( my $external_content = $storage->Get($digest) ) {
+ my ( $encoding, $content ) = $self->_EncodeLOB( $external_content, $self->ContentType, $self->Filename );
+ my ( $ret, $msg ) = $self->__Set( Field => 'ContentEncoding', Value => $encoding );
+ if ( !$ret ) {
+ RT->Logger->error("Could not set ContentEncoding to $encoding: $msg");
+ RT::Shredder::Exception::Info->throw('CouldntInternalizeObject');
+ }
+
+ ( $ret, $msg ) = $self->__Set( Field => 'Content', Value => $content );
+ if ( !$ret ) {
+ RT->Logger->error("Could not set Content: $msg");
+ RT::Shredder::Exception::Info->throw('CouldntInternalizeObject');
+ }
+ $self->{_internalized} = 1;
+ }
+ else {
+ # Can't internalize since content is absent somehow
+ RT->Logger->error("Could not get content of $digest");
+ return 1;
+ }
+
+ my ( $ret, $msg ) = $storage->Delete($digest);
+ if ( !$ret ) {
+ RT->Logger->error("Failed to delete $digest from external storage: $msg");
+ RT::Shredder::Exception::Info->throw('CouldntDeleteExternalObject');
+ }
+ }
+ return 1;
+}
=head2 Dependencies
diff --git a/lib/RT/Shredder/Exceptions.pm b/lib/RT/Shredder/Exceptions.pm
index a679dd7382..75a8b1d4dc 100644
--- a/lib/RT/Shredder/Exceptions.pm
+++ b/lib/RT/Shredder/Exceptions.pm
@@ -90,6 +90,17 @@ resolve, so the requested object was not removed. Some plugins
do not automatically shred dependent objects for safety, but you
may be able to shred the dependent objects directly using other plugins.
The documentation for this plugin may have more information.
+END
+
+ CouldntInternalizeObject => <<END,
+Shredder couldn't internalize objects. Check out RT logs for details.
+END
+
+ InvalidExternalStorage => <<END,
+ExternalStorage is not configured correctly, please check your configuration.
+
+ CouldntDeleteExternalObject => <<END,
+Shredder couldn't delete external objects. Check out RT logs for details.
END
);
diff --git a/lib/RT/ExternalStorage/Backend.pm b/lib/RT/Shredder/Plugin/ExternalStorageDump.pm
similarity index 59%
copy from lib/RT/ExternalStorage/Backend.pm
copy to lib/RT/Shredder/Plugin/ExternalStorageDump.pm
index 6ec8cf38b5..6a3f90d66f 100644
--- a/lib/RT/ExternalStorage/Backend.pm
+++ b/lib/RT/Shredder/Plugin/ExternalStorageDump.pm
@@ -46,47 +46,57 @@
#
# END BPS TAGGED BLOCK }}}
-use warnings;
+package RT::Shredder::Plugin::ExternalStorageDump;
+
use strict;
+use warnings;
-package RT::ExternalStorage::Backend;
+use base qw(RT::Shredder::Plugin::Base::Dump);
+use RT::Shredder;
-use Role::Basic;
+sub AppliesToStates { return 'after wiping dependencies' }
-requires 'Init';
-requires 'Get';
-requires 'Store';
-requires 'DownloadURLFor';
+sub SupportArgs
+{
+ my $self = shift;
+ return $self->SUPER::SupportArgs, qw(file_name from_storage);
+}
-sub new {
- my $class = shift;
+sub TestArgs
+{
+ my $self = shift;
my %args = @_;
+ $args{'from_storage'} = 1 unless defined $args{'from_storage'};
+ my $file = $args{'file_name'} = RT::Shredder->GetFileName(
+ FileName => $args{'file_name'} || '%t-XXXX.external-storage.sh',
+ FromStorage => delete $args{'from_storage'},
+ );
+ open $args{'file_handle'}, ">:raw", $file
+ or return (0, "Couldn't open '$file' for write: $!");
+
+ return $self->SUPER::TestArgs( %args );
+}
- $class = delete $args{Type};
- if (not $class) {
- RT->Logger->error("No storage engine type provided");
- return undef;
- } elsif ($class->require) {
- # no action needed; $class was loaded
- } else {
- my $long = "RT::ExternalStorage::$class";
- if ($long->require) {
- $class = $long;
- } else {
- RT->Logger->error("Can't load external storage engine $class: $@");
- return undef;
- }
- }
+sub FileName { return $_[0]->{'opt'}{'file_name'} }
+sub FileHandle { return $_[0]->{'opt'}{'file_handle'} }
- unless ($class->DOES("RT::ExternalStorage::Backend")) {
- RT->Logger->error("External storage engine $class doesn't implement RT::ExternalStorage::Backend");
- return undef;
- }
+sub Run {
+ my $self = shift;
+ return ( 0, 'no handle' ) unless my $fh = $self->{'opt'}{'file_handle'};
- my $self = bless \%args, $class;
- $self->Init;
-}
+ my %args = ( Object => undef, @_ );
+
+ return 1 unless $args{'Object'}{'_internalized'};
+ my $query = sprintf(
+ "$RT::SbinPath/rt-externalize-attachments --class %s --ids %d\n",
+ ref( $args{'Object'} ) . 's',
+ $args{'Object'}->Id
+ );
-RT::Base->_ImportOverlays();
+ utf8::encode($query) if utf8::is_utf8($query);
+
+ return 1 if print $fh $query;
+ return ( 0, "Couldn't write to filehandle" );
+}
1;
diff --git a/sbin/rt-externalize-attachments.in b/sbin/rt-externalize-attachments.in
index 6f23f668e1..20d8649d39 100644
--- a/sbin/rt-externalize-attachments.in
+++ b/sbin/rt-externalize-attachments.in
@@ -76,6 +76,8 @@ GetOptions( \%opts,
"age=s",
"batchsize=s",
"dry-run",
+ "class=s",
+ "ids=s",
);
if ($opts{'help'}) {
@@ -113,17 +115,29 @@ my $last = RT->System->FirstAttribute("ExternalStorage");
$last = $last ? $last->Content : {};
for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+ next if $opts{class} && $opts{class} ne $class;
+
my $column = $class eq 'RT::Attachments' ? "Content" : "LargeContent";
my $id = $last->{$class} || 0;
my $batchsize = $opts{'batchsize'} || 1;
while (1) {
my $attach = $class->new($RT::SystemUser);
- $attach->Limit(
- FIELD => 'id',
- OPERATOR => '>',
- VALUE => $id,
- );
+ if ($opts{ids}) {
+ $attach->Limit(
+ FIELD => 'id',
+ OPERATOR => 'IN',
+ VALUE => [ split /,/, $opts{ids} ],
+ );
+ }
+ else {
+ $attach->Limit(
+ FIELD => 'id',
+ OPERATOR => '>',
+ VALUE => $id,
+ );
+ }
+
$attach->Limit(
FIELD => 'ContentEncoding',
OPERATOR => '!=',
@@ -209,7 +223,8 @@ for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
last unless $attach->Count and $batchsize > 0;
}
- $last->{$class} = $id;
+ # Do not affect the normal externalize process when handing specific ones
+ $last->{$class} = $id unless $opts{ids};
}
if (!$opts{'dry-run'}) {
@@ -252,6 +267,14 @@ By default everything is moved.
If batchsize is given, then only C<NUM> number of attachments will be moved.
By default everything is moved.
+=item --class=RT::Attachments or --class=RT::ObjectCustomFieldValues
+
+If class is given, only objects of the specificed class will be externalized.
+
+=item --ids=NUM,NUM
+
+Use C<--ids> along with C<--class> to explicitly externalize specified records.
+
=item -h
=item --help
diff --git a/sbin/rt-shredder.in b/sbin/rt-shredder.in
index b880d7e12b..0269b21d8d 100644
--- a/sbin/rt-shredder.in
+++ b/sbin/rt-shredder.in
@@ -112,6 +112,16 @@ Outputs help for specified plugin.
Don't prompt with questions before shredding objects.
+=head2 --include-external-storage
+
+Shred externalized contents too. When this is enabled, a separate dump file
+that contains commands to re-externalize objects will be generated, along with
+SQL dump file, so you can fully "undo" if needed.
+
+By default this is not enabled.
+
+See also L<RT_Config/%ExternalStorage>.
+
=head1 SEE ALSO
L<RT::Shredder>
@@ -170,6 +180,34 @@ if ( !$opt{'no-sqldump'} ) {
} else {
print "SQL dump file is '". $plugin->FileName ."'\n";
}
+
+ if ($RT::Shredder::IncludeExternalStorage) {
+
+ # As external storage is bound quite tightly with sqldump(users need to
+ # import the sql dump before re-externalizing), here we create external
+ # storage dump file based on sqldump
+
+ my $file_name = $opt{'sqldump'} || '';
+ if ( $file_name ) {
+ if ( not $file_name =~ s!\.sql$!.externalstorage.sh! ) {
+ $file_name .= '.externalstorage.sh';
+ }
+ }
+
+ my $plugin = eval {
+ $shredder->AddDumpPlugin(
+ Name => 'ExternalStorageDump',
+ Arguments => { file_name => $file_name, from_storage => 0, }
+ );
+ };
+ if ($@) {
+ print STDERR "ERROR: Couldn't open ExternalStorage dump file: $@\n";
+ exit 1 if $file_name;
+ }
+ else {
+ print "ExternalStorage dump file is '" . $plugin->FileName . "'\n";
+ }
+ }
}
my @objs = process_plugins( $shredder );
@@ -258,6 +296,11 @@ sub parse_args
if( GetOptions( 'no-sqldump' => \$tmp ) && $tmp ) {
$opt{'no-sqldump'} = $tmp;
}
+
+ if( GetOptions( 'include-external-storage' => \$tmp ) && $tmp ) {
+ $RT::Shredder::IncludeExternalStorage = 1;
+ }
+ $tmp = undef;
return;
}
diff --git a/share/html/Admin/Tools/Shredder/Elements/DumpFileLink b/share/html/Admin/Tools/Shredder/Elements/DumpFileLink
index 9cc5cd0491..9d032267e9 100644
--- a/share/html/Admin/Tools/Shredder/Elements/DumpFileLink
+++ b/share/html/Admin/Tools/Shredder/Elements/DumpFileLink
@@ -46,10 +46,11 @@
%#
%# END BPS TAGGED BLOCK }}}
<div id="shredder-dump-file-link" class="shredder-help">
-<a href="<% RT->Config->Get('WebPath') %>/Admin/Tools/Shredder/Dumps/<% $File %>"><% loc('Download dumpfile') %></a>
+<a href="<% RT->Config->Get('WebPath') %>/Admin/Tools/Shredder/Dumps/<% $File %>"><% $Label %></a>
</div>
<%ARGS>
$File => ''
+$Label => loc('Download dumpfile')
</%ARGS>
<%INIT>
return unless $File;
diff --git a/share/html/Admin/Tools/Shredder/Elements/SelectObjects b/share/html/Admin/Tools/Shredder/Elements/SelectObjects
index 206ac25c8a..9c4eead554 100644
--- a/share/html/Admin/Tools/Shredder/Elements/SelectObjects
+++ b/share/html/Admin/Tools/Shredder/Elements/SelectObjects
@@ -60,6 +60,9 @@
<& ObjectCheckBox, Object => $o &>
% }
</div>
+% if ( RT->Config->Get('ExternalStorage') ) {
+<& /Elements/Submit, Name => 'WipeoutIncludingExternalStorage', Label => loc('Wipeout Including External Storage') &>
+% }
<& /Elements/Submit, Name => 'Wipeout', Label => loc('Wipeout') &>
% }
</div>
diff --git a/share/html/Admin/Tools/Shredder/index.html b/share/html/Admin/Tools/Shredder/index.html
index 96ca48efa4..8c80e5a831 100644
--- a/share/html/Admin/Tools/Shredder/index.html
+++ b/share/html/Admin/Tools/Shredder/index.html
@@ -49,6 +49,7 @@
$Plugin => ''
$Search => ''
$Wipeout => ''
+$WipeoutIncludingExternalStorage => ''
@WipeoutObject => ()
</%ARGS>
<& /Admin/Elements/Header, Title => $title &>
@@ -58,13 +59,16 @@ $Wipeout => ''
<& /Elements/ListActions, actions => $messages{'Errors'} &>
<& /Elements/ListActions, actions => $messages{'Success'} &>
<& Elements/DumpFileLink, File => $dump_file &>
+% if ( $WipeoutIncludingExternalStorage ) {
+<& Elements/DumpFileLink, File => $external_storage_dump_file, Label => loc('Download external storage dump file') &>
+% }
<& Elements/SelectPlugin, Plugin => $Plugin, %ARGS &>
<div id="shredder-submit-button" class="<% $Plugin? '': 'hidden' %>">
<& /Elements/Submit, Name => 'Search', Label => loc('Search') &>
</div>
</div>
<br />
-% if( $Search || $Wipeout ) {
+% if( $Search || $Wipeout || $WipeoutIncludingExternalStorage ) {
<& Elements/SelectObjects, Objects => \@objs &>
% }
</form>
@@ -122,8 +126,9 @@ if( $Plugin ) { { # use additional block({}) to effectively exit block on errors
} }
my $dump_file = '';
+my $external_storage_dump_file = '';
-if( $Plugin && $Wipeout ) { { # use additional block({}) to effectively exit block on errors
+if( $Plugin && ($Wipeout || $WipeoutIncludingExternalStorage) ) { { # use additional block({}) to effectively exit block on errors
my $shredder = RT::Shredder->new( force => 1 );
my $backup_plugin = RT::Shredder::Plugin->new;
my ($status, $msg) = $backup_plugin->LoadByName('SQLDump');
@@ -144,6 +149,31 @@ if( $Plugin && $Wipeout ) { { # use additional block({}) to effectively exit blo
$shredder->AddDumpPlugin( Object => $backup_plugin );
+ local $RT::Shredder::IncludeExternalStorage;
+
+ if ($WipeoutIncludingExternalStorage) {
+ $RT::Shredder::IncludeExternalStorage = 1;
+
+ my $external_storage_plugin = RT::Shredder::Plugin->new;
+ my ( $status, $msg ) = $external_storage_plugin->LoadByName('ExternalStorageDump');
+ unless ($status) {
+ push @{ $messages{Errors} }, $msg;
+ $Search = '';
+ @objs = ();
+ last;
+ }
+ ( $status, $msg ) = $external_storage_plugin->TestArgs;
+ unless ($status) {
+ push @{ $messages{Errors} }, $msg;
+ $Search = '';
+ @objs = ();
+ last;
+ }
+ ($external_storage_dump_file) = $external_storage_plugin->FileName;
+ push @{ $messages{'Success'} }, "External storage dump file is '$external_storage_dump_file'";
+ $shredder->AddDumpPlugin( Object => $external_storage_plugin );
+ }
+
$shredder->PutObjects( Objects => \@WipeoutObject );
($status, $msg) = $plugin_obj->SetResolvers( Shredder => $shredder );
unless( $status ) {
-----------------------------------------------------------------------
hooks/post-receive
--
rt
More information about the rt-commit
mailing list