[Rt-commit] rt branch 4.4/shred-external-storage created. rt-4.4.6-75-g0240a44737
BPS Git Server
git at git.bestpractical.com
Tue Jul 25 19:36:09 UTC 2023
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "rt".
The branch, 4.4/shred-external-storage has been created
at 0240a44737d981028f0a541bc2f9865cd29db075 (commit)
- Log -----------------------------------------------------------------
commit 0240a44737d981028f0a541bc2f9865cd29db075
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Jul 25 15:33:54 2023 -0400
Test shredding external contents
diff --git a/lib/RT/Test/Shredder.pm b/lib/RT/Test/Shredder.pm
index 5ed78f4f65..0735c8868c 100644
--- a/lib/RT/Test/Shredder.pm
+++ b/lib/RT/Test/Shredder.pm
@@ -166,6 +166,17 @@ sub shredder_new
from_storage => 0,
} );
+ if ($RT::Shredder::IncludeExternalStorage) {
+ my $file = File::Spec->catfile( $self->temp_directory, 'dump.XXXX.external-storage.sh' );
+ $obj->AddDumpPlugin(
+ Name => 'ExternalStorageDump',
+ Arguments => {
+ file_name => $file,
+ from_storage => 0,
+ }
+ );
+ }
+
return $obj;
}
diff --git a/t/shredder/05external_storage.t b/t/shredder/05external_storage.t
new file mode 100644
index 0000000000..14bac53885
--- /dev/null
+++ b/t/shredder/05external_storage.t
@@ -0,0 +1,143 @@
+use strict;
+use warnings;
+
+use Test::Deep;
+use Digest::SHA 'sha256_hex';
+use RT::Test::Shredder tests => undef, config => <<'EOF';
+my $storage_path = File::Spec->catdir(RT->Config->Get('LogDir'), 'attachments');
+use File::Path 'mkpath';
+mkpath($storage_path);
+Set(%ExternalStorage,
+ Type => 'Disk',
+ Path => $storage_path,
+);
+Set($ExternalStorageCutoffSize, 20*1024);
+EOF
+
+my $test = "RT::Test::Shredder";
+$test->create_savepoint('clean');
+
+my $image_mime = MIME::Entity->build(
+ Type => 'text/plain',
+ Subject => 'Test external storage',
+ Data => <<END,
+This is a test
+END
+);
+
+my $image_path = RT::Test::get_relocatable_file( 'owls.jpg', '..', 'data' );
+my $image_content = RT::Test->file_content($image_path);
+my $image_sha = sha256_hex($image_content);
+
+$image_mime->attach(
+ Path => $image_path,
+ Type => "image/gif",
+ Encoding => "base64",
+);
+
+my $ticket = RT::Test->create_ticket(
+ Subject => 'Test',
+ Queue => 'General',
+ MIMEObj => $image_mime,
+);
+
+# Get rid of the warning of "TransactionBatch was fired on a ticket that no longer exists"
+$ticket->ApplyTransactionBatch;
+
+ok( RT::Test->run_singleton_command('sbin/rt-externalize-attachments'), "Ran rt-externalize-attachments successfully" );
+
+my $attach = RT::Attachment->new( RT->SystemUser );
+$attach->LoadByCols( Filename => 'owls.jpg' );
+ok( $attach->Id, 'Found owls.jpg' );
+
+is( $attach->_Value('Content'), $image_sha, 'owls.jpg is externalized' );
+
+# Clean up the additional attribute for easier comparison
+( RT->System->Attributes->Named("ExternalStorage") )[0]->Delete;
+
+$test->create_savepoint('owls');
+
+diag "Test shredder without external storage included";
+{
+ my $shredder = $test->shredder_new();
+ $shredder->PutObjects( Objects => $ticket );
+ $shredder->WipeoutAll;
+
+ $test->db_is_valid;
+ cmp_deeply( $test->dump_current_and_savepoint('clean'), "Shredded successfully" );
+
+ is( RT->System->ExternalStorage->Get($image_sha), $image_content, 'External content is not deleted' );
+
+ # Undo
+ my $sql_file = $shredder->{dump_plugins}[0]->FileName;
+ RT->DatabaseHandle->dbh->do($_) for split /^(?=INSERT)/m, RT::Test->file_content($sql_file);
+
+ cmp_deeply( $test->dump_current_and_savepoint('owls'), "Undid successfully" );
+}
+
+diag "Test shredder with external storage included";
+{
+ $test->restore_savepoint('owls');
+
+ local $RT::Shredder::IncludeExternalStorage = 1;
+ my $shredder = $test->shredder_new();
+ $shredder->PutObjects( Objects => $ticket );
+ $shredder->WipeoutAll;
+
+ my ($ret) = RT->System->ExternalStorage->Get($image_sha);
+ ok( !$ret, 'External content is deleted' );
+
+ $test->db_is_valid;
+ cmp_deeply( $test->dump_current_and_savepoint('clean'), "Shredded successfully" );
+
+ # Undo
+ my $sql_file = $shredder->{dump_plugins}[0]->FileName;
+ RT->DatabaseHandle->dbh->do($_) for split /^(?=INSERT)/m, RT::Test->file_content($sql_file);
+
+ my $external_file = $shredder->{dump_plugins}[1]->FileName;
+ ok( RT::Test->run_singleton_command($_), "$_ ran successfully" )
+ for split /\n/, RT::Test->file_content($external_file);
+
+ # Clean up the additional attribute
+ ( RT->System->Attributes->Named("ExternalStorage") )[0]->Delete;
+ cmp_deeply( $test->dump_current_and_savepoint('owls'), "Undid successfully" );
+}
+
+my $cf = RT::Test->load_or_create_custom_field( Name => 'Upload', Queue => 'General', Type => 'BinarySingle' );
+ok( $ticket->AddCustomFieldValue( Field => $cf, Value => 'owls.jpg', LargeContent => $image_content ) );
+
+ok( RT::Test->run_singleton_command('sbin/rt-externalize-attachments'), "Ran rt-externalize-attachments successfully" );
+
+my $ocfv = $ticket->CustomFieldValues('Upload')->First;
+is( $ocfv->LargeContent, $image_content );
+is( $ocfv->_Value('LargeContent'), $image_sha, 'CF owls.jpg is externalized' );
+
+# Clean up the additional attribute for easier comparison
+( RT->System->Attributes->Named("ExternalStorage") )[0]->Delete;
+$test->create_savepoint('2 owls');
+
+diag "Test shredder with external content referenced by multiple times";
+{
+ local $RT::Shredder::IncludeExternalStorage = 1;
+ my $shredder = $test->shredder_new();
+ $shredder->PutObjects( Objects => $ocfv );
+ $shredder->WipeoutAll;
+
+ is( RT->System->ExternalStorage->Get($image_sha), $image_content, 'External content is not deleted' );
+
+ $shredder = $test->shredder_new();
+ $shredder->PutObjects( Objects => $ticket );
+ $shredder->WipeoutAll;
+
+ $shredder = $test->shredder_new();
+ $shredder->PutObjects( Objects => $cf );
+ $shredder->WipeoutAll;
+
+ $test->db_is_valid;
+ cmp_deeply( $test->dump_current_and_savepoint('clean'), "Shredded successfully" );
+
+ my ($ret) = RT->System->ExternalStorage->Get($image_sha);
+ ok( !$ret, 'External content is deleted' );
+}
+
+done_testing;
diff --git a/t/web/shredder.t b/t/web/shredder.t
index 47ea111c5f..4303042d1a 100644
--- a/t/web/shredder.t
+++ b/t/web/shredder.t
@@ -1,10 +1,25 @@
use strict;
use warnings;
-use RT::Test tests => undef;
+# Set ExternalStorage in config file to run rt-externalize-attachments
+use RT::Test tests => undef, config => <<'EOF';
+my $storage_path = File::Spec->catdir(RT->Config->Get('LogDir'), 'attachments');
+use File::Path 'mkpath';
+mkpath($storage_path);
+Set(%ExternalStorage,
+ Type => 'Disk',
+ Path => $storage_path,
+);
+Set($ExternalStorageCutoffSize, 20*1024);
+EOF
RT::Config->Set('ShredderStoragePath', RT::Test->temp_directory . '');
+# Disable ExternalStorage for old tests
+my %storage_config = RT->Config->Get( 'ExternalStorage' );
+RT->Config->Set( 'ExternalStorage' );
+RT->Config->PostLoadCheck;
+
my ( $baseurl, $agent ) = RT::Test->started_ok;
diag("Test server running at $baseurl");
@@ -34,6 +49,7 @@ my $ticket_id;
},
button => 'Search',
}, "Search for ticket object");
+ $agent->content_lacks('Wipeout Including External Storage', 'No External Storage button' );
$agent->submit_form_ok({
form_id => 'shredder-search-form',
@@ -71,6 +87,7 @@ my $ticket_id;
},
button => 'Search',
}, "Search for user");
+ $agent->content_lacks('Wipeout Including External Storage', 'No External Storage button' );
$agent->submit_form_ok({
form_id => 'shredder-search-form',
@@ -85,4 +102,142 @@ my $ticket_id;
ok !$ret, 'User successfully shredded';
}
+# Shred RT::Ticket with external storage
+use Digest::SHA 'sha256_hex';
+RT->Config->Set( 'ExternalStorage', %storage_config, );
+RT->Config->PostLoadCheck;
+
+my $image_mime = MIME::Entity->build(
+ Type => 'text/plain',
+ Subject => 'Test external storage',
+ Data => <<END,
+This is a test
+END
+);
+
+my $image_path = RT::Test::get_relocatable_file( 'owls.jpg', '..', 'data' );
+my $image_content = RT::Test->file_content($image_path);
+my $image_sha = sha256_hex($image_content);
+
+$image_mime->attach(
+ Path => $image_path,
+ Type => "image/gif",
+ Encoding => "base64",
+);
+
+RT::Test->stop_server;
+( $baseurl, $agent ) = RT::Test->started_ok;
+$agent->login( 'root' => 'password' );
+
+diag "Test shredder without external storage included";
+{
+ my $ticket_id = create_image_ticket();
+
+ $agent->get_ok( $baseurl . '/Admin/Tools/Shredder/' );
+ $agent->submit_form_ok(
+ {
+ form_id => 'shredder-search-form',
+ fields => { Plugin => 'Tickets' },
+ },
+ "Select Tickets shredder plugin"
+ );
+
+ $agent->submit_form_ok(
+ {
+ form_id => 'shredder-search-form',
+ fields => {
+ 'Tickets:query' => 'id=' . $ticket_id,
+ },
+ button => 'Search',
+ },
+ "Search for ticket object"
+ );
+ $agent->content_contains( 'Wipeout Including External Storage', 'Found External Storage button' );
+
+ $agent->submit_form_ok(
+ {
+ form_id => 'shredder-search-form',
+ fields => {
+ 'WipeoutObject' => 'RT::Ticket-example.com-' . $ticket_id,
+ },
+ button => 'Wipeout',
+ },
+ "Select and destroy ticket object"
+ );
+ $agent->text_contains( 'objects were successfuly removed', 'Found success message' );
+ ok( $agent->find_link( text => 'Download dumpfile' ), 'Found dumpfile' );
+ ok( !$agent->find_link( text => 'Download external storage dumpfile' ), 'No external storage dumpfile' );
+
+ my $ticket = RT::Ticket->new( RT->SystemUser );
+ my ($ret) = $ticket->Load($ticket_id);
+ ok !$ret, 'Ticket successfully shredded';
+ is( RT->System->ExternalStorage->Get($image_sha), $image_content, 'External content is not deleted' );
+}
+
+diag "Test shredder with external storage included";
+{
+ my $ticket_id = create_image_ticket();
+ $agent->get_ok( $baseurl . '/Admin/Tools/Shredder/' );
+ $agent->submit_form_ok(
+ {
+ form_id => 'shredder-search-form',
+ fields => { Plugin => 'Tickets' },
+ },
+ "Select Tickets shredder plugin"
+ );
+
+ $agent->submit_form_ok(
+ {
+ form_id => 'shredder-search-form',
+ fields => {
+ 'Tickets:query' => 'id=' . $ticket_id,
+ },
+ button => 'Search',
+ },
+ "Search for ticket object"
+ );
+ $agent->content_contains( 'Wipeout Including External Storage', 'No External Storage button' );
+
+ $agent->submit_form_ok(
+ {
+ form_id => 'shredder-search-form',
+ fields => {
+ 'WipeoutObject' => 'RT::Ticket-example.com-' . $ticket_id,
+ },
+ button => 'WipeoutIncludingExternalStorage',
+ },
+ "Select and destroy ticket object"
+ );
+ $agent->text_contains( 'objects were successfuly removed', 'Found success message' );
+
+ ok( $agent->find_link( text => 'Download dumpfile' ), 'Found dumpfile' );
+ ok( $agent->find_link( text => 'Download external storage dumpfile' ), 'Found external storage dumpfile' );
+
+ my $ticket = RT::Ticket->new( RT->SystemUser );
+ my ($ret) = $ticket->Load($ticket_id);
+ ok !$ret, 'Ticket successfully shredded';
+ ($ret) = RT->System->ExternalStorage->Get($image_sha);
+ ok( !$ret, 'External content is deleted' );
+}
+
+sub create_image_ticket {
+ my $ticket = RT::Test->create_ticket(
+ Subject => 'Test',
+ Queue => 'General',
+ MIMEObj => $image_mime,
+ );
+
+ ok( RT::Test->run_singleton_command('sbin/rt-externalize-attachments'),
+ "Ran rt-externalize-attachments successfully" );
+
+ # reset to re-externalize all later
+ ( RT->System->Attributes->Named("ExternalStorage") )[0]->Delete;
+
+ my $attach = RT::Attachment->new( RT->SystemUser );
+ $attach->LoadByCols( Filename => 'owls.jpg' );
+ ok( $attach->Id, 'Found owls.jpg' );
+ is( $attach->_Value('Content'), $image_sha, 'owls.jpg is externalized' );
+ return $ticket->Id;
+}
+
done_testing();
commit 33aa35b03a056ca8a71fd21209b12a6ee31f31ca
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Jul 21 12:49:17 2023 -0400
Implement Delete for Dropbox external storage
The Delete method is used by shredder when running with
--include-external-storage.
diff --git a/lib/RT/ExternalStorage/Dropbox.pm b/lib/RT/ExternalStorage/Dropbox.pm
index 2b4e790b6f..3a5639e61f 100644
--- a/lib/RT/ExternalStorage/Dropbox.pm
+++ b/lib/RT/ExternalStorage/Dropbox.pm
@@ -161,6 +161,17 @@ sub DownloadURLFor {
return;
}
+sub Delete {
+ my $self = shift;
+ my $sha = shift;
+ my $path = $self->_FilePath($sha);
+
+ if ( $self->_PathExists($path) && !$self->Dropbox->delete($path) ) {
+ return ( undef, "Delete $sha from dropbox failed: " . $self->Dropbox->error );
+ }
+ return ($sha);
+}
+
=head1 NAME
RT::ExternalStorage::Dropbox - Store files in the Dropbox cloud
commit 44ebae28f29d2e94012e48aca020cca14f43d62a
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Jul 21 12:39:15 2023 -0400
Switch to WebService::Dropbox to use Dropbox API v2
Dropbox API v1 has been turned off for years, sadly that File::Dropbox
doesn't support v2.
As there are no persistent access tokens any more(tokens are only valid
in 4 hours), we need to configure RefreshToken instead to automatically
generate new access tokens.
diff --git a/lib/RT/ExternalStorage/Dropbox.pm b/lib/RT/ExternalStorage/Dropbox.pm
index 8f8a965e27..2b4e790b6f 100644
--- a/lib/RT/ExternalStorage/Dropbox.pm
+++ b/lib/RT/ExternalStorage/Dropbox.pm
@@ -62,78 +62,99 @@ sub Dropbox {
return $self->{Dropbox};
}
-sub AccessToken {
+sub AppKey {
my $self = shift;
- return $self->{AccessToken};
+ return $self->{AppKey};
}
-sub Init {
+sub AppSecret {
my $self = shift;
+ return $self->{AppSecret};
+}
- {
- # suppress given/warn is experimental warnings from File::Dropbox 0.6
- # https://rt.cpan.org/Ticket/Display.html?id=108107
-
- my $original_warn_handler = $SIG{__WARN__};
- local $SIG{__WARN__} = sub {
- return if $_[0] =~ /(given|when) is experimental/;
+sub RefreshToken {
+ my $self = shift;
+ return $self->{RefreshToken};
+}
- # Avoid reporting this anonymous call frame as the source of the warning.
- goto &$original_warn_handler;
- };
+sub Init {
+ my $self = shift;
- if (not File::Dropbox->require) {
- RT->Logger->error("Required module File::Dropbox is not installed");
- return;
- } elsif (not $self->AccessToken) {
- RT->Logger->error("Required option 'AccessToken' not provided for Dropbox external storage. See the documentation for " . __PACKAGE__ . " for setting up this integration.");
- return;
- }
+ if ( not WebService::Dropbox->require ) {
+ RT->Logger->error("Required module WebService::Dropbox is not installed");
+ return;
+ }
+ WebService::Dropbox->import;
+ for my $item (qw/AppKey AppSecret RefreshToken/) {
+ next if $self->$item;
+ RT->Logger->error(
+ "Required option '$item' not provided for Dropbox external storage. See the documentation for "
+ . __PACKAGE__
+ . " for setting up this integration." );
+ return;
}
-
- my $dropbox = File::Dropbox->new(
- oauth2 => 1,
- access_token => $self->AccessToken,
- root => 'sandbox',
- furlopts => { timeout => 60 },
+ my $dropbox = WebService::Dropbox->new(
+ {
+ key => $self->AppKey,
+ secret => $self->AppSecret,
+ }
);
+
+ $dropbox->refresh_access_token( $self->RefreshToken );
$self->Dropbox($dropbox);
return $self;
}
-sub Get {
+# Dropbox requires the "/" prefix
+sub _FilePath {
my $self = shift;
- my ($sha) = @_;
+ my $sha = shift;
+ return "/$sha";
+}
- my $dropbox = $self->Dropbox;
+sub _PathExists {
+ my $self = shift;
+ my $path = shift;
- open( $dropbox, "<", $sha)
- or return (undef, "Failed to retrieve file from dropbox: $!");
- my $content = do {local $/; <$dropbox>};
- close $dropbox;
+ # Get rid of expected warnings when path doesn't exist
+ local $SIG{__WARN__} = sub {};
+ return $self->Dropbox->get_metadata($path);
+}
- return ($content);
+sub Get {
+ my $self = shift;
+ my ($sha) = @_;
+ my $path = $self->_FilePath($sha);
+
+ my $content;
+ open my $fh, '>', \$content;
+ $self->Dropbox->download($path, $fh);
+ close $fh;
+ if ( $content ) {
+ return ($content);
+ }
+ else {
+ return ( undef, "Read $sha from dropbox failed: " . $self->Dropbox->error );
+ }
}
sub Store {
my $self = shift;
my ($sha, $content, $attachment) = @_;
- my $dropbox = $self->Dropbox;
+ my $path = $self->_FilePath($sha);
# No-op if the path exists already. This forces a metadata read.
- return ($sha) if open( $dropbox, "<", $sha);
-
- open( $dropbox, ">", $sha )
- or return (undef, "Open for write on dropbox failed: $!");
- print $dropbox $content
- or return (undef, "Write to dropbox failed: $!");
- close $dropbox
- or return (undef, "Flush to dropbox failed: $!");
+ return ($sha) if $self->_PathExists($path);
- return ($sha);
+ if ( $self->Dropbox->upload( $path, $content ) ) {
+ return ($sha);
+ }
+ else {
+ return ( undef, "Write $sha to dropbox failed: " . $self->Dropbox->error );
+ }
}
sub DownloadURLFor {
@@ -147,8 +168,10 @@ RT::ExternalStorage::Dropbox - Store files in the Dropbox cloud
=head1 SYNOPSIS
Set(%ExternalStorage,
- Type => 'Dropbox',
- AccessToken => '...',
+ Type => 'Dropbox',
+ AccessKey => '...',
+ AccessSecret => '...',
+ RefreshToken => '...',
);
=head1 DESCRIPTION
@@ -180,11 +203,11 @@ Click C<Create app> on L<https://www.dropbox.com/developers/apps>
=item 3.
-Choose B<Dropbox API app> as the type of app.
+Choose B<Scoped access> as the API.
=item 4.
-Choose B<Yes>, your application only needs access to files it creates.
+Choose B<App folder> as the type of access.
=item 5.
@@ -192,16 +215,38 @@ Enter a descriptive name -- C<Request Tracker files> is fine.
=item 6.
-Under C<Generated access token>, click the C<Generate> button.
+After creation, grant the following permissions on Permissions tab:
+
+ files.metadata.write
+ files.metadata.read
+ files.content.write
+ files.content.read
=item 7.
-Copy the provided value into your F<RT_SiteConfig.pm> file as the
-C<AccessToken>:
+On Settings tab, get C<App key>/C<App secret> and then access the following
+URL:
+
+ https://www.dropbox.com/oauth2/authorize?token_access_type=offline&response_type=code&client_id=<App key>
+
+Where <App key> is the one you got earlier.
+
+After a confirmation page, you will receive a code, use it along with C<App
+key> and C<App secret> in the following command and run it:
+
+ curl https://api.dropbox.com/oauth2/token -d code=<received code> -d grant_type=authorization_code -u <App key>:<App secret>
+
+The response shall contain C<refresh_token> value.
+
+=item 8.
+
+Copy the provided values into your F<RT_SiteConfig.pm>:
Set(%ExternalStorage,
- Type => 'Dropbox',
- AccessToken => '...', # Replace the value here, between the quotes
+ Type => 'Dropbox',
+ AccessKey => '...',
+ AccessSecret => '...',
+ RefreshToken => '...', # Replace the value here, between the quotes
);
=back
diff --git a/sbin/rt-test-dependencies.in b/sbin/rt-test-dependencies.in
index 14dd5a8e26..2365f599bb 100644
--- a/sbin/rt-test-dependencies.in
+++ b/sbin/rt-test-dependencies.in
@@ -304,7 +304,7 @@ Amazon::S3
.
$deps{'DROPBOX'} = [ text_to_hash( <<'.') ];
-File::Dropbox
+WebService::Dropbox
.
my %AVOID = (
commit c8c7bc8fff994615c10508dadcbe7f67fa35a571
Author: sunnavy <sunnavy at bestpractical.com>
Date: Thu Jul 20 03:48:02 2023 -0400
Update tests as we added ExternalStorageDump plugin
diff --git a/t/shredder/03plugin.t b/t/shredder/03plugin.t
index de5d44fa7d..19633fb92a 100644
--- a/t/shredder/03plugin.t
+++ b/t/shredder/03plugin.t
@@ -3,10 +3,10 @@ use strict;
use warnings;
use Test::Deep;
-use RT::Test::Shredder nodb => 1, tests => 28;
+use RT::Test::Shredder nodb => 1, tests => undef;
my $test = "RT::Test::Shredder";
-my @PLUGINS = sort qw(Attachments Base Objects SQLDump Summary Tickets Users);
+my @PLUGINS = sort qw(Attachments Base ExternalStorageDump Objects SQLDump Summary Tickets Users);
use_ok('RT::Shredder::Plugin');
{
@@ -37,3 +37,4 @@ use_ok('RT::Shredder::Plugin');
ok(!$status, "not loaded plugin - not exist");
}
+done_testing;
commit 3769701cc62fd20827a40f4ee4a9c799d23ce55d
Author: sunnavy <sunnavy at bestpractical.com>
Date: Thu Jul 20 02:00:47 2023 -0400
Support to shred external contents of attachments/objectcustomfieldvalues
To undo changes, here we insert external contents back to RT database
before wiping them out, so the SQL dump contains all the data. Then
users can re-externalize them using commands saved in a separate file
*.external-storage.sh that is also generated by shredder.
Note that this commit doesn't touch the broken Dropbox storage, because
File::Dropbox(last released in 2016) doesn't support Dropbox's current
API v2. We will migrate it in a separate commit later.
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index a72aa1d0dd..3ba250ab0f 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -172,6 +172,18 @@ sub DownloadURLFor {
return "https://" . $self->Bucket . ".s3.amazonaws.com/" . $digest;
}
+sub Delete {
+ my $self = shift;
+ my $sha = shift;
+
+ if ( $self->BucketObj->head_key($sha) ) {
+ $self->BucketObj->delete_key($sha)
+ or return ( undef, "Failed to delete $sha from AmazonS3: " . $self->S3->errstr );
+ }
+
+ return ($sha);
+}
+
=head1 NAME
RT::ExternalStorage::AmazonS3 - Store files in Amazon's S3 cloud
diff --git a/lib/RT/ExternalStorage/Backend.pm b/lib/RT/ExternalStorage/Backend.pm
index 6ec8cf38b5..e3fd0c9bf9 100644
--- a/lib/RT/ExternalStorage/Backend.pm
+++ b/lib/RT/ExternalStorage/Backend.pm
@@ -57,6 +57,7 @@ requires 'Init';
requires 'Get';
requires 'Store';
requires 'DownloadURLFor';
+requires 'Delete';
sub new {
my $class = shift;
diff --git a/lib/RT/ExternalStorage/Disk.pm b/lib/RT/ExternalStorage/Disk.pm
index bd505906cf..b51fe81797 100644
--- a/lib/RT/ExternalStorage/Disk.pm
+++ b/lib/RT/ExternalStorage/Disk.pm
@@ -52,7 +52,7 @@ use strict;
package RT::ExternalStorage::Disk;
use File::Path qw//;
-
+use File::Basename 'dirname';
use Role::Basic qw/with/;
with 'RT::ExternalStorage::Backend';
@@ -85,12 +85,21 @@ sub IsWriteable {
return (1);
}
+sub _FilePath {
+ my $self = shift;
+ my $sha = shift;
+
+ # fan out to avoid one gigantic directory which slows down all file access
+ $sha =~ m{^(...)(...)(.*)};
+ return $self->Path . "/$1/$2/$3";
+}
+
sub Get {
my $self = shift;
my ($sha) = @_;
$sha =~ m{^(...)(...)(.*)};
- my $path = $self->Path . "/$1/$2/$3";
+ my $path = $self->_FilePath($sha);
return (undef, "File does not exist") unless -e $path;
@@ -105,15 +114,11 @@ sub Get {
sub Store {
my $self = shift;
my ($sha, $content, $attachment) = @_;
-
- # fan out to avoid one gigantic directory which slows down all file access
- $sha =~ m{^(...)(...)(.*)};
- my $dir = $self->Path . "/$1/$2";
- my $path = "$dir/$3";
+ my $path = $self->_FilePath($sha);
return ($sha) if -f $path;
- File::Path::make_path($dir, {error => \my $err});
+ File::Path::make_path(dirname($path), {error => \my $err});
return (undef, "Making directory failed") if @{$err};
open( my $fh, ">:raw", $path ) or return (undef, "Cannot write file on disk: $!");
@@ -127,6 +132,17 @@ sub DownloadURLFor {
return;
}
+sub Delete {
+ my $self = shift;
+ my $sha = shift;
+ my $path = $self->_FilePath($sha);
+
+ if ( -f $path ) {
+ unlink $path or return ( undef, "Cannot delete file: $!" );
+ }
+ return ($sha);
+}
+
=head1 NAME
RT::ExternalStorage::Disk - On-disk storage of attachments
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 6dc972542a..e565258631 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -2615,7 +2615,62 @@ sub _AsInsertQuery
return $res;
}
-sub BeforeWipeout { return 1 }
+sub BeforeWipeout {
+ my $self = shift;
+ if ( $RT::Shredder::IncludeExternalStorage
+ && ( $self->isa('RT::Attachment') || $self->isa('RT::ObjectCustomFieldValue') )
+ && ( $self->ContentEncoding // '' ) eq 'external' )
+ {
+ my $digest = $self->ExternalStoreDigest;
+
+ # Delete external resource only if there are no objects referenced to it
+ for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+ my $objects = $class->new( $self->CurrentUser );
+ $objects->Limit( FIELD => 'ContentEncoding', VALUE => 'external' );
+ $objects->Limit( FIELD => 'Content', VALUE => $digest );
+ $objects->Limit( FIELD => 'id', VALUE => $self->Id, OPERATOR => '!=' );
+ if ( $objects->First ) {
+ RT->Logger->info("$digest is referenced by other objects, skipping");
+ return 1;
+ }
+ }
+
+ my $storage = RT->System->ExternalStorage;
+ unless ($storage) {
+ RT->Logger->error("External storage not configured");
+ RT::Shredder::Exception::Info->throw('InvalidExternalStorage');
+ }
+
+ # Internalize content so we can re-create objects easily from generated SQL
+ if ( my $external_content = $storage->Get($digest) ) {
+ my ( $encoding, $content ) = $self->_EncodeLOB( $external_content, $self->ContentType, $self->Filename );
+ my ( $ret, $msg ) = $self->__Set( Field => 'ContentEncoding', Value => $encoding );
+ if ( !$ret ) {
+ RT->Logger->error("Could not set ContentEncoding to $encoding: $msg");
+ RT::Shredder::Exception::Info->throw('CouldntInternalizeObject');
+ }
+
+ ( $ret, $msg ) = $self->__Set( Field => 'Content', Value => $content );
+ if ( !$ret ) {
+ RT->Logger->error("Could not set Content: $msg");
+ RT::Shredder::Exception::Info->throw('CouldntInternalizeObject');
+ }
+ $self->{_internalized} = 1;
+ }
+ else {
+ # Can't internalize since content is absent somehow
+ RT->Logger->error("Could not get content of $digest");
+ return 1;
+ }
+
+ my ( $ret, $msg ) = $storage->Delete($digest);
+ if ( !$ret ) {
+ RT->Logger->error("Failed to delete $digest from external storage: $msg");
+ RT::Shredder::Exception::Info->throw('CouldntDeleteExternalObject');
+ }
+ }
+ return 1;
+}
=head2 Dependencies
diff --git a/lib/RT/Shredder/Exceptions.pm b/lib/RT/Shredder/Exceptions.pm
index a679dd7382..75a8b1d4dc 100644
--- a/lib/RT/Shredder/Exceptions.pm
+++ b/lib/RT/Shredder/Exceptions.pm
@@ -90,6 +90,17 @@ resolve, so the requested object was not removed. Some plugins
do not automatically shred dependent objects for safety, but you
may be able to shred the dependent objects directly using other plugins.
The documentation for this plugin may have more information.
+END
+
+ CouldntInternalizeObject => <<END,
+Shredder couldn't internalize objects. Check out RT logs for details.
+END
+
+ InvalidExternalStorage => <<END,
+ExternalStorage is not configured correctly, please check your configuration.
+
+ CouldntDeleteExternalObject => <<END,
+Shredder couldn't delete external objects. Check out RT logs for details.
END
);
diff --git a/lib/RT/ExternalStorage/Backend.pm b/lib/RT/Shredder/Plugin/ExternalStorageDump.pm
similarity index 59%
copy from lib/RT/ExternalStorage/Backend.pm
copy to lib/RT/Shredder/Plugin/ExternalStorageDump.pm
index 6ec8cf38b5..6a3f90d66f 100644
--- a/lib/RT/ExternalStorage/Backend.pm
+++ b/lib/RT/Shredder/Plugin/ExternalStorageDump.pm
@@ -46,47 +46,57 @@
#
# END BPS TAGGED BLOCK }}}
-use warnings;
+package RT::Shredder::Plugin::ExternalStorageDump;
+
use strict;
+use warnings;
-package RT::ExternalStorage::Backend;
+use base qw(RT::Shredder::Plugin::Base::Dump);
+use RT::Shredder;
-use Role::Basic;
+sub AppliesToStates { return 'after wiping dependencies' }
-requires 'Init';
-requires 'Get';
-requires 'Store';
-requires 'DownloadURLFor';
+sub SupportArgs
+{
+ my $self = shift;
+ return $self->SUPER::SupportArgs, qw(file_name from_storage);
+}
-sub new {
- my $class = shift;
+sub TestArgs
+{
+ my $self = shift;
my %args = @_;
+ $args{'from_storage'} = 1 unless defined $args{'from_storage'};
+ my $file = $args{'file_name'} = RT::Shredder->GetFileName(
+ FileName => $args{'file_name'} || '%t-XXXX.external-storage.sh',
+ FromStorage => delete $args{'from_storage'},
+ );
+ open $args{'file_handle'}, ">:raw", $file
+ or return (0, "Couldn't open '$file' for write: $!");
+
+ return $self->SUPER::TestArgs( %args );
+}
- $class = delete $args{Type};
- if (not $class) {
- RT->Logger->error("No storage engine type provided");
- return undef;
- } elsif ($class->require) {
- # no action needed; $class was loaded
- } else {
- my $long = "RT::ExternalStorage::$class";
- if ($long->require) {
- $class = $long;
- } else {
- RT->Logger->error("Can't load external storage engine $class: $@");
- return undef;
- }
- }
+sub FileName { return $_[0]->{'opt'}{'file_name'} }
+sub FileHandle { return $_[0]->{'opt'}{'file_handle'} }
- unless ($class->DOES("RT::ExternalStorage::Backend")) {
- RT->Logger->error("External storage engine $class doesn't implement RT::ExternalStorage::Backend");
- return undef;
- }
+sub Run {
+ my $self = shift;
+ return ( 0, 'no handle' ) unless my $fh = $self->{'opt'}{'file_handle'};
- my $self = bless \%args, $class;
- $self->Init;
-}
+ my %args = ( Object => undef, @_ );
+
+ return 1 unless $args{'Object'}{'_internalized'};
+ my $query = sprintf(
+ "$RT::SbinPath/rt-externalize-attachments --class %s --ids %d\n",
+ ref( $args{'Object'} ) . 's',
+ $args{'Object'}->Id
+ );
-RT::Base->_ImportOverlays();
+ utf8::encode($query) if utf8::is_utf8($query);
+
+ return 1 if print $fh $query;
+ return ( 0, "Couldn't write to filehandle" );
+}
1;
diff --git a/sbin/rt-externalize-attachments.in b/sbin/rt-externalize-attachments.in
index 6f23f668e1..20d8649d39 100644
--- a/sbin/rt-externalize-attachments.in
+++ b/sbin/rt-externalize-attachments.in
@@ -76,6 +76,8 @@ GetOptions( \%opts,
"age=s",
"batchsize=s",
"dry-run",
+ "class=s",
+ "ids=s",
);
if ($opts{'help'}) {
@@ -113,17 +115,29 @@ my $last = RT->System->FirstAttribute("ExternalStorage");
$last = $last ? $last->Content : {};
for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
+ next if $opts{class} && $opts{class} ne $class;
+
my $column = $class eq 'RT::Attachments' ? "Content" : "LargeContent";
my $id = $last->{$class} || 0;
my $batchsize = $opts{'batchsize'} || 1;
while (1) {
my $attach = $class->new($RT::SystemUser);
- $attach->Limit(
- FIELD => 'id',
- OPERATOR => '>',
- VALUE => $id,
- );
+ if ($opts{ids}) {
+ $attach->Limit(
+ FIELD => 'id',
+ OPERATOR => 'IN',
+ VALUE => [ split /,/, $opts{ids} ],
+ );
+ }
+ else {
+ $attach->Limit(
+ FIELD => 'id',
+ OPERATOR => '>',
+ VALUE => $id,
+ );
+ }
+
$attach->Limit(
FIELD => 'ContentEncoding',
OPERATOR => '!=',
@@ -209,7 +223,8 @@ for my $class (qw/RT::Attachments RT::ObjectCustomFieldValues/) {
last unless $attach->Count and $batchsize > 0;
}
- $last->{$class} = $id;
+ # Do not affect the normal externalize process when handing specific ones
+ $last->{$class} = $id unless $opts{ids};
}
if (!$opts{'dry-run'}) {
@@ -252,6 +267,14 @@ By default everything is moved.
If batchsize is given, then only C<NUM> number of attachments will be moved.
By default everything is moved.
+=item --class=RT::Attachments or --class=RT::ObjectCustomFieldValues
+
+If class is given, only objects of the specificed class will be externalized.
+
+=item --ids=NUM,NUM
+
+Use C<--ids> along with C<--class> to explicitly externalize specified records.
+
=item -h
=item --help
diff --git a/sbin/rt-shredder.in b/sbin/rt-shredder.in
index b880d7e12b..0269b21d8d 100644
--- a/sbin/rt-shredder.in
+++ b/sbin/rt-shredder.in
@@ -112,6 +112,16 @@ Outputs help for specified plugin.
Don't prompt with questions before shredding objects.
+=head2 --include-external-storage
+
+Shred externalized contents too. When this is enabled, a separate dump file
+that contains commands to re-externalize objects will be generated, along with
+SQL dump file, so you can fully "undo" if needed.
+
+By default this is not enabled.
+
+See also L<RT_Config/%ExternalStorage>.
+
=head1 SEE ALSO
L<RT::Shredder>
@@ -170,6 +180,34 @@ if ( !$opt{'no-sqldump'} ) {
} else {
print "SQL dump file is '". $plugin->FileName ."'\n";
}
+
+ if ($RT::Shredder::IncludeExternalStorage) {
+
+ # As external storage is bound quite tightly with sqldump(users need to
+ # import the sql dump before re-externalizing), here we create external
+ # storage dump file based on sqldump
+
+ my $file_name = $opt{'sqldump'} || '';
+ if ( $file_name ) {
+ if ( not $file_name =~ s!\.sql$!.externalstorage.sh! ) {
+ $file_name .= '.externalstorage.sh';
+ }
+ }
+
+ my $plugin = eval {
+ $shredder->AddDumpPlugin(
+ Name => 'ExternalStorageDump',
+ Arguments => { file_name => $file_name, from_storage => 0, }
+ );
+ };
+ if ($@) {
+ print STDERR "ERROR: Couldn't open ExternalStorage dump file: $@\n";
+ exit 1 if $file_name;
+ }
+ else {
+ print "ExternalStorage dump file is '" . $plugin->FileName . "'\n";
+ }
+ }
}
my @objs = process_plugins( $shredder );
@@ -258,6 +296,11 @@ sub parse_args
if( GetOptions( 'no-sqldump' => \$tmp ) && $tmp ) {
$opt{'no-sqldump'} = $tmp;
}
+
+ if( GetOptions( 'include-external-storage' => \$tmp ) && $tmp ) {
+ $RT::Shredder::IncludeExternalStorage = 1;
+ }
+ $tmp = undef;
return;
}
diff --git a/share/html/Admin/Tools/Shredder/Elements/DumpFileLink b/share/html/Admin/Tools/Shredder/Elements/DumpFileLink
index 9cc5cd0491..9d032267e9 100644
--- a/share/html/Admin/Tools/Shredder/Elements/DumpFileLink
+++ b/share/html/Admin/Tools/Shredder/Elements/DumpFileLink
@@ -46,10 +46,11 @@
%#
%# END BPS TAGGED BLOCK }}}
<div id="shredder-dump-file-link" class="shredder-help">
-<a href="<% RT->Config->Get('WebPath') %>/Admin/Tools/Shredder/Dumps/<% $File %>"><% loc('Download dumpfile') %></a>
+<a href="<% RT->Config->Get('WebPath') %>/Admin/Tools/Shredder/Dumps/<% $File %>"><% $Label %></a>
</div>
<%ARGS>
$File => ''
+$Label => loc('Download dumpfile')
</%ARGS>
<%INIT>
return unless $File;
diff --git a/share/html/Admin/Tools/Shredder/Elements/SelectObjects b/share/html/Admin/Tools/Shredder/Elements/SelectObjects
index 206ac25c8a..f9eaed6d8c 100644
--- a/share/html/Admin/Tools/Shredder/Elements/SelectObjects
+++ b/share/html/Admin/Tools/Shredder/Elements/SelectObjects
@@ -60,6 +60,9 @@
<& ObjectCheckBox, Object => $o &>
% }
</div>
+% if ( keys %{RT->Config->Get('ExternalStorage') ||{}} ) {
+<& /Elements/Submit, Name => 'WipeoutIncludingExternalStorage', Label => loc('Wipeout Including External Storage') &>
+% }
<& /Elements/Submit, Name => 'Wipeout', Label => loc('Wipeout') &>
% }
</div>
diff --git a/share/html/Admin/Tools/Shredder/index.html b/share/html/Admin/Tools/Shredder/index.html
index 96ca48efa4..4c8b4b655b 100644
--- a/share/html/Admin/Tools/Shredder/index.html
+++ b/share/html/Admin/Tools/Shredder/index.html
@@ -49,6 +49,7 @@
$Plugin => ''
$Search => ''
$Wipeout => ''
+$WipeoutIncludingExternalStorage => ''
@WipeoutObject => ()
</%ARGS>
<& /Admin/Elements/Header, Title => $title &>
@@ -58,13 +59,16 @@ $Wipeout => ''
<& /Elements/ListActions, actions => $messages{'Errors'} &>
<& /Elements/ListActions, actions => $messages{'Success'} &>
<& Elements/DumpFileLink, File => $dump_file &>
+% if ( $WipeoutIncludingExternalStorage ) {
+<& Elements/DumpFileLink, File => $external_storage_dump_file, Label => loc('Download external storage dumpfile') &>
+% }
<& Elements/SelectPlugin, Plugin => $Plugin, %ARGS &>
<div id="shredder-submit-button" class="<% $Plugin? '': 'hidden' %>">
<& /Elements/Submit, Name => 'Search', Label => loc('Search') &>
</div>
</div>
<br />
-% if( $Search || $Wipeout ) {
+% if( $Search || $Wipeout || $WipeoutIncludingExternalStorage ) {
<& Elements/SelectObjects, Objects => \@objs &>
% }
</form>
@@ -122,8 +126,9 @@ if( $Plugin ) { { # use additional block({}) to effectively exit block on errors
} }
my $dump_file = '';
+my $external_storage_dump_file = '';
-if( $Plugin && $Wipeout ) { { # use additional block({}) to effectively exit block on errors
+if( $Plugin && ($Wipeout || $WipeoutIncludingExternalStorage) ) { { # use additional block({}) to effectively exit block on errors
my $shredder = RT::Shredder->new( force => 1 );
my $backup_plugin = RT::Shredder::Plugin->new;
my ($status, $msg) = $backup_plugin->LoadByName('SQLDump');
@@ -144,6 +149,31 @@ if( $Plugin && $Wipeout ) { { # use additional block({}) to effectively exit blo
$shredder->AddDumpPlugin( Object => $backup_plugin );
+ local $RT::Shredder::IncludeExternalStorage;
+
+ if ($WipeoutIncludingExternalStorage) {
+ $RT::Shredder::IncludeExternalStorage = 1;
+
+ my $external_storage_plugin = RT::Shredder::Plugin->new;
+ my ( $status, $msg ) = $external_storage_plugin->LoadByName('ExternalStorageDump');
+ unless ($status) {
+ push @{ $messages{Errors} }, $msg;
+ $Search = '';
+ @objs = ();
+ last;
+ }
+ ( $status, $msg ) = $external_storage_plugin->TestArgs;
+ unless ($status) {
+ push @{ $messages{Errors} }, $msg;
+ $Search = '';
+ @objs = ();
+ last;
+ }
+ ($external_storage_dump_file) = $external_storage_plugin->FileName;
+ push @{ $messages{'Success'} }, "External storage dump file is '$external_storage_dump_file'";
+ $shredder->AddDumpPlugin( Object => $external_storage_plugin );
+ }
+
$shredder->PutObjects( Objects => \@WipeoutObject );
($status, $msg) = $plugin_obj->SetResolvers( Shredder => $shredder );
unless( $status ) {
-----------------------------------------------------------------------
hooks/post-receive
--
rt
More information about the rt-commit
mailing list