[Rt-commit] rt branch, 4.4/external-storage, updated. rt-4.2.11-51-gebfef40
Shawn Moore
shawn at bestpractical.com
Fri May 22 16:23:47 EDT 2015
The branch, 4.4/external-storage has been updated
via ebfef40b559b4ce927907e016635f974df9d832a (commit)
via aabdb18d6ab181c5c816a74991b9f2b5f91593e2 (commit)
via c89ddab8e6d9b47a14337333d6d66369dc81725f (commit)
via 7262974c46ea87942c3746a7703b230525d75bc4 (commit)
via 61a9dc18cd672d02a8e965e1b212c569018b53bd (commit)
via de60f1ce5280e1a0ab95695ef7b7e1b51603fefe (commit)
via 94b7ee4a8bb651d66b71ade2bfd7001b079ae1b7 (commit)
via 4cd20632b44d61293bf8e3b6e32d2fa01e4f0075 (commit)
via db2889e10b379f880701570925f9630640e93d8c (commit)
via 20c99d6709532da1eaea4909f0525d5d509e807e (commit)
from 7f3ffb014d05f125163be0d806a1eb36cd5b8b03 (commit)
Summary of changes:
lib/RT/ExternalStorage.pm | 4 +--
lib/RT/ExternalStorage/AmazonS3.pm | 63 ++++++++++++++++++++++++++++----------
lib/RT/ExternalStorage/Backend.pm | 2 ++
lib/RT/ExternalStorage/Disk.pm | 3 ++
lib/RT/ExternalStorage/Dropbox.pm | 37 ++++++++++++++--------
sbin/rt-externalize-attachments.in | 2 ++
6 files changed, 80 insertions(+), 31 deletions(-)
- Log -----------------------------------------------------------------
commit 20c99d6709532da1eaea4909f0525d5d509e807e
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 19:37:18 2015 +0000
Import overlays for each ExternalStorage PM
diff --git a/lib/RT/ExternalStorage.pm b/lib/RT/ExternalStorage.pm
index 5c4a9f1..9324ef9 100644
--- a/lib/RT/ExternalStorage.pm
+++ b/lib/RT/ExternalStorage.pm
@@ -131,4 +131,6 @@ storage.
=cut
+RT::Base->_ImportOverlays();
+
1;
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index c43f647..95dd3b4 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -123,4 +123,6 @@ sub Store {
return (1);
}
+RT::Base->_ImportOverlays();
+
1;
diff --git a/lib/RT/ExternalStorage/Backend.pm b/lib/RT/ExternalStorage/Backend.pm
index b753321..f69403d 100644
--- a/lib/RT/ExternalStorage/Backend.pm
+++ b/lib/RT/ExternalStorage/Backend.pm
@@ -86,4 +86,6 @@ sub new {
$self->Init;
}
+RT::Base->_ImportOverlays();
+
1;
diff --git a/lib/RT/ExternalStorage/Disk.pm b/lib/RT/ExternalStorage/Disk.pm
index d01e28a..57fa2e2 100644
--- a/lib/RT/ExternalStorage/Disk.pm
+++ b/lib/RT/ExternalStorage/Disk.pm
@@ -151,4 +151,6 @@ internal inconsistency.
=cut
+RT::Base->_ImportOverlays();
+
1;
diff --git a/lib/RT/ExternalStorage/Dropbox.pm b/lib/RT/ExternalStorage/Dropbox.pm
index ee1eaf6..c7482e0 100644
--- a/lib/RT/ExternalStorage/Dropbox.pm
+++ b/lib/RT/ExternalStorage/Dropbox.pm
@@ -180,4 +180,6 @@ C<AccessToken>:
=cut
+RT::Base->_ImportOverlays();
+
1;
commit db2889e10b379f880701570925f9630640e93d8c
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 19:38:44 2015 +0000
Move Digest::SHA load to externalize-attachments
diff --git a/lib/RT/ExternalStorage.pm b/lib/RT/ExternalStorage.pm
index 9324ef9..1bc843b 100644
--- a/lib/RT/ExternalStorage.pm
+++ b/lib/RT/ExternalStorage.pm
@@ -51,8 +51,6 @@ use strict;
package RT::ExternalStorage;
-use Digest::SHA qw//;
-
require RT::ExternalStorage::Backend;
=head1 NAME
diff --git a/sbin/rt-externalize-attachments.in b/sbin/rt-externalize-attachments.in
index d846e80..9ca5d97 100644
--- a/sbin/rt-externalize-attachments.in
+++ b/sbin/rt-externalize-attachments.in
@@ -72,6 +72,8 @@ use RT -init;
use Fcntl ':flock';
exit unless flock main::DATA, LOCK_EX | LOCK_NB;
+use Digest::SHA qw//;
+
my $ExternalStorage = RT->System->ExternalStorage;
die "\%ExternalStorage is not configured\n"
commit 4cd20632b44d61293bf8e3b6e32d2fa01e4f0075
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 19:47:48 2015 +0000
Comment explaining the fan-out for ExternalStorage on disk
diff --git a/lib/RT/ExternalStorage/Disk.pm b/lib/RT/ExternalStorage/Disk.pm
index 57fa2e2..80eca68 100644
--- a/lib/RT/ExternalStorage/Disk.pm
+++ b/lib/RT/ExternalStorage/Disk.pm
@@ -106,6 +106,7 @@ sub Store {
my $self = shift;
my ($sha, $content) = @_;
+ # fan out to avoid one gigantic directory which slows down all file access
$sha =~ m{^(...)(...)(.*)};
my $dir = $self->Path . "/$1/$2";
my $path = "$dir/$3";
commit 94b7ee4a8bb651d66b71ade2bfd7001b079ae1b7
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 19:48:04 2015 +0000
Use an accessor for Dropbox instead of global $DROPBOX
diff --git a/lib/RT/ExternalStorage/Dropbox.pm b/lib/RT/ExternalStorage/Dropbox.pm
index c7482e0..29c61c3 100644
--- a/lib/RT/ExternalStorage/Dropbox.pm
+++ b/lib/RT/ExternalStorage/Dropbox.pm
@@ -54,27 +54,34 @@ package RT::ExternalStorage::Dropbox;
use Role::Basic qw/with/;
with 'RT::ExternalStorage::Backend';
-our $DROPBOX;
+sub Dropbox {
+ my $self = shift;
+ if (@_) {
+ $self->{Dropbox} = shift;
+ }
+ return $self->{Dropbox};
+}
+
sub Init {
my $self = shift;
- my %self = %{$self};
if (not File::Dropbox->require) {
RT->Logger->error("Required module File::Dropbox is not installed");
return;
- } elsif (not $self{AccessToken}) {
+ } elsif (not $self->{AccessToken}) {
RT->Logger->error("AccessToken not provided for Dropbox. Register a new application"
. " at https://www.dropbox.com/developers/apps and generate an access token.");
return;
}
- $DROPBOX = File::Dropbox->new(
+ my $dropbox = File::Dropbox->new(
oauth2 => 1,
- access_token => $self{AccessToken},
+ access_token => $self->{AccessToken},
root => 'sandbox',
furlopts => { timeout => 60 },
);
+ $self->Dropbox($dropbox);
return $self;
}
@@ -83,10 +90,12 @@ sub Get {
my $self = shift;
my ($sha) = @_;
- open( $DROPBOX, "<", $sha)
+ my $dropbox = $self->Dropbox;
+
+ open( $dropbox, "<", $sha)
or return (undef, "Failed to retrieve file from dropbox: $!");
- my $content = do {local $/; <$DROPBOX>};
- close $DROPBOX;
+ my $content = do {local $/; <$dropbox>};
+ close $dropbox;
return ($content);
}
@@ -95,14 +104,16 @@ sub Store {
my $self = shift;
my ($sha, $content) = @_;
+ my $dropbox = $self->Dropbox;
+
# No-op if the path exists already. This forces a metadata read.
- return (1) if open( $DROPBOX, "<", $sha);
+ return (1) if open( $dropbox, "<", $sha);
- open( $DROPBOX, ">", $sha )
+ open( $dropbox, ">", $sha )
or return (undef, "Open for write on dropbox failed: $!");
- print $DROPBOX $content
+ print $dropbox $content
or return (undef, "Write to dropbox failed: $!");
- close $DROPBOX
+ close $dropbox
or return (undef, "Flush to dropbox failed: $!");
return (1);
commit de60f1ce5280e1a0ab95695ef7b7e1b51603fefe
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 19:51:36 2015 +0000
Remove unused $BUCKET
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index 95dd3b4..a804178 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -54,7 +54,7 @@ package RT::ExternalStorage::AmazonS3;
use Role::Basic qw/with/;
with 'RT::ExternalStorage::Backend';
-our( $S3, $BUCKET);
+our( $S3 );
sub Init {
my $self = shift;
my %self = %{$self};
commit 61a9dc18cd672d02a8e965e1b212c569018b53bd
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 19:52:43 2015 +0000
Use S3 accessor instead of global $S3
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index a804178..cbe03a8 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -54,7 +54,14 @@ package RT::ExternalStorage::AmazonS3;
use Role::Basic qw/with/;
with 'RT::ExternalStorage::Backend';
-our( $S3 );
+sub S3 {
+ my $self = shift;
+ if (@_) {
+ $self->{S3} = shift;
+ }
+ return $self->{S3};
+}
+
sub Init {
my $self = shift;
my %self = %{$self};
@@ -74,11 +81,12 @@ sub Init {
}
- $S3 = Amazon::S3->new( {
+ my $S3 = Amazon::S3->new( {
aws_access_key_id => $self{AccessKeyId},
aws_secret_access_key => $self{SecretAccessKey},
retry => 1,
} );
+ $self->S3($S3);
my $buckets = $S3->bucket( $self{Bucket} );
unless ( $buckets ) {
@@ -103,8 +111,8 @@ sub Get {
my $self = shift;
my ($sha) = @_;
- my $ok = $S3->bucket($self->{Bucket})->get_key( $sha );
- return (undef, "Could not retrieve from AmazonS3:" . $S3->errstr)
+ my $ok = $self->S3->bucket($self->{Bucket})->get_key( $sha );
+ return (undef, "Could not retrieve from AmazonS3:" . $self->S3->errstr)
unless $ok;
return ($ok->{value});
}
@@ -114,11 +122,11 @@ sub Store {
my ($sha, $content) = @_;
# No-op if the path exists already
- return (1) if $S3->bucket($self->{Bucket})->head_key( $sha );
+ return (1) if $self->S3->bucket($self->{Bucket})->head_key( $sha );
- $S3->bucket($self->{Bucket})->add_key(
+ $self->S3->bucket($self->{Bucket})->add_key(
$sha => $content
- ) or return (undef, "Failed to write to AmazonS3: " . $S3->errstr);
+ ) or return (undef, "Failed to write to AmazonS3: " . $self->S3->errstr);
return (1);
}
commit 7262974c46ea87942c3746a7703b230525d75bc4
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 20:19:56 2015 +0000
Factor out AmazonS3->BucketObj
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index cbe03a8..bab7f8b 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -62,6 +62,11 @@ sub S3 {
return $self->{S3};
}
+sub BucketObj {
+ my $self = shift;
+ return $self->S3->bucket($self->{Bucket});
+}
+
sub Init {
my $self = shift;
my %self = %{$self};
@@ -111,7 +116,7 @@ sub Get {
my $self = shift;
my ($sha) = @_;
- my $ok = $self->S3->bucket($self->{Bucket})->get_key( $sha );
+ my $ok = $self->BucketObj->get_key( $sha );
return (undef, "Could not retrieve from AmazonS3:" . $self->S3->errstr)
unless $ok;
return ($ok->{value});
@@ -122,9 +127,9 @@ sub Store {
my ($sha, $content) = @_;
# No-op if the path exists already
- return (1) if $self->S3->bucket($self->{Bucket})->head_key( $sha );
+ return (1) if $self->BucketObj->head_key( $sha );
- $self->S3->bucket($self->{Bucket})->add_key(
+ $self->BucketObj->add_key(
$sha => $content
) or return (undef, "Failed to write to AmazonS3: " . $self->S3->errstr);
commit c89ddab8e6d9b47a14337333d6d66369dc81725f
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 20:21:03 2015 +0000
Switch from %self to $self
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index bab7f8b..4befc8c 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -69,42 +69,41 @@ sub BucketObj {
sub Init {
my $self = shift;
- my %self = %{$self};
if (not Amazon::S3->require) {
RT->Logger->error("Required module Amazon::S3 is not installed");
return;
- } elsif (not $self{AccessKeyId}) {
+ } elsif (not $self->{AccessKeyId}) {
RT->Logger->error("AccessKeyId not provided for AmazonS3");
return;
- } elsif (not $self{SecretAccessKey}) {
+ } elsif (not $self->{SecretAccessKey}) {
RT->Logger->error("SecretAccessKey not provided for AmazonS3");
return;
- } elsif (not $self{Bucket}) {
+ } elsif (not $self->{Bucket}) {
RT->Logger->error("Bucket not provided for AmazonS3");
return;
}
my $S3 = Amazon::S3->new( {
- aws_access_key_id => $self{AccessKeyId},
- aws_secret_access_key => $self{SecretAccessKey},
+ aws_access_key_id => $self->{AccessKeyId},
+ aws_secret_access_key => $self->{SecretAccessKey},
retry => 1,
} );
$self->S3($S3);
- my $buckets = $S3->bucket( $self{Bucket} );
+ my $buckets = $S3->bucket( $self->{Bucket} );
unless ( $buckets ) {
RT->Logger->error("Can't list buckets of AmazonS3: ".$S3->errstr);
return;
}
- unless ( grep {$_->bucket eq $self{Bucket}} @{$buckets->{buckets}} ) {
+ unless ( grep {$_->bucket eq $self->{Bucket}} @{$buckets->{buckets}} ) {
my $ok = $S3->add_bucket( {
- bucket => $self{Bucket},
+ bucket => $self->{Bucket},
acl_short => 'private',
} );
unless ($ok) {
- RT->Logger->error("Can't create new bucket '$self{Bucket}' on AmazonS3: ".$S3->errstr);
+ RT->Logger->error("Can't create new bucket '$self->{Bucket}' on AmazonS3: ".$S3->errstr);
return;
}
}
commit aabdb18d6ab181c5c816a74991b9f2b5f91593e2
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 20:22:11 2015 +0000
Make an accessor for Bucket
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index 4befc8c..c856e37 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -62,9 +62,14 @@ sub S3 {
return $self->{S3};
}
+sub Bucket {
+ my $self = shift;
+ return $self->{Bucket};
+}
+
sub BucketObj {
my $self = shift;
- return $self->S3->bucket($self->{Bucket});
+ return $self->S3->bucket($self->Bucket);
}
sub Init {
@@ -79,7 +84,7 @@ sub Init {
} elsif (not $self->{SecretAccessKey}) {
RT->Logger->error("SecretAccessKey not provided for AmazonS3");
return;
- } elsif (not $self->{Bucket}) {
+ } elsif (not $self->Bucket) {
RT->Logger->error("Bucket not provided for AmazonS3");
return;
}
@@ -92,18 +97,18 @@ sub Init {
} );
$self->S3($S3);
- my $buckets = $S3->bucket( $self->{Bucket} );
+ my $buckets = $S3->bucket( $self->Bucket );
unless ( $buckets ) {
RT->Logger->error("Can't list buckets of AmazonS3: ".$S3->errstr);
return;
}
- unless ( grep {$_->bucket eq $self->{Bucket}} @{$buckets->{buckets}} ) {
+ unless ( grep {$_->bucket eq $self->Bucket} @{$buckets->{buckets}} ) {
my $ok = $S3->add_bucket( {
- bucket => $self->{Bucket},
+ bucket => $self->Bucket,
acl_short => 'private',
} );
unless ($ok) {
- RT->Logger->error("Can't create new bucket '$self->{Bucket}' on AmazonS3: ".$S3->errstr);
+ RT->Logger->error("Can't create new bucket '".$self->Bucket."' on AmazonS3: ".$S3->errstr);
return;
}
}
commit ebfef40b559b4ce927907e016635f974df9d832a
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Fri May 22 20:23:15 2015 +0000
Use accessors for AccessKeyId and SecretAccessKey too
diff --git a/lib/RT/ExternalStorage/AmazonS3.pm b/lib/RT/ExternalStorage/AmazonS3.pm
index c856e37..866d5f4 100644
--- a/lib/RT/ExternalStorage/AmazonS3.pm
+++ b/lib/RT/ExternalStorage/AmazonS3.pm
@@ -67,6 +67,16 @@ sub Bucket {
return $self->{Bucket};
}
+sub AccessKeyId {
+ my $self = shift;
+ return $self->{AccessKeyId};
+}
+
+sub SecretAccessKey {
+ my $self = shift;
+ return $self->{SecretAccessKey};
+}
+
sub BucketObj {
my $self = shift;
return $self->S3->bucket($self->Bucket);
@@ -78,10 +88,10 @@ sub Init {
if (not Amazon::S3->require) {
RT->Logger->error("Required module Amazon::S3 is not installed");
return;
- } elsif (not $self->{AccessKeyId}) {
+ } elsif (not $self->AccessKeyId) {
RT->Logger->error("AccessKeyId not provided for AmazonS3");
return;
- } elsif (not $self->{SecretAccessKey}) {
+ } elsif (not $self->SecretAccessKey) {
RT->Logger->error("SecretAccessKey not provided for AmazonS3");
return;
} elsif (not $self->Bucket) {
@@ -91,8 +101,8 @@ sub Init {
my $S3 = Amazon::S3->new( {
- aws_access_key_id => $self->{AccessKeyId},
- aws_secret_access_key => $self->{SecretAccessKey},
+ aws_access_key_id => $self->AccessKeyId,
+ aws_secret_access_key => $self->SecretAccessKey,
retry => 1,
} );
$self->S3($S3);
-----------------------------------------------------------------------
More information about the rt-commit
mailing list