[Bps-public-commit] cpan2rt branch, alternate-bugtracker-sync, created. deployed-5-gf8eb29b
Thomas Sibley
trs at bestpractical.com
Tue Mar 12 22:25:22 EDT 2013
The branch, alternate-bugtracker-sync has been created
at f8eb29b5847b049f3e9d7a35caaf33a7cf4b8bdf (commit)
- Log -----------------------------------------------------------------
commit 0614a9889526baef9a8701b927bbfdeba5dcdb29
Author: Ian Norton <i.norton at shadowcat.co.uk>
Date: Tue Mar 12 17:30:57 2013 -0700
Import alternate bug tracker info from MetaCPAN into RT [rt.cpan.org #80020]
This patch adds the queue Attribute 'DistributionBugtracker' which is
populated using data from the MetaCPAN API.
A two parse approach is taken, one to set or update the information from
MetaCPAN into the RT database and a second that removes the information
from queues that no longer require it.
This work has been sponsored by Shadowcat Systems.
[TRS: Relies on complimentary patches to RT::BugTracker and
RT::BugTracker::Public. I replaced a call to the 3.8-era LimitAttribute
with the explicit Join and Limits.]
Signed-off-by: Thomas Sibley <trs at bestpractical.com>
diff --git a/Makefile.PL b/Makefile.PL
index 5efe4f2..ab890c1 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -15,6 +15,7 @@ requires 'LWP::UserAgent';
requires 'HTTP::Date';
requires 'File::Spec';
requires 'XML::SAX';
+requires 'ElasticSearch'; # For querying MetaCPAN
auto_install();
WriteAll();
diff --git a/bin/cpan2rt b/bin/cpan2rt
index cff3165..9cc172b 100755
--- a/bin/cpan2rt
+++ b/bin/cpan2rt
@@ -89,6 +89,7 @@ sub cmd_update {
$importer->sync_distributions( $opt{'force'} ) unless $opt{'skip'}{'distributions'};
$importer->sync_versions( $opt{'force'} ) unless $opt{'skip'}{'versions'};
$importer->sync_maintainers( $opt{'force'} ) unless $opt{'skip'}{'maintainers'};
+ $importer->sync_bugtracker( $opt{'force'} ) unless $opt{'skip'}{'bugtrackers'};
}
sub usage {
diff --git a/lib/CPAN2RT.pm b/lib/CPAN2RT.pm
index 1a1b0c8..2575fbd 100644
--- a/lib/CPAN2RT.pm
+++ b/lib/CPAN2RT.pm
@@ -144,6 +144,72 @@ sub fetch_file {
return 1;
}
+=head2 fetch_bugtracker
+
+Retrieve bugtracker information from the meta CPAN API.
+
+=cut
+
+sub fetch_bugtracker {
+ my $self = shift;
+
+ require ElasticSearch;
+ my $es = ElasticSearch->new(
+ servers => 'api.metacpan.org',
+ no_refresh => 1,
+ );
+
+ # Pull the details of distribution bugtrackers
+ my $scroller = $es->scrolled_search(
+ query => { match_all => {} },
+ size => 100,
+ search_type => 'scan',
+ scroll => '5m',
+ index => 'v0',
+ type => 'release',
+ fields => [ "distribution" , "resources.bugtracker" ],
+ filter => {
+ and => [{
+ or => [
+ { exists => { field => "resources.bugtracker.mailto" }},
+ { exists => { field => "resources.bugtracker.web" }},
+ ]},
+ { term => { "release.status" => "latest" }},
+ { term => { "release.maturity" => "released" }},
+ ],
+ },
+ );
+
+ unless ( defined($scroller) ) {
+ die("Request to api.metacpan.org failed.\n");
+ }
+
+ debug { "Fetched data from api.metacpan.org\n" };
+
+ my $data = {};
+
+ # Iterate the results from MetaCPAN
+ while ( my $result = $scroller->next ) {
+
+ # Record data
+ my $distribution = $result->{"fields"}->{"distribution"};
+ my $mailto = $result->{"fields"}->{"resources.bugtracker"}->{"mailto"};
+ my $web = $result->{"fields"}->{"resources.bugtracker"}->{"web"};
+
+ # Email based alternative - we don't care if this is rt.cpan.org
+ if(defined($mailto) && !($mailto =~ m/rt\.cpan\.org/)) {
+ $data->{$distribution}->{"mailto"} = $mailto;
+ }
+
+ # Web based alternative - we don't care if this is rt.cpan.org
+ if(defined($web) && !($web =~ m/rt\.cpan\.org/)) {
+ $data->{$distribution}->{"web"} = $web;
+ }
+ }
+
+ return $data;
+}
+
{ my $cache;
sub authors {
my $self = shift;
@@ -287,6 +353,143 @@ sub sync_authors {
return (1);
}
+sub sync_bugtracker {
+ my $self = shift;
+
+ debug { "Fetching alternate bug tracker data\n" };
+
+ my $data = $self->fetch_bugtracker();
+
+ debug { "Syncing alternate bug trackers\n" };
+
+ $self->_sync_bugtracker_cpan2rt({ data => $data });
+
+ $self->_sync_bugtracker_rt2cpan({ data => $data });
+
+}
+
+=head2 _sync_bugtracker_cpan2rt
+
+Sync DistributionBugtracker info from CPAN to RT.
+This updates and adds to existing queues.
+
+=cut
+
+sub _sync_bugtracker_cpan2rt {
+ my $self = shift;
+ my $args = shift;
+
+ my $data = $args->{"data"};
+
+ # Iterate through the ditributions.
+ foreach my $dist (keys(%{$data})) {
+ my $bugtracker = {};
+
+ # Build the text to set in the queue attribute.
+ foreach my $method (keys(%{$data->{$dist}})) {
+ my $uri = $data->{$dist}->{$method};
+
+ if( $method eq "mailto" || $method eq "web" ) {
+ $bugtracker->{$method} = $uri;
+ }
+ }
+
+ # Fetch the queue
+ my $queue = $self->load_queue( $dist );
+ unless( $queue ) {
+ debug { "No queue for dist '$dist'" };
+ next;
+ }
+
+ # Get the existing bugtracker from the queue and log if it's changing
+ my $attr = $queue->DistributionBugtracker();
+
+ # Set this if we need to update when we're done
+ my $update = 0;
+
+ # If the attr is defined, then check it hasn't changed.
+ if(defined($attr)) {
+
+ debug { "Bugtracker set for distribution '$dist'. Has it changed?\n" };
+
+ foreach my $method (keys(%{$bugtracker})) {
+
+ if(ref($attr) eq "HASH") {
+ # If this method has changed, log it
+ if(defined($attr->{$method}) && $attr->{$method} ne $bugtracker->{$method}) {
+ debug { "Changing DistributionBugtracker for $dist from '" . $attr->{$method} . "' to '" . $bugtracker->{$method} . "'\n" };
+ $update = 1;
+ }
+ }
+
+ else {
+ # Hmm, something odd happened. Data in the db is wrong, fix it.
+ $update = 1;
+ }
+ }
+ }
+
+ else {
+ debug { "Setting DistributionBugtracker for $dist from nothing\n" };
+ $update = 1;
+ }
+
+
+ if($update) {
+ # Set the queue bugtracker
+ $queue->SetDistributionBugtracker( $bugtracker );
+ }
+ }
+
+ return 1;
+}
+
+=head2 _sync_bugtracker_rt2cpan
+
+Sync DistributionBugtracker info from RT to CPAN.
+This deletes records that are no longer needed or missing in the source.
+
+=cut
+
+sub _sync_bugtracker_rt2cpan {
+ my $self = shift;
+ my $args = shift;
+
+ my $data = $args->{"data"};
+ my $name = "DistributionBugtracker";
+
+ # Find queues with a DistributionBugtracker attribute
+ my $queues = RT::Queues->new( $RT::SystemUser );
+ my $attributes = $queues->Join(
+ ALIAS1 => 'main',
+ FIELD1 => 'id',
+ TABLE2 => 'Attributes',
+ FIELD2 => 'ObjectId',
+ );
+ $queues->Limit(
+ ALIAS => $attributes,
+ FIELD => "ObjectType",
+ VALUE => "RT::Queue",
+ );
+ $queues->Limit(
+ ALIAS => $attributes,
+ FIELD => "Name",
+ VALUE => $name,
+ );
+
+ # Iterate over queues from RT
+ while(my $queue = $queues->Next()) {
+
+ my $dist = $queue->Name();
+
+ # Check that the source defines this queue as having an external tracker
+ unless(defined($data->{$dist})) {
+ # Delete the attribute, it's no longer needed.
+ $queue->DeleteAttribute( $name );
+ }
+ }
+}
+
sub sync_distributions {
my $self = shift;
my $force = shift;
commit 4f129ae789b2afb26b321b15d19ce3fbb278772e
Author: Thomas Sibley <trs at bestpractical.com>
Date: Tue Mar 12 17:38:49 2013 -0700
Add a note from [rt.cpan.org #80020] so we have a chance of remembering
Otherwise I suspect we'll never remember to check in on the CPAN-API
issue.
diff --git a/lib/CPAN2RT.pm b/lib/CPAN2RT.pm
index 2575fbd..75da4ec 100644
--- a/lib/CPAN2RT.pm
+++ b/lib/CPAN2RT.pm
@@ -159,6 +159,17 @@ sub fetch_bugtracker {
no_refresh => 1,
);
+ # Ian Norton wrote:
+ # > Thomas Sibley wrote:
+ # >> 2) Is it feasible to further limit returned [MetaCPAN] results to those where
+ # >> .web or .mailto lacks "rt.cpan.org"?
+ # >
+ # > Spoke to the metacpan guys on irc and seemingly it would be expensive to
+ # > do this server side. Request submitted to have the fields added as full
+ # > text searchable - https://github.com/CPAN-API/cpan-api/issues/238
+ # > following a chat with clintongormley. Once that's done then we can
+ # > improve this.
+
# Pull the details of distribution bugtrackers
my $scroller = $es->scrolled_search(
query => { match_all => {} },
commit f8eb29b5847b049f3e9d7a35caaf33a7cf4b8bdf
Author: Thomas Sibley <trs at bestpractical.com>
Date: Tue Mar 12 19:09:17 2013 -0700
More debug messages when updating bugtracker data
diff --git a/lib/CPAN2RT.pm b/lib/CPAN2RT.pm
index 75da4ec..2936d41 100644
--- a/lib/CPAN2RT.pm
+++ b/lib/CPAN2RT.pm
@@ -430,11 +430,14 @@ sub _sync_bugtracker_cpan2rt {
if(defined($attr->{$method}) && $attr->{$method} ne $bugtracker->{$method}) {
debug { "Changing DistributionBugtracker for $dist from '" . $attr->{$method} . "' to '" . $bugtracker->{$method} . "'\n" };
$update = 1;
+ } else {
+ debug { "Bugtracker $method for $dist is the same. Skipping.\n" };
}
}
else {
# Hmm, something odd happened. Data in the db is wrong, fix it.
+ debug { "Bugtracker data in database looks corrupt? Updating." };
$update = 1;
}
}
-----------------------------------------------------------------------
More information about the Bps-public-commit
mailing list