[Rt-commit] rtir branch, 4.2/rss-feed-reader, created. 4.0.1rc1-142-g9cbbe900
Blaine Motsinger
blaine at bestpractical.com
Tue May 26 17:29:17 EDT 2020
The branch, 4.2/rss-feed-reader has been created
at 9cbbe900969d6386655797970d98c592f8af729c (commit)
- Log -----------------------------------------------------------------
commit 2d2fee444abd47feec07c953ed7ea527f7d8836f
Author: Aaron Trevena <aaron at aarontrevena.co.uk>
Date: Fri Feb 21 08:33:56 2020 +0000
Added external feeds to tools to create incidents from RSS
Added new page and class to fetch external feeds, initially RSS
based on configuration, and create pre-populated incidents from items
diff --git a/Makefile.PL b/Makefile.PL
index 9f3f91e8..7150607e 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -31,6 +31,10 @@ requires('Regexp::Common');
# queries parsing
requires('Parse::BooleanLogic');
+# For RSS feeds
+requires('HTML::Entities');
+requires('XML::RSS');
+
# for tests
build_requires('Test::More');
build_requires('File::Find');
diff --git a/html/Callbacks/RTIR/Elements/Tabs/Privileged b/html/Callbacks/RTIR/Elements/Tabs/Privileged
index 2349d71a..7942a0a6 100644
--- a/html/Callbacks/RTIR/Elements/Tabs/Privileged
+++ b/html/Callbacks/RTIR/Elements/Tabs/Privileged
@@ -124,6 +124,7 @@ $tools->child( reporting => title => loc('Reporting'), path => RT::IR->HREFTo('R
my $scripted_actions = $tools->child( scripted_actions => title => loc('Scripted Action') );
$scripted_actions->child( email => title => loc('By Email address'), path => RT::IR->HREFTo('Tools/ScriptedAction.html', IncludeWebPath => 0) );
$scripted_actions->child( ip => title => loc('By IP address'), path => RT::IR->HREFTo('Tools/ScriptedAction.html?loop=IP', IncludeWebPath => 0) );
+my $external_feeds = $tools->child( 'external_feeds', title => loc('External Feeds'), path => RT::IR->HREFTo('Tools//ExternalFeeds.html') );
my $request_path = $HTML::Mason::Commands::r->path_info;
$request_path =~ s!/{2,}!/!g;
diff --git a/html/RTIR/Tools/ExternalFeeds.html b/html/RTIR/Tools/ExternalFeeds.html
new file mode 100644
index 00000000..ae7cb545
--- /dev/null
+++ b/html/RTIR/Tools/ExternalFeeds.html
@@ -0,0 +1,138 @@
+%# BEGIN BPS TAGGED BLOCK {{{
+%#
+%# COPYRIGHT:
+%#
+%# This software is Copyright (c) 1996-2018 Best Practical Solutions, LLC
+%# <sales at bestpractical.com>
+%#
+%# (Except where explicitly superseded by other copyright notices)
+%#
+%#
+%# LICENSE:
+%#
+%# This work is made available to you under the terms of Version 2 of
+%# the GNU General Public License. A copy of that license should have
+%# been provided with this software, but in any event can be snarfed
+%# from www.gnu.org.
+%#
+%# This work is distributed in the hope that it will be useful, but
+%# WITHOUT ANY WARRANTY; without even the implied warranty of
+%# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+%# General Public License for more details.
+%#
+%# You should have received a copy of the GNU General Public License
+%# along with this program; if not, write to the Free Software
+%# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+%# 02110-1301 or visit their web page on the internet at
+%# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+%#
+%#
+%# CONTRIBUTION SUBMISSION POLICY:
+%#
+%# (The following paragraph is not intended to limit the rights granted
+%# to you to modify and distribute this software under the terms of
+%# the GNU General Public License and is only of importance to you if
+%# you choose to contribute your changes and enhancements to the
+%# community by submitting them to Best Practical Solutions, LLC.)
+%#
+%# By intentionally submitting any modifications, corrections or
+%# derivatives to this work, or any other work intended for use with
+%# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+%# you are the copyright holder for those contributions and you grant
+%# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+%# royalty-free, perpetual, license to use, copy, create derivative
+%# works based on those contributions, and sublicense and distribute
+%# those contributions and any derivatives thereof.
+%#
+%# END BPS TAGGED BLOCK }}}
+
+<& /RTIR/Elements/Header, Title => $title &>
+<& /Elements/Tabs &>
+% my $i = 0;
+% if ($FeedName) {
+% my $feed = $ExternalFeeds->fetch_rss_feed($FeedName);
+<&|/Widgets/TitleBox,
+ title => $feed->{Title},
+ class => "fullwidth",
+ bodyclass => "",
+&>
+<div class="table-responsive">
+ <% $feed->{Description} %>
+% if ( $feed->{PubDate} || $feed->{LastBuildDate}) {
+ <i>updated <% $feed->{PubDate} || $feed->{LastBuildDate} || '-'%></i>
+% }
+ <table cellspacing="0" class="table collection collection-as-table">
+ <tr class="collection-as-table">
+ <th class="collection-as-table"><&|/l&>Name</&></th>
+ <th class="collection-as-table"><&|/l&>Created</&></th>
+ <th class="collection-as-table"><% loc('Create a new [_1]', $ticket_type) %></th>
+</tr>
+<tbody class="list-item">
+% foreach my $item (@{ $feed->{items} }) {
+% my $GeneratedSubject = sprintf('Incident from RSS feed %s : %s', $feed->{Title}, $item->{Title});
+% my $GeneratedMessage = join("\n",
+% sprintf('Incident created from RSS feed %s : %s', $feed->{Title}, $item->{Title}),
+% sprintf('Source : %s on %s', $item->{Link} , $item->{PubDate} || $item->{LastBuildDate} || '-'),
+% $item->{Description} || '' );
+ <tr class="<% $i%2 ? 'oddline' : 'evenline'%>" >
+ <td class="collection-as-table align-text-top"><strong><% $item->{Title} %></strong> <a href="<% $item->{Link} %>" target="_New_<% $i %>"><% $item->{Link} %></a> </td>
+ <td class="collection-as-table align-text-top"><i><% $item->{PubDate} || $item->{LastBuildDate} %></i></td>
+ <td class="collection-as-table align-text-top">
+ <form action="<% $CreateURI %>" name="CreateIncident-<% $i %>" id="CreateIncident-<% $i %>" method="post">
+ <input type="hidden" value="<% $GeneratedSubject %>" name="Subject">
+ <input type="hidden" value="<% $GeneratedMessage %>" name="Content">
+ <& /RTIR/Elements/SelectRTIRQueue,
+ Name => 'Queue',
+ ShowNullOption => 0,
+ Lifecycle => $Lifecycle,
+ LimitToConstituency => 1,
+ Constituency => $m->{'RTIR_ConstituencyFilter'} &>
+ <& /Elements/Submit, Label => loc("Go"), Caption => loc("This will take you to a partially prefilled [_1] creation form.", $ticket_type) &>
+ </form>
+ </td>
+ </tr>
+
+
+ <tr class="<% $i%2 ? 'oddline' : 'evenline' %>">
+ <td class="collection-as-table" colspan="3"><small><% $item->{Description} %></small></td>
+ </tr>
+% $i++;
+% }
+</tbody>
+</table>
+</div>
+</&>
+% }
+% else {
+<&|/Widgets/TitleBox,
+ title => loc("RSS"),
+ class => "fullwidth",
+ bodyclass => ""
+&>
+<table border="0" cellspacing="0" cellpadding="1" width="100%" class="table queue-summary">
+<tr>
+<th class="collection-as-table"><&|/l&>Name</&></th>
+<th class="collection-as-table"><&|/l&>Description</&></th>
+</tr>
+% foreach my $feed ($ExternalFeeds->rss_feeds) {
+<tr class="<% $i%2 ? 'oddline' : 'evenline'%>" >
+ <td class="collection-as-table"><a href="/RTIR/Tools/ExternalFeeds.html?FeedName=<% uri_escape($feed->{Name}) %>"><%$feed->{Name}%></a></td>
+ <td class="collection-as-table"><%$feed->{Description}%></td>
+</tr>
+% $i++;
+% }
+</table>
+</&>
+% }
+<%INIT>
+use URI::Escape;
+use RT::IR::ExternalFeeds;
+my $CreateURI = RT::IR->HREFTo('Incident/Create.html');
+my $ExternalFeeds = new RT::IR::ExternalFeeds;
+my $Lifecycle = 'incidents';
+my $ticket_type = lc RT::IR::TicketType( Lifecycle => $Lifecycle );
+my $title = loc('External Feeds');
+</%INIT>
+<%ARGS>
+$FeedName => undef
+</%ARGS>
diff --git a/lib/RT/IR/ExternalFeeds.pm b/lib/RT/IR/ExternalFeeds.pm
new file mode 100644
index 00000000..e2da97dd
--- /dev/null
+++ b/lib/RT/IR/ExternalFeeds.pm
@@ -0,0 +1,116 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2018 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::IR::ExternalFeeds;
+use strict;
+use warnings;
+
+use LWP::UserAgent;
+use XML::RSS;
+use HTML::Entities;
+
+sub new {
+ my $proto = shift;
+ my $class = ref($proto) || $proto;
+ my $self = {};
+ bless( $self, $class );
+ $self->_Init(@_);
+ return $self;
+}
+
+sub _Init {
+ my $self = shift;
+ my %args = (
+ Constituency => undef,
+ @_,
+ );
+ $self->{ua} = LWP::UserAgent->new(timeout => 20);
+ $self->{rss_feeds} = {
+ map { $_->{Name} => $_ }
+ @{RT->Config->Get('ExternalFeeds')->{RSS}}
+ };
+ $self->{_rss_parser} = XML::RSS->new();
+}
+
+sub rss_feeds {
+ my $self = shift;
+ return values %{$self->{rss_feeds}};
+}
+
+sub fetch_rss_feed {
+ my ($self, $name) = @_;
+ my $url = $self->{rss_feeds}{$name}{URI};
+ my $response = $self->{ua}->get($url);
+ return $self->_parse_rss_feed($response);
+}
+
+######
+
+sub _parse_rss_feed {
+ my ($self, $response) = @_;
+ return { } unless ($response->is_success);
+ $self->{_rss_parser}->parse($response->content);
+ my $parsed_feed = { map { ucfirst($_) => $self->{_rss_parser}{channel}{$_} }
+ ( qw(title description pubDate lastBuildDate) ) };
+ foreach my $item (@{$self->{_rss_parser}{items}}) {
+ my $item_values = {
+ map { ucfirst($_) => $item->{$_} }
+ (qw(title link url guid pubDate) )
+ };
+ $item_values->{Link} //= $item_values->{Url};
+ if (defined( $item->{'description'} ) ) {
+ $item_values->{Description} = decode_entities($item->{'description'});
+ } else {
+ $item_values->{Description} = 'No content/description for this item';
+ }
+ push (@{$parsed_feed->{items}}, $item_values);
+ }
+ return $parsed_feed;
+}
+
+1;
commit 9cbbe900969d6386655797970d98c592f8af729c
Author: Aaron Trevena <ast at bestpractical.com>
Date: Fri Apr 17 21:06:41 2020 +0100
Code review improvements to RSS Feeds
diff --git a/etc/RTIR_Config.pm b/etc/RTIR_Config.pm
index 1f02cdcf..0546dcb5 100644
--- a/etc/RTIR_Config.pm
+++ b/etc/RTIR_Config.pm
@@ -783,8 +783,36 @@ to true value return back old behaviour.
Set($RunWhoisRequestByDefault, 0);
+=item %ExternalFeeds
+
+Sources for the External Feeds tool, currently RSS is supported. Provide a Name and URI for each source and you can also provide an optional Description.
+
+
+Set( %ExternalFeeds, (
+ 'RSS' => [
+ { Name => 'US Cert Alerts', URI => 'https://www.us-cert.gov/ncas/alerts.xml', Description => 'US Cert Alerts' },
+ ....
+ ]
+ )
+);
+
+The initial list is US Cert Alerts, UK NCSC Security News, Security Focus Vulnerability Alerts, Threatpost Vulnerability Alerts and Bugtraq.
+
+=cut
+
+Set( %ExternalFeeds, (
+ 'RSS' => [
+ { Name => 'US Cert Alerts', URI => 'https://www.us-cert.gov/ncas/alerts.xml', Description => 'US Cert Alerts' },
+ { Name => 'UK NCSC Security News', URI => 'https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml', Description => 'UK NCSC Security News' },
+ { Name => 'Security Focus Vulnerability Alerts', URI => 'https://www.securityfocus.com/rss/vulnerabilities.xml', Description => 'Security Focus Vulnerability Alerts' },
+ { Name => 'Threatpost Vulnerability Alerts', URI => 'https://threatpost.com/category/vulnerabilities/feed/', Description => 'Threatpost Vulnerability Alerts' },
+ { Name => 'Bugtraq', URI => 'https://seclists.org/rss/bugtraq.rss', Description => 'Bugtraq feed' },
+ ]
+));
+
=back
+
=head1 Service Level Agreements (SLA)
Read F<docs/AdministrationTutorial.pod>.
diff --git a/html/RTIR/Incident/Create.html b/html/RTIR/Incident/Create.html
index a4ecd5a6..2c3956a7 100644
--- a/html/RTIR/Incident/Create.html
+++ b/html/RTIR/Incident/Create.html
@@ -71,6 +71,8 @@ if ( $ChildObj && $ChildObj->id && !$ChildObj->CurrentUserHasRight('ModifyTicket
% }
<input type="hidden" name="id" value="new" />
+<input type="hidden" class="hidden" name="Queue" value="<% $QueueObj->Id %>" />
+<input type="hidden" class="hidden" name="new-RefersTo" value="<% $ARGS{'new-RefersTo'} %>" />
<input type="hidden" class="hidden" name="Token" value="<% $ARGS{'Token'} %>" />
<input type="hidden" name="QueueChanged" value="0" />
% if ( $ChildObj ) {
@@ -443,6 +445,8 @@ if ( $CreateIncident ) {
$checks_failure = 1;
}
+ my $links = ProcessLinksForCreate(ARGSRef => \%ARGS);
+
$checks_failure += RT::IR->FilterRTAddresses(
ARGSRef => \%ARGS,
Fields => { Requestors => 'Requestor', Cc => 'Cc', AdminCc => 'AdminCc' },
diff --git a/html/RTIR/Tools/ExternalFeeds.html b/html/RTIR/Tools/ExternalFeeds.html
index ae7cb545..d8a1ef59 100644
--- a/html/RTIR/Tools/ExternalFeeds.html
+++ b/html/RTIR/Tools/ExternalFeeds.html
@@ -55,7 +55,11 @@
title => $feed->{Title},
class => "fullwidth",
bodyclass => "",
-&>
+ &>
+% if ($feed->{__error}) {
+<p> <% $feed->{__error} %> </p>
+% }
+% else {
<div class="table-responsive">
<% $feed->{Description} %>
% if ( $feed->{PubDate} || $feed->{LastBuildDate}) {
@@ -69,7 +73,7 @@
</tr>
<tbody class="list-item">
% foreach my $item (@{ $feed->{items} }) {
-% my $GeneratedSubject = sprintf('Incident from RSS feed %s : %s', $feed->{Title}, $item->{Title});
+% my $GeneratedSubject = sprintf('%s : %s', $feed->{Title}, $item->{Title});
% my $GeneratedMessage = join("\n",
% sprintf('Incident created from RSS feed %s : %s', $feed->{Title}, $item->{Title}),
% sprintf('Source : %s on %s', $item->{Link} , $item->{PubDate} || $item->{LastBuildDate} || '-'),
@@ -81,6 +85,7 @@
<form action="<% $CreateURI %>" name="CreateIncident-<% $i %>" id="CreateIncident-<% $i %>" method="post">
<input type="hidden" value="<% $GeneratedSubject %>" name="Subject">
<input type="hidden" value="<% $GeneratedMessage %>" name="Content">
+ <input type="hidden" value="<% $item->{Link} %>" Name="new-RefersTo">
<& /RTIR/Elements/SelectRTIRQueue,
Name => 'Queue',
ShowNullOption => 0,
@@ -92,25 +97,25 @@
</td>
</tr>
-
<tr class="<% $i%2 ? 'oddline' : 'evenline' %>">
- <td class="collection-as-table" colspan="3"><small><% $item->{Description} %></small></td>
+ <td class="collection-as-table" colspan="3"><small><% $item->{scrubbed_description} |n%></small></td>
</tr>
% $i++;
% }
</tbody>
</table>
</div>
-</&>
% }
-% else {
+</&>
+% } else {
<&|/Widgets/TitleBox,
title => loc("RSS"),
class => "fullwidth",
bodyclass => ""
-&>
-<table border="0" cellspacing="0" cellpadding="1" width="100%" class="table queue-summary">
-<tr>
+ &>
+% if ($ExternalFeeds->have_rss_feeds) {
+<table cellspacing="0" class="table collection collection-as-table">
+<tr class="collection-as-table">
<th class="collection-as-table"><&|/l&>Name</&></th>
<th class="collection-as-table"><&|/l&>Description</&></th>
</tr>
@@ -122,8 +127,16 @@
% $i++;
% }
</table>
+% }
+% else {
+<p>
+ No RSS feeds currently configured, you can configure feeds in the %ExternalFeeds option of etc/RT_SiteConfig.pm or etc/RT_SiteConfig.d/RT_SiteConfig.pm, a default set of security feeds is included in the inital RTIR configuration.
+</p>
+
+% }
</&>
% }
+
<%INIT>
use URI::Escape;
use RT::IR::ExternalFeeds;
diff --git a/lib/RT/IR/ExternalFeeds.pm b/lib/RT/IR/ExternalFeeds.pm
index e2da97dd..6b5f41ff 100644
--- a/lib/RT/IR/ExternalFeeds.pm
+++ b/lib/RT/IR/ExternalFeeds.pm
@@ -70,21 +70,35 @@ sub _Init {
@_,
);
$self->{ua} = LWP::UserAgent->new(timeout => 20);
- $self->{rss_feeds} = {
- map { $_->{Name} => $_ }
- @{RT->Config->Get('ExternalFeeds')->{RSS}}
- };
- $self->{_rss_parser} = XML::RSS->new();
+ $self->{rss_feeds} = { };
+ $self->{have_rss_feeds} = 0;
+
+ if (RT->Config->Get('ExternalFeeds')->{RSS}) {
+ my $i = 1;
+ foreach my $rss_feed ( @{RT->Config->Get('ExternalFeeds')->{RSS}} ) {
+ next unless (ref $rss_feed eq 'HASH');
+ $rss_feed->{index} = $i++;
+ $self->{rss_feeds}{$rss_feed->{Name}} = $rss_feed;
+ $self->{have_rss_feeds} ||= 1;
+ }
+ }
+ $self->{_rss_parser} = XML::RSS->new();
+
}
sub rss_feeds {
my $self = shift;
- return values %{$self->{rss_feeds}};
+ return sort { $a->{index} <=> $b->{index} } values %{$self->{rss_feeds}};
+}
+
+sub have_rss_feeds {
+ return shift()->{have_rss_feeds};
}
sub fetch_rss_feed {
my ($self, $name) = @_;
my $url = $self->{rss_feeds}{$name}{URI};
+ # make sure we have a fairly short timeout so page doesn't get apache timeout.
my $response = $self->{ua}->get($url);
return $self->_parse_rss_feed($response);
}
@@ -93,8 +107,12 @@ sub fetch_rss_feed {
sub _parse_rss_feed {
my ($self, $response) = @_;
- return { } unless ($response->is_success);
- $self->{_rss_parser}->parse($response->content);
+ return { __error => "Can't reach feed : " . $response->status_line } unless ($response->is_success);
+ eval { $self->{_rss_parser}->parse($response->content); };
+ unless ( $self->{_rss_parser}{channel}{title} && $self->{_rss_parser}{items}[0] ) {
+ return { __error => "Couldn't parse RSS response "};
+ }
+
my $parsed_feed = { map { ucfirst($_) => $self->{_rss_parser}{channel}{$_} }
( qw(title description pubDate lastBuildDate) ) };
foreach my $item (@{$self->{_rss_parser}{items}}) {
@@ -105,12 +123,31 @@ sub _parse_rss_feed {
$item_values->{Link} //= $item_values->{Url};
if (defined( $item->{'description'} ) ) {
$item_values->{Description} = decode_entities($item->{'description'});
+ $item_values->{scrubbed_description} = $self->_scrub_html($item_values->{Description});
} else {
- $item_values->{Description} = 'No content/description for this item';
+ $item_values->{scrubbed_description} = $item_values->{Description} = 'No content/description for this item';
}
push (@{$parsed_feed->{items}}, $item_values);
}
return $parsed_feed;
}
+sub _scrub_html {
+ my ($self, $html) = @_;
+ unless ($self->{_scrubber}) {
+ my $scrubber = HTML::Scrubber->new( script => 0, allow => [ qw[ p b i u br ] ] );
+ $scrubber->rules(
+ a => {
+ 'href' => qr{^(?:http|https)://}i,
+ '*' => 0
+ },
+ '*' => 0
+ );
+ $self->{_scrubber} = $scrubber;
+ }
+ my $scrubbed_html = $self->{_scrubber}->scrub($html);
+ $scrubbed_html =~ s|<\/?p>|<br>|gi;
+ return $scrubbed_html;
+}
+
1;
-----------------------------------------------------------------------
More information about the rt-commit
mailing list