[Rt-commit] rtir branch, 4.2/rss-feed-reader, updated. 4.0.1rc1-114-gdbe3d183

Aaron Trevena ast at bestpractical.com
Fri Apr 17 16:06:56 EDT 2020


The branch, 4.2/rss-feed-reader has been updated
       via  dbe3d183e5821bd127c336a5777cedd2f20bd0c6 (commit)
      from  60487edf210085a894a742aead3ba2dca5633b6b (commit)

Summary of changes:
 etc/RTIR_Config.pm                 | 28 +++++++++++++++++++
 html/RTIR/Incident/Create.html     |  3 +++
 html/RTIR/Tools/ExternalFeeds.html | 31 ++++++++++++++-------
 lib/RT/IR/ExternalFeeds.pm         | 55 +++++++++++++++++++++++++++++++-------
 4 files changed, 99 insertions(+), 18 deletions(-)

- Log -----------------------------------------------------------------
commit dbe3d183e5821bd127c336a5777cedd2f20bd0c6
Author: Aaron Trevena <ast at bestpractical.com>
Date:   Fri Apr 17 21:06:41 2020 +0100

    Code review improvements to RSS Feeds

diff --git a/etc/RTIR_Config.pm b/etc/RTIR_Config.pm
index b0534476..6c44e284 100644
--- a/etc/RTIR_Config.pm
+++ b/etc/RTIR_Config.pm
@@ -732,8 +732,36 @@ to true value return back old behaviour.
 
 Set($RunWhoisRequestByDefault, 0);
 
+=item %ExternalFeeds
+
+Sources for the External Feeds tool, currently RSS is supported. Provide a Name and URI for each source and you can also provide an optional Description.
+
+
+Set( %ExternalFeeds, (
+    'RSS' => [
+        { Name => 'US Cert Alerts', URI => 'https://www.us-cert.gov/ncas/alerts.xml', Description => 'US Cert Alerts' },
+     ....
+     ]
+  )
+);
+
+The initial list is US Cert Alerts, UK NCSC Security News, Security Focus Vulnerability Alerts, Threatpost Vulnerability Alerts and Bugtraq.
+
+=cut
+
+Set( %ExternalFeeds, (
+    'RSS' => [
+        { Name => 'US Cert Alerts', URI => 'https://www.us-cert.gov/ncas/alerts.xml', Description => 'US Cert Alerts' },
+        { Name => 'UK NCSC Security News', URI => 'https://www.ncsc.gov.uk/api/1/services/v1/all-rss-feed.xml', Description => 'UK NCSC Security News' },
+        { Name => 'Security Focus Vulnerability Alerts', URI => 'https://www.securityfocus.com/rss/vulnerabilities.xml', Description => 'Security Focus Vulnerability Alerts' },
+        { Name => 'Threatpost Vulnerability Alerts', URI => 'https://threatpost.com/category/vulnerabilities/feed/', Description => 'Threatpost Vulnerability Alerts' },
+        { Name => 'Bugtraq', URI => 'https://seclists.org/rss/bugtraq.rss', Description => 'Bugtraq feed' },
+    ]
+));
+
 =back
 
+
 =head1 Service Level Agreements (SLA)
 
 Read F<docs/AdministrationTutorial.pod>.
diff --git a/html/RTIR/Incident/Create.html b/html/RTIR/Incident/Create.html
index 0a711e8c..f6c131a3 100644
--- a/html/RTIR/Incident/Create.html
+++ b/html/RTIR/Incident/Create.html
@@ -72,6 +72,7 @@ if ( $ChildObj && $ChildObj->id && !$ChildObj->CurrentUserHasRight('ModifyTicket
 
 <input type="hidden" name="id"           value="new" />
 <input type="hidden" class="hidden" name="Queue" value="<% $QueueObj->Id %>" />
+<input type="hidden" class="hidden" name="new-RefersTo" value="<% $ARGS{'new-RefersTo'} %>" />
 <input type="hidden" class="hidden" name="Token" value="<% $ARGS{'Token'} %>" />
 % if ( $ChildObj ) {
 <input type="hidden" name="Child"        value="<% $Child %>" />
@@ -413,6 +414,8 @@ if ( $CreateIncident ) {
         $checks_failure = 1;
     }
 
+    my $links = ProcessLinksForCreate(ARGSRef => \%ARGS);
+
     $checks_failure += RT::IR->FilterRTAddresses(
         ARGSRef => \%ARGS,
         Fields => { Requestors => 'Requestor', Cc => 'Cc', AdminCc => 'AdminCc' },
diff --git a/html/RTIR/Tools/ExternalFeeds.html b/html/RTIR/Tools/ExternalFeeds.html
index ae7cb545..d8a1ef59 100644
--- a/html/RTIR/Tools/ExternalFeeds.html
+++ b/html/RTIR/Tools/ExternalFeeds.html
@@ -55,7 +55,11 @@
     title => $feed->{Title},
     class => "fullwidth",
     bodyclass => "",
-&>
+  &>
+% if ($feed->{__error}) {
+<p> <% $feed->{__error} %> </p>  
+% }
+% else {
 <div class="table-responsive">
   <% $feed->{Description} %>
 % if ( $feed->{PubDate} || $feed->{LastBuildDate}) {
@@ -69,7 +73,7 @@
 </tr>
 <tbody class="list-item">
 %   foreach my $item (@{ $feed->{items} }) {
-%      my $GeneratedSubject = sprintf('Incident from RSS feed %s : %s', $feed->{Title}, $item->{Title});
+%      my $GeneratedSubject = sprintf('%s : %s', $feed->{Title}, $item->{Title});
 %      my $GeneratedMessage = join("\n",
 %         sprintf('Incident created from RSS feed %s : %s', $feed->{Title}, $item->{Title}),
 %         sprintf('Source : %s on %s', $item->{Link} , $item->{PubDate} || $item->{LastBuildDate} || '-'),
@@ -81,6 +85,7 @@
       <form action="<% $CreateURI %>" name="CreateIncident-<% $i %>" id="CreateIncident-<% $i %>"  method="post">
         <input type="hidden" value="<% $GeneratedSubject %>" name="Subject">
         <input type="hidden" value="<% $GeneratedMessage %>" name="Content">
+        <input type="hidden" value="<% $item->{Link} %>" Name="new-RefersTo">
         <& /RTIR/Elements/SelectRTIRQueue,
                                 Name => 'Queue',
                                 ShowNullOption => 0,
@@ -92,25 +97,25 @@
     </td>
   </tr>
 
-
   <tr class="<% $i%2 ? 'oddline' : 'evenline' %>">
-    <td class="collection-as-table" colspan="3"><small><% $item->{Description} %></small></td>
+    <td class="collection-as-table" colspan="3"><small><% $item->{scrubbed_description} |n%></small></td>
   </tr>
 % $i++;
 %   }
 </tbody>
 </table>
 </div>
-</&>
 % }
-% else {
+</&>
+% } else {
 <&|/Widgets/TitleBox,
     title => loc("RSS"),
     class => "fullwidth",
     bodyclass => ""
-&>
-<table border="0" cellspacing="0" cellpadding="1" width="100%" class="table queue-summary">
-<tr>
+  &>
+% if ($ExternalFeeds->have_rss_feeds) {
+<table cellspacing="0" class="table collection collection-as-table">
+<tr class="collection-as-table">
 <th class="collection-as-table"><&|/l&>Name</&></th>
 <th class="collection-as-table"><&|/l&>Description</&></th>
 </tr>
@@ -122,8 +127,16 @@
 % $i++;
 %    }
 </table>
+% }
+% else {
+<p>
+  No RSS feeds currently configured, you can configure feeds in the %ExternalFeeds option of etc/RT_SiteConfig.pm or etc/RT_SiteConfig.d/RT_SiteConfig.pm, a default set of security feeds is included in the inital RTIR configuration.
+</p>
+
+% }
 </&>
 % }
+
 <%INIT>
 use URI::Escape;
 use RT::IR::ExternalFeeds;
diff --git a/lib/RT/IR/ExternalFeeds.pm b/lib/RT/IR/ExternalFeeds.pm
index e2da97dd..6b5f41ff 100644
--- a/lib/RT/IR/ExternalFeeds.pm
+++ b/lib/RT/IR/ExternalFeeds.pm
@@ -70,21 +70,35 @@ sub _Init {
         @_,
     );
     $self->{ua} = LWP::UserAgent->new(timeout => 20);
-    $self->{rss_feeds} = {
-        map { $_->{Name} => $_ }
-        @{RT->Config->Get('ExternalFeeds')->{RSS}}
-    };
-    $self->{_rss_parser} =  XML::RSS->new();
+    $self->{rss_feeds} = { };
+    $self->{have_rss_feeds} = 0;
+
+    if (RT->Config->Get('ExternalFeeds')->{RSS}) {
+        my $i = 1;
+        foreach my $rss_feed ( @{RT->Config->Get('ExternalFeeds')->{RSS}} ) {
+            next unless (ref $rss_feed eq 'HASH');
+            $rss_feed->{index} = $i++;
+            $self->{rss_feeds}{$rss_feed->{Name}} = $rss_feed;
+            $self->{have_rss_feeds} ||= 1;
+        }
+    }
+    $self->{_rss_parser} = XML::RSS->new();
+    
 }
 
 sub rss_feeds {
     my $self = shift;
-    return values %{$self->{rss_feeds}};
+    return sort { $a->{index} <=> $b->{index} } values %{$self->{rss_feeds}};
+}
+
+sub have_rss_feeds {
+    return shift()->{have_rss_feeds};
 }
 
 sub fetch_rss_feed {
     my ($self, $name) = @_;
     my $url = $self->{rss_feeds}{$name}{URI};
+    # make sure we have a fairly short timeout so page doesn't get apache timeout.
     my $response = $self->{ua}->get($url);
     return $self->_parse_rss_feed($response);
 }
@@ -93,8 +107,12 @@ sub fetch_rss_feed {
 
 sub _parse_rss_feed {
     my ($self, $response) = @_;
-    return { }  unless ($response->is_success);
-    $self->{_rss_parser}->parse($response->content);
+    return { __error => "Can't reach feed : " . $response->status_line } unless ($response->is_success);
+    eval { $self->{_rss_parser}->parse($response->content); };
+    unless ( $self->{_rss_parser}{channel}{title} && $self->{_rss_parser}{items}[0] ) {
+        return { __error => "Couldn't parse RSS response "};
+    }
+
     my $parsed_feed = { map { ucfirst($_) => $self->{_rss_parser}{channel}{$_} }
                             ( qw(title description pubDate lastBuildDate) ) };
     foreach my $item (@{$self->{_rss_parser}{items}}) {
@@ -105,12 +123,31 @@ sub _parse_rss_feed {
         $item_values->{Link} //= $item_values->{Url};
         if (defined( $item->{'description'} ) ) {
             $item_values->{Description} = decode_entities($item->{'description'});
+            $item_values->{scrubbed_description} = $self->_scrub_html($item_values->{Description});
         } else {
-            $item_values->{Description} = 'No content/description for this item';
+            $item_values->{scrubbed_description} = $item_values->{Description} = 'No content/description for this item';
         }
         push (@{$parsed_feed->{items}}, $item_values);
     }
     return $parsed_feed;
 }
 
+sub _scrub_html {
+    my ($self, $html) = @_;
+    unless ($self->{_scrubber}) {
+        my $scrubber = HTML::Scrubber->new( script => 0, allow => [ qw[ p b i u br ] ] );
+        $scrubber->rules(
+            a => {
+                'href' => qr{^(?:http|https)://}i,
+                '*' =>  0
+            },
+            '*' => 0
+        );
+        $self->{_scrubber} = $scrubber;
+    }
+    my $scrubbed_html = $self->{_scrubber}->scrub($html);
+    $scrubbed_html =~ s|<\/?p>|<br>|gi;
+    return $scrubbed_html;
+}
+
 1;

-----------------------------------------------------------------------


More information about the rt-commit mailing list