[Rt-commit] rt branch, 4.4-trunk, updated. rt-4.4.3-85-g476a6c31f

? sunnavy sunnavy at bestpractical.com
Tue Dec 4 17:15:47 EST 2018


The branch, 4.4-trunk has been updated
       via  476a6c31fdf765defb93c872808027c4fb1cb653 (commit)
       via  6da1e5d082930c0e2683cc1e36af2127f550773c (commit)
       via  9fbb7d1da72ae0cfaf205b6c51e108f39b9d8b0f (commit)
       via  2c4b5da65d0ed2536a6a9aa3524aa259e0cc138f (commit)
      from  d2dc702caa1a79807039707accfbc51e86270558 (commit)

Summary of changes:
 .gitignore                                         |  1 +
 Makefile.in                                        |  1 +
 configure.ac                                       |  1 +
 lib/RT/Attachment.pm                               | 56 +++++++++++++
 lib/RT/Attachments.pm                              | 75 +++++++++++++++++
 lib/RT/Transaction.pm                              |  4 +
 ...t-session-viewer.in => rt-munge-attachments.in} | 69 ++++++++++------
 t/api/attachment.t                                 | 94 +++++++++++++++++++++-
 8 files changed, 275 insertions(+), 26 deletions(-)
 copy sbin/{rt-session-viewer.in => rt-munge-attachments.in} (62%)

- Log -----------------------------------------------------------------
commit 2c4b5da65d0ed2536a6a9aa3524aa259e0cc138f
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Fri Sep 7 08:44:33 2018 -0400

    Create ReplaceAttachments, ReplaceHeader and ReplaceContent Methods
    
    Quickly search on the attachments table and remove or replace matching
    data found.

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 4c544301c..5708e3142 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -743,6 +743,62 @@ sub SetHeader {
     $self->__Set( Field => 'Headers', Value => $newheader);
 }
 
+=head2 ReplaceHeaders ( Search => 'SEARCH', Replacement => 'Replacement' )
+
+Search the attachments table's Header column for the search string provided.
+When a match is found call the SetHeader() method on the header with the match,
+either set the header to empty or a replacement value.
+
+=cut
+
+sub ReplaceHeaders {
+    my $self = shift;
+    my %args = (
+        Search      => undef,
+        Replacement => '',
+        @_,
+    );
+
+    return ( 0, $self->loc('No Search string provided') ) unless $args{Search};
+
+    foreach my $header ( $self->SplitHeaders ) {
+        my ( $tag, $value ) = split /:/, $header, 2;
+        if ( $value =~ s/\Q$args{Search}\E/$args{Replacement}/ig ) {
+            my $ret = $self->SetHeader( $tag, $value );
+            RT::Logger->error("Could not set header: $tag to $value") unless $ret;
+        }
+    }
+    return ( 1, $self->loc('Headers cleared') );
+}
+
+=head2 ReplaceContent ( Search => 'SEARCH', Replacement => 'Replacement' )
+
+Search the attachments table's Content column for the search string provided.
+When a match is found either replace it with the provided replacement string or an
+empty string.
+
+=cut
+
+sub ReplaceContent {
+    my $self = shift;
+    my %args = (
+        Search      => undef,
+        Replacement => '',
+        @_,
+    );
+
+    return ( 0, $self->loc('No search string provided') ) unless $args{Search};
+
+    my $content = $self->Content;
+
+    if ( $content && $content =~ s/\Q$args{Search}\E/$args{Replacement}/ig ) {
+        my ( $ret, $msg ) = $self->_Set( Field => 'Content', Value => $content );
+        return ( $ret, 'Content replaced' );
+    }
+    return ( 1, $self->loc('No content matches found') );
+}
+
+
 sub _CanonicalizeHeaderValue {
     my $self  = shift;
     my $value = shift;
diff --git a/lib/RT/Attachments.pm b/lib/RT/Attachments.pm
index 8b0608867..a4b070e2c 100644
--- a/lib/RT/Attachments.pm
+++ b/lib/RT/Attachments.pm
@@ -252,6 +252,81 @@ sub AddRecord {
     return $self->SUPER::AddRecord( $record );
 }
 
+=head2 ReplaceAttachments ( Search => 'SEARCH', Replacement => 'Replacement', Header => 1, Content => 1 )
+
+Provide a search string to search the attachments table for, by default the Headers and Content
+columns will both be searched for matches.
+
+=cut
+
+sub ReplaceAttachments {
+    my $self = shift;
+    my %args = (
+        Search      => undef,
+        Replacement => '',
+        Headers     => 1,
+        Content     => 1,
+        @_,
+    );
+
+    return ( 0, $self->loc('Provide a search string to search on') ) unless $args{Search};
+
+    $self->Limit(
+        ENTRYAGGREGATOR => 'OR',
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => $args{Search},
+        SUBCLAUSE       => 'Attachments',
+    ) if $args{Headers};
+    $self->Limit(
+        ENTRYAGGREGATOR => 'OR',
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => $args{Search},
+        SUBCLAUSE       => 'Attachments',
+    ) if $args{Content};
+    $self->Limit(
+        FIELD           => 'ContentType',
+        OPERATOR        => 'IN',
+        VALUE           => ['text/plain', 'text/html'],
+    );
+    $self->Limit(
+        FIELD           => 'ContentEncoding',
+        VALUE           => 'none',
+    );
+
+    my %tickets;
+    my ($ret, $msg);
+    while (my $attachment = $self->Next) {
+        my $content_replaced;
+        if ( $args{Headers} ) {
+            ($ret, $msg) = $attachment->ReplaceHeaders(Search => $args{Search}, Replacement => $args{Replacement});
+            $content_replaced ||= $ret;
+
+            RT::Logger->error($msg) unless $ret;
+        }
+
+        if ( $args{Content} ) {
+            ($ret, $msg) = $attachment->ReplaceContent(Search => $args{Search}, Replacement => $args{Replacement});
+            $content_replaced ||= $ret;
+
+            RT::Logger->error($msg) unless $ret;
+        }
+
+        my $ticket = $attachment->TransactionObj->TicketObj;
+        $tickets{$ticket->Id} ||= $ticket if $content_replaced;
+    }
+
+    foreach my $id ( sort { $a <=> $b } keys %tickets ) {
+        (my $transaction, $msg, my $trans) = $tickets{$id}->_NewTransaction (
+            Type     => "Munge",
+        );
+        RT::Logger->error($msg) unless $transaction;
+    }
+    my $count = scalar keys %tickets;
+    return ( 1, $self->loc( "Updated [_1] ticket's attachment content", $count ) );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Transaction.pm b/lib/RT/Transaction.pm
index 9f897c307..c88f43f9d 100644
--- a/lib/RT/Transaction.pm
+++ b/lib/RT/Transaction.pm
@@ -1373,6 +1373,10 @@ sub _CanonicalizeRoleName {
         $principal->Load($self->Field);
         return ("Removed from group '[_1]'", $principal->Object->Name); #loc()
     },
+    Munge => sub {
+        my $self = shift;
+        return "Attachment content modified";
+    },
 );
 
 

commit 9fbb7d1da72ae0cfaf205b6c51e108f39b9d8b0f
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Fri Sep 7 08:45:39 2018 -0400

    Create rt-munge-attachments executable

diff --git a/.gitignore b/.gitignore
index 1159ee89c..ff1a7110d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,6 +46,7 @@
 /sbin/rt-ldapimport
 /sbin/rt-passwd
 /sbin/standalone_httpd
+/sbin/rt-munge-attachments
 /var/mason_data/
 /autom4te.cache/
 /configure
diff --git a/Makefile.in b/Makefile.in
index cc418241f..e70406927 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -139,6 +139,7 @@ BINARIES		=	$(RT_MAILGATE_BIN) \
 				$(RT_CRON_BIN)
 
 SYSTEM_BINARIES		=	rt-attributes-viewer \
+				rt-munge-attachments \
 				rt-clean-sessions \
 				rt-dump-metadata \
 				rt-email-dashboards \
diff --git a/configure.ac b/configure.ac
index d7685d80d..4c6e34b6a 100755
--- a/configure.ac
+++ b/configure.ac
@@ -486,6 +486,7 @@ AC_CONFIG_FILES([
                  sbin/rt-serializer
                  sbin/rt-importer
                  sbin/rt-passwd
+                 sbin/rt-munge-attachments
                  bin/rt-crontool
                  bin/rt-mailgate
                  bin/rt],
diff --git a/sbin/rt-munge-attachments.in b/sbin/rt-munge-attachments.in
new file mode 100644
index 000000000..2a3eec969
--- /dev/null
+++ b/sbin/rt-munge-attachments.in
@@ -0,0 +1,123 @@
+#!@PERL@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2018 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+use strict;
+use warnings;
+
+# fix lib paths, some may be relative
+BEGIN {    # BEGIN RT CMD BOILERPLATE
+    require File::Spec;
+    require Cwd;
+    my @libs = ( "@RT_LIB_PATH@", "@LOCAL_LIB_PATH@" );
+    my $bin_path;
+
+    for my $lib (@libs) {
+        unless ( File::Spec->file_name_is_absolute($lib) ) {
+            $bin_path
+                ||= ( File::Spec->splitpath( Cwd::abs_path(__FILE__) ) )[1];
+            $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
+        }
+        unshift @INC, $lib;
+    }
+
+}
+
+# Read in the options
+my %opts;
+use Getopt::Long;
+GetOptions( \%opts, "help|h", "search=s", "replacement=s", );
+
+if ( $opts{'help'} || !$opts{'search'} ) {
+    require Pod::Usage;
+    print Pod::Usage::pod2usage( -verbose => 2 );
+    exit;
+}
+
+use RT -init;
+
+my $replacement = $opts{'replacement'} || '';
+
+my $search = $opts{'search'};
+
+my $attachments = RT::Attachments->new( RT->SystemUser );
+my ($ret, $msg) = $attachments->ReplaceAttachments(Search => $search, Replacement => $replacement);
+
+print STDERR $msg . "\n";
+
+
+=head1 rt-munge-attachments
+
+rt-munge-attachments - Remove or replace string from attachments table.
+
+=head1 SYNOPSIS
+
+    rt-munge-attachments --search="user1 at example.com" --replace="Ex-Employee"
+
+=cut
+
+=head1 DESCRIPTION
+
+When a match is found in the Headers column, the header is deleted unless a replacement
+value was provided. If a match is found in the Content column then the matching substring
+will be replaced with a blank or provided value.
+
+=head1 OPTIONS
+
+=over 2
+
+=item --search=SEARCH
+
+Provide a string to search the header and content columns in the attachments table, if a match
+is found the content will be removed.
+
+=item --replace=REPLACEMENT
+
+Provide a string to replace strings matching the search string for the
+attachments table.
+
+=back

commit 6da1e5d082930c0e2683cc1e36af2127f550773c
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Fri Sep 7 08:45:59 2018 -0400

    Add tests for methods that munge the attachments table

diff --git a/t/api/attachment.t b/t/api/attachment.t
index 52e3c3f16..25d736b8b 100644
--- a/t/api/attachment.t
+++ b/t/api/attachment.t
@@ -2,7 +2,7 @@
 use strict;
 use warnings;
 use RT;
-use RT::Test tests => 7;
+use RT::Test tests => undef;
 
 
 {
@@ -64,3 +64,95 @@ is ($#headers, 2, "testing a bunch of singline multiple headers" );
         'body of ContentAsMIME is original'
     );
 }
+
+diag 'Test clearing and replacing header and content in attachments table';
+{
+    my $queue = RT::Test->load_or_create_queue( Name => 'General' );
+    ok $queue && $queue->id, 'loaded or created queue';
+
+    my $t = RT::Test->create_ticket( Queue => 'General', Subject => 'test' );
+    ok $t && $t->id, 'created a ticket';
+
+    $t->Comment( Content => 'test' );
+
+    my $attachments = RT::Attachments->new(RT->SystemUser);
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'test',
+    );
+    is $attachments->Count, 1, 'Found content with "test"';
+
+    # Replace attachment value for 'test' in Conetent col
+    my ($ret, $msg) = $attachments->ReplaceAttachments(Search => 'test', Replacement => 'new_value', Headers => 0);
+    ok $ret, $msg;
+
+    $attachments->CleanSlate;
+
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'test',
+    );
+    is $attachments->Count, 0, 'Found no content with "test"';
+
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'new_value',
+    );
+    is $attachments->Count, 1, 'Found content with "new_value"';
+
+    $attachments->CleanSlate;
+
+    $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'API',
+    );
+    is $attachments->Count, 1, 'Found header with content "API"';
+
+    # Replace attachment value for 'API' in Header col
+    ($ret, $msg) = $attachments->ReplaceAttachments(Search => 'API', Replacement => 'replacement', Content => 0);
+    ok $ret, $msg;
+    $attachments->CleanSlate;
+
+    $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'API',
+    );
+    is $attachments->Count, 0, 'Found no header with content "API"';
+    $attachments->CleanSlate;
+
+    $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'replacement',
+    );
+    is $attachments->Count, 1, 'Found header with content "replacement"';
+
+    ($ret, $msg) = $attachments->ReplaceAttachments(Search => 'new_value', Replacement => 'replacement', Content => 0);
+    ok $ret, $msg;
+
+    $attachments->CleanSlate;
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'new_value',
+    );
+    is $attachments->Count, 1, 'Content is not changed when flagged as false';
+
+    ($ret, $msg) = $attachments->ReplaceAttachments(Search => 'replacement', Replacement => 'new_value', Headers => 0);
+    ok $ret, $msg;
+
+    $attachments->CleanSlate;
+    $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'replacement',
+    );
+    is $attachments->Count, 1, 'Headers are not replaced when flagged as false';
+}
+
+done_testing();

commit 476a6c31fdf765defb93c872808027c4fb1cb653
Merge: d2dc702ca 6da1e5d08
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Wed Dec 5 06:05:18 2018 +0800

    Merge branch '4.4/rt-munge-attachments' into 4.4-trunk


-----------------------------------------------------------------------


More information about the rt-commit mailing list