[Rt-commit] rt branch, 4.4/rt-munge-attachments, created. rt-4.4.3-45-g5be71a6d2

Craig Kaiser craig at bestpractical.com
Wed Nov 7 12:25:07 EST 2018


The branch, 4.4/rt-munge-attachments has been created
        at  5be71a6d23a6452ff98a173f4ebf5114dfe4129d (commit)

- Log -----------------------------------------------------------------
commit 51b1225b676366eb53f96770f062cb10f0c75f3f
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Fri Sep 7 08:44:33 2018 -0400

    Create ReplaceAttachments, ReplaceHeader and ReplaceContent Methods
    
    Quickly search on the attachments table and remove or replace matching
    data found.

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 4c544301c..d0df800b0 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -743,6 +743,63 @@ sub SetHeader {
     $self->__Set( Field => 'Headers', Value => $newheader);
 }
 
+=head2 ReplaceHeaders ( Search => 'SEARCH', Replacement => 'Replacement' )
+
+Search the attachments table's Header column for the search string provided.
+When a match is found call the SetHeader() method on the header with the match,
+either set the header to empty or a replacement value.
+
+=cut
+
+sub ReplaceHeaders {
+    my $self = shift;
+    my %args = (
+        Search   => undef,
+        Replacement => '',
+        @_,
+    );
+
+    return (0, 'No Search string provided') unless $args{Search};
+
+    foreach my $header ($self->SplitHeaders) {
+        if ( $header =~ /\Q$args{Search}\E/ig ) {
+            (my $tag) = $header =~ /^([^\:]+)/;
+            my ( $ret, $msg ) = $self->SetHeader($tag, $args{Replacement});
+            return ($ret, $msg) unless $ret;
+        }
+    }
+    return (1, 'Headers cleared');
+}
+
+=head2 ReplaceContent ( Search => 'SEARCH', Replacement => 'Replacement' )
+
+Search the attachments table's Content column for the search string provided.
+When a match is found either replace it with the provided replacement string or an
+empty string.
+
+=cut
+
+sub ReplaceContent {
+    my $self = shift;
+    my %args = (
+        Search      => undef,
+        Replacement => '',
+        @_,
+    );
+
+    return (0, 'No search string provided') unless $args{Search};
+
+    my $content = $self->Content;
+
+    if ( $content ) {
+        $content =~ s/\Q$args{Search}\E/$args{Replacement}/ig;
+        my ($ret, $msg) = $self->_Set( Field => 'Content', Value => $content );
+        return ($ret, $msg);
+    }
+    return(1, 'No content matches found');
+}
+
+
 sub _CanonicalizeHeaderValue {
     my $self  = shift;
     my $value = shift;
diff --git a/lib/RT/Attachments.pm b/lib/RT/Attachments.pm
index 8b0608867..9389c49c7 100644
--- a/lib/RT/Attachments.pm
+++ b/lib/RT/Attachments.pm
@@ -252,6 +252,54 @@ sub AddRecord {
     return $self->SUPER::AddRecord( $record );
 }
 
+=head2 ReplaceAttachments ( Search => 'SEARCH', Replacement => 'Replacement', Header => 1, Content => 1 )
+
+Provide a search string to search the attachments table for, by default the Headers and Content
+columns will both be searched for matches.
+
+=cut
+
+sub ReplaceAttachments {
+    my $self = shift;
+    my %args = (
+        Search      => undef,
+        Replacement => '',
+        Headers     => 1,
+        Content     => 1,
+        @_,
+    );
+
+    return (0, 'Provide a search string to search on') unless $args{Search};
+
+    $self->Limit(
+        ENTRYAGGREGATOR => 'OR',
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => $args{Search},
+        SUBCLAUSE       => 'Attachments',
+    ) unless !$args{Headers};
+
+    $self->Limit(
+        ENTRYAGGREGATOR => 'OR',
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => $args{Search},
+        SUBCLAUSE       => 'Attachments',
+    ) unless !$args{Content};
+
+    while (my $attachment = $self->Next) {
+        if ( $args{Headers} ) {
+            my ($ret, $msg) = $attachment->ReplaceHeaders(Search => $args{Search}, Replacement => $args{Replacement});
+            RT::Logger->error($msg) unless $ret;
+        }
+        if ( $args{Content} ) {
+            my ($ret, $msg) = $attachment->ReplaceContent(Search => $args{Search}, Replacement => $args{Replacement});
+            RT::Logger->error($msg) unless $ret;
+        }
+    }
+    return;
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit c9cfaacecc71934ba398d77c6a0c7f16520f5619
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Fri Sep 7 08:45:39 2018 -0400

    Create rt-munge-attachments executable

diff --git a/.gitignore b/.gitignore
index 1159ee89c..ff1a7110d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,6 +46,7 @@
 /sbin/rt-ldapimport
 /sbin/rt-passwd
 /sbin/standalone_httpd
+/sbin/rt-munge-attachments
 /var/mason_data/
 /autom4te.cache/
 /configure
diff --git a/Makefile.in b/Makefile.in
index cc418241f..e70406927 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -139,6 +139,7 @@ BINARIES		=	$(RT_MAILGATE_BIN) \
 				$(RT_CRON_BIN)
 
 SYSTEM_BINARIES		=	rt-attributes-viewer \
+				rt-munge-attachments \
 				rt-clean-sessions \
 				rt-dump-metadata \
 				rt-email-dashboards \
diff --git a/configure.ac b/configure.ac
index d7685d80d..4c6e34b6a 100755
--- a/configure.ac
+++ b/configure.ac
@@ -486,6 +486,7 @@ AC_CONFIG_FILES([
                  sbin/rt-serializer
                  sbin/rt-importer
                  sbin/rt-passwd
+                 sbin/rt-munge-attachments
                  bin/rt-crontool
                  bin/rt-mailgate
                  bin/rt],
diff --git a/sbin/rt-munge-attachments.in b/sbin/rt-munge-attachments.in
new file mode 100644
index 000000000..3d9203af6
--- /dev/null
+++ b/sbin/rt-munge-attachments.in
@@ -0,0 +1,120 @@
+#!@PERL@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2018 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+use strict;
+use warnings;
+
+# fix lib paths, some may be relative
+BEGIN {    # BEGIN RT CMD BOILERPLATE
+    require File::Spec;
+    require Cwd;
+    my @libs = ( "@RT_LIB_PATH@", "@LOCAL_LIB_PATH@" );
+    my $bin_path;
+
+    for my $lib (@libs) {
+        unless ( File::Spec->file_name_is_absolute($lib) ) {
+            $bin_path
+                ||= ( File::Spec->splitpath( Cwd::abs_path(__FILE__) ) )[1];
+            $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
+        }
+        unshift @INC, $lib;
+    }
+
+}
+
+# Read in the options
+my %opts;
+use Getopt::Long;
+GetOptions( \%opts, "help|h", "search=s", "replacement=s", );
+
+if ( $opts{'help'} ) {
+    require Pod::Usage;
+    print Pod::Usage::pod2usage( -verbose => 2 );
+    exit;
+}
+
+use RT -init;
+
+if ( $opts{'search'} ) {
+    my $replacement = $opts{'replacement'} || '';
+
+    my $search = $opts{'search'};
+
+    my $attachments = RT::Attachments->new( RT->SystemUser );
+    my ($ret, $msg) = $attachments->ReplaceAttachments(Search => $search, Replacement => $replacement);
+
+    print STDERR $msg . "\n";
+} else {
+    print STDERR "Provide a search string \n";
+}
+
+=head1 rt-munge-attachments
+
+rt-munge-attachments - Remove or replace string from attachments table.
+
+=head1 DESCRIPTION
+
+When a match is found in the Headers column, the header is deleted unless a replacement
+value was provided. If a match is found in the Content column then the matching substing
+will be replaced with a blank or provided value.
+
+=head1 OPTIONS
+
+=over 2
+
+=item --search=SEARCH
+
+Provide a string to search the header and content columns in the attachments table, if a match
+is found the content will be removed.
+
+=item --replace=REPLACEMENT
+
+Provide a string to replace strings matching the search string for the
+attachments table.
+
+=back

commit 4f1157ffcc8ecb22a9e3dce49c17a314037f4467
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Fri Sep 7 08:45:59 2018 -0400

    Add tests for methods that munge the attachments table

diff --git a/t/api/attachment.t b/t/api/attachment.t
index 52e3c3f16..3f876a9a6 100644
--- a/t/api/attachment.t
+++ b/t/api/attachment.t
@@ -2,7 +2,7 @@
 use strict;
 use warnings;
 use RT;
-use RT::Test tests => 7;
+use RT::Test tests => undef;
 
 
 {
@@ -64,3 +64,75 @@ is ($#headers, 2, "testing a bunch of singline multiple headers" );
         'body of ContentAsMIME is original'
     );
 }
+
+diag 'Test clearing and replacing header and content in attachments table';
+{
+    my $queue = RT::Test->load_or_create_queue( Name => 'General' );
+    ok $queue && $queue->id, 'loaded or created queue';
+
+    my $t = RT::Test->create_ticket( Queue => 'General', Subject => 'test' );
+    ok $t && $t->id, 'created a ticket';
+
+    $t->Comment( Content => 'test' );
+
+    my $attachments = RT::Attachments->new(RT->SystemUser);
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'test',
+    );
+    is $attachments->Count, 1, 'Found content with "test"';
+
+    # Replace attachment value for 'test' in Conetent col
+    my ($ret, $msg) = $attachments->ReplaceAttachments(Search => 'test', Replacement => 'new_value', Header => 0);
+    ok $ret, $msg;
+
+    $attachments->CleanSlate;
+
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'test',
+    );
+    is $attachments->Count, 0, 'Found no content with "test"';
+
+    $attachments->Limit(
+        FIELD           => 'Content',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'new_value',
+    );
+    is $attachments->Count, 1, 'Found content with "new_value"';
+
+    $attachments->CleanSlate;
+
+     $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'API',
+    );
+    is $attachments->Count, 1, 'Found header with content "API"';
+
+    # Replace attachment value for 'API' in Header col
+    ($ret, $msg) = $attachments->ReplaceAttachments(Search => 'API', Replacement => 'replacement', Content => 0);
+    ok $ret, $msg;
+
+    $attachments->CleanSlate;
+
+     $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'API',
+    );
+    is $attachments->Count, 0, 'Found no header with content "API"';
+
+    $attachments->CleanSlate;
+
+     $attachments->Limit(
+        FIELD           => 'Headers',
+        OPERATOR        => 'LIKE',
+        VALUE           => 'replacement',
+    );
+    is $attachments->Count, 1, 'Found header with content "replacement"';
+}
+
+done_testing();

commit 8ff98c5c5b8ba7787cd6d515924ef082a5da853c
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Tue Nov 6 17:37:54 2018 -0500

    Add new transaction type for munging attachments

diff --git a/lib/RT/Transaction.pm b/lib/RT/Transaction.pm
index 9f897c307..39cb3b71c 100644
--- a/lib/RT/Transaction.pm
+++ b/lib/RT/Transaction.pm
@@ -1373,6 +1373,10 @@ sub _CanonicalizeRoleName {
         $principal->Load($self->Field);
         return ("Removed from group '[_1]'", $principal->Object->Name); #loc()
     },
+    Munge => sub {
+        my $self = shift;
+        return $self->loc("Attachment content modified");
+    },
 );
 
 

commit b50ab76e93d01b8d4d37c4bbdc3582a9111b3816
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Tue Nov 6 17:38:31 2018 -0500

    Record transaction when munging attachment content

diff --git a/lib/RT/Attachments.pm b/lib/RT/Attachments.pm
index 9389c49c7..188edfe69 100644
--- a/lib/RT/Attachments.pm
+++ b/lib/RT/Attachments.pm
@@ -287,15 +287,26 @@ sub ReplaceAttachments {
         SUBCLAUSE       => 'Attachments',
     ) unless !$args{Content};
 
+    my %tickets;
     while (my $attachment = $self->Next) {
+        my $content_replaced;
         if ( $args{Headers} ) {
             my ($ret, $msg) = $attachment->ReplaceHeaders(Search => $args{Search}, Replacement => $args{Replacement});
             RT::Logger->error($msg) unless $ret;
+            $content_replaced = $ret;
         }
         if ( $args{Content} ) {
             my ($ret, $msg) = $attachment->ReplaceContent(Search => $args{Search}, Replacement => $args{Replacement});
             RT::Logger->error($msg) unless $ret;
+            $content_replaced = $ret || $content_replaced;
         }
+        my $ticket = $attachment->TransactionObj->TicketObj;
+        $tickets{$ticket->Id} = $ticket if $content_replaced;
+    }
+    foreach my $ticket (values %tickets){
+        $ticket->_NewTransaction(
+            Type     => "Munge",
+        );
     }
     return;
 }

commit 5be71a6d23a6452ff98a173f4ebf5114dfe4129d
Author: Craig Kaiser <craig at bestpractical.com>
Date:   Wed Nov 7 11:44:11 2018 -0500

    Return $ret and $msg from "ReplaceAttachments"

diff --git a/lib/RT/Attachments.pm b/lib/RT/Attachments.pm
index 188edfe69..846a4c0ef 100644
--- a/lib/RT/Attachments.pm
+++ b/lib/RT/Attachments.pm
@@ -288,18 +288,21 @@ sub ReplaceAttachments {
     ) unless !$args{Content};
 
     my %tickets;
+    my ($ret, $msg);
     while (my $attachment = $self->Next) {
         my $content_replaced;
         if ( $args{Headers} ) {
-            my ($ret, $msg) = $attachment->ReplaceHeaders(Search => $args{Search}, Replacement => $args{Replacement});
+            ($ret, $msg) = $attachment->ReplaceHeaders(Search => $args{Search}, Replacement => $args{Replacement});
             RT::Logger->error($msg) unless $ret;
             $content_replaced = $ret;
         }
+        return ($ret, $msg) unless $ret;
         if ( $args{Content} ) {
-            my ($ret, $msg) = $attachment->ReplaceContent(Search => $args{Search}, Replacement => $args{Replacement});
+            ($ret, $msg) = $attachment->ReplaceContent(Search => $args{Search}, Replacement => $args{Replacement});
             RT::Logger->error($msg) unless $ret;
             $content_replaced = $ret || $content_replaced;
         }
+        return ($ret, $msg) unless $ret;
         my $ticket = $attachment->TransactionObj->TicketObj;
         $tickets{$ticket->Id} = $ticket if $content_replaced;
     }
@@ -308,7 +311,8 @@ sub ReplaceAttachments {
             Type     => "Munge",
         );
     }
-    return;
+    my $count = scalar keys %tickets;
+    return (1, "Updated $count ticket's attachment content");
 }
 
 RT::Base->_ImportOverlays();

-----------------------------------------------------------------------


More information about the rt-commit mailing list