[Rt-commit] rt branch 5.0/shred-attachments-fts-index created. rt-5.0.4-20-g6bcd3ab7c6
BPS Git Server
git at git.bestpractical.com
Sun May 28 10:39:29 UTC 2023
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "rt".
The branch, 5.0/shred-attachments-fts-index has been created
at 6bcd3ab7c6a9b28ccd72f3e0bda80edccb29ff81 (commit)
- Log -----------------------------------------------------------------
commit 6bcd3ab7c6a9b28ccd72f3e0bda80edccb29ff81
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Wed May 24 08:39:52 2023 +0300
Wipeout full text index records during shredding
* Shredder::RawRecord allows us to delete a record in a table without
RT::Record mapping, for example AttachmentsIndex
* wipeout AttachmentsIndex if it's enabled
* write a test that works with mysql and Pg
diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 2c50447a91..08466e54d4 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -1345,6 +1345,19 @@ sub __DependsOn {
);
push( @$list, $objs );
+ # fulltext index
+ my $fts = RT->Config->Get('FullTextSearch');
+ if ( $fts && $fts->{Indexed} && $fts->{Table} ) {
+ require RT::Shredder::RawRecord;
+ push @$list, RT::Shredder::RawRecord->new(
+ CurrentUser => $self->CurrentUser,
+ Table => $fts->{Table},
+ Columns => {
+ id => $self->Id,
+ },
+ );
+ }
+
$deps->_PushDependencies(
BaseObject => $self,
Flags => RT::Shredder::Constants::DEPENDS_ON,
diff --git a/lib/RT/Shredder.pm b/lib/RT/Shredder.pm
index 6baaf42e30..362c8b7058 100644
--- a/lib/RT/Shredder.pm
+++ b/lib/RT/Shredder.pm
@@ -351,6 +351,8 @@ sub CastObjectsToRecords
while( my $tmp = $targets->Next ) { push @res, $tmp };
} elsif ( UNIVERSAL::isa( $targets, 'RT::Record' ) ) {
push @res, $targets;
+ } elsif ( UNIVERSAL::isa( $targets, 'RT::Shredder::RawRecord' ) ) {
+ push @res, $targets;
} elsif ( UNIVERSAL::isa( $targets, 'ARRAY' ) ) {
foreach( @$targets ) {
push @res, $self->CastObjectsToRecords( Objects => $_ );
@@ -439,7 +441,7 @@ sub PutObject
my %args = ( Object => undef, @_ );
my $obj = $args{'Object'};
- unless( UNIVERSAL::isa( $obj, 'RT::Record' ) ) {
+ if( !UNIVERSAL::isa( $obj, 'RT::Record' ) && !UNIVERSAL::isa( $obj, 'RT::Shredder::RawRecord' ) ) {
RT::Shredder::Exception->throw( "Unsupported type '". (ref $obj || $obj || '(undef)')."'" );
}
diff --git a/lib/RT/Shredder/Plugin/SQLDump.pm b/lib/RT/Shredder/Plugin/SQLDump.pm
index c0366d5572..1e3f1f4644 100644
--- a/lib/RT/Shredder/Plugin/SQLDump.pm
+++ b/lib/RT/Shredder/Plugin/SQLDump.pm
@@ -87,6 +87,8 @@ sub Run
my %args = ( Object => undef, @_ );
my $query = $args{'Object'}->_AsInsertQuery;
+ return 1 unless $query;
+
$query .= "\n" unless $query =~ /\n$/;
utf8::encode($query) if utf8::is_utf8($query);
diff --git a/lib/RT/Shredder/Plugin/SQLDump.pm b/lib/RT/Shredder/RawRecord.pm
similarity index 50%
copy from lib/RT/Shredder/Plugin/SQLDump.pm
copy to lib/RT/Shredder/RawRecord.pm
index c0366d5572..621a5dbc7c 100644
--- a/lib/RT/Shredder/Plugin/SQLDump.pm
+++ b/lib/RT/Shredder/RawRecord.pm
@@ -46,53 +46,100 @@
#
# END BPS TAGGED BLOCK }}}
-package RT::Shredder::Plugin::SQLDump;
+package RT::Shredder::RawRecord;
use strict;
-use warnings;
+use warnings FATAL => 'all';
-use base qw(RT::Shredder::Plugin::Base::Dump);
-use RT::Shredder;
-
-sub AppliesToStates { return 'after wiping dependencies' }
+sub new
+{
+ my $proto = shift;
+ my $self = bless( {}, ref $proto || $proto );
+ $self->Set( @_ );
+ return $self;
+}
-sub SupportArgs
+sub Set
{
my $self = shift;
- return $self->SUPER::SupportArgs, qw(file_name from_storage);
+ my %args = ( @_ );
+ my @keys = qw(Table Columns);
+ @$self{ @keys } = @args{ @keys };
+
+ $self->Load;
+
+ return;
}
-sub TestArgs
-{
+sub UID {
my $self = shift;
- my %args = @_;
- $args{'from_storage'} = 1 unless defined $args{'from_storage'};
- my $file = $args{'file_name'} = RT::Shredder->GetFileName(
- FileName => $args{'file_name'},
- FromStorage => delete $args{'from_storage'},
- );
- open $args{'file_handle'}, ">:raw", $file
- or return (0, "Couldn't open '$file' for write: $!");
+ return $self->{'UID'} if $self->{'UID'};
- return $self->SUPER::TestArgs( %args );
+ my $cols = map { "$_:".$self->{'Columns'}{$_} } sort keys %{$self->{'Columns'}};
+ my $uid = join '-', ref $self, $RT::Organization, $self->Table, $cols;
+ return $self->{'UID'} = $uid;
}
-sub FileName { return $_[0]->{'opt'}{'file_name'} }
-sub FileHandle { return $_[0]->{'opt'}{'file_handle'} }
+sub Load {
+ my $self = shift;
+
+ my @cols = keys %{$self->{'Columns'}};
+
+ my $dbh = $RT::Handle->dbh;
+ my $res = $dbh->selectall_arrayref(
+ "SELECT * FROM ". $self->Table ." WHERE "
+ . join( " AND ", map $dbh->quote_identifier($_) ." = ?", @cols ),
+ { Slice => {} }, @{ $self->{'Columns'} }{@cols},
+ );
+ unless ( $res ) {
+ die "Failed to load ". $self->UID .": ". $dbh->errstr;
+ }
+
+ $self->{'records'} = $res;
+}
-sub Run
+sub _AsInsertQuery
{
my $self = shift;
- return (0, 'no handle') unless my $fh = $self->{'opt'}{'file_handle'};
+ return "" unless $self->{'records'} && scalar @{$self->{'records'}};
- my %args = ( Object => undef, @_ );
- my $query = $args{'Object'}->_AsInsertQuery;
- $query .= "\n" unless $query =~ /\n$/;
+ my $dbh = $RT::Handle->dbh;
+ my @cols = keys %{$self->{'records'}[0]};
- utf8::encode($query) if utf8::is_utf8($query);
+ my $res = "INSERT INTO ". $self->Table;
+ $res .= "(". join( ", ", map $dbh->quote_identifier( $_ ), @cols ) .")";
+ $res .= " VALUES\n";
+ for my $rec (@{$self->{'records'}}) {
+ $res .= "\t(". join( ", ", map { $dbh->quote( $rec->{$_} ) } @cols ) ."),\n";
+ }
+ $res =~ s/,\n$/;\n/;
+
+ return $res;
+}
+
+sub BeforeWipeout {
+ return 1;
+}
- return 1 if print $fh $query;
- return (0, "Couldn't write to filehandle");
+sub Dependencies {
+ return RT::Shredder::Dependencies->new()
}
-1;
+sub __Wipeout {
+ my $self = shift;
+ my $msg = $self->UID ." wiped out";
+
+ my $dbh = $RT::Handle->dbh;
+ my $query = "DELETE FROM ". $self->Table ." WHERE "
+ . join( " AND ", map { $dbh->quote_identifier( $_ ) ."=". $dbh->quote( $self->{'Columns'}{$_} ) } keys %{$self->{'Columns'}} )
+ .";";
+
+ $dbh->do( $query );
+
+ $RT::Logger->info( $msg );
+}
+
+
+sub Table { return $_[0]->{'Table'} }
+
+1;
\ No newline at end of file
diff --git a/lib/RT/Shredder/Plugin/SQLDump.pm b/lib/RT/Test/FTS.pm
similarity index 62%
copy from lib/RT/Shredder/Plugin/SQLDump.pm
copy to lib/RT/Test/FTS.pm
index c0366d5572..5304de6499 100644
--- a/lib/RT/Shredder/Plugin/SQLDump.pm
+++ b/lib/RT/Test/FTS.pm
@@ -46,53 +46,60 @@
#
# END BPS TAGGED BLOCK }}}
-package RT::Shredder::Plugin::SQLDump;
-
use strict;
use warnings;
-use base qw(RT::Shredder::Plugin::Base::Dump);
-use RT::Shredder;
+package RT::Test::FTS;
-sub AppliesToStates { return 'after wiping dependencies' }
+require Test::More;
+require RT::Test;
-sub SupportArgs
-{
- my $self = shift;
- return $self->SUPER::SupportArgs, qw(file_name from_storage);
-}
+=head1 DESCRIPTION
+
+RT::Test::FTS - test suite utilities for testing with Full Text Search enabled
+
+=head1 FUNCTIONS
+
+=head2 setup_indexing
+
+ RT::Test::FTS->setup_indexing;
+
+Runs rt-setup-fulltext-index in silent mode with defaults.
+
+=cut
+
+sub setup_indexing {
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
-sub TestArgs
-{
my $self = shift;
- my %args = @_;
- $args{'from_storage'} = 1 unless defined $args{'from_storage'};
- my $file = $args{'file_name'} = RT::Shredder->GetFileName(
- FileName => $args{'file_name'},
- FromStorage => delete $args{'from_storage'},
+ my %args = (
+ 'no-ask' => 1,
+ command => $RT::SbinPath . '/rt-setup-fulltext-index',
+ dba => $ENV{'RT_DBA_USER'},
+ 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
);
- open $args{'file_handle'}, ">:raw", $file
- or return (0, "Couldn't open '$file' for write: $!");
-
- return $self->SUPER::TestArgs( %args );
+ my ( $exit_code, $output ) = RT::Test->run_and_capture(%args);
+ Test::More::ok( !$exit_code, "setted up index" )
+ or Test::More::diag("output: $output");
}
-sub FileName { return $_[0]->{'opt'}{'file_name'} }
-sub FileHandle { return $_[0]->{'opt'}{'file_handle'} }
+=head2 sync_index
-sub Run
-{
- my $self = shift;
- return (0, 'no handle') unless my $fh = $self->{'opt'}{'file_handle'};
+ RT::Test::FTS->sync_index;
+
+Runs rt-fulltext-indexer to update index, run after creating attachments
+before executing searches.
- my %args = ( Object => undef, @_ );
- my $query = $args{'Object'}->_AsInsertQuery;
- $query .= "\n" unless $query =~ /\n$/;
+=cut
- utf8::encode($query) if utf8::is_utf8($query);
+sub sync_index {
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
- return 1 if print $fh $query;
- return (0, "Couldn't write to filehandle");
+ my $self = shift;
+ my %args = ( command => $RT::SbinPath . '/rt-fulltext-indexer', );
+ my ( $exit_code, $output ) = RT::Test->run_and_capture(%args);
+ Test::More::ok( !$exit_code, "setted up index" )
+ or Test::More::diag("output: $output");
}
1;
diff --git a/lib/RT/Test/Shredder.pm b/lib/RT/Test/Shredder.pm
index 4c18a54d70..6360ffc8b4 100644
--- a/lib/RT/Test/Shredder.pm
+++ b/lib/RT/Test/Shredder.pm
@@ -59,46 +59,30 @@ require Cwd;
RT::Shredder test suite utilities
-=head1 TESTING
-
-Since RT:Shredder 0.01_03 we have a test suite. You
-can run tests and see if everything works as expected
-before you try shredder on your actual data.
-Tests also help in the development process.
-
-The test suite uses SQLite databases to store data in individual files,
-so you could sun tests on your production servers without risking
-damage to your production data.
-
-You'll want to run the test suite almost every time you install or update
-the shredder distribution, especialy if you have local customizations of
-the DB schema and/or RT code.
-
-Tests are one thing you can write even if you don't know much perl,
-but want to learn more about RT's internals. New tests are very welcome.
-
=head2 WRITING TESTS
The shredder distribution has several files to help write new tests.
- t/shredder/utils.pl - this file, utilities
- t/00skeleton.t - skeleteton .t file for new tests
+ lib/RT/Test/Shredder.pm - this file, utilities
+ t/shredder/00skeleton.t - skeleteton .t file for new tests
All tests follow this algorithm:
- require "t/shredder/utils.pl"; # plug in utilities
- init_db(); # create new tmp RT DB and init RT API
+ use RT::Test::Shredder tests => undef; # plug in utilities
+ my $test = "RT::Test::Shredder"; # alias for RT::Test::Shredder
# create RT data you want to be always in the RT DB
# ...
- create_savepoint('mysp'); # create DB savepoint
+ $test->create_savepoint('clean'); # create DB savepoint
# create data you want delete with shredder
# ...
# run shredder on the objects you've created
# ...
# check that shredder deletes things you want
# this command will compare savepoint DB with current
- cmp_deeply( dump_current_and_savepoint('mysp'), "current DB equal to savepoint");
+ cmp_deeply( $test->dump_current_and_savepoint('mysp'), "current DB equal to savepoint");
# then you can create another object and delete it, then check again
+ # ...
+ done_testing();
Savepoints are named and you can create two or more savepoints.
diff --git a/t/fts/indexed_mysql.t b/t/fts/indexed_mysql.t
index 672b220723..658cff125c 100644
--- a/t/fts/indexed_mysql.t
+++ b/t/fts/indexed_mysql.t
@@ -5,33 +5,16 @@ use warnings;
use RT::Test tests => undef;
plan skip_all => 'Not mysql' unless RT->Config->Get('DatabaseType') eq 'mysql';
+use RT::Test::FTS;
+
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Table => 'AttachmentsIndex' );
-setup_indexing();
+RT::Test::FTS->setup_indexing();
my $q = RT::Test->load_or_create_queue( Name => 'General' );
ok $q && $q->id, 'loaded or created queue';
my $queue = $q->Name;
-sub setup_indexing {
- my %args = (
- 'no-ask' => 1,
- command => $RT::SbinPath .'/rt-setup-fulltext-index',
- dba => $ENV{'RT_DBA_USER'},
- 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
-sub sync_index {
- my %args = (
- command => $RT::SbinPath .'/rt-fulltext-indexer',
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
sub run_tests {
my @test = @_;
while ( my ($query, $checks) = splice @test, 0, 2 ) {
@@ -71,7 +54,7 @@ sub run_test {
{ Subject => 'third', Content => 'spanish' },
{ Subject => 'fourth', Content => 'german' },
);
-sync_index();
+RT::Test::FTS->sync_index();
run_tests(
"Content LIKE 'english'" => { first => 1, second => 0, third => 0, fourth => 0 },
diff --git a/t/fts/indexed_oracle.t b/t/fts/indexed_oracle.t
index a5b15bd825..bd634eb400 100644
--- a/t/fts/indexed_oracle.t
+++ b/t/fts/indexed_oracle.t
@@ -8,31 +8,12 @@ plan tests => 13;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, IndexName => 'rt_fts_index' );
-setup_indexing();
+RT::Test::FTS->setup_indexing();
my $q = RT::Test->load_or_create_queue( Name => 'General' );
ok $q && $q->id, 'loaded or created queue';
my $queue = $q->Name;
-sub setup_indexing {
- my %args = (
- 'no-ask' => 1,
- command => $RT::SbinPath .'/rt-setup-fulltext-index',
- dba => $ENV{'RT_DBA_USER'},
- 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
-sub sync_index {
- my %args = (
- command => $RT::SbinPath .'/rt-fulltext-indexer',
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "synced the index") or diag "output: $output";
-}
-
sub run_tests {
my @test = @_;
while ( my ($query, $checks) = splice @test, 0, 2 ) {
@@ -70,7 +51,7 @@ sub run_test {
{ Subject => 'book', Content => 'book' },
{ Subject => 'bar', Content => 'bar' },
);
-sync_index();
+RT::Test::FTS->sync_index();
run_tests(
"Content LIKE 'book'" => { book => 1, bar => 0 },
diff --git a/t/fts/indexed_pg.t b/t/fts/indexed_pg.t
index 1494fded25..4ec2fd40b2 100644
--- a/t/fts/indexed_pg.t
+++ b/t/fts/indexed_pg.t
@@ -5,37 +5,20 @@ use warnings;
use RT::Test tests => undef;
plan skip_all => 'Not Pg' unless RT->Config->Get('DatabaseType') eq 'Pg';
+use RT::Test::FTS;
+
my ($major, $minor) = $RT::Handle->dbh->get_info(18) =~ /^0*(\d+)\.0*(\d+)/;
plan skip_all => "Need Pg 8.2 or higher; we have $major.$minor"
if "$major.$minor" < 8.2;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Column => 'ContentIndex', Table => 'AttachmentsIndex' );
-setup_indexing();
+RT::Test::FTS->setup_indexing();
my $q = RT::Test->load_or_create_queue( Name => 'General' );
ok $q && $q->id, 'loaded or created queue';
my $queue = $q->Name;
-sub setup_indexing {
- my %args = (
- 'no-ask' => 1,
- command => $RT::SbinPath .'/rt-setup-fulltext-index',
- dba => $ENV{'RT_DBA_USER'},
- 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
-sub sync_index {
- my %args = (
- command => $RT::SbinPath .'/rt-fulltext-indexer',
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
sub run_tests {
my @test = @_;
while ( my ($query, $checks) = splice @test, 0, 2 ) {
@@ -74,7 +57,7 @@ my $blase = Encode::decode_utf8("blasé");
{ Subject => 'fts test 1', Content => "book $blase" },
{ Subject => 'fts test 2', Content => "bars blasé", ContentType => 'text/html' },
);
-sync_index();
+RT::Test::FTS->sync_index();
my $book = $tickets[0];
my $bars = $tickets[1];
diff --git a/t/shredder/04fts.t b/t/shredder/04fts.t
new file mode 100644
index 0000000000..8cc3fc92c2
--- /dev/null
+++ b/t/shredder/04fts.t
@@ -0,0 +1,82 @@
+
+use strict;
+use warnings;
+
+use RT::Test tests => undef;
+plan skip_all => 'Only on Pg and mysql' unless RT->Config->Get('DatabaseType') =~ /mysql|Pg/;
+
+use RT::Test::FTS;
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Column => 'ContentIndex', Table => 'AttachmentsIndex' );
+RT::Test::FTS->setup_indexing();
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+
+# don't want to import as we are not on SQLite, but we want to use couple of utils
+require RT::Test::Shredder;
+
+{
+ my @tickets = RT::Test->create_tickets(
+ { Queue => $q->id },
+ { Subject => 'first', Content => 'english' },
+ );
+ $_->ApplyTransactionBatch for @tickets;
+
+ RT::Test::FTS->sync_index;
+
+ my @index_ids = index_ids(@tickets);
+ is scalar @index_ids, 1, 'one attachment indexed';
+
+ my $shredder = RT::Test::Shredder->shredder_new();
+ $shredder->PutObjects( Objects => \@tickets );
+ $shredder->WipeoutAll;
+
+ my $count = count_indexes(@index_ids);
+ is $count, 0, 'no attachment indexed';
+
+ RT::Test::Shredder->db_is_valid;
+
+ like get_dump($shredder), qr/AttachmentsIndex/, 'dump contains AttachmentsIndex';
+}
+
+# select directly from FTS table and get ids of indexed attachments
+sub index_ids {
+ my @tickets = @_;
+ my @ids = map { $_->id } @tickets;
+
+ my $dbh = $RT::Handle->dbh;
+ my $res = $dbh->selectcol_arrayref(
+ "SELECT a.id FROM AttachmentsIndex ai
+ JOIN Attachments a ON a.id = ai.id
+ JOIN Transactions txn ON a.TransactionId = txn.id AND txn.ObjectType = ?
+ WHERE txn.ObjectId IN (" . join( ',', ('?') x @ids ) . ")",
+ undef, 'RT::Ticket', @ids,
+ );
+ return @$res;
+}
+
+sub count_indexes {
+ my @ids = @_;
+ my $dbh = $RT::Handle->dbh;
+ my ($res) = $dbh->selectrow_array(
+ "SELECT COUNT(*) FROM AttachmentsIndex ai WHERE ai.id IN (" . join( ',', ('?') x @ids ) . ")",
+ undef, @ids,
+ );
+ return $res;
+}
+
+sub slurp {
+ my $fname = shift;
+
+ open my $fh, '<', $fname or die "Can't open $fname: $!";
+ return do { local $/; <$fh> };
+}
+
+sub get_dump {
+ my $shredder = shift;
+ my $dplugin = $shredder->{'dump_plugins'}[0];
+ my $fname = $dplugin->FileName;
+ return slurp($fname);
+}
+
+done_testing();
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
rt
More information about the rt-commit
mailing list