[Rt-commit] rt branch 5.0/split-fulltext-searches created. rt-5.0.5-150-gbce33fe8a4
BPS Git Server
git at git.bestpractical.com
Wed Jan 24 21:33:40 UTC 2024
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "rt".
The branch, 5.0/split-fulltext-searches has been created
at bce33fe8a423ca0fbef395606340f545c497621a (commit)
- Log -----------------------------------------------------------------
commit bce33fe8a423ca0fbef395606340f545c497621a
Author: sunnavy <sunnavy at bestpractical.com>
Date: Wed Jan 24 16:33:14 2024 -0500
Add more fulltext search tests for the new split feature
diff --git a/t/fts/indexed_mysql.t b/t/fts/indexed_mysql.t
index 672b220723..4ce8d83efb 100644
--- a/t/fts/indexed_mysql.t
+++ b/t/fts/indexed_mysql.t
@@ -66,7 +66,7 @@ sub run_test {
@tickets = RT::Test->create_tickets(
{ Queue => $q->id },
- { Subject => 'first', Content => 'english' },
+ { Subject => 'first', Content => 'english american' },
{ Subject => 'second', Content => 'french' },
{ Subject => 'third', Content => 'spanish' },
{ Subject => 'fourth', Content => 'german' },
@@ -76,8 +76,73 @@ sync_index();
run_tests(
"Content LIKE 'english'" => { first => 1, second => 0, third => 0, fourth => 0 },
"Content LIKE 'french'" => { first => 0, second => 1, third => 0, fourth => 0 },
+ "Subject LIKE 'first' OR Content LIKE 'french'" => { first => 1, second => 1, third => 0, fourth => 0 },
+ "Content LIKE 'english' AND Content LIKE 'american'" => { first => 1, second => 0, third => 0, fourth => 0 },
);
+my ( $ret, $msg ) = $tickets[0]->Correspond( Content => 'chinese' );
+ok( $ret, 'Corresponded' ) or diag $msg;
+
+( $ret, $msg ) = $tickets[0]->SetSubject('updated');
+ok( $ret, 'Updated subject' ) or diag $msg;
+
+sync_index();
+
+run_tests(
+ "Content LIKE 'english' AND Content LIKE 'chinese'" => { updated => 1, second => 0, third => 0, fourth => 0 },
+ "Subject LIKE 'updated' OR Content LIKE 'french'" => { updated => 1, second => 1, third => 0, fourth => 0 },
+ "( Subject LIKE 'updated' OR Content LIKE 'english' ) AND ( Content LIKE 'french' OR Content LIKE 'chinese' )"
+ => { updated => 1, second => 0, third => 0, fourth => 0 },
+);
+
+diag "Checking SQL query";
+
+my $tickets = RT::Tickets->new( RT->SystemUser );
+$tickets->FromSQL(q{Content LIKE 'english' AND Content LIKE 'chinese'});
+like( $tickets->BuildSelectQuery(), qr{ INTERSECT }, 'AND query contains INTERSECT' );
+
+$tickets->FromSQL(q{Subject LIKE 'updated' OR Content LIKE 'french'});
+like( $tickets->BuildSelectQuery(), qr{ UNION }, 'OR query contains UNION' );
+
+$tickets->FromSQL(
+ q{(Subject LIKE 'updated' OR Content LIKE 'english') AND ( Content LIKE 'french' OR Content LIKE 'chinese' )});
+like(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR query contains both INTERSECT and UNION'
+);
+
+diag "Checking transaction searches";
+
+my $txns = RT::Transactions->new( RT->SystemUser );
+$txns->FromSQL(q{Content LIKE 'english' AND Content LIKE 'american'});
+is( $txns->Count, 1, 'Found one transaction' );
+my $txn = $txns->First;
+like( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query contains INTERSECT' );
+like( $txn->Content, qr/english american/, 'Transaction content' );
+
+$txns->FromSQL(q{Content LIKE 'english' AND Content LIKE 'chinese'});
+like( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query contains INTERSECT' );
+is( $txns->Count, 0, 'Found 0 transactions' );
+
+$txns->FromSQL(q{Content LIKE 'english' OR Content LIKE 'chinese'});
+like( $txns->BuildSelectQuery(), qr{ UNION }, 'OR transaction query contains UNION' );
+is( $txns->Count, 2, 'Found 2 transactions' );
+my @txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/english/, 'Transaction content' );
+like( $txns[1]->Content, qr/chinese/, 'Transaction content' );
+
+$txns->FromSQL(q{( Content LIKE 'english' AND Content LIKE 'american' ) OR Content LIKE 'chinese'});
+like(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR transaction query contains both INTERSECT and UNION'
+);
+is( $txns->Count, 2, 'Found 2 transactions' );
+ at txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/english/, 'Transaction content' );
+like( $txns[1]->Content, qr/chinese/, 'Transaction content' );
+
@tickets = ();
done_testing;
diff --git a/t/fts/indexed_oracle.t b/t/fts/indexed_oracle.t
index a5b15bd825..9ac7f80a26 100644
--- a/t/fts/indexed_oracle.t
+++ b/t/fts/indexed_oracle.t
@@ -4,7 +4,6 @@ use warnings;
use RT::Test tests => undef;
plan skip_all => 'Not Oracle' unless RT->Config->Get('DatabaseType') eq 'Oracle';
-plan tests => 13;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, IndexName => 'rt_fts_index' );
@@ -67,7 +66,7 @@ sub run_test {
@tickets = RT::Test->create_tickets(
{ Queue => $q->id },
- { Subject => 'book', Content => 'book' },
+ { Subject => 'book', Content => 'book initial' },
{ Subject => 'bar', Content => 'bar' },
);
sync_index();
@@ -77,5 +76,70 @@ run_tests(
"Content LIKE 'bar'" => { book => 0, bar => 1 },
);
+my $book = $tickets[0];
+my ( $ret, $msg ) = $book->Correspond( Content => 'hobbit' );
+ok( $ret, 'Corresponded' ) or diag $msg;
+
+( $ret, $msg ) = $book->SetSubject('updated');
+ok( $ret, 'Updated subject' ) or diag $msg;
+
+sync_index();
+
+run_tests(
+ "Content LIKE 'book' AND Content LIKE 'hobbit'" => { updated => 1, bar => 0 },
+ "Subject LIKE 'updated' OR Content LIKE 'bar'" => { updated => 1, bar => 1 },
+ "( Subject LIKE 'updated' OR Content LIKE 'hobbit' ) AND ( Content LIKE 'book' OR Content LIKE 'bar' )" =>
+ { updated => 1, bar => 0 },
+);
+
+diag "Checking SQL query";
+
+my $tickets = RT::Tickets->new( RT->SystemUser );
+$tickets->FromSQL(q{Content LIKE 'book' AND Content LIKE 'hobbit'});
+like( $tickets->BuildSelectQuery(), qr{ INTERSECT }, 'AND query contains INTERSECT' );
+
+$tickets->FromSQL(q{Subject LIKE 'updated' OR Content LIKE 'bar'});
+like( $tickets->BuildSelectQuery(), qr{ UNION }, 'OR query contains UNION' );
+
+$tickets->FromSQL(
+ q{( Subject LIKE 'updated' OR Content LIKE 'hobbit' ) AND ( Content LIKE 'book' OR Content LIKE 'bar' )});
+like(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR query contains both INTERSECT and UNION'
+);
+
+diag "Checking transaction searches";
+
+my $txns = RT::Transactions->new( RT->SystemUser );
+$txns->FromSQL(qq{Content LIKE 'book' AND Content LIKE 'initial'});
+is( $txns->Count, 1, 'Found one transaction' );
+my $txn = $txns->First;
+like( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query contains INTERSECT' );
+like( $txn->Content, qr/book initial/, 'Transaction content' );
+
+$txns->FromSQL(q{Content LIKE 'book' AND Content LIKE 'hobbit'});
+like( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query contains INTERSECT' );
+is( $txns->Count, 0, 'Found 0 transactions' );
+
+$txns->FromSQL(q{Content LIKE 'book' OR Content LIKE 'hobbit'});
+like( $txns->BuildSelectQuery(), qr{ UNION }, 'OR transaction query contains UNION' );
+is( $txns->Count, 2, 'Found 2 transactions' );
+my @txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/book/, 'Transaction content' );
+like( $txns[1]->Content, qr/hobbit/, 'Transaction content' );
+
+$txns->FromSQL(qq{( Content LIKE 'book' AND Content LIKE 'initial' ) OR Content LIKE 'hobbit'});
+like(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR transaction query contains both INTERSECT and UNION'
+);
+is( $txns->Count, 2, 'Found 2 transactions' );
+ at txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/book/, 'Transaction content' );
+like( $txns[1]->Content, qr/hobbit/, 'Transaction content' );
+
@tickets = ();
+done_testing;
diff --git a/t/fts/indexed_pg.t b/t/fts/indexed_pg.t
index 1494fded25..a81c9b2c3d 100644
--- a/t/fts/indexed_pg.t
+++ b/t/fts/indexed_pg.t
@@ -97,7 +97,72 @@ run_tests(
"Content LIKE 'pubs'" => { $book->id => 0, $bars->id => 0 },
);
-# Test the "ts_vector too long" skip
+my ( $ret, $msg ) = $book->Correspond( Content => 'hobbit' );
+ok( $ret, 'Corresponded' ) or diag $msg;
+
+( $ret, $msg ) = $book->SetSubject('updated');
+ok( $ret, 'Updated subject' ) or diag $msg;
+
+sync_index();
+
+run_tests(
+ "Content LIKE 'book' AND Content LIKE 'hobbit'" => { $book->id => 1, $bars->id => 0 },
+ "Subject LIKE 'updated' OR Content LIKE 'bars'" => { $book->id => 1, $bars->id => 1 },
+ "( Subject LIKE 'updated' OR Content LIKE 'hobbit' ) AND ( Content LIKE 'book' OR Content LIKE 'bars' )" =>
+ { $book->id => 1, $bars->id => 0 },
+);
+
+diag "Checking SQL query";
+
+my $tickets = RT::Tickets->new( RT->SystemUser );
+$tickets->FromSQL(q{Content LIKE 'book' AND Content LIKE 'hobbit'});
+like( $tickets->BuildSelectQuery(), qr{ INTERSECT }, 'AND query contains INTERSECT' );
+
+$tickets->FromSQL(q{Subject LIKE 'updated' OR Content LIKE 'bars'});
+like( $tickets->BuildSelectQuery(), qr{ UNION }, 'OR query contains UNION' );
+
+$tickets->FromSQL(
+ q{( Subject LIKE 'updated' OR Content LIKE 'hobbit' ) AND ( Content LIKE 'book' OR Content LIKE 'bars' )});
+like(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR query contains both INTERSECT and UNION'
+);
+
+diag "Checking transaction searches";
+
+my $txns = RT::Transactions->new( RT->SystemUser );
+$txns->FromSQL(qq{Content LIKE 'book' AND Content LIKE '$blase'});
+is( $txns->Count, 1, 'Found one transaction' );
+my $txn = $txns->First;
+like( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query contains INTERSECT' );
+like( $txn->Content, qr/book $blase/, 'Transaction content' );
+
+$txns->FromSQL(q{Content LIKE 'book' AND Content LIKE 'hobbit'});
+like( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query contains INTERSECT' );
+is( $txns->Count, 0, 'Found 0 transactions' );
+
+$txns->FromSQL(q{Content LIKE 'book' OR Content LIKE 'hobbit'});
+like( $txns->BuildSelectQuery(), qr{ UNION }, 'OR transaction query contains UNION' );
+is( $txns->Count, 2, 'Found 2 transactions' );
+my @txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/book/, 'Transaction content' );
+like( $txns[1]->Content, qr/hobbit/, 'Transaction content' );
+
+$txns->FromSQL(qq{( Content LIKE 'book' AND Content LIKE '$blase' ) OR Content LIKE 'hobbit'});
+like(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR transaction query contains both INTERSECT and UNION'
+);
+is( $txns->Count, 2, 'Found 2 transactions' );
+ at txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/book/, 'Transaction content' );
+like( $txns[1]->Content, qr/hobbit/, 'Transaction content' );
+
+
+diag q{Test the "ts_vector too long" skip};
+
my $content = "";
$content .= "$_\n" for 1..200_000;
@tickets = RT::Test->create_tickets(
diff --git a/t/fts/not_indexed.t b/t/fts/not_indexed.t
index 9ec77ebfbd..c8333e4fa9 100644
--- a/t/fts/not_indexed.t
+++ b/t/fts/not_indexed.t
@@ -2,7 +2,7 @@
use strict;
use warnings;
-use RT::Test tests => 20;
+use RT::Test tests => undef;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 0 );
@@ -44,7 +44,7 @@ sub run_test {
@tickets = RT::Test->create_tickets(
{ Queue => $q->id },
- { Subject => 'book', Content => 'book' },
+ { Subject => 'book', Content => 'book initial' },
{ Subject => 'bar', Content => 'bar' },
{ Subject => 'no content', Content => undef },
);
@@ -57,4 +57,67 @@ run_tests(
"(Content LIKE 'bar' OR Subject LIKE 'missing')" => { bar => 1 },
);
+my $book = $tickets[0];
+my ( $ret, $msg ) = $book->Correspond( Content => 'hobbit' );
+ok( $ret, 'Corresponded' ) or diag $msg;
+( $ret, $msg ) = $book->SetSubject('updated');
+ok( $ret, 'Updated subject' ) or diag $msg;
+
+run_tests(
+ "Subject LIKE 'updated' OR Content LIKE 'bar'" => { updated => 1, bar => 1 },
+ "( Subject LIKE 'updated' OR Content LIKE 'hobbit' ) AND ( Content LIKE 'book' OR Content LIKE 'bar' )" =>
+ { updated => 1, bar => 0 },
+);
+
+diag "Checking SQL query";
+
+my $tickets = RT::Tickets->new( RT->SystemUser );
+$tickets->FromSQL(q{Content LIKE 'book' AND Content LIKE 'hobbit'});
+unlike( $tickets->BuildSelectQuery(), qr{ INTERSECT }, 'AND query does not contain INTERSECT' );
+
+$tickets->FromSQL(q{Subject LIKE 'updated' OR Content LIKE 'bar'});
+unlike( $tickets->BuildSelectQuery(), qr{ UNION }, 'OR query does not contain UNION' );
+
+$tickets->FromSQL(
+ q{( Subject LIKE 'updated' OR Content LIKE 'hobbit' ) AND ( Content LIKE 'book' OR Content LIKE 'bar' )});
+unlike(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR query does not contain both INTERSECT and UNION'
+);
+
+diag "Checking transaction searches";
+
+my $txns = RT::Transactions->new( RT->SystemUser );
+$txns->FromSQL(qq{Content LIKE 'book' AND Content LIKE 'initial'});
+is( $txns->Count, 1, 'Found one transaction' );
+my $txn = $txns->First;
+unlike( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query does not contain INTERSECT' );
+like( $txn->Content, qr/book initial/, 'Transaction content' );
+
+$txns->FromSQL(q{Content LIKE 'book' AND Content LIKE 'hobbit'});
+unlike( $txns->BuildSelectQuery(), qr{ INTERSECT }, 'AND transaction query does not contain INTERSECT' );
+is( $txns->Count, 0, 'Found 0 transactions' );
+
+$txns->FromSQL(q{Content LIKE 'book' OR Content LIKE 'hobbit'});
+unlike( $txns->BuildSelectQuery(), qr{ UNION }, 'OR transaction query does not contain UNION' );
+is( $txns->Count, 2, 'Found 2 transactions' );
+my @txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/book/, 'Transaction content' );
+like( $txns[1]->Content, qr/hobbit/, 'Transaction content' );
+
+$txns->FromSQL(qq{( Content LIKE 'book' AND Content LIKE 'initial' ) OR Content LIKE 'hobbit'});
+unlike(
+ $tickets->BuildSelectQuery(),
+ qr{ (?:INTERSECT|UNION) .+ (?:INTERSECT|UNION) },
+ 'AND&OR transaction query does not contain both INTERSECT and UNION'
+);
+is( $txns->Count, 2, 'Found 2 transactions' );
+ at txns = @{ $txns->ItemsArrayRef };
+like( $txns[0]->Content, qr/book/, 'Transaction content' );
+like( $txns[1]->Content, qr/hobbit/, 'Transaction content' );
+
+ at tickets = ();
+
+done_testing;
commit f1b92767939238e28684f0661a3ce4e22842d262
Author: sunnavy <sunnavy at bestpractical.com>
Date: Wed Jan 24 16:32:48 2024 -0500
Fix aggregator of no-indexed content fields for transaction searches
We should use the one in %rest instead of "AND". We copied the "AND" from
Tickets.pm, but forgot to change it when commenting out the Limit call above
that excludes "EmailRecord" and "CommentEmailRecord" type.
diff --git a/lib/RT/Transactions.pm b/lib/RT/Transactions.pm
index 8e128dacad..5e6d6e3173 100644
--- a/lib/RT/Transactions.pm
+++ b/lib/RT/Transactions.pm
@@ -834,7 +834,7 @@ sub _AttachContentLimit {
# );
$self->Limit(
- ENTRYAGGREGATOR => 'AND',
+ %rest,
ALIAS => $self->{_sql_aliases}{attach},
FIELD => $field,
OPERATOR => $op,
commit 5404f1f8b642987d7d85538612d22e43dc8cbdee
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Jan 23 17:29:02 2024 -0500
Revert "Only search Content, not Content and Subject, for better indexing"
This reverts commit 8450f0a9f233d6a761ac22dbdf14926abc54d7fa.
As we support to separate fulltext search terms from others in a49a079655,
the performance penalty is gone for searches like
Subject LIKE 'foo' OR Content LIKE 'foo'
Via searching both Subject and Content, now we can cover cases where
modified Subject is not included in any attachments.
diff --git a/lib/RT/Search/Simple.pm b/lib/RT/Search/Simple.pm
index f9702361a0..5da09c36d7 100644
--- a/lib/RT/Search/Simple.pm
+++ b/lib/RT/Search/Simple.pm
@@ -280,7 +280,7 @@ sub GuessType {
sub HandleDefault {
my $fts = RT->Config->Get('FullTextSearch');
if ($fts->{Enable} and $fts->{Indexed}) {
- return default => "Content LIKE '$_[1]'";
+ return default => "(Subject LIKE '$_[1]' OR Content LIKE '$_[1]')";
} else {
return default => "Subject LIKE '$_[1]'";
}
commit a49a0796553cc821cef0212286bf3ea045e5b490
Author: sunnavy <sunnavy at bestpractical.com>
Date: Sat Jan 20 03:52:11 2024 -0500
Split fulltext search terms for both correctness and performance
Previously for searches like:
Content LIKE "foo" AND Content LIKE "bar"
If a ticket has "foo" saved in one attachment and "bar" in another, it
wouldn't match because these fulltext criteria needed to match on a single
attachment. What we want is to match on ticket level, i.e. as long as the
ticket has "foo" and "bar" saved in some attachments of the given ticket, it
should match.
Thus we split the query into 2 and intersect them:
(Content LIKE "foo") INTERSECT (Content LIKE "bar")
For searches like:
Content LIKE "foo" OR Subject LIKE "bar"
We split the query and unite them accordingly:
(Content LIKE "foo") UNION (Subject LIKE "bar")
The latter version has much better performance as it makes use of fulltext
indexes, unfortunately the former version can not right now.
See also 8450f0a9f2
diff --git a/lib/RT/Interface/Web/QueryBuilder/Tree.pm b/lib/RT/Interface/Web/QueryBuilder/Tree.pm
index 2b1f8cb5c1..8b1d4b7564 100644
--- a/lib/RT/Interface/Web/QueryBuilder/Tree.pm
+++ b/lib/RT/Interface/Web/QueryBuilder/Tree.pm
@@ -404,6 +404,147 @@ sub ParseSQL {
return @results;
}
+
+=head2 Split Type => intersect|union, Fields => [FIELD1, FIELD2, ...]
+
+E.g. to split "AND" Content terms: Type => 'insersect', Fields => ['Content']
+
+ Status = "open" AND Content LIKE "foo" AND Content LIKE "bar"
+
+will be split into 2 subqueries:
+
+ Status = "open" AND Content LIKE "foo"
+ Status = "open" AND Content LIKE "bar"
+
+then they can be joined via "INTERSECT".
+
+To split "OR" Content terms: Type => 'union', Fields => ['Content']
+
+ Content LIKE "foo" OR Subject LIKE "foo"
+
+will be split into 2 subqueries:
+
+ Content LIKE "foo"
+ Subject LIKE "foo"
+
+then they can be joined via "UNION". Unlike the original version, the new SQL
+can make use of fulltext indexes.
+
+Note that queries like:
+
+ Content LIKE "foo" AND Subject LIKE "foo"
+
+will not be split as there are no benifits, unlike the C<OR> example above.
+
+=cut
+
+sub Split {
+ my $self = shift;
+ my %args = (
+ Type => undef,
+ Fields => undef,
+ @_,
+ );
+
+ if ( !$args{Type} ) {
+ RT->Logger->warning("Missing Type, skipping");
+ return $self;
+ }
+
+ if ( $args{Type} !~ /^(?:intersect|union)$/i ) {
+ RT->Logger->warning("Unsupported type $args{Type}, should be 'intersect' or 'union', skipping");
+ return $self;
+ }
+
+ if ( !$args{Fields} || @{ $args{Fields} } == 0 ) {
+ RT->Logger->warning("Missing Fields, skipping");
+ return $self;
+ }
+
+ my @items;
+
+ my $relation = lc $args{Type} eq 'intersect' ? 'and' : 'or';
+
+ $self->traverse(
+ sub {
+ my $node = shift;
+ return unless $node->isLeaf;
+
+ if ( grep { lc $node->getNodeValue->{Key} eq lc $_ } @{ $args{Fields} } ) {
+ $node = $node->getParent;
+ if ( lc( $node->getNodeValue // '' ) eq $relation ) {
+ my @children = $node->getAllChildren;
+
+ my @splits;
+ my @others;
+ for my $child (@children) {
+ if ( $child->isLeaf && grep { lc $child->getNodeValue->{Key} eq lc $_ } @{ $args{Fields} } )
+ {
+ push @splits, $child;
+ }
+ else {
+ push @others, $child;
+ }
+ }
+
+ # Split others from split fields only if it's "OR" like "Content LIKE 'foo' OR Subject LIKE 'foo'"
+ return unless @splits > 1 || ( $relation eq 'or' && @splits + @others > 1 );
+
+ my $parent = $node->getParent;
+
+ my @list;
+
+ if ( $relation eq 'and' ) {
+ if ( @others ) {
+ for my $item ( @splits ) {
+ my $new = RT::Interface::Web::QueryBuilder::Tree->new( $relation, 'root');
+ $new->addChild($item);
+ $new->addChild($_->clone) for @others;
+ push @list, $new;
+ }
+ }
+ else {
+ @list = @splits;
+ }
+ }
+ else {
+ @list = @splits;
+ if (@others) {
+ my $others = RT::Interface::Web::QueryBuilder::Tree->new( $relation, 'root' );
+ $others->addChild( $_->clone ) for @others;
+ push @list, $others;
+ }
+ }
+
+ if ( $parent eq 'root' ) {
+ for my $item ( @list ) {
+ my $new = RT::Interface::Web::QueryBuilder::Tree->new( $relation, 'root');
+ $new->addChild($item->clone);
+ push @items, $new->clone->Split(%args);
+ }
+ }
+ else {
+ my $index = $node->getIndex;
+ $parent->removeChild($node);
+
+ for my $item ( @list ) {
+ $parent->insertChild( $index, $item );
+ push @items, $self->clone->Split(%args);
+ $parent->removeChild($item);
+ }
+ }
+
+ return 'ABORT';
+ }
+ }
+ }
+ );
+
+ return @items ? @items : $self;
+}
+
+
+
RT::Base->_ImportOverlays();
1;
diff --git a/lib/RT/Report.pm b/lib/RT/Report.pm
index 5e76fa29ed..eb11f7aa68 100644
--- a/lib/RT/Report.pm
+++ b/lib/RT/Report.pm
@@ -583,7 +583,7 @@ sub SetupGroupings {
# within the matching tickets grouped by what is wanted.
$self->Columns( 'id' );
if ( RT->Config->Get('UseSQLForACLChecks') ) {
- my $query = $self->BuildSelectQuery( PreferBind => 0 );
+ my $query = $self->{_split_query} || $self->BuildSelectQuery( PreferBind => 0 );
$self->CleanSlate;
$self->Limit( FIELD => 'Id', OPERATOR => 'IN', VALUE => "($query)", QUOTEVALUE => 0 );
}
diff --git a/lib/RT/SearchBuilder.pm b/lib/RT/SearchBuilder.pm
index 6a707ace01..78844ccdfe 100644
--- a/lib/RT/SearchBuilder.pm
+++ b/lib/RT/SearchBuilder.pm
@@ -96,6 +96,8 @@ sub CleanSlate {
$self->{'_sql_aliases'} = {};
delete $self->{'handled_disabled_column'};
delete $self->{'find_disabled_rows'};
+ delete $self->{'_split_query'};
+ delete $self->{'_untamed_order_by'};
return $self->SUPER::CleanSlate(@_);
}
@@ -1190,6 +1192,106 @@ sub CurrentUserCanSeeAll {
return $self->CurrentUser->HasRight( Right => 'SuperUser', Object => RT->System ) ? 1 : 0;
}
+sub BuildSelectQuery {
+ my $self = shift;
+ return $self->_BuildQuery('BuildSelectQuery', @_);
+}
+
+sub BuildSelectCountQuery {
+ my $self = shift;
+ return $self->_BuildQuery('BuildSelectCountQuery', @_);
+}
+
+sub BuildSelectAndCountQuery {
+ my $self = shift;
+ return $self->_BuildQuery('BuildSelectAndCountQuery', @_);
+}
+
+sub _BuildQuery {
+ my $self = shift;
+ my $method = shift;
+ if ( my $query = $self->{_split_query} ) {
+ my $objects = $self->new( $self->CurrentUser );
+ $objects->Limit( FIELD => 'id', VALUE => "($query)", OPERATOR => 'IN', QUOTEVALUE => 0 );
+
+ # Sync page and columns related info
+ $objects->{$_} = $self->{$_} for qw/first_row show_rows columns/;
+
+ # Sync order by
+ if ( $self->{_untamed_order_by} ) {
+ $objects->OrderByCols( @{ $self->{_untamed_order_by} } );
+ }
+ elsif ( $self->{_order_by} ) {
+ $objects->OrderByCols( @{ $self->{_order_by} } );
+ }
+
+ my $query = $objects->$method(@_);
+ $self->{_bind_values} = $objects->{_bind_values};
+ return $query;
+ }
+ my $super = "SUPER::$method";
+ return $self->$super(@_);
+}
+
+sub _SplitQuery {
+ my $self = shift;
+
+ # SQLite doesn't support user defined precedences via parens for INTERSECT/UNION operations.
+ return if RT->Config->Get('DatabaseType') eq 'SQLite';
+
+ my $query = shift;
+ return unless $query;
+
+ return unless $self->can('SplitFields');
+ my @fields = $self->SplitFields;
+ return unless @fields;
+
+ my $tree = RT::Interface::Web::QueryBuilder::Tree->new;
+ $tree->ParseSQL(
+ Query => $query,
+ CurrentUser => $self->CurrentUser,
+ Class => ref $self,
+ );
+
+ my $split;
+ my %intersect_exists;
+ my @union_queries;
+ for my $intersect ( $tree->Split( Type => 'intersect', Fields => \@fields ) ) {
+ my $query = join ' ', map { $_->{TEXT} } @{ $intersect->__LinearizeTree };
+ next if $intersect_exists{$query}++;
+
+ my @union_selects;
+ my %union_exists;
+ for my $union ( $intersect->Split( Type => 'union', Fields => \@fields ) ) {
+ my $query = join ' ', map { $_->{TEXT} } @{ $union->__LinearizeTree };
+ next if $union_exists{$query}++;
+
+ my @intersects = $union->Split( Type => 'intersect', Fields => \@fields );
+ if ( @intersects > 1 ) {
+ push @union_selects, $self->_SplitQuery($query);
+ }
+ else {
+ my $collection = $self->new( $self->CurrentUser );
+ $collection->{_no_split} = 1;
+
+ $collection->FromSQL($query);
+ $collection->OrderByCols();
+ $collection->CurrentUserCanSee
+ if RT->Config->Get('UseSQLForACLChecks') && $collection->can('CurrentUserCanSee');
+ $collection->Columns('id');
+ push @union_selects, $collection->BuildSelectQuery( PreferBind => 0 );
+ }
+ }
+ push @union_queries, '( ' . join( ' UNION ', @union_selects ) . ' )';
+ $split = 1 if @union_selects > 1;
+ }
+
+ $split = 1 if @union_queries > 1;
+ return unless $split; # Return empty if the query is not split.
+
+ return '(' . join( ' INTERSECT ', @union_queries ) . ')';
+}
+
RT::Base->_ImportOverlays();
1;
diff --git a/lib/RT/Tickets.pm b/lib/RT/Tickets.pm
index 275fe6653a..c7f1851bd7 100644
--- a/lib/RT/Tickets.pm
+++ b/lib/RT/Tickets.pm
@@ -271,6 +271,23 @@ sub SortFields {
return (@SORTFIELDS);
}
+=head2 SplitFields
+
+Returns the list of fields that are supposed to be split into individual
+subqueries and then combined later.
+
+If fulltext search is enabled and indexed, it returns C<Content>, otherwise it
+returns an empty list.
+
+=cut
+
+sub SplitFields {
+ my $self = shift;
+ my $config = RT->Config->Get('FullTextSearch') || {};
+ return 'Content' if $config->{Enable} && $config->{Indexed};
+ return;
+}
+
# BEGIN SQL STUFF *********************************
@@ -1617,6 +1634,9 @@ sub OrderByCols {
my @res = ();
my $order = 0;
+ # Save original args so we can redo OrderByCols later for split queries, especially to joins tables if needed.
+ $self->{_untamed_order_by} = \@_;
+
foreach my $row (@args) {
if ( $row->{ALIAS} ) {
push @res, $row;
@@ -3719,6 +3739,12 @@ sub FromSQL {
);
}
+ if ( !$self->{_no_split} ) {
+ if ( my $split_query = $self->_SplitQuery($query) ) {
+ $self->{_split_query} = $split_query;
+ }
+ }
+
# set SB's dirty flag
$self->{'must_redo_search'} = 1;
$self->{'RecalcTicketLimits'} = 0;
diff --git a/lib/RT/Transactions.pm b/lib/RT/Transactions.pm
index e03d2995a8..8e128dacad 100644
--- a/lib/RT/Transactions.pm
+++ b/lib/RT/Transactions.pm
@@ -253,6 +253,23 @@ sub SortFields {
return (@SORTFIELDS);
}
+=head2 SplitFields
+
+Returns the list of fields that are supposed to be split into individual
+subqueries and then combined later.
+
+If fulltext search is enabled and indexed, it returns C<Content>, otherwise
+it returns an empty list.
+
+=cut
+
+sub SplitFields {
+ my $self = shift;
+ my $config = RT->Config->Get('FullTextSearch') || {};
+ return 'Content' if $config->{Enable} && $config->{Indexed};
+ return;
+}
+
=head1 Limit Helper Routines
These routines are the targets of a dispatch table depending on the
@@ -1179,6 +1196,12 @@ sub FromSQL {
return (0, $error);
}
+ if ( !$self->{_no_split} ) {
+ if ( my $split_query = $self->_SplitQuery($query) ) {
+ $self->{_split_query} = $split_query;
+ }
+ }
+
# set SB's dirty flag
$self->{'must_redo_search'} = 1;
-----------------------------------------------------------------------
hooks/post-receive
--
rt
More information about the rt-commit
mailing list