[Rt-commit] rt branch, experimental/fts-mysql-multiple-queries, created. rt-4.0.4-283-g19e8a32
Alex Vandiver
alexmv at bestpractical.com
Fri Oct 11 19:19:28 EDT 2013
The branch, experimental/fts-mysql-multiple-queries has been created
at 19e8a32a332ca69ba6af5980a5465c75a315cba2 (commit)
- Log -----------------------------------------------------------------
commit 221b7b1aac34aa3222f5ff3547b37dcd1161b249
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Feb 3 18:34:35 2012 +0800
test multiple "Content LIKE ..." search
diff --git a/t/fts/indexed_mysql.t b/t/fts/indexed_mysql.t
index 8966f1c..c8d4c0f 100644
--- a/t/fts/indexed_mysql.t
+++ b/t/fts/indexed_mysql.t
@@ -14,7 +14,7 @@ $sphinx{'indexer'} = RT::Test->find_executable('indexer');
plan skip_all => "No searchd and indexer under PATH"
unless $sphinx{'searchd'} && $sphinx{'indexer'};
-plan tests => 15;
+plan tests => 25;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Table => 'AttachmentsIndex', MaxMatches => 1000 );
@@ -116,12 +116,17 @@ sub run_test {
{ Queue => $q->id },
{ Subject => 'book', Content => 'book' },
{ Subject => 'bar', Content => 'bar' },
+ { Subject => 'foobar', Content => 'foo bar' },
);
sync_index();
run_tests(
- "Content LIKE 'book'" => { book => 1, bar => 0 },
- "Content LIKE 'bar'" => { book => 0, bar => 1 },
+ "Content LIKE 'book'" => { book => 1, bar => 0, foobar => 0 },
+ "Content LIKE 'bar'" => { book => 0, bar => 1, foobar => 1 },
+ "Content LIKE 'foo' OR Content LIKE 'bar' " =>
+ { book => 0, bar => 1, foobar => 1 },
+ "Content LIKE 'foo' AND Content LIKE 'bar' " =>
+ { book => 0, bar => 0, foobar => 1 },
);
END {
commit 0e04cd4861159f1231d82ab492048a9a443d9913
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Feb 7 10:15:29 2012 +0800
tests number fix
diff --git a/t/fts/indexed_mysql.t b/t/fts/indexed_mysql.t
index c8d4c0f..5a1e4fd 100644
--- a/t/fts/indexed_mysql.t
+++ b/t/fts/indexed_mysql.t
@@ -14,7 +14,7 @@ $sphinx{'indexer'} = RT::Test->find_executable('indexer');
plan skip_all => "No searchd and indexer under PATH"
unless $sphinx{'searchd'} && $sphinx{'indexer'};
-plan tests => 25;
+plan tests => 22;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Table => 'AttachmentsIndex', MaxMatches => 1000 );
commit 63582c77233510e54e34e865aeb78646c11e6783
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Feb 7 15:33:36 2012 +0800
test of mutiple queries with not operator
diff --git a/t/fts/indexed_mysql.t b/t/fts/indexed_mysql.t
index 5a1e4fd..15eee54 100644
--- a/t/fts/indexed_mysql.t
+++ b/t/fts/indexed_mysql.t
@@ -14,7 +14,7 @@ $sphinx{'indexer'} = RT::Test->find_executable('indexer');
plan skip_all => "No searchd and indexer under PATH"
unless $sphinx{'searchd'} && $sphinx{'indexer'};
-plan tests => 22;
+plan tests => 24;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Table => 'AttachmentsIndex', MaxMatches => 1000 );
@@ -127,6 +127,8 @@ run_tests(
{ book => 0, bar => 1, foobar => 1 },
"Content LIKE 'foo' AND Content LIKE 'bar' " =>
{ book => 0, bar => 0, foobar => 1 },
+ "Content NOT LIKE 'foo' AND Content LIKE 'bar' " =>
+ { book => 0, bar => 1, foobar => 0 },
);
END {
commit 97e4fed09b396885e5b9db461f8ea50cb78b8265
Author: sunnavy <sunnavy at bestpractical.com>
Date: Thu Feb 9 18:31:08 2012 +0800
support continuous "content like ..." queries for mysql+sphinx
by combining them into one query="..." in sphinx's boolean mode
diff --git a/lib/RT/Tickets.pm b/lib/RT/Tickets.pm
index 7ed43f7..89657a4 100644
--- a/lib/RT/Tickets.pm
+++ b/lib/RT/Tickets.pm
@@ -844,12 +844,13 @@ sub _TransContentLimit {
$value =~ s/;/\\;/g;
my $max = $config->{'MaxMatches'};
+ my $mode = $self->{_sql_sphinx_mode} || 'all';
$self->_SQLLimit(
%rest,
- ALIAS => $alias,
- FIELD => 'query',
- OPERATOR => '=',
- VALUE => "$value;limit=$max;maxmatches=$max",
+ ALIAS => $alias,
+ FIELD => 'query',
+ OPERATOR => '=',
+ VALUE => "$value;limit=$max;maxmatches=$max;mode=$mode",
);
}
} else {
diff --git a/lib/RT/Tickets_SQL.pm b/lib/RT/Tickets_SQL.pm
index ec1bb49..ec51775 100644
--- a/lib/RT/Tickets_SQL.pm
+++ b/lib/RT/Tickets_SQL.pm
@@ -53,6 +53,7 @@ use warnings;
use RT::SQL;
+use Regexp::Common qw/delimited/;
# Import configuration data from the lexcial scope of __PACKAGE__ (or
# at least where those two Subroutines are defined.)
@@ -291,8 +292,65 @@ sub FromSQL {
$self->_InitSQL();
return (1, $self->loc("No Query")) unless $query;
-
$self->{_sql_query} = $query;
+
+
+ my $config = RT->Config->Get('FullTextSearch') || {};
+ if ( $config->{Indexed} && RT->Config->Get('DatabaseType') eq 'mysql' ) {
+
+# mysql doesn't bother asking sphinx if there are different query clauses.
+# I guess it's because mysql thinks there can't be a field who can meet
+# both "query='foo'' and "query='bar'", so it returns empty set directly.
+# that's why we combine those query into one here.
+# currently only continuous content queries are supported.
+
+ my $re_delim = $RE{delimited}{ -delim => qq{\'\"} };
+ my $re_content = '(?:^|\s+)Content\s+(?:(NOT)\s+)?LIKE\s+(' . $re_delim . ')';
+ my $first_escaped;
+ while ( $query =~ /($re_content\s+(AND|OR)\s*$re_content)/i ) {
+ my $whole = $1;
+ my $first_neg = $2;
+ my $first = $3;
+ my $rel = $4;
+ my $second_neg = $5;
+ my $second = $6;
+ $rel = $rel =~ /and/i ? '&' : '|';
+
+ my $first_quote = substr $first, 0, 1;
+ $first =~ s!^$first_quote!!;
+ $first =~ s!$first_quote$!!;
+
+ if ( !$first_escaped ) {
+ $first =~ s/(&|\||!|-|\(|\))/\\$1/g;
+ $first = "!($first)" if $first_neg;
+ }
+
+ # we will quote value with '
+ $first =~ s!'!\\'!g if $first_quote eq '"';
+
+ my $second_quote = substr $second, 0, 1;
+ $second =~ s!^$second_quote!!;
+ $second =~ s!$second_quote$!!;
+ $second =~ s/(&|\||!|-|\(|\))/\\$1/g;
+ $second = "!($second)" if $second_neg;
+
+ # we will quote value with '
+ $second =~ s!'!\\'!g if $second_quote eq '"';
+
+ $query =~ s!\Q$whole!Content LIKE '($first$rel$second)'!;
+
+ $self->{_sql_sphinx_mode} = 'boolean';
+ $first_escaped = 1;
+ }
+
+ if ( $query =~ /$re_content.*$re_content/s ) {
+ return (0, $self->loc("Incontinuous Content queries"));
+ }
+ elsif ( $query =~ /$re_content/ && $1 ) {
+ return (0, $self->loc("Single NOT operator is not supported"));
+ }
+ }
+
eval { $self->_parser( $query ); };
if ( $@ ) {
$RT::Logger->error( $@ );
commit 19e8a32a332ca69ba6af5980a5465c75a315cba2
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Feb 10 12:57:59 2012 +0800
put "Content LIKE" combination into method PrepareSQL
diff --git a/lib/RT/Tickets_SQL.pm b/lib/RT/Tickets_SQL.pm
index ec51775..c3c83a7 100644
--- a/lib/RT/Tickets_SQL.pm
+++ b/lib/RT/Tickets_SQL.pm
@@ -294,18 +294,80 @@ sub FromSQL {
return (1, $self->loc("No Query")) unless $query;
$self->{_sql_query} = $query;
+ ( my $ret, $query ) = $self->PrepareSQL( $query );
+ return ( $ret, $query ) unless $ret;
+
+ eval { $self->_parser( $query ); };
+ if ( $@ ) {
+ $RT::Logger->error( $@ );
+ return (0, $@);
+ }
+
+ # We only want to look at EffectiveId's (mostly) for these searches.
+ unless ( exists $self->{_sql_looking_at}{'effectiveid'} ) {
+ #TODO, we shouldn't be hard #coding the tablename to main.
+ $self->SUPER::Limit( FIELD => 'EffectiveId',
+ VALUE => 'main.id',
+ ENTRYAGGREGATOR => 'AND',
+ QUOTEVALUE => 0,
+ );
+ }
+ # FIXME: Need to bring this logic back in
+
+ # if ($self->_isLimited && (! $self->{'looking_at_effective_id'})) {
+ # $self->SUPER::Limit( FIELD => 'EffectiveId',
+ # OPERATOR => '=',
+ # QUOTEVALUE => 0,
+ # VALUE => 'main.id'); #TODO, we shouldn't be hard coding the tablename to main.
+ # }
+ # --- This is hardcoded above. This comment block can probably go.
+ # Or, we need to reimplement the looking_at_effective_id toggle.
+
+ # Unless we've explicitly asked to look at a specific Type, we need
+ # to limit to it.
+ unless ( $self->{looking_at_type} ) {
+ $self->SUPER::Limit( FIELD => 'Type', VALUE => 'ticket' );
+ }
+
+ # We don't want deleted tickets unless 'allow_deleted_search' is set
+ unless( $self->{'allow_deleted_search'} ) {
+ $self->SUPER::Limit( FIELD => 'Status',
+ OPERATOR => '!=',
+ VALUE => 'deleted',
+ );
+ }
+
+ # set SB's dirty flag
+ $self->{'must_redo_search'} = 1;
+ $self->{'RecalcTicketLimits'} = 0;
+
+ return (1, $self->loc("Valid Query"));
+}
+
+=head2 PrepareSQL
+
+Prepare the query.
+
+Returns (1, $query ) on success and (0, 'Error Message') on failure.
+
+=cut
+
+sub PrepareSQL {
+ my $self = shift;
+ my $query = shift;
my $config = RT->Config->Get('FullTextSearch') || {};
if ( $config->{Indexed} && RT->Config->Get('DatabaseType') eq 'mysql' ) {
-# mysql doesn't bother asking sphinx if there are different query clauses.
-# I guess it's because mysql thinks there can't be a field who can meet
-# both "query='foo'' and "query='bar'", so it returns empty set directly.
-# that's why we combine those query into one here.
-# currently only continuous content queries are supported.
+ # mysql doesn't bother asking sphinx if there are different query clauses.
+ # I guess it's because mysql thinks there can't be a field who can meet
+ # both "query='foo'' and "query='bar'", so it returns empty set directly.
+ # that's why we combine those query into one here.
+ # currently only continuous content queries are supported.
my $re_delim = $RE{delimited}{ -delim => qq{\'\"} };
- my $re_content = '(?:^|\s+)Content\s+(?:(NOT)\s+)?LIKE\s+(' . $re_delim . ')';
+ my $re_content =
+ '(?:^|\s+)Content\s+(?:(NOT)\s+)?LIKE\s+(' . $re_delim . ')';
my $first_escaped;
while ( $query =~ /($re_content\s+(AND|OR)\s*$re_content)/i ) {
my $whole = $1;
@@ -344,58 +406,13 @@ sub FromSQL {
}
if ( $query =~ /$re_content.*$re_content/s ) {
- return (0, $self->loc("Incontinuous Content queries"));
+ return ( 0, $self->loc("Incontinuous Content queries") );
}
elsif ( $query =~ /$re_content/ && $1 ) {
- return (0, $self->loc("Single NOT operator is not supported"));
+ return ( 0, $self->loc("Single NOT operator is not supported") );
}
}
-
- eval { $self->_parser( $query ); };
- if ( $@ ) {
- $RT::Logger->error( $@ );
- return (0, $@);
- }
-
- # We only want to look at EffectiveId's (mostly) for these searches.
- unless ( exists $self->{_sql_looking_at}{'effectiveid'} ) {
- #TODO, we shouldn't be hard #coding the tablename to main.
- $self->SUPER::Limit( FIELD => 'EffectiveId',
- VALUE => 'main.id',
- ENTRYAGGREGATOR => 'AND',
- QUOTEVALUE => 0,
- );
- }
- # FIXME: Need to bring this logic back in
-
- # if ($self->_isLimited && (! $self->{'looking_at_effective_id'})) {
- # $self->SUPER::Limit( FIELD => 'EffectiveId',
- # OPERATOR => '=',
- # QUOTEVALUE => 0,
- # VALUE => 'main.id'); #TODO, we shouldn't be hard coding the tablename to main.
- # }
- # --- This is hardcoded above. This comment block can probably go.
- # Or, we need to reimplement the looking_at_effective_id toggle.
-
- # Unless we've explicitly asked to look at a specific Type, we need
- # to limit to it.
- unless ( $self->{looking_at_type} ) {
- $self->SUPER::Limit( FIELD => 'Type', VALUE => 'ticket' );
- }
-
- # We don't want deleted tickets unless 'allow_deleted_search' is set
- unless( $self->{'allow_deleted_search'} ) {
- $self->SUPER::Limit( FIELD => 'Status',
- OPERATOR => '!=',
- VALUE => 'deleted',
- );
- }
-
- # set SB's dirty flag
- $self->{'must_redo_search'} = 1;
- $self->{'RecalcTicketLimits'} = 0;
-
- return (1, $self->loc("Valid Query"));
+ return ( 1, $query );
}
=head2 Query
-----------------------------------------------------------------------
More information about the Rt-commit
mailing list