[Rt-commit] rt branch, 3.9-fts, updated. rt-3.9.4-48-gc36bbd1
Ruslan Zakirov
ruz at bestpractical.com
Mon Oct 18 18:43:47 EDT 2010
The branch, 3.9-fts has been updated
via c36bbd1e9fe0e03f2f3516995fad017502018abd (commit)
via 0140bbd04f3141b0be37884d6cb4c7a3d78cbc84 (commit)
via 70bead3660f3c23bcb1f0b32b523e2cfcc85d128 (commit)
via cb03b3576a731c98a7dfeb1068f3b4d508e7cc24 (commit)
from b82cf713dbce7e63db029b6a4b23bfdaa08be2e9 (commit)
Summary of changes:
sbin/rt-fulltext-indexer.in | 10 +++++
sbin/rt-setup-fulltext-index.in | 60 +++++++++++++++++++++---------
t/fts/{indexed_pg.t => indexed_oracle.t} | 17 +++++---
3 files changed, 62 insertions(+), 25 deletions(-)
copy t/fts/{indexed_pg.t => indexed_oracle.t} (88%)
- Log -----------------------------------------------------------------
commit cb03b3576a731c98a7dfeb1068f3b4d508e7cc24
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Sat Oct 16 03:03:59 2010 +0400
fix oracle fulltext indexing
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 4585846..4606a86 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -111,6 +111,16 @@ unless ( $fts_config->{'Indexed'} ) {
my $db_type = RT->Config->Get('DatabaseType');
+if ( $db_type eq 'Oracle' ) {
+ my $index = $fts_config->{'IndexName'} || 'rt_fts_index';
+ my $memory = '2M';
+ $RT::Handle->dbh->do(
+ "begin ctx_ddl.sync_index(?, ?); end;", undef,
+ $index, $memory
+ );
+ exit;
+}
+
my @types = qw(text html);
foreach my $type ( @types ) {
my $attaches = attachments($type);
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 348a2d4..96647f7 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -154,7 +154,7 @@ CREATE TABLE $table (
) ENGINE=SPHINX CONNECTION="$url"
END
- print_rt_config( table => $table, column => $DEFAULT{'column'} );
+ print_rt_config( Table => $table, Column => $DEFAULT{'column'} );
insert_schema( $schema );
require URI;
@@ -245,7 +245,7 @@ elsif ( $DB{'type'} eq 'Pg' ) {
."$column tsvector )";
}
- print_rt_config( table => $table, column => $column );
+ print_rt_config( Table => $table, Column => $column );
insert_schema_as_dba( $schema );
@@ -321,14 +321,16 @@ elsif ( $DB{'type'} eq 'Oracle' ) {
push @params, ora_create_storage( %{ $PREFERENCES{'storage'} } );
my $index_params = join "\n", @params;
- do_error_is_ok( $dbh => "DROP INDEX $DEFAULT{prefix}index" );
+ my $index_name = $DEFAULT{prefix} .'index';
+ do_error_is_ok( $dbh => "DROP INDEX $index_name" );
$dbh->do(
- "CREATE INDEX $DEFAULT{prefix}index ON Attachments(Content)
+ "CREATE INDEX $index_name ON Attachments(Content)
indextype is ctxsys.context parameters('
$index_params
')",
);
+ print_rt_config( IndexName => $index_name );
}
else {
die "FTS on $DB{type} is not yet supported";
@@ -346,7 +348,7 @@ sub check_sphinx {
}
sub ora_create_datastore {
- return sprintf 'datastore %s', create_preference(
+ return sprintf 'datastore %s', ora_create_preference(
@_,
name => 'datastore',
);
@@ -354,8 +356,8 @@ sub ora_create_datastore {
sub ora_create_filter {
my $res = '';
- $res .= sprintf "format column %s\n", create_format_column();
- $res .= sprintf 'filter %s', create_preference(
+ $res .= sprintf "format column %s\n", ora_create_format_column();
+ $res .= sprintf 'filter %s', ora_create_preference(
@_,
name => 'filter',
);
@@ -363,14 +365,14 @@ sub ora_create_filter {
}
sub ora_create_lexer {
- return sprintf 'lexer %s', create_preference(
+ return sprintf 'lexer %s', ora_create_preference(
@_,
name => 'lexer',
);
}
sub ora_create_word_list {
- return sprintf 'wordlist %s', create_preference(
+ return sprintf 'wordlist %s', ora_create_preference(
@_,
name => 'word_list',
);
@@ -378,6 +380,7 @@ sub ora_create_word_list {
sub ora_create_stop_list {
my $file = shift || 'etc/stopwords/en.txt';
+ return '' unless -e $file;
my $name = $DEFAULT{'prefix'} .'stop_list';
do_error_is_ok( $dbh => 'begin ctx_ddl.drop_stoplist(?); end;', $name );
@@ -412,7 +415,7 @@ sub ora_create_section_group {
}
sub ora_create_storage {
- return sprintf 'storage %s', create_preference(
+ return sprintf 'storage %s', ora_create_preference(
@_,
name => 'storage',
);
@@ -469,7 +472,7 @@ sub ora_create_format_column {
RETURN format;
END;
};
- create_procedure( $detect_format );
+ ora_create_procedure( $detect_format );
$dbh->do(qq{
UPDATE Attachments
@@ -609,8 +612,8 @@ Set( %FullTextSearch,
Indexed => 1,
END
- $config .= " Table => '$args{'table'}',\n" if $args{'table'};
- $config .= " Column => '$args{'column'}',\n" if $args{'column'};
+ $config .= " $_ => '$args{$_}',\n"
+ foreach grep defined $args{$_}, keys %args;
$config .= ");\n";
print $config;
commit 70bead3660f3c23bcb1f0b32b523e2cfcc85d128
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Sat Oct 16 03:04:59 2010 +0400
add tests for FTS on Oracle. they pass
diff --git a/t/fts/indexed_oracle.t b/t/fts/indexed_oracle.t
new file mode 100644
index 0000000..5a843fd
--- /dev/null
+++ b/t/fts/indexed_oracle.t
@@ -0,0 +1,106 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+
+use RT::Test tests => undef;
+plan skip_all => 'Not Pg' unless RT->Config->Get('DatabaseType') eq 'Oracle';
+plan tests => 9;
+
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1 );
+
+setup_indexing();
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+my $queue = $q->Name;
+
+my ($total, @data, @tickets, @test, @conditions) = (0, ());
+
+sub setup_indexing {
+ my %args = (
+ 'no-ask' => 1,
+ command => $RT::SbinPath .'/rt-setup-fulltext-index',
+ dba => $ENV{'RT_DBA_USER'},
+ 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
+ );
+ my ($exit_code, $output) = RT::Test->run_and_capture( %args );
+ ok(!$exit_code, "setted up index") or diag "output: $output";
+}
+
+sub sync_index {
+ my %args = (
+ command => $RT::SbinPath .'/rt-fulltext-indexer',
+ );
+ my ($exit_code, $output) = RT::Test->run_and_capture( %args );
+ ok(!$exit_code, "synced the index") or diag "output: $output";
+}
+
+sub add_tix_from_data {
+ my @res = ();
+ while (@data) {
+ my $t = RT::Ticket->new(RT->SystemUser);
+ my %args = %{ shift(@data) };
+
+ if ( my $content = delete $args{'Content'} ) {
+ $args{'MIMEObj'} = MIME::Entity->build(
+ From => $args{'Requestor'},
+ To => $q->CorrespondAddress,
+ Subject => $args{'Subject'},
+ Data => $content,
+ );
+ }
+ my ( $id, undef, $msg ) = $t->Create(
+ Queue => $q->id,
+ %args,
+ );
+ ok( $id, "ticket created" ) or diag("error: $msg");
+ push @res, $t;
+ $total++;
+ }
+ sync_index();
+ return @res;
+}
+
+sub run_tests {
+ while ( my ($query, $checks) = splice @test, 0, 2 ) {
+ run_test( $query, %$checks );
+ }
+}
+
+sub run_test {
+ my ($query, %checks) = @_;
+ my $query_prefix = join ' OR ', map 'id = '. $_->id, @tickets;
+
+ my $tix = RT::Tickets->new(RT->SystemUser);
+ $tix->FromSQL( "( $query_prefix ) AND ( $query )" );
+
+ my $error = 0;
+
+ my $count = 0;
+ $count++ foreach grep $_, values %checks;
+ is($tix->Count, $count, "found correct number of ticket(s) by '$query'") or $error = 1;
+
+ my $good_tickets = ($tix->Count == $count);
+ while ( my $ticket = $tix->Next ) {
+ next if $checks{ $ticket->Subject };
+ diag $ticket->Subject ." ticket has been found when it's not expected";
+ $good_tickets = 0;
+ }
+ ok( $good_tickets, "all tickets are good with '$query'" ) or $error = 1;
+
+ diag "Wrong SQL query for '$query':". $tix->BuildSelectQuery if $error;
+}
+
+ at data = (
+ { Subject => 'book', Content => 'book' },
+ { Subject => 'bar', Content => 'bar' },
+);
+ at tickets = add_tix_from_data();
+
+ at test = (
+ "Content LIKE 'book'" => { book => 1, bar => 0 },
+ "Content LIKE 'bar'" => { book => 0, bar => 1 },
+);
+run_tests();
+
commit 0140bbd04f3141b0be37884d6cb4c7a3d78cbc84
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Sat Oct 16 06:44:26 2010 +0400
documentation for rt-setup-fulltext-index
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 96647f7..8383f53 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -596,8 +596,8 @@ sub show_help {
RT::Interface::CLI->ShowHelp(
ExitValue => $error,
Sections => $error
- ? 'NAME|'. lc($DB{'type'}) .'/USAGE'
- : 'NAME|'. lc($DB{'type'})
+ ? 'NAME|'. uc($DB{'type'}) .'/USAGE'
+ : 'NAME|'. uc($DB{'type'})
,
);
}
@@ -645,6 +645,15 @@ sub _insert_schema {
rt-setup-fulltext-index - Helps create indexes for full text search
+=head1 DESCRIPTION
+
+Full text search is very database specific. This script setups indexing
+for Oracle, Pg and mysql. Below you can find specifics for your current
+DB.
+
+At the end script prints piece that you should put into RT config to enable
+created index. Don't forget to restart the web-server after changing config.
+
=head1 ORACLE
=head2 USAGE
@@ -654,18 +663,26 @@ rt-setup-fulltext-index - Helps create indexes for full text search
=head2 DESCRIPTION
-This utility grants CTXAPP role to RT's user and rights to execute functions
-from CTX_DDL package. Also, it creates several prefernces, functions and triggers
-all starting with 'rt_fts_' prefix in the name. After all an index is created.
+Creates Oracle CONTEXT index on the Content column in the Attachments table.
+As well, it creates several preferences, functions and triggers to support
+this index.
+
+CONTEXT index needs periodic synchronization after any updates. Either use
+F<sbin/rt-fulltext-indexer> or read its documentation for alternatives.
=head1 PG
=head2 USAGE
rt-setup-fulltext-index --help
+ rt-setup-fulltext-index --dba postgres --dba-password 'secrete'
=head2 DESCRIPTION
+Creates an additional column to store a ts_vector and then creates either
+GiN or GiST index on it. Use F<sbin/rt-fulltext-indexer> to keep index in
+sync.
+
=head1 MYSQL
=head2 USAGE
@@ -674,6 +691,10 @@ all starting with 'rt_fts_' prefix in the name. After all an index is created.
=head2 DESCRIPTION
+Full text search in mysql implemented through Sphinx and MySQL storage
+engine (SphinxSE), so you need mysql compiled with it. Use
+F<sbin/rt-fulltext-indexer> to keep index in sync.
+
=head1 AUTHOR
Ruslan Zakirov E<lt>ruz at bestpractical.comE<gt>
commit c36bbd1e9fe0e03f2f3516995fad017502018abd
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Tue Oct 19 02:42:56 2010 +0400
show generic description when --help is used
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 8383f53..bae18ef 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -597,7 +597,7 @@ sub show_help {
ExitValue => $error,
Sections => $error
? 'NAME|'. uc($DB{'type'}) .'/USAGE'
- : 'NAME|'. uc($DB{'type'})
+ : 'NAME|DESCRIPTION|'. uc($DB{'type'})
,
);
}
-----------------------------------------------------------------------
More information about the Rt-commit
mailing list