[Rt-commit] rt branch, 4.0/fulltext-search, created. rt-4.0.0rc8-189-gbce3688
Alex Vandiver
alexmv at bestpractical.com
Thu Apr 28 13:36:18 EDT 2011
The branch, 4.0/fulltext-search has been created
at bce3688d864a83f9bf51642770fa6f09b488ed7d (commit)
- Log -----------------------------------------------------------------
commit e7f378895ec06e64bd056e1c966277aeee2ef6bd
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 12:34:40 2011 -0400
Pull default DBA from autoconf
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index a386390..1536413 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -84,7 +84,7 @@ use RT::Interface::CLI ();
my %DB = (
type => scalar RT->Config->Get('DatabaseType'),
user => scalar RT->Config->Get('DatabaseUser'),
- admin => undef,
+ admin => '@DB_DBA@',
admin_password => undef,
);
commit 14492925702ddeee76d165e27d57693a98137248
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 12:35:04 2011 -0400
Add --dry-run option, which is already respected in the code
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 1536413..596c162 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -91,6 +91,7 @@ my %DB = (
my %OPT = (
help => 0,
ask => 1,
+ dryrun => 0,
);
my %DEFAULT;
@@ -116,6 +117,7 @@ use Getopt::Long qw(GetOptions);
GetOptions(
'h|help!' => \$OPT{'help'},
'ask!' => \$OPT{'ask'},
+ 'dry-run!' => \$OPT{'dryrun'},
'dba=s' => \$DB{'admin'},
'dba-password=s' => \$DB{'admin_password'},
commit 8a4a5d5bf6c498f5803d86e23344a1713ffeb8ed
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 12:35:13 2011 -0400
Show help if options fail to parse
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 596c162..a1ed7ef 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -121,7 +121,7 @@ GetOptions(
'dba=s' => \$DB{'admin'},
'dba-password=s' => \$DB{'admin_password'},
-);
+) or show_help();
if ( $OPT{'help'} || (!$DB{'admin'} && $DB{'type'} eq 'Oracle' ) ) {
show_help( !$OPT{'help'} );
commit 5a85181b49ff95f6f241bbf14524d66f0d920d86
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 12:36:18 2011 -0400
Group questions, then schema changes, then configuration suggestion
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index a1ed7ef..01ccda8 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -156,9 +156,10 @@ CREATE TABLE $table (
) ENGINE=SPHINX CONNECTION="$url"
END
- print_rt_config( Table => $table, Column => $DEFAULT{'column'} );
insert_schema( $schema );
+ print_rt_config( Table => $table, Column => $DEFAULT{'column'} );
+
require URI;
my $urlo = URI->new( $url );
my $host = $urlo->host;
@@ -247,10 +248,6 @@ elsif ( $DB{'type'} eq 'Pg' ) {
."$column tsvector )";
}
- print_rt_config( Table => $table, Column => $column );
-
- insert_schema_as_dba( $schema );
-
my $index_type;
do {
$index_type = lc prompt(
@@ -263,7 +260,11 @@ elsif ( $DB{'type'} eq 'Pg' ) {
silent => !$OPT{'ask'},
);
} while ( $index_type ne 'gist' && $index_type ne 'gin' );
+
+ insert_schema_as_dba( $schema );
insert_schema_as_dba("CREATE INDEX ${column}_idx ON $table USING $index_type($column)");
+
+ print_rt_config( Table => $table, Column => $column );
}
elsif ( $DB{'type'} eq 'Oracle' ) {
{
commit 50a856c6d7e13881af48b98a4bec157bfd6fe739
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 12:37:14 2011 -0400
Wording and spacing fixes
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 01ccda8..34d7952 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -137,12 +137,14 @@ if ( $DB{'type'} eq 'mysql' ) {
check_sphinx();
my $table = prompt(
- message => 'Enter name of a DB table that will be used to connect to the sphinx server',
+ message => "Enter name of a new Mysql table that will be used to connect to the\n"
+ . "Sphinx server:",
default => $DEFAULT{'table'},
silent => !$OPT{'ask'},
);
my $url = prompt(
- message => 'Enter URL of the sphinx search server, it should be sphinx://<server>:<port>/<index name>. Simple config for this sphinx instance will be generated for you.',
+ message => "Enter URL of the sphinx search server; this should be of the form\n"
+ . "sphinx://<server>:<port>/<index name>",
default => 'sphinx://localhost:3312/rt',
silent => !$OPT{'ask'},
);
@@ -174,10 +176,13 @@ END
$sphinx_conf{'user'} = RT->config->get('DatabaseUser');
$sphinx_conf{'pass'} = RT->config->get('DatabasePassword');
- print "Here is simple sphinx config, you can use it to index text/plain attachments in your DB."
- ." This config is not ideal. You should read Sphinx docs to get better ideas.";
print <<END
+Below is a simple Sphinx configuration which can be used to index all
+text/plain attachments in your database. This configuration is not
+ideal; you should read the Sphinx documentation to understand how to
+confiture it to better suit your needs.
+
source rt {
type = mysql
@@ -226,15 +231,14 @@ elsif ( $DB{'type'} eq 'Pg' ) {
my $dbh = $RT::Handle->dbh;
my $table = prompt(
- message =>
- 'Enter name of a DB table that will be used to store Pg tsvector.'
- . ' Attachments table can be used.'
- ,
+ message => "Enter the name of a DB table that will be used to store the Pg tsvector.\n"
+ . "You may either use the existing Attachments table, or create a new\n"
+ . "table.",
default => $DEFAULT{'table'},
silent => !$OPT{'ask'},
);
my $column = prompt(
- message => 'Enter name of a column that will be used to store Pg tsvector',
+ message => 'Enter the name of a column that will be used to store the Pg tsvector:',
default => $DEFAULT{'column'},
silent => !$OPT{'ask'},
);
@@ -251,11 +255,8 @@ elsif ( $DB{'type'} eq 'Pg' ) {
my $index_type;
do {
$index_type = lc prompt(
- message =>
- 'You have choice between GiST or GIN index,'
- .' the first is times slower to search, but'
- .' it takes less place and faster to update.'
- ,
+ message => "You may choose between GiST or GIN indexes; the former is several times\n"
+ . "slower to search, but takes less space on disk and is faster to update.",
default => 'GiST',
silent => !$OPT{'ask'},
);
@@ -336,16 +337,21 @@ elsif ( $DB{'type'} eq 'Oracle' ) {
print_rt_config( IndexName => $index_name );
}
else {
- die "FTS on $DB{type} is not yet supported";
+ die "Full-text indexes on $DB{type} are not yet supported";
}
sub check_sphinx {
my $dbh = $RT::Handle->dbh;
my $sphinx = ($dbh->selectrow_array("show variables like 'have_sphinx'"))[1];
unless ( lc $sphinx eq 'yes' ) {
- print STDERR "Mysql server you have compiled without sphinx storage engine (sphinxse).\n";
- print STDERR "Either use system packages with sphinxse, binaries from Sphinx site
- or compile mysql according to instructions in Sphinx's docs.\n";
+ print STDERR <<EOT;
+
+Your Mysql server has not been compiled with the Sphinx storage engine
+(sphinxse). Either install your system's sphinxse package, the binaries
+from http://sphinxsearch.org/, or recompile mysql according to the
+instructions in Sphinx's documentation.
+
+EOT
exit 1;
}
}
@@ -585,6 +591,7 @@ sub prompt {
my $res = <STDIN>;
chomp $res;
+ print "\n";
return $args{'default'} if !$res && $args{'default'};
return $res;
}
@@ -609,13 +616,15 @@ sub print_rt_config {
my %args = @_;
my $config = <<END;
-Configure your RT via site config:
+You can now configure RT to use the newly-created full-text index by
+adding the following to your RT_SiteConfig.pm:
+
Set( %FullTextSearch,
- Enable => 1,
- Indexed => 1,
+ Enable => 1,
+ Indexed => 1,
END
- $config .= " $_ => '$args{$_}',\n"
+ $config .= sprintf(" %-9s => '$args{$_}',\n",$_)
foreach grep defined $args{$_}, keys %args;
$config .= ");\n";
@@ -623,11 +632,11 @@ END
}
sub insert_schema {
- return _insert_schema( $dbh, "Going to do the following change in the DB:", @_);
+ return _insert_schema( $dbh, "Going to run the following in the DB:", @_);
}
sub insert_schema_as_dba {
- return _insert_schema( dba_handle(), "Going to do the following change in the DB:", @_);
+ return _insert_schema( dba_handle(), "Going to run the following in the DB:", @_);
}
sub _insert_schema {
@@ -635,7 +644,7 @@ sub _insert_schema {
my $message = shift;
my $schema = shift;
print "$message\n";
- print $schema;
+ print " $schema\n\n";
return if $OPT{'dryrun'};
my $res = $dbh->do( $schema );
commit d351275efc28e26b7b6bfd9e5166bef3fddd935a
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 14:28:21 2011 -0400
Update index setup POD
diff --git a/sbin/rt-setup-fulltext-index.in b/sbin/rt-setup-fulltext-index.in
index 34d7952..3bc78f0 100644
--- a/sbin/rt-setup-fulltext-index.in
+++ b/sbin/rt-setup-fulltext-index.in
@@ -655,16 +655,15 @@ sub _insert_schema {
=head1 NAME
-rt-setup-fulltext-index - Helps create indexes for full text search
+rt-setup-fulltext-index - Create indexes for full text search
=head1 DESCRIPTION
-Full text search is very database specific. This script setups indexing
-for Oracle, Pg and mysql. Below you can find specifics for your current
-DB.
-
-At the end script prints piece that you should put into RT config to enable
-created index. Don't forget to restart the web-server after changing config.
+Full text indexes are very database specific; this script sets up
+indexing for Oracle, Pg and mysql; specifics for each are below. After
+creating the indexes, it will print a short section that should be
+inserted into your your RT_SiteConfig.pm to enable the index. You will
+need to restart the web-server after making those changes.
=head1 ORACLE
@@ -675,37 +674,39 @@ created index. Don't forget to restart the web-server after changing config.
=head2 DESCRIPTION
-Creates Oracle CONTEXT index on the Content column in the Attachments table.
-As well, it creates several preferences, functions and triggers to support
-this index.
+Creates an Oracle CONTEXT index on the Content column in the Attachments
+table. It also creates several preferences, functions and triggers to
+support this index.
-CONTEXT index needs periodic synchronization after any updates. Either use
-F<sbin/rt-fulltext-indexer> or read its documentation for alternatives.
+CONTEXT indexes needperiodic synchronization after any updates; either
+use F<sbin/rt-fulltext-indexer> via cron, or read its documentation for
+alternatives.
=head1 PG
=head2 USAGE
rt-setup-fulltext-index --help
- rt-setup-fulltext-index --dba postgres --dba-password 'secrete'
+ rt-setup-fulltext-index --dba postgres --dba-password 'secret'
=head2 DESCRIPTION
-Creates an additional column to store a ts_vector and then creates either
-GiN or GiST index on it. Use F<sbin/rt-fulltext-indexer> to keep index in
-sync.
+Creates an additional column to store a ts_vector, and then creates
+either a GiN or GiST index on it. Use F<sbin/rt-fulltext-indexer> via
+cron to keep the index in sync.
=head1 MYSQL
=head2 USAGE
rt-setup-fulltext-index --help
+ rt-setup-fulltext-index --dba root
=head2 DESCRIPTION
-Full text search in mysql implemented through Sphinx and MySQL storage
-engine (SphinxSE), so you need mysql compiled with it. Use
-F<sbin/rt-fulltext-indexer> to keep index in sync.
+Full text search in mysql is implemented through the Sphinx storage
+engine (SphinxSE), which your mysql must be compiled with support
+for. Use F<sbin/rt-fulltext-indexer> via cron to keep the index in sync.
=head1 AUTHOR
commit feb6d53a6c08b2c3a6f8d08267dab61f216dd3c6
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 18:44:41 2011 -0400
Update indexer POD
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index a12e1f6..8c9ed3f 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -120,15 +120,22 @@ if ( $OPT{'help'} ) {
my $fts_config = RT->Config->Get('FullTextSearch') || {};
unless ( $fts_config->{'Enable'} ) {
- print STDERR "Full text search disabled in the RT config."
- ." Read documentation for %FullTextSearch config option.";
+ print STDERR <<EOT;
+
+Full text search is disabled in your RT configuration. Run
+ at RT_SBIN_PATH_R@/rt-setup-fulltext-index to configure and enable it.
+
+EOT
exit 1;
}
unless ( $fts_config->{'Indexed'} ) {
- print STDERR "Full text search is enabled in the RT config,"
- ." however full text search works without special index,"
- ." so this tool is not required."
- ." Read documentation for %FullTextSearch config option.";
+ print STDERR <<EOT;
+
+Full text search is enabled in your RT configuration, but not with any
+full-text database indexing -- hence this tool is not required. Read
+the documentation for %FullTextSearch in your RT_Config for more details.
+
+EOT
exit 1;
}
@@ -404,10 +411,10 @@ rt-fulltext-indexer - Indexer for full text search
=head1 DESCRIPTION
-This is a helper script to keep full text index in sync with data.
-
-Start from F<sbin/rt-setup-fulltext-index>, adjust config and run
-this script via cron.
+This is a helper script to keep full text indexes in sync with data. It
+relies on indexes having been properly configured by
+F<sbin/rt-setup-fulltext-index> first. This script should be run
+occasionally via C<cron>.
=head1 ORACLE
@@ -419,18 +426,19 @@ this script via cron.
=head2 DESCRIPTION
-Basicly this script runs the following query:
+Basicly, this script simply runs the following query:
begin
ctx_ddl.sync_index('rt_fts_index', '2M');
end;
-Ammount of memory used for the sync can be controlled with --memory option.
+The ammount of memory used for the sync can be controlled with --memory option.
-There is way to do all of this on Oracle side by setting up a DBMS_JOB. Read
-"Managing DML Operations for a CONTEXT Index" chapter from "Text Application
-Developer's Guide" for more info on the topic and how to keep index optimized,
-collect garbage and other stuff.
+There is way to do all of this by setting up a DBMS_JOB; read the
+"Managing DML Operations for a CONTEXT Index" chapter from Oracle's
+"Text Application Developer's Guide" for more info on the topic, and
+details how to keep the index optimized, perform garbage collection, and
+other tasks.
=head1 PG
@@ -442,8 +450,9 @@ collect garbage and other stuff.
=head2 DESCRIPTION
-This script find attachments that should be indexed and process content if required
-and stores in the index. Use --limit option to specify how many attachments to process.
+This script finds attachments that should be indexed and stores them in
+the index. You can use the --limit option to specify how many
+attachments to process at once; defaults to 100.
=head1 MYSQL
@@ -455,11 +464,12 @@ and stores in the index. Use --limit option to specify how many attachments to p
=head2 DESCRIPTION
-This script find attachments that should be indexed and process content if required
-and generates xmlpipe2 document stream that can processed by sphinx. Read more in
-sphinx reference manual.
+This script finds attachments that should be indexed and prints an
+xmlpipe2 document stream that can processed by sphinx; the output should
+be piped into. For more details, see the sphinx reference manual.
-Use --limit option to specify how many attachments to process.
+You can use the --limit option to specify how many attachments to
+process at once; defaults to 100.
=cut
commit 11a447a11bdd7ae697e0f0d77e2f35c649db7e91
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 18:45:01 2011 -0400
Expand the configuration description of %FullTextSearch
diff --git a/etc/RT_Config.pm.in b/etc/RT_Config.pm.in
index 60e951b..b152325 100755
--- a/etc/RT_Config.pm.in
+++ b/etc/RT_Config.pm.in
@@ -1170,14 +1170,22 @@ Set($DefaultSelfServiceSearchResultFormat, qq{
=item C<%FullTextSearch>
-Full text search (FTS) without indexes is a very slow operation and by
-default is disabled at all. To enable FTS set key 'Enable' to 1.
-
-Setup of indexes and filling them with data requires additional steps
-that vary from DB to DB. Use F<sbin/rt-setup-fulltext-index> helper
-for quick start. This script creates required structures in the DB and
-gives some ideas on next steps. Use F<sbin/rt-fulltext-indexer> to
-keep the index in sync.
+Full text search (FTS) without database indexing is a very slow
+operation, and is thus disabled by default.
+
+To enable and configure database support for full-text indexing, run
+F<sbin/rt-setup-fulltext-index> and follow the prompts it presents.
+This script will create the necessary indexes and inform you how to
+set the %FullTextSearch configuration.
+
+You will also need to update the index by running
+F<sbin/rt-fulltext-indexer> to run at regular intervals; how frequently
+depends entirely on your workload.
+
+It is possible to enable FTS without database indexing support, simply
+by setting the C<Enable> key to 1, while leaving C<Indexed> set to 0.
+This is not generally suggested, as unindexed full-text searching can
+cause severe performance problems.
=cut
commit 8703f1c3246fab5315c78760f131808739a5bc3f
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:06:47 2011 -0400
Actually set the table and columns like users will when testing
diff --git a/t/fts/indexed_pg.t b/t/fts/indexed_pg.t
index 84cbaae..6310631 100644
--- a/t/fts/indexed_pg.t
+++ b/t/fts/indexed_pg.t
@@ -12,7 +12,7 @@ plan skip_all => "Need Pg 8.2 or higher; we have $major.$minor"
plan tests => 11;
-RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1 );
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Column => 'ContentIndex', Table => 'Attachments' );
setup_indexing();
commit 1c258cbacc756f405240a31a3f53973727fe570c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:07:43 2011 -0400
Don't do an extra join if our index is on the Attachments table (like we default to)
diff --git a/lib/RT/Tickets.pm b/lib/RT/Tickets.pm
index 6148f58..6689b7c 100644
--- a/lib/RT/Tickets.pm
+++ b/lib/RT/Tickets.pm
@@ -784,7 +784,7 @@ sub _TransContentLimit {
my $db_type = RT->Config->Get('DatabaseType');
my $alias;
- if ( $config->{'Table'} ) {
+ if ( $config->{'Table'} and $config->{'Table'} ne "Attachments") {
$alias = $self->{'_sql_aliases'}{'full_text'} ||= $self->_SQLJoin(
TYPE => 'LEFT',
ALIAS1 => $self->{'_sql_trattachalias'},
commit f619b0462883d3c5d2e331881e9c91be3fd2fe61
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:08:05 2011 -0400
Actually use the tsvector column and index that we added
Failing to do this means that we fail to actually use either our
existing tokenizations or our full-text index over them, causing
postgres to re-tokenize the content of every row in the process of doing
the search. This causes the search to be two orders of magnitude slower
than simply using LIKE.
It incidentally also skips the (not-yet-used, but stubbed in
rt-fulltext-indexer) step where we would cleverly downgrade text/html
attachments before tokenizing them.
diff --git a/lib/RT/Tickets.pm b/lib/RT/Tickets.pm
index 6689b7c..99a0199 100644
--- a/lib/RT/Tickets.pm
+++ b/lib/RT/Tickets.pm
@@ -825,7 +825,7 @@ sub _TransContentLimit {
$self->_SQLLimit(
%rest,
ALIAS => $alias,
- FIELD => $field,
+ FIELD => $config->{'Column'},
OPERATOR => '@@',
VALUE => 'plainto_tsquery('. $dbh->quote($value) .')',
QUOTEVALUE => 0,
commit 2a7bc4f0b3d16ebc4867e772f8833dded0e11f9c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:13:44 2011 -0400
lc(...) will always be ne 'Attachments'; remove the unnecessary and incorrect lc
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 8c9ed3f..03a645d 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -303,7 +303,7 @@ sub last_indexed_pg {
my $type = shift;
my $attachments = attachments( $type );
my $alias = 'main';
- if ( $fts_config->{'Table'} && lc $fts_config->{'Table'} ne 'Attachments' ) {
+ if ( $fts_config->{'Table'} && $fts_config->{'Table'} ne 'Attachments' ) {
$alias = $attachments->Join(
TYPE => 'left',
FIELD1 => 'id',
commit 0e6012143b4856359d79987b9537c5ae4d266b19
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:14:16 2011 -0400
The configuration key is 'Table', not 'table'; join to an existant table
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 03a645d..f051332 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -307,7 +307,7 @@ sub last_indexed_pg {
$alias = $attachments->Join(
TYPE => 'left',
FIELD1 => 'id',
- TABLE2 => $fts_config->{'table'},
+ TABLE2 => $fts_config->{'Table'},
FIELD2 => 'id',
);
}
commit fab3d7af082bae8b20e5cb62351029310f13b9e2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:14:51 2011 -0400
The content type field is ContentType, not content_type; attachments_html gets it right, however
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index f051332..6c73f41 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -349,7 +349,7 @@ sub process_pg {
sub attachments_text {
my $res = shift;
- $res->Limit( FIELD => 'content_type', VALUE => 'text/plain' );
+ $res->Limit( FIELD => 'ContentType', VALUE => 'text/plain' );
return $res;
}
commit 5bff1d032162d72b95ff765930c830ef1df243cf
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:16:01 2011 -0400
RT::Attachments->JoinTransactions finds transactions _on_ attachments, not vice versa
This caused the query to never return any rows, the previous commits not-withstanding.
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 6c73f41..cc57ad8 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -178,12 +178,22 @@ foreach my $type ( @types ) {
sub attachments {
my $type = shift;
my $res = RT::Attachments->new( RT->SystemUser );
-
- my $txn_alias = $res->JoinTransactions;
- $res->Limit( ALIAS => $txn_alias, FIELD => 'ObjectType', VALUE => 'RT::Ticket' );
+ my $txn_alias = $res->Join(
+ ALIAS1 => 'main',
+ FIELD1 => 'TransactionId',
+ TABLE2 => 'Transactions',
+ FIELD2 => 'id',
+ );
+ $res->Limit(
+ ALIAS => $txn_alias,
+ FIELD => 'ObjectType',
+ VALUE => 'RT::Ticket',
+ );
my $ticket_alias = $res->Join(
- ALIAS1 => $txn_alias, FIELD1 => 'ObjectId',
- TABLE2 => 'Tickets', FIELD2 => 'id'
+ ALIAS1 => $txn_alias,
+ FIELD1 => 'ObjectId',
+ TABLE2 => 'Tickets',
+ FIELD2 => 'id',
);
$res->Limit(
ALIAS => $ticket_alias,
commit c967e2dca96c50c354b9047c23944e3d63870fda
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:19:47 2011 -0400
The localization sub _ was not imported into the script
Skip the localization for now as there are no existing translations for
the strings, and they interpolate variables without placeholders
anyways.
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index cc57ad8..2ca65a6 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -410,8 +410,8 @@ sub goto_specific {
# helper functions
-sub verbose { print _(@_), "\n" if $OPT{verbose} || $OPT{verbose}; 1 }
-sub debug { print _(@_), "\n" if $OPT{debug}; 1 }
+sub verbose { print @_, "\n" if $OPT{verbose} || $OPT{verbose}; 1 }
+sub debug { print @_, "\n" if $OPT{debug}; 1 }
sub error { $RT::Logger->error(_(@_)); verbose(@_); 1 }
sub warning { $RT::Logger->warn(_(@_)); verbose(@_); 1 }
commit cf6ca2967e5d325cb4064fa868d5b2effcb3824b
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:23:16 2011 -0400
Accept the --debug flag which is respected later in the code
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 2ca65a6..6d17c00 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -83,8 +83,9 @@ use RT::Interface::CLI ();
my %OPT = (
help => 0,
+ debug => 0,
);
-my @OPT_LIST = qw(help|h!);
+my @OPT_LIST = qw(help|h! debug!);
my $db_type = RT->Config->Get('DatabaseType');
if ( $db_type eq 'Pg' ) {
commit 0a028a14c1e06959da61fce716549b1851787c72
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:26:10 2011 -0400
Remove some debugging output
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 6d17c00..41230cd 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -162,12 +162,9 @@ foreach my $type ( @types ) {
my $found = 0;
while ( my $a = $attaches->Next ) {
- print "bla\n";
- debug("Found attachment #". $a->id );
next if filter( $type, $a );
- debug("Attachment #". $a->id ." hasn't been filtered" );
+ debug("Found attachment #". $a->id );
my $txt = extract($type, $a) or next;
- debug("Extracted text from attachment #". $a->id );
$found++;
process( $type, $a, $txt );
debug("Processed attachment #". $a->id );
commit 6040de1c294fe72c34aacce55832d337f0cb4abc
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Wed Apr 27 21:29:31 2011 -0400
Rename attachment collection to be a little more grammatical
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 41230cd..9e76c00 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -151,17 +151,17 @@ if ( $db_type eq 'Oracle' ) {
my @types = qw(text html);
foreach my $type ( @types ) {
- my $attaches = attachments($type);
- $attaches->Limit(
+ my $attachments = attachments($type);
+ $attachments->Limit(
FIELD => 'id',
OPERATOR => '>',
VALUE => last_indexed($type)
);
- $attaches->OrderBy( FIELD => 'id', ORDER => 'asc' );
- $attaches->RowsPerPage( $OPT{'limit'} || 100 );
+ $attachments->OrderBy( FIELD => 'id', ORDER => 'asc' );
+ $attachments->RowsPerPage( $OPT{'limit'} || 100 );
my $found = 0;
- while ( my $a = $attaches->Next ) {
+ while ( my $a = $attachments->Next ) {
next if filter( $type, $a );
debug("Found attachment #". $a->id );
my $txt = extract($type, $a) or next;
@@ -169,7 +169,7 @@ foreach my $type ( @types ) {
process( $type, $a, $txt );
debug("Processed attachment #". $a->id );
}
- finalize( $type, $attaches ) if $found;
+ finalize( $type, $attachments ) if $found;
clean( $type );
}
commit bce3688d864a83f9bf51642770fa6f09b488ed7d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Thu Apr 28 12:21:19 2011 -0400
We only need to fetch the one last indexed row
diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 9e76c00..758b14a 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -326,6 +326,7 @@ sub last_indexed_pg {
VALUE => 'NULL',
);
$attachments->OrderBy( FIELD => 'id', ORDER => 'desc' );
+ $attachments->RowsPerPage( 1 );
my $res = $attachments->First;
return 0 unless $res;
return $res->id;
-----------------------------------------------------------------------
More information about the Rt-commit
mailing list