[Bps-public-commit] dbix-searchbuilder branch combine-search-and-count created. 1.71-5-g7d7844b
BPS Git Server
git at git.bestpractical.com
Fri Aug 26 17:32:02 UTC 2022
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "dbix-searchbuilder".
The branch, combine-search-and-count has been created
at 7d7844b2713da76eb3564c645e24b9cd74832163 (commit)
- Log -----------------------------------------------------------------
commit 7d7844b2713da76eb3564c645e24b9cd74832163
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Aug 26 22:43:41 2022 +0800
Test combined SQL that does searching and counting in a single query
diff --git a/t/03searches_combine.t b/t/03searches_combine.t
new file mode 100644
index 0000000..25c06e6
--- /dev/null
+++ b/t/03searches_combine.t
@@ -0,0 +1,339 @@
+use strict;
+use Test::More;
+
+BEGIN { require "./t/utils.pl" }
+our (@AvailableDrivers);
+
+use constant TESTS_PER_DRIVER => 14;
+
+my $total = scalar(@AvailableDrivers) * TESTS_PER_DRIVER;
+plan tests => $total;
+
+foreach my $d (@AvailableDrivers) {
+SKIP: {
+ unless ( has_schema( 'TestApp', $d ) ) {
+ skip "No schema for '$d' driver", TESTS_PER_DRIVER;
+ }
+ unless ( should_test($d) ) {
+ skip "ENV is not defined for driver '$d'", TESTS_PER_DRIVER;
+ }
+
+ if ( $d eq 'SQLite' && version->parse($DBD::SQLite::VERSION) <= 1.6 ) {
+ skip "Require DBD::SQLite 1.60+ to enable combine search", TESTS_PER_DRIVER;
+ }
+
+ my $handle = get_handle($d);
+ connect_handle($handle);
+ isa_ok( $handle->dbh, 'DBI::db' );
+
+ my $ret = init_schema( 'TestApp', $handle );
+ isa_ok( $ret, 'DBI::st', "Inserted the schema. got a statement handle back" );
+
+ my $count_users = init_data( 'TestApp::User', $handle );
+ ok( $count_users, "init users data" );
+ my $count_groups = init_data( 'TestApp::Group', $handle );
+ ok( $count_groups, "init groups data" );
+ my $count_us2gs = init_data( 'TestApp::UsersToGroup', $handle );
+ ok( $count_us2gs, "init users&groups relations data" );
+
+ my $users_obj = TestApp::Users->new($handle);
+ $users_obj->CombineSearchAndCount(1);
+
+ $users_obj->Limit( FIELD => 'Login', VALUE => 'Gandalf' );
+ is( $users_obj->Count, 1, 'only one value' );
+ is( $users_obj->CountAll, 1, 'only one value' );
+ is( $users_obj->First->Login, 'Gandalf', 'found Gandalf' );
+
+ $users_obj->CleanSlate;
+ $users_obj->CombineSearchAndCount(1);
+ $users_obj->Limit(
+ FIELD => 'Login',
+ VALUE => [ "Bilbo\\Baggins", "Baggins' Frodo" ],
+ OPERATOR => 'IN',
+ );
+ $users_obj->RowsPerPage(1);
+ is( $users_obj->Count, 1, '1 value' );
+ is( $users_obj->CountAll, 2, '2 total values' );
+ $users_obj->OrderByCols( { FIELD => 'Login' } );
+ is_deeply(
+ [ map { $_->Login } @{ $users_obj->ItemsArrayRef } ],
+ [ "Baggins' Frodo" ],
+ '1 Baggin',
+ );
+
+ $users_obj->CleanSlate;
+ $users_obj->CombineSearchAndCount(1);
+ $users_obj->OrderByCols( { FIELD => 'Login' } );
+
+ my $alias = $users_obj->Join(
+ FIELD1 => 'id',
+ TABLE2 => 'UsersToGroups',
+ FIELD2 => 'UserId'
+ );
+
+ my $group_alias = $users_obj->Join(
+ ALIAS1 => $alias,
+ FIELD1 => 'GroupID',
+ ALIAS2 => $users_obj->NewAlias('Groups'),
+ FIELD2 => 'id'
+ );
+ $users_obj->Limit(
+ LEFTJOIN => $group_alias,
+ FIELD => 'Name',
+ VALUE => "Shire's Bag End",
+ );
+
+ is( $users_obj->CountAll, 2, "2 total values" );
+ is( $users_obj->Count, 2, "2 values" );
+ is_deeply(
+ [ sort map { $_->Login } @{ $users_obj->ItemsArrayRef } ],
+ [ "Baggins' Frodo", "Bilbo\\Baggins" ],
+ '2 Baggins',
+ );
+
+ cleanup_schema( 'TestApp', $handle );
+ }
+} # SKIP, foreach blocks
+
+1;
+
+package TestApp;
+
+sub schema_sqlite {
+ [ q{
+CREATE TABLE Users (
+ id integer primary key,
+ Login varchar(36)
+) },
+ q{
+CREATE TABLE UsersToGroups (
+ id integer primary key,
+ UserId integer,
+ GroupId integer
+) },
+ q{
+CREATE TABLE Groups (
+ id integer primary key,
+ Name varchar(36)
+) },
+ ]
+}
+
+sub schema_mysql {
+ [ q{
+CREATE TEMPORARY TABLE Users (
+ id integer primary key AUTO_INCREMENT,
+ Login varchar(36)
+) },
+ q{
+CREATE TEMPORARY TABLE UsersToGroups (
+ id integer primary key AUTO_INCREMENT,
+ UserId integer,
+ GroupId integer
+) },
+ q{
+CREATE TEMPORARY TABLE `Groups` (
+ id integer primary key AUTO_INCREMENT,
+ Name varchar(36)
+) },
+ ]
+}
+
+sub schema_pg {
+ [ q{
+CREATE TEMPORARY TABLE Users (
+ id serial primary key,
+ Login varchar(36)
+) },
+ q{
+CREATE TEMPORARY TABLE UsersToGroups (
+ id serial primary key,
+ UserId integer,
+ GroupId integer
+) },
+ q{
+CREATE TEMPORARY TABLE Groups (
+ id serial primary key,
+ Name varchar(36)
+) },
+ ]
+}
+
+sub schema_oracle {
+ [ "CREATE SEQUENCE Users_seq",
+ "CREATE TABLE Users (
+ id integer CONSTRAINT Users_Key PRIMARY KEY,
+ Login varchar(36)
+ )",
+ "CREATE SEQUENCE UsersToGroups_seq",
+ "CREATE TABLE UsersToGroups (
+ id integer CONSTRAINT UsersToGroups_Key PRIMARY KEY,
+ UserId integer,
+ GroupId integer
+ )",
+ "CREATE SEQUENCE Groups_seq",
+ "CREATE TABLE Groups (
+ id integer CONSTRAINT Groups_Key PRIMARY KEY,
+ Name varchar(36)
+ )",
+ ]
+}
+
+sub cleanup_schema_oracle {
+ [ "DROP SEQUENCE Users_seq",
+ "DROP TABLE Users",
+ "DROP SEQUENCE Groups_seq",
+ "DROP TABLE Groups",
+ "DROP SEQUENCE UsersToGroups_seq",
+ "DROP TABLE UsersToGroups",
+ ]
+}
+
+package TestApp::User;
+
+use base $ENV{SB_TEST_CACHABLE}
+ ? qw/DBIx::SearchBuilder::Record::Cachable/
+ : qw/DBIx::SearchBuilder::Record/;
+
+sub _Init {
+ my $self = shift;
+ my $handle = shift;
+ $self->Table('Users');
+ $self->_Handle($handle);
+}
+
+sub _ClassAccessible {
+ {
+
+ id => { read => 1, type => 'int(11)' },
+ Login => { read => 1, write => 1, type => 'varchar(36)' },
+
+ }
+}
+
+sub init_data {
+ return (
+ ['Login'],
+
+ ['Gandalf'],
+ ["Bilbo\\Baggins"],
+ ["Baggins' Frodo"],
+ );
+}
+
+package TestApp::SearchBuilder;
+use base qw/DBIx::SearchBuilder/;
+
+sub BuildSelectQuery { die 'should not be called' }
+sub BuildSelectCountQuery { die 'should not be called' }
+
+1;
+
+package TestApp::Users;
+
+use base qw/TestApp::SearchBuilder/;
+
+sub _Init {
+ my $self = shift;
+ $self->SUPER::_Init( Handle => shift );
+ $self->Table('Users');
+}
+
+sub NewItem {
+ my $self = shift;
+ return TestApp::User->new( $self->_Handle );
+}
+
+1;
+
+package TestApp::Group;
+
+use base $ENV{SB_TEST_CACHABLE}
+ ? qw/DBIx::SearchBuilder::Record::Cachable/
+ : qw/DBIx::SearchBuilder::Record/;
+
+sub _Init {
+ my $self = shift;
+ my $handle = shift;
+ $self->Table('Groups');
+ $self->_Handle($handle);
+}
+
+sub _ClassAccessible {
+ { id => { read => 1, type => 'int(11)' },
+ Name => { read => 1, write => 1, type => 'varchar(36)' },
+ }
+}
+
+sub init_data {
+ return (
+ ['Name'],
+
+ ["Shire's Bag End"],
+ ['The Fellowship of the Ring'],
+ );
+}
+
+package TestApp::Groups;
+
+use base qw/TestApp::SearchBuilder/;
+
+sub _Init {
+ my $self = shift;
+ $self->SUPER::_Init( Handle => shift );
+ $self->Table('Groups');
+}
+
+sub NewItem { return TestApp::Group->new( (shift)->_Handle ) }
+
+1;
+
+package TestApp::UsersToGroup;
+
+use base $ENV{SB_TEST_CACHABLE}
+ ? qw/DBIx::SearchBuilder::Record::Cachable/
+ : qw/DBIx::SearchBuilder::Record/;
+
+sub _Init {
+ my $self = shift;
+ my $handle = shift;
+ $self->Table('UsersToGroups');
+ $self->_Handle($handle);
+}
+
+sub _ClassAccessible {
+ {
+
+ id => { read => 1, type => 'int(11)' },
+ UserId => { read => 1, type => 'int(11)' },
+ GroupId => { read => 1, type => 'int(11)' },
+ }
+}
+
+sub init_data {
+ return (
+ [ 'GroupId', 'UserId' ],
+
+ # Shire
+ [ 1, 2 ],
+ [ 1, 3 ],
+
+ # Fellowship of the Ring
+ [ 2, 1 ],
+ [ 2, 3 ],
+ );
+}
+
+package TestApp::UsersToGroups;
+
+use base qw/TestApp::SearchBuilder/;
+
+sub _Init {
+ my $self = shift;
+ $self->Table('UsersToGroups');
+ return $self->SUPER::_Init( Handle => shift );
+}
+
+sub NewItem { return TestApp::UsersToGroup->new( (shift)->_Handle ) }
+
+1;
commit 4b01deb8876393528c5d4d38098a488077b198a8
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Aug 23 05:40:29 2022 +0800
Support to search and count in a single query
As an optional feature, user needs to call CombineSearchAndCount(1) to
enable it.
diff --git a/lib/DBIx/SearchBuilder.pm b/lib/DBIx/SearchBuilder.pm
index 2c878a6..0fb6e1f 100755
--- a/lib/DBIx/SearchBuilder.pm
+++ b/lib/DBIx/SearchBuilder.pm
@@ -153,6 +153,7 @@ sub CleanSlate {
query_hint
_bind_values
_prefer_bind
+ _combine_search_and_count
);
#we have no limit statements. DoSearch won't work.
@@ -234,19 +235,34 @@ it is called automatically the first time that you actually need results
sub _DoSearch {
my $self = shift;
+ if ( $self->{_combine_search_and_count} ) {
+ my ($count) = $self->_DoSearchAndCount;
+ return $count;
+ }
+
my $QueryString = $self->BuildSelectQuery();
+ my $records = $self->_Handle->SimpleQuery( $QueryString, @{ $self->{_bind_values} || [] } );
+ return $self->__DoSearch($records);
+}
+
+sub __DoSearch {
+ my $self = shift;
+ my $records = shift;
# If we're about to redo the search, we need an empty set of items and a reset iterator
delete $self->{'items'};
$self->{'itemscount'} = 0;
- my $records = $self->_Handle->SimpleQuery( $QueryString, @{ $self->{_bind_values} || [] } );
return 0 unless $records;
while ( my $row = $records->fetchrow_hashref() ) {
- my $item = $self->NewItem();
- $item->LoadFromHash($row);
- $self->AddRecord($item);
+ # search_builder_count_all is from combine search
+ if ( !$self->{count_all} && $row->{search_builder_count_all} ) {
+ $self->{count_all} = $row->{search_builder_count_all};
+ }
+ my $item = $self->NewItem();
+ $item->LoadFromHash($row);
+ $self->AddRecord($item);
}
return $self->_RecordCount if $records->err;
@@ -294,6 +310,11 @@ it is used by C<Count> and C<CountAll>.
sub _DoCount {
my $self = shift;
+ if ( $self->{_combine_search_and_count} ) {
+ (undef, my $count_all) = $self->_DoSearchAndCount;
+ return $count_all;
+ }
+
my $QueryString = $self->BuildSelectCountQuery();
my $records = $self->_Handle->SimpleQuery( $QueryString, @{ $self->{_bind_values} || [] } );
return 0 unless $records;
@@ -306,7 +327,23 @@ sub _DoCount {
return ( $row[0] );
}
+=head2 _DoSearchAndCount
+
+This internal private method actually executes the search and also counting on the database;
+
+=cut
+sub _DoSearchAndCount {
+ my $self = shift;
+
+ my $QueryString = $self->BuildSelectAndCountQuery();
+ my $records = $self->_Handle->SimpleQuery( $QueryString, @{ $self->{_bind_values} || [] } );
+
+ $self->{count_all} = 0;
+ # __DoSearch updates count_all
+ my $count = $self->__DoSearch($records);
+ return ( $count, $self->{count_all} );
+}
=head2 _ApplyLimits STATEMENTREF
@@ -344,6 +381,21 @@ sub _DistinctQuery {
$self->_Handle->DistinctQuery($statementref, $self)
}
+=head2 _DistinctQueryAndCount STATEMENTREF
+
+This routine takes a reference to a scalar containing an SQL statement.
+It massages the statement to ensure a distinct result set and total number
+of potential records are returned.
+
+=cut
+
+sub _DistinctQueryAndCount {
+ my $self = shift;
+ my $statementref = shift;
+
+ $self->_Handle->DistinctQueryAndCount($statementref, $self);
+}
+
=head2 _BuildJoins
Build up all of the joins we need to perform this query.
@@ -504,7 +556,43 @@ sub BuildSelectCountQuery {
return ($QueryString);
}
+=head2 BuildSelectAndCountQuery PreferBind => 1|0
+
+Builds a query string that is a combination of BuildSelectQuery and
+BuildSelectCountQuery.
+
+=cut
+sub BuildSelectAndCountQuery {
+ my $self = shift;
+
+ # Generally it's BuildSelectQuery plus extra COUNT part.
+ my $QueryString = $self->_BuildJoins . " ";
+ $QueryString .= $self->_WhereClause . " "
+ if ( $self->_isLimited > 0 );
+
+ $self->_OptimizeQuery( \$QueryString, @_ );
+
+ my $QueryHint = $self->QueryHintFormatted;
+
+ if ( my $clause = $self->_GroupClause ) {
+ $QueryString
+ = "SELECT" . $QueryHint . "main.*, COUNT(main.id) OVER() AS search_builder_count_all FROM $QueryString";
+ $QueryString .= $clause;
+ $QueryString .= $self->_OrderClause;
+ }
+ elsif ( !$self->{'joins_are_distinct'} && $self->_isJoined ) {
+ $self->_DistinctQueryAndCount( \$QueryString );
+ }
+ else {
+ $QueryString
+ = "SELECT" . $QueryHint . "main.*, COUNT(main.id) OVER() AS search_builder_count_all FROM $QueryString";
+ $QueryString .= $self->_OrderClause;
+ }
+
+ $self->_ApplyLimits( \$QueryString );
+ return ($QueryString);
+}
=head2 Next
@@ -710,7 +798,20 @@ sub RedoSearch {
$self->{'must_redo_search'} = 1;
}
+=head2 CombineSearchAndCount 1|0
+Tells DBIx::SearchBuilder if it shall search both records and the total count
+in a single query.
+
+=cut
+
+sub CombineSearchAndCount {
+ my $self = shift;
+ if ( @_ ) {
+ $self->{'_combine_search_and_count'} = shift;
+ }
+ return $self->{'_combine_search_and_count'};
+}
=head2 UnLimit
diff --git a/lib/DBIx/SearchBuilder/Handle.pm b/lib/DBIx/SearchBuilder/Handle.pm
index d08faa5..b18d9b9 100755
--- a/lib/DBIx/SearchBuilder/Handle.pm
+++ b/lib/DBIx/SearchBuilder/Handle.pm
@@ -1457,6 +1457,31 @@ sub DistinctQuery {
$$statementref .= $sb->_OrderClause;
}
+=head2 DistinctQueryAndCount STATEMENTREF
+
+takes an incomplete SQL SELECT statement and massages it to return a
+DISTINCT result set and the total count of potential records.
+
+=cut
+
+sub DistinctQueryAndCount {
+ my $self = shift;
+ my $statementref = shift;
+ my $sb = shift;
+
+ $self->DistinctQuery($statementref, $sb);
+
+ # Add the count part.
+ if ( $sb->_OrderClause !~ /(?<!main)\./ ) {
+ # Wrap it with another SELECT to get distinct count.
+ $$statementref
+ = 'SELECT main.*, COUNT(main.id) OVER() AS search_builder_count_all FROM (' . $$statementref . ') main';
+ }
+ else {
+ # if order by other tables, then DistinctQuery already has an outer SELECT, which we can reuse
+ $$statementref =~ s!(?= FROM)!, COUNT(main.id) OVER() AS search_builder_count_all!;
+ }
+}
commit 5d263298ec784fc10dae02c1fe2a4b3f842f1e5a
Author: sunnavy <sunnavy at bestpractical.com>
Date: Fri Aug 26 10:28:22 2022 +0800
Test Count with RowsPerPage set before searches
diff --git a/t/01searches.t b/t/01searches.t
index 1276544..6bbd273 100644
--- a/t/01searches.t
+++ b/t/01searches.t
@@ -7,7 +7,7 @@ use Test::More;
BEGIN { require "./t/utils.pl" }
our (@AvailableDrivers);
-use constant TESTS_PER_DRIVER => 150;
+use constant TESTS_PER_DRIVER => 151;
my $total = scalar(@AvailableDrivers) * TESTS_PER_DRIVER;
plan tests => $total;
@@ -313,6 +313,7 @@ SKIP: {
$users_obj->UnLimit;
$users_obj->OrderBy(FIELD => 'Login');
$users_obj->RowsPerPage(2);
+ is($users_obj->Count, 2, 'user count on first page' );
{
my %seen;
my $saw_on_page = 0;
commit 53d5b36b7fcb771b062de4e04f5741477aa4fac0
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Aug 23 08:31:31 2022 +0800
Fix Count method to always returns count in selected page
Previously if Count was called first(before search), it actually
returned total count and ignored RowsPerPage setting, which is wrong.
diff --git a/lib/DBIx/SearchBuilder.pm b/lib/DBIx/SearchBuilder.pm
index da7209a..2c878a6 100755
--- a/lib/DBIx/SearchBuilder.pm
+++ b/lib/DBIx/SearchBuilder.pm
@@ -1484,23 +1484,17 @@ sub Count {
# An unlimited search returns no tickets
return 0 unless ($self->_isLimited);
-
- # If we haven't actually got all objects loaded in memory, we
- # really just want to do a quick count from the database.
if ( $self->{'must_redo_search'} ) {
-
- # If we haven't already asked the database for the row count, do that
- $self->_DoCount unless ( $self->{'count_all'} );
-
- #Report back the raw # of rows in the database
- return ( $self->{'count_all'} );
+ if ( $self->RowsPerPage ) {
+ $self->_DoSearch;
+ }
+ else {
+ # No RowsPerPage means Count == CountAll
+ return $self->CountAll;
+ }
}
- # If we have loaded everything from the DB we have an
- # accurate count already.
- else {
- return $self->_RecordCount;
- }
+ return $self->_RecordCount;
}
commit cbc4709894bf235e02968e81e7f93ccf65e135c1
Author: sunnavy <sunnavy at bestpractical.com>
Date: Tue Aug 23 01:35:51 2022 +0800
Simplify count's internal logic to always use the "count_all" key
Both "raw_rows" and "count_all" are the count of *all* matched rows,
there is no need to use 2 keys for it.
diff --git a/lib/DBIx/SearchBuilder.pm b/lib/DBIx/SearchBuilder.pm
index 9d72e32..da7209a 100755
--- a/lib/DBIx/SearchBuilder.pm
+++ b/lib/DBIx/SearchBuilder.pm
@@ -143,7 +143,6 @@ sub CleanSlate {
delete $self->{$_} for qw(
items
left_joins
- raw_rows
count_all
subclauses
restrictions
@@ -294,7 +293,6 @@ it is used by C<Count> and C<CountAll>.
sub _DoCount {
my $self = shift;
- my $all = shift || 0;
my $QueryString = $self->BuildSelectCountQuery();
my $records = $self->_Handle->SimpleQuery( $QueryString, @{ $self->{_bind_values} || [] } );
@@ -303,7 +301,7 @@ sub _DoCount {
my @row = $records->fetchrow_array();
return 0 if $records->err;
- $self->{ $all ? 'count_all' : 'raw_rows' } = $row[0];
+ $self->{'count_all'} = $row[0];
return ( $row[0] );
}
@@ -1492,10 +1490,10 @@ sub Count {
if ( $self->{'must_redo_search'} ) {
# If we haven't already asked the database for the row count, do that
- $self->_DoCount unless ( $self->{'raw_rows'} );
+ $self->_DoCount unless ( $self->{'count_all'} );
#Report back the raw # of rows in the database
- return ( $self->{'raw_rows'} );
+ return ( $self->{'count_all'} );
}
# If we have loaded everything from the DB we have an
@@ -1546,7 +1544,7 @@ sub CountAll {
# or if we have paging enabled then we count as well and store it in count_all
if ( $self->{'must_redo_search'} || ( $self->RowsPerPage && !$self->{'count_all'} ) ) {
# If we haven't already asked the database for the row count, do that
- $self->_DoCount(1);
+ $self->_DoCount;
#Report back the raw # of rows in the database
return ( $self->{'count_all'} );
-----------------------------------------------------------------------
hooks/post-receive
--
dbix-searchbuilder
More information about the Bps-public-commit
mailing list