[Bps-public-commit] smokingit branch, full-tap, updated. fe8393bce0bde6a585aa9b584a5ab643c65266c4

Alex Vandiver alexmv at bestpractical.com
Mon Jul 22 22:56:00 EDT 2013


The branch, full-tap has been updated
       via  fe8393bce0bde6a585aa9b584a5ab643c65266c4 (commit)
       via  198bb13b1a2ba16699910bda67465f6ad7a58e62 (commit)
       via  72b30d25c063509059a187bfa2fb333b9806948d (commit)
       via  f20cffa5d236edbf88b7b4f8819fed2517f548d4 (commit)
       via  349e0782b07f559ea50a3c77d359445ad3074b35 (commit)
       via  2836a8bc3053ca32faf1ef0d05780ba79e4b6e86 (commit)
      from  4cfca5e2cecfd31c051ddb68adedd9f834755e4f (commit)

Summary of changes:
 lib/Smokingit/Upgrade.pm     | 79 +++++++++++++++++++++++++++-----------------
 lib/Smokingit/View/Commit.pm | 30 +++++++++--------
 2 files changed, 64 insertions(+), 45 deletions(-)

- Log -----------------------------------------------------------------
commit 2836a8bc3053ca32faf1ef0d05780ba79e4b6e86
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 22 02:32:12 2013 -0400

    Switch to batching the upgrade, soas to not attempt to fit everything in memory

diff --git a/lib/Smokingit/Upgrade.pm b/lib/Smokingit/Upgrade.pm
index f56a6df..1941b84 100644
--- a/lib/Smokingit/Upgrade.pm
+++ b/lib/Smokingit/Upgrade.pm
@@ -38,42 +38,53 @@ since '0.0.8' => sub {
     # Time elapsed became a float
     Jifty->handle->simple_query("ALTER TABLE smoke_results ALTER COLUMN elapsed TYPE float");
 
-    # Go through and inflate all of the old aggregators into test result rows
-    my $tests = Smokingit::Model::SmokeResultCollection->new( current_user => $super );
-    $tests->unlimit;
-    $tests->columns( "id", "aggregator" );
     require TAP::Parser;
     require TAP::Parser::Aggregator;
-    while (my $test = $tests->next) {
-        my $aggregator = $test->_value('aggregator');
-        for my $filename ($aggregator->descriptions) {
-            my ($parser) = $aggregator->parsers($filename);
 
-            my $tap = "";
-            if ($parser->skip_all) {
-                $tap = "1..0 # skipped\n";
-            } else {
-                $tap = $parser->plan . "\n";
-                my %lines;
-                $lines{$_}   = "ok $_ # skip" for $parser->skipped;
-                $lines{$_} ||= "ok $_ # TODO" for $parser->todo_passed;
-                $lines{$_} ||= "not ok $_ # TODO" for $parser->todo;
-                $lines{$_} ||= "ok $_" for $parser->actual_passed;
-                $lines{$_} ||= "not ok $_" for $parser->actual_failed;
-                $tap .= "$lines{$_}\n" for sort {$a <=> $b} keys %lines;
-            }
+    # Go through and inflate all of the old aggregators into test result
+    # rows; do this in batches of 100, to save on memory.
+    my $max = 0;
+    do {
+        my $tests = Smokingit::Model::SmokeResultCollection->new( current_user => $super );
+        $tests->limit( column => "id", operator => ">", value => $max );
+        $tests->order_by( { column => "id", order  => "asc" } );
+        $tests->rows_per_page(100);
+        $tests->columns( "id", "aggregator" );
+        $max = 0;
+        while (my $test = $tests->next) {
+            $max = $test->id;
+            warn "$max\n";
+            my $aggregator = $test->_value('aggregator');
+            next unless $aggregator;
+            for my $filename ($aggregator->descriptions) {
+                my ($parser) = $aggregator->parsers($filename);
 
-            my $filetest = Smokingit::Model::SmokeFileResult->new( current_user => $super );
-            $filetest->create(
-                smoke_result_id => $test->id,
-                filename        => $filename,
-                elapsed         => ($parser->end_time - $parser->start_time),
-                is_ok           => !$parser->has_problems,
-                tests_run       => $parser->tests_run,
-                raw_tap         => $tap,
-            );
+                my $tap = "";
+                if ($parser->skip_all) {
+                    $tap = "1..0 # skipped\n";
+                } else {
+                    $tap = $parser->plan . "\n";
+                    my %lines;
+                    $lines{$_}   = "ok $_ # skip" for $parser->skipped;
+                    $lines{$_} ||= "ok $_ # TODO" for $parser->todo_passed;
+                    $lines{$_} ||= "not ok $_ # TODO" for $parser->todo;
+                    $lines{$_} ||= "ok $_" for $parser->actual_passed;
+                    $lines{$_} ||= "not ok $_" for $parser->actual_failed;
+                    $tap .= "$lines{$_}\n" for sort {$a <=> $b} keys %lines;
+                }
+
+                my $filetest = Smokingit::Model::SmokeFileResult->new( current_user => $super );
+                $filetest->create(
+                    smoke_result_id => $test->id,
+                    filename        => $filename,
+                    elapsed         => ($parser->end_time - $parser->start_time),
+                    is_ok           => !$parser->has_problems,
+                    tests_run       => $parser->tests_run,
+                    raw_tap         => $tap,
+                );
+            }
         }
-    }
+    } while ($max);
 };
 
 1;

commit 349e0782b07f559ea50a3c77d359445ad3074b35
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 22 12:45:24 2013 -0400

    Use DBI-level calls because Jifty adds too much overhead for this tight loop

diff --git a/lib/Smokingit/Upgrade.pm b/lib/Smokingit/Upgrade.pm
index 1941b84..1716bc1 100644
--- a/lib/Smokingit/Upgrade.pm
+++ b/lib/Smokingit/Upgrade.pm
@@ -41,6 +41,13 @@ since '0.0.8' => sub {
     require TAP::Parser;
     require TAP::Parser::Aggregator;
 
+    # We use low-level DBI calls to speed up the creation
+    my $table = Smokingit::Model::SmokeFileResultCollection->new( current_user => $super )->table;
+    my $sth = Jifty->handle->dbh->prepare(
+        "INSERT INTO $table (smoke_result_id, filename, elapsed, is_ok, tests_run, raw_tap) "
+        ."VALUES (?,?,?,?,?,?)"
+    );
+
     # Go through and inflate all of the old aggregators into test result
     # rows; do this in batches of 100, to save on memory.
     my $max = 0;
@@ -73,14 +80,13 @@ since '0.0.8' => sub {
                     $tap .= "$lines{$_}\n" for sort {$a <=> $b} keys %lines;
                 }
 
-                my $filetest = Smokingit::Model::SmokeFileResult->new( current_user => $super );
-                $filetest->create(
-                    smoke_result_id => $test->id,
-                    filename        => $filename,
-                    elapsed         => ($parser->end_time - $parser->start_time),
-                    is_ok           => !$parser->has_problems,
-                    tests_run       => $parser->tests_run,
-                    raw_tap         => $tap,
+                $sth->execute(
+                    $test->id,
+                    $filename,
+                    ($parser->end_time - $parser->start_time),
+                    ($parser->has_problems ? 'f' : 't'),
+                    $parser->tests_run,
+                    $tap,
                 );
             }
         }

commit f20cffa5d236edbf88b7b4f8819fed2517f548d4
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 22 12:45:47 2013 -0400

    Micro-optimize TAP line generation

diff --git a/lib/Smokingit/Upgrade.pm b/lib/Smokingit/Upgrade.pm
index 1716bc1..cd07478 100644
--- a/lib/Smokingit/Upgrade.pm
+++ b/lib/Smokingit/Upgrade.pm
@@ -71,13 +71,13 @@ since '0.0.8' => sub {
                     $tap = "1..0 # skipped\n";
                 } else {
                     $tap = $parser->plan . "\n";
-                    my %lines;
-                    $lines{$_}   = "ok $_ # skip" for $parser->skipped;
-                    $lines{$_} ||= "ok $_ # TODO" for $parser->todo_passed;
-                    $lines{$_} ||= "not ok $_ # TODO" for $parser->todo;
-                    $lines{$_} ||= "ok $_" for $parser->actual_passed;
-                    $lines{$_} ||= "not ok $_" for $parser->actual_failed;
-                    $tap .= "$lines{$_}\n" for sort {$a <=> $b} keys %lines;
+                    my @lines;
+                    $lines[$_]   = "ok $_ # skip" for $parser->skipped;
+                    $lines[$_] ||= "ok $_ # TODO" for $parser->todo_passed;
+                    $lines[$_] ||= "not ok $_ # TODO" for $parser->todo;
+                    $lines[$_] ||= "ok $_" for $parser->actual_passed;
+                    $lines[$_] ||= "not ok $_" for $parser->actual_failed;
+                    $tap .= "$_\n" for grep {defined} @lines;
                 }
 
                 $sth->execute(

commit 72b30d25c063509059a187bfa2fb333b9806948d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 22 18:57:28 2013 -0400

    Add commitlist class to pick up okbox styling

diff --git a/lib/Smokingit/View/Commit.pm b/lib/Smokingit/View/Commit.pm
index 28589ee..c4a9875 100644
--- a/lib/Smokingit/View/Commit.pm
+++ b/lib/Smokingit/View/Commit.pm
@@ -10,19 +10,18 @@ template '/commit' => page {
 
     my $commit = get('commit');
 
-    ul {
-        my $configs = $b->project->configurations;
+    span {
+        class is "commitlist";
+        my $configs = $commit->project->configurations;
         while (my $config = $configs->next) {
-            li {
-                my $smoke = Smokingit::Model::SmokeResult->new;
-                $smoke->load_by_cols(
-                    project_id       => $commit->project->id,
-                    configuration_id => $config->id,
-                    commit_id        => $commit->id,
-                );
-                next unless $smoke->id;
-                Smokingit::View::test_result( $smoke );
-            }
+            my $smoke = Smokingit::Model::SmokeResult->new;
+            $smoke->load_by_cols(
+                project_id       => $commit->project->id,
+                configuration_id => $config->id,
+                commit_id        => $commit->id,
+            );
+            next unless $smoke->id;
+            Smokingit::View::test_result( $smoke );
         }
     }
 };

commit 198bb13b1a2ba16699910bda67465f6ad7a58e62
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 22 18:57:56 2013 -0400

    The id columns is necessary to be able to get columns not in ->columns

diff --git a/lib/Smokingit/View/Commit.pm b/lib/Smokingit/View/Commit.pm
index c4a9875..ef962be 100644
--- a/lib/Smokingit/View/Commit.pm
+++ b/lib/Smokingit/View/Commit.pm
@@ -40,7 +40,7 @@ template '/smoke' => page {
     my $results = Smokingit::Model::SmokeFileResultCollection->new;
     $results->limit( column => "smoke_result_id", value => $s->id );
     $results->order_by( column => "filename" );
-    $results->columns( "filename", "is_ok", "elapsed" );
+    $results->columns( "id", "filename", "is_ok", "elapsed" );
     while (my $result = $results->next) {
         div {
             class is ($result->is_ok ? "passingfile" : "failingfile");

commit fe8393bce0bde6a585aa9b584a5ab643c65266c4
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Jul 22 18:58:13 2013 -0400

    Sort by non-OK first, and display their raw TAP

diff --git a/lib/Smokingit/View/Commit.pm b/lib/Smokingit/View/Commit.pm
index ef962be..14695d8 100644
--- a/lib/Smokingit/View/Commit.pm
+++ b/lib/Smokingit/View/Commit.pm
@@ -39,7 +39,7 @@ template '/smoke' => page {
 
     my $results = Smokingit::Model::SmokeFileResultCollection->new;
     $results->limit( column => "smoke_result_id", value => $s->id );
-    $results->order_by( column => "filename" );
+    $results->order_by( { column => "is_ok" }, { column => "filename" } );
     $results->columns( "id", "filename", "is_ok", "elapsed" );
     while (my $result = $results->next) {
         div {
@@ -49,6 +49,9 @@ template '/smoke' => page {
                 class is "elapsed";
                 outs sprintf "(%.2fs)", $result->elapsed;
             };
+            unless ($result->is_ok) {
+                pre { $result->raw_tap };
+            }
         };
     }
 };

-----------------------------------------------------------------------



More information about the Bps-public-commit mailing list