The ultimate prize and final step to close the issue: do not group by…

… the whole set of matched rows, way too slow on huge datasets!
d19dotca · Dec 5, 2014 · a37254a · a37254a
1 parent dac71ca
commit a37254a
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 6 deletions.
diff --git a/core/DataAccess/LogQueryBuilder.php b/core/DataAccess/LogQueryBuilder.php
@@ -184,14 +184,18 @@ private function buildWrappedSelectQuery($select, $from, $where, $groupBy, $orde
         $innerSelect = implode(", \n", $neededFields);
         $innerFrom = $from;
         $innerWhere = $where;
-        $innerGroupBy = "log_visit.idvisit";
-        $innerLimit = $limit;
 
+        $innerLimit = $limit;
+        $innerGroupBy = "log_visit.idvisit";
         $innerOrderBy = "NULL";
-        // Only when there is a LIMIT then we can apply to the inner query the same ORDER BY as the parent query
         if($innerLimit && $orderBy) {
+            // only When LIMITing we can apply to the inner query the same ORDER BY as the parent query
             $innerOrderBy = $orderBy;
         }
+        if($innerLimit) {
+            // When LIMITing, no need to GROUP BY (GROUPing by is done before the LIMIT which is super slow when large amount of rows is matched)
+            $innerGroupBy = false;
+        }
 
         $innerQuery = $this->buildSelectQuery($innerSelect, $innerFrom, $innerWhere, $innerGroupBy, $innerOrderBy, $innerLimit);
 

diff --git a/plugins/Live/tests/System/ModelTest.php b/plugins/Live/tests/System/ModelTest.php
@@ -91,7 +91,6 @@ public function test_makeLogVisitsQueryString_whenSegment()
                           AND log_visit.visit_last_action_time >= ?
                           AND log_visit.visit_last_action_time <= ? )
                           AND ( log_link_visit_action.custom_var_k1 = ? )
-                        GROUP BY log_visit.idvisit
                         ORDER BY idsite, visit_last_action_time DESC
                         LIMIT 100
                         ) AS log_inner

diff --git a/tests/PHPUnit/Integration/SegmentTest.php b/tests/PHPUnit/Integration/SegmentTest.php
@@ -458,7 +458,7 @@ public function test_bogusSegment_shouldThrowException($segment)
     }
 
 
-    public function test_getSelectQuery_whenLimit_innerQueryShouldHaveLimit()
+    public function test_getSelectQuery_whenLimit_innerQueryShouldHaveLimitAndNoGroupBy()
     {
         $select = 'sum(log_visit.visit_total_time) as sum_visit_length';
         $from = 'log_visit';
@@ -489,7 +489,6 @@ public function test_getSelectQuery_whenLimit_innerQueryShouldHaveLimit()
                     ( log_visit.idvisit = ? )
                     AND
                     ( log_link_visit_action.custom_var_k1 = ? )
-                GROUP BY log_visit.idvisit
                 ORDER BY NULL
                 LIMIT 33
                     ) AS log_inner