Skip to content

Commit

Permalink
Refactor Live SQL into Model
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattab
mattab committed Dec 5, 2014
1 parent 64bc69c commit 7f509b0
Show file tree
Hide file tree
Showing 4 changed files with 292 additions and 199 deletions.
20 changes: 13 additions & 7 deletions core/DataAccess/LogQueryBuilder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,25 @@

class LogQueryBuilder
{
public function getSelectQueryString(SegmentExpression $segmentExpression, $select, $from, $where, $bind, $orderBy, $groupBy)
public function __construct(SegmentExpression $segmentExpression)
{
$this->segmentExpression = $segmentExpression;
}

public function getSelectQueryString($select, $from, $where, $bind, $orderBy, $groupBy)
{
if (!is_array($from)) {
$from = array($from);
}

if(!$segmentExpression->isEmpty()) {
$segmentExpression->parseSubExpressionsIntoSqlExpressions($from);
$segmentSql = $segmentExpression->getSql();
if(!$this->segmentExpression->isEmpty()) {
$this->segmentExpression->parseSubExpressionsIntoSqlExpressions($from);
$segmentSql = $this->segmentExpression->getSql();
$where = $this->getWhereMatchBoth($where, $segmentSql['where']);
$bind = array_merge($bind, $segmentSql['bind']);
}

$joins = $this->generateJoins($from);
$joins = $this->generateJoinsString($from);
$joinWithSubSelect = $joins['joinWithSubSelect'];
$from = $joins['sql'];

Expand All @@ -51,7 +56,7 @@ public function getSelectQueryString(SegmentExpression $segmentExpression, $sele
* @throws Exception if tables can't be joined
* @return array
*/
private function generateJoins($tables)
private function generateJoinsString($tables)
{
$knownTables = array("log_visit", "log_link_visit_action", "log_conversion", "log_conversion_item");
$visitsAvailable = $actionsAvailable = $conversionsAvailable = $conversionItemAvailable = false;
Expand Down Expand Up @@ -184,7 +189,8 @@ private function buildWrappedSelectQuery($select, $from, $where, $orderBy, $grou
$innerQuery = $this->buildSelectQuery($innerSelect, $innerFrom, $innerWhere, $innerOrderBy, $innerGroupBy);

$select = preg_replace('/'.$matchTables.'\./', 'log_inner.', $select);
$from = "(
$from = "
(
$innerQuery
) AS log_inner";
$where = false;
Expand Down
4 changes: 2 additions & 2 deletions core/Segment.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,8 @@ public function getHash()
public function getSelectQuery($select, $from, $where = false, $bind = array(), $orderBy = false, $groupBy = false)
{
$segmentExpression = $this->segmentExpression;
$segmentQuery = new LogQueryBuilder();
return $segmentQuery->getSelectQueryString($segmentExpression, $select, $from, $where, $bind, $orderBy, $groupBy);
$segmentQuery = new LogQueryBuilder($segmentExpression);
return $segmentQuery->getSelectQueryString($select, $from, $where, $bind, $orderBy, $groupBy);
}
/**
* Returns the segment string.
Expand Down
205 changes: 15 additions & 190 deletions plugins/Live/API.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,49 +69,8 @@ class API extends \Piwik\Plugin\API
public function getCounters($idSite, $lastMinutes, $segment = false)
{
Piwik::checkUserHasViewAccess($idSite);
$lastMinutes = (int) $lastMinutes;

$counters = array(
'visits' => 0,
'actions' => 0,
'visitors' => 0,
'visitsConverted' => 0,
);

if (empty($lastMinutes)) {
return array($counters);
}

list($whereIdSites, $idSites) = $this->getIdSitesWhereClause($idSite);

$select = "count(*) as visits, COUNT(DISTINCT log_visit.idvisitor) as visitors";
$where = $whereIdSites . "AND log_visit.visit_last_action_time >= ?";
$bind = $idSites;
$bind[] = Date::factory(time() - $lastMinutes * 60)->toString('Y-m-d H:i:s');

$segment = new Segment($segment, $idSite);
$query = $segment->getSelectQuery($select, 'log_visit', $where, $bind);

$data = Db::fetchAll($query['sql'], $query['bind']);

$counters['visits'] = $data[0]['visits'];
$counters['visitors'] = $data[0]['visitors'];

$select = "count(*)";
$from = 'log_link_visit_action';
list($whereIdSites) = $this->getIdSitesWhereClause($idSite, $from);
$where = $whereIdSites . "AND log_link_visit_action.server_time >= ?";
$query = $segment->getSelectQuery($select, $from, $where, $bind);
$counters['actions'] = Db::fetchOne($query['sql'], $query['bind']);

$select = "count(*)";
$from = 'log_conversion';
list($whereIdSites) = $this->getIdSitesWhereClause($idSite, $from);
$where = $whereIdSites . "AND log_conversion.server_time >= ?";
$query = $segment->getSelectQuery($select, $from, $where, $bind);
$counters['visitsConverted'] = Db::fetchOne($query['sql'], $query['bind']);

return array($counters);
$model = new Model();
return $model->queryCounters($idSite, $lastMinutes, $segment);
}

/**
Expand Down Expand Up @@ -455,38 +414,8 @@ public function getMostRecentVisitorId($idSite, $segment = false)
*/
private function getAdjacentVisitorId($idSite, $visitorId, $visitLastActionTime, $segment, $getNext)
{
if ($getNext) {
$visitLastActionTimeCondition = "sub.visit_last_action_time <= ?";
$orderByDir = "DESC";
} else {
$visitLastActionTimeCondition = "sub.visit_last_action_time >= ?";
$orderByDir = "ASC";
}

$visitLastActionDate = Date::factory($visitLastActionTime);
$dateOneDayAgo = $visitLastActionDate->subDay(1);
$dateOneDayInFuture = $visitLastActionDate->addDay(1);

$select = "log_visit.idvisitor, MAX(log_visit.visit_last_action_time) as visit_last_action_time";
$from = "log_visit";
$where = "log_visit.idsite = ? AND log_visit.idvisitor <> ? AND visit_last_action_time >= ? and visit_last_action_time <= ?";
$whereBind = array($idSite, @Common::hex2bin($visitorId), $dateOneDayAgo->toString('Y-m-d H:i:s'), $dateOneDayInFuture->toString('Y-m-d H:i:s'));
$orderBy = "MAX(log_visit.visit_last_action_time) $orderByDir";
$groupBy = "log_visit.idvisitor";

$segment = new Segment($segment, $idSite);
$queryInfo = $segment->getSelectQuery($select, $from, $where, $whereBind, $orderBy, $groupBy);

$sql = "SELECT sub.idvisitor, sub.visit_last_action_time FROM ({$queryInfo['sql']}) as sub
WHERE $visitLastActionTimeCondition
LIMIT 1";
$bind = array_merge($queryInfo['bind'], array($visitLastActionTime));

$visitorId = Db::fetchOne($sql, $bind);
if (!empty($visitorId)) {
$visitorId = bin2hex($visitorId);
}
return $visitorId;
$model = new Model();
return $model->queryAdjacentVisitorId($idSite, $visitorId, $visitLastActionTime, $segment, $getNext);
}

/**
Expand Down Expand Up @@ -614,112 +543,21 @@ private function addFilterToCleanVisitors(DataTable $dataTable, $idSite, $flat =

private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment = false, $countVisitorsToFetch = 100, $visitorId = false, $minTimestamp = false, $filterSortOrder = false)
{
$where = $whereBind = array();

list($whereClause, $idSites) = $this->getIdSitesWhereClause($idSite);
$model = new Model();
$data = $model->queryLogVisits($idSite, $period, $date, $segment, $countVisitorsToFetch, $visitorId, $minTimestamp, $filterSortOrder);

$where[] = $whereClause;
$whereBind = $idSites;

if (strtolower($filterSortOrder) !== 'asc') {
$filterSortOrder = 'DESC';
}

$orderBy = "idsite, visit_last_action_time " . $filterSortOrder;
$orderByParent = "sub.visit_last_action_time " . $filterSortOrder;

if (!empty($visitorId)) {
$where[] = "log_visit.idvisitor = ? ";
$whereBind[] = @Common::hex2bin($visitorId);
}

if (!empty($minTimestamp)) {
$where[] = "log_visit.visit_last_action_time > ? ";
$whereBind[] = date("Y-m-d H:i:s", $minTimestamp);
}

// If no other filter, only look at the last 24 hours of stats
if (empty($visitorId)
&& empty($countVisitorsToFetch)
&& empty($period)
&& empty($date)
) {
$period = 'day';
$date = 'yesterdaySameTime';
}

// SQL Filter with provided period
if (!empty($period) && !empty($date)) {
$currentSite = new Site($idSite);
$currentTimezone = $currentSite->getTimezone();

$dateString = $date;
if ($period == 'range') {
$processedPeriod = new Range('range', $date);
if ($parsedDate = Range::parseDateRange($date)) {
$dateString = $parsedDate[2];
}
} else {
$processedDate = Date::factory($date);
if ($date == 'today'
|| $date == 'now'
|| $processedDate->toString() == Date::factory('now', $currentTimezone)->toString()
) {
$processedDate = $processedDate->subDay(1);
}
$processedPeriod = Period\Factory::build($period, $processedDate);
}
$dateStart = $processedPeriod->getDateStart()->setTimezone($currentTimezone);
$where[] = "log_visit.visit_last_action_time >= ?";
$whereBind[] = $dateStart->toString('Y-m-d H:i:s');

if (!in_array($date, array('now', 'today', 'yesterdaySameTime'))
&& strpos($date, 'last') === false
&& strpos($date, 'previous') === false
&& Date::factory($dateString)->toString('Y-m-d') != Date::factory('now', $currentTimezone)->toString()
) {
$dateEnd = $processedPeriod->getDateEnd()->setTimezone($currentTimezone);
$where[] = " log_visit.visit_last_action_time <= ?";
$dateEndString = $dateEnd->addDay(1)->toString('Y-m-d H:i:s');
$whereBind[] = $dateEndString;
}
}

if (count($where) > 0) {
$where = join("
AND ", $where);
} else {
$where = false;
}

$segment = new Segment($segment, $idSite);

// Subquery to use the indexes for ORDER BY
$select = "log_visit.*";
$from = "log_visit";
$subQuery = $segment->getSelectQuery($select, $from, $where, $whereBind, $orderBy);

$sqlLimit = $countVisitorsToFetch >= 1 ? " LIMIT 0, " . (int)$countVisitorsToFetch : "";

// Group by idvisit so that a visitor converting 2 goals only appears once
$sql = "
SELECT sub.* FROM (
" . $subQuery['sql'] . "
$sqlLimit
) AS sub
GROUP BY sub.idvisit
ORDER BY $orderByParent
";
try {
$data = Db::fetchAll($sql, $subQuery['bind']);
} catch (Exception $e) {
echo $e->getMessage();
exit;
}
return $this->makeVisitorTableFromArray($data);
}

/**
* @param $data
* @return DataTable
* @throws Exception
*/
private function makeVisitorTableFromArray($data)
{
$dataTable = new DataTable();
$dataTable->addRowsFromSimpleArray($data);
// $dataTable->disableFilter('Truncate');

if (!empty($data[0])) {
$columnsToNotAggregate = array_map(function () {
Expand All @@ -732,18 +570,5 @@ private function loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $se
return $dataTable;
}

/**
* @param $idSite
* @param string $table
* @return array
*/
private function getIdSitesWhereClause($idSite, $table = 'log_visit')
{
$idSites = array($idSite);
Piwik::postEvent('Live.API.getIdSitesString', array(&$idSites));

$idSitesBind = Common::getSqlStringFieldsArray($idSites);
$whereClause = $table . ".idsite in ($idSitesBind) ";
return array($whereClause, $idSites);
}
}
Loading

0 comments on commit 7f509b0

Please sign in to comment.