Eagerly fill LogFilteredData::filteredItemsCache_

src/crawlerwidget.cpp

@@ -386,7 +386,7 @@ void CrawlerWidget::markLineFromFiltered( qint64 line )
     if ( line < logFilteredData_->getNbLine() ) {
         qint64 line_in_file = logFilteredData_->getMatchingLineNumber( line );
         if ( logFilteredData_->filteredLineTypeByIndex( line )
-                == LogFilteredData::Mark )
+                & LogFilteredData::Mark )
             logFilteredData_->deleteMark( line_in_file );
         else
             logFilteredData_->addMark( line_in_file );

src/data/logfiltereddata.cpp

@@ -23,9 +23,11 @@
 
 #include "log.h"
 
+#include <QtGlobal>
 #include <QString>
-#include <cassert>
+#include <algorithm>
 #include <limits>
+#include <utility>
 
 #include "utils.h"
 #include "logdata.h"
@@ -47,8 +49,6 @@ LogFilteredData::LogFilteredData() : AbstractLogData(),
     maxLengthMarks_ = 0;
     searchDone_ = true;
     visibility_ = MarksAndMatches;
-
-    filteredItemsCacheDirty_ = true;
 }
 
 // Usual constructor: just copy the data, the search is started by runSearch()
@@ -72,8 +72,6 @@ LogFilteredData::LogFilteredData( const LogData* logData )
 
     visibility_ = MarksAndMatches;
 
-    filteredItemsCacheDirty_ = true;
-
     // Forward the update signal
     connect( &workerThread_, SIGNAL( searchProgressed( int, int, qint64 ) ),
             this, SLOT( handleSearchProgressed( int, int, qint64 ) ) );
@@ -123,7 +121,7 @@ void LogFilteredData::clearSearch()
     matching_lines_.clear();
     maxLength_        = 0;
     nbLinesProcessed_ = 0;
-    filteredItemsCacheDirty_ = true;
+    removeAllFromFilteredItemsCache( Match );
 }
 
 qint64 LogFilteredData::getMatchingLineNumber( int matchNum ) const
@@ -172,11 +170,6 @@ LogFilteredData::FilteredLineType
     else if ( visibility_ == MarksOnly )
         return Mark;
     else {
-        // If it is MarksAndMatches, we have to look.
-        // Regenerate the cache if needed
-        if ( filteredItemsCacheDirty_ )
-            regenerateFilteredItemsCache();
-
         return filteredItemsCache_[ index ].type();
     }
 }
@@ -186,10 +179,10 @@ LogFilteredData::FilteredLineType
 void LogFilteredData::addMark( qint64 line, QChar mark )
 {
     if ( ( line >= 0 ) && ( line < sourceLogData_->getNbLine() ) ) {
-        marks_.addMark( line, mark );
+        int index = marks_.addMark( line, mark );
         maxLengthMarks_ = qMax( maxLengthMarks_,
                 sourceLogData_->getLineLength( line ) );
-        filteredItemsCacheDirty_ = true;
+        insertIntoFilteredItemsCache( index, { static_cast<LineNumber>( line ), Mark } );
     }
     else
         LOG(logERROR) << "LogFilteredData::addMark\
@@ -236,53 +229,80 @@ qint64 LogFilteredData::getMarkBefore( qint64 line ) const
 
 void LogFilteredData::deleteMark( QChar mark )
 {
-    marks_.deleteMark( mark );
-    filteredItemsCacheDirty_ = true;
+    int index = marks_.findMark( mark );
+    qint64 line = marks_.getLineMarkedByIndex( index );
+    marks_.deleteMarkAt( index );
+
+    if ( line < 0 ) {
+        // LOG(logWARNING)?
+        return;
+    }
 
-    // FIXME: maxLengthMarks_
+    updateMaxLengthMarks( line );
+    removeFromFilteredItemsCache( index, { static_cast<LineNumber>( line ), Mark } );
 }
 
 void LogFilteredData::deleteMark( qint64 line )
 {
-    marks_.deleteMark( line );
-    filteredItemsCacheDirty_ = true;
+    int index = marks_.deleteMark( line );
+
+    updateMaxLengthMarks( line );
+    removeFromFilteredItemsCache( index, { static_cast<LineNumber>( line ), Mark } );
+}
 
+void LogFilteredData::updateMaxLengthMarks( qint64 removed_line )
+{
+    if ( removed_line < 0 ) {
+        LOG(logWARNING) << "updateMaxLengthMarks called with negative line-number";
+        return;
+    }
     // Now update the max length if needed
-    if ( sourceLogData_->getLineLength( line ) >= maxLengthMarks_ ) {
+    if ( sourceLogData_->getLineLength( removed_line ) >= maxLengthMarks_ ) {
         LOG(logDEBUG) << "deleteMark recalculating longest mark";
         maxLengthMarks_ = 0;
-        for ( Marks::const_iterator i = marks_.begin();
-                i != marks_.end(); ++i ) {
-            LOG(logDEBUG) << "line " << i->lineNumber();
+        for ( auto& mark : marks_ ) {
+            LOG(logDEBUG) << "line " << mark.lineNumber();
             maxLengthMarks_ = qMax( maxLengthMarks_,
-                    sourceLogData_->getLineLength( i->lineNumber() ) );
+                    sourceLogData_->getLineLength( mark.lineNumber() ) );
         }
     }
 }
 
 void LogFilteredData::clearMarks()
 {
     marks_.clear();
-    filteredItemsCacheDirty_ = true;
     maxLengthMarks_ = 0;
+    removeAllFromFilteredItemsCache( Mark );
 }
 
 void LogFilteredData::setVisibility( Visibility visi )
 {
     visibility_ = visi;
+
+    if ( visibility_ == MarksAndMatches )
+        regenerateFilteredItemsCache();
 }
 
 //
 // Slots
 //
 void LogFilteredData::handleSearchProgressed( int nbMatches, int progress, qint64 initial_position )
 {
+    using std::begin;
+    using std::end;
+    using std::next;
+
     LOG(logDEBUG) << "LogFilteredData::handleSearchProgressed matches="
         << nbMatches << " progress=" << progress;
 
     // searchDone_ = true;
-    workerThread_.getSearchResult( &maxLength_, &matching_lines_, &nbLinesProcessed_ );
-    filteredItemsCacheDirty_ = true;
+    Q_ASSERT( nbMatches >= 0 );
+
+    size_t start_index = matching_lines_.size();
+
+    workerThread_.updateSearchResult( &maxLength_, &matching_lines_, &nbLinesProcessed_ );
+
+    insertMatchesIntoFilteredItemsCache( start_index );
 
     emit searchProgressed( nbMatches, progress, initial_position );
 }
@@ -305,10 +325,6 @@ LineNumber LogFilteredData::findLogDataLine( LineNumber lineNum ) const
             LOG(logERROR) << "Index too big in LogFilteredData: " << lineNum;
     }
     else {
-        // Regenerate the cache if needed
-        if ( filteredItemsCacheDirty_ )
-            regenerateFilteredItemsCache();
-
         if ( lineNum < filteredItemsCache_.size() )
             line = filteredItemsCache_[ lineNum ].lineNumber();
         else
@@ -333,11 +349,6 @@ LineNumber LogFilteredData::findFilteredLine( LineNumber lineNum ) const
                                       lineNum );
     }
     else {
-      // Regenerate the cache if needed
-        if ( filteredItemsCacheDirty_ ) {
-            regenerateFilteredItemsCache();
-        }
-
         lineIndex = lookupLineNumber( filteredItemsCache_.begin(),
                                       filteredItemsCache_.end(),
                                       lineNum );
@@ -398,10 +409,6 @@ qint64 LogFilteredData::doGetNbLine() const
     else if ( visibility_ == MarksOnly )
         nbLines = marks_.size();
     else {
-        // Regenerate the cache if needed (hopefully most of the time
-        // it won't be necessarily)
-        if ( filteredItemsCacheDirty_ )
-            regenerateFilteredItemsCache();
         nbLines = filteredItemsCache_.size();
     }
 
@@ -439,13 +446,16 @@ void LogFilteredData::doSetMultibyteEncodingOffsets( int, int )
 {
 }
 
-// TODO: We might be a bit smarter and not regenerate the whole thing when
-// e.g. stuff is added at the end of the search.
 void LogFilteredData::regenerateFilteredItemsCache() const
 {
     LOG(logDEBUG) << "regenerateFilteredItemsCache";
 
-    filteredItemsCache_.clear();
+    if ( filteredItemsCache_.size() > 0 ) {
+        // the cache was not invalidated, so we can keep it
+        LOG(logDEBUG) << "cache was not invalidated";
+        return;
+    }
+
     filteredItemsCache_.reserve( matching_lines_.size() + marks_.size() );
     // (it's an overestimate but probably not by much so it's fine)
 
@@ -457,24 +467,136 @@ void LogFilteredData::regenerateFilteredItemsCache() const
             ( j != marks_.end() ) ? j->lineNumber() : std::numeric_limits<qint64>::max();
         qint64 next_match =
             ( i != matching_lines_.cend() ) ? i->lineNumber() : std::numeric_limits<qint64>::max();
-        // We choose a Mark over a Match if a line is both, just an arbitrary choice really.
+        FilteredLineType type = static_cast<FilteredLineType>( 0 );
+        LineNumber line;
         if ( next_mark <= next_match ) {
             // LOG(logDEBUG) << "Add mark at " << next_mark;
-            filteredItemsCache_.push_back( FilteredItem( next_mark, Mark ) );
-            if ( j != marks_.end() )
-                ++j;
-            if ( ( next_mark == next_match ) && ( i != matching_lines_.cend() ) )
-                ++i;  // Case when it's both match and mark.
+            type |= Mark;
+            line = next_mark;
+            ++j;
         }
-        else {
+        if ( next_mark >= next_match ) {
             // LOG(logDEBUG) << "Add match at " << next_match;
-            filteredItemsCache_.push_back( FilteredItem( next_match, Match ) );
-            if ( i != matching_lines_.cend() )
-                ++i;
+            type |= Match;
+            line = next_match;
+            ++i;
         }
+        filteredItemsCache_.emplace_back( line, type );
     }
 
-    filteredItemsCacheDirty_ = false;
-
     LOG(logDEBUG) << "finished regenerateFilteredItemsCache";
 }
+
+void LogFilteredData::insertIntoFilteredItemsCache( size_t insert_index, FilteredItem &&item )
+{
+    using std::begin;
+    using std::end;
+    using std::next;
+
+    if ( visibility_ != MarksAndMatches ) {
+        // this is invalidated and will be regenerated when we need it
+        filteredItemsCache_.clear();
+        LOG(logDEBUG) << "cache is invalidated";
+    }
+
+    // Search for the corresponding index.
+    // We can start the search from insert_index, since lineNumber >= index is always true.
+    auto found = std::lower_bound( next( begin( filteredItemsCache_ ), insert_index ), end( filteredItemsCache_ ), item );
+    if ( found == end( filteredItemsCache_ ) || found->lineNumber() > item.lineNumber() ) {
+        filteredItemsCache_.emplace( found, std::move( item ) );
+    } else {
+        Q_ASSERT( found->lineNumber() == item.lineNumber() );
+        found->add( item.type() );
+    }
+}
+
+void LogFilteredData::insertIntoFilteredItemsCache( FilteredItem &&item )
+{
+    return insertIntoFilteredItemsCache( 0, std::move( item ) );
+}
+
+void LogFilteredData::insertMatchesIntoFilteredItemsCache( size_t start_index )
+{
+    using std::begin;
+    using std::end;
+    using std::next;
+
+    Q_ASSERT( start_index <= matching_lines_.size() );
+
+    if ( visibility_ != MarksAndMatches ) {
+        // this is invalidated and will be regenerated when we need it
+        filteredItemsCache_.clear();
+        LOG(logDEBUG) << "cache is invalidated";
+        return;
+    }
+
+    Q_ASSERT( start_index <= filteredItemsCache_.size() );
+
+    filteredItemsCache_.reserve( matching_lines_.size() + marks_.size() );
+    // (it's an overestimate but probably not by much so it's fine)
+
+    // Search for the corresponding index.
+    // We can start the search from insert_index, since lineNumber >= index is always true.
+    auto filteredIt = next( begin( filteredItemsCache_ ), start_index );
+    for ( auto matchesIt = next( begin( matching_lines_ ), start_index ); matchesIt != end( matching_lines_ ); ++matchesIt ) {
+        filteredIt = std::lower_bound( filteredIt, end( filteredItemsCache_ ), matchesIt->lineNumber() );
+        if ( filteredIt == end( filteredItemsCache_ ) || filteredIt->lineNumber() > matchesIt->lineNumber() ) {
+            filteredIt = filteredItemsCache_.emplace( filteredIt, matchesIt->lineNumber(), Match );
+        } else {
+            Q_ASSERT( filteredIt->lineNumber() == matchesIt->lineNumber() );
+            filteredIt->add( Match );
+        }
+    }
+}
+
+void LogFilteredData::removeFromFilteredItemsCache( size_t remove_index, const FilteredItem &item )
+{
+    using std::begin;
+    using std::end;
+    using std::next;
+
+    if ( visibility_ != MarksAndMatches ) {
+        // this is invalidated and will be regenerated when we need it
+        filteredItemsCache_.clear();
+        LOG(logDEBUG) << "cache is invalidated";
+        return;
+    }
+
+    // Search for the corresponding index.
+    // We can start the search from remove_index, since lineNumber >= index is always true.
+    auto found = std::equal_range( next( begin( filteredItemsCache_ ), remove_index ), end( filteredItemsCache_ ), item );
+    if( found.first == found.second ) {
+        LOG(logERROR) << "Attempt to remove line " << item.lineNumber() << " from filteredItemsCache_ failed, since it was not found";
+        return;
+    }
+
+    if ( next( found.first ) != found.second ) {
+        LOG(logERROR) << "Multiple matches found for line " << item.lineNumber() << " in filteredItemsCache_";
+        // FIXME: collapse them?
+    }
+
+    if ( !found.first->remove( item.type() ) ) {
+        filteredItemsCache_.erase( found.first );
+    }
+}
+
+void LogFilteredData::removeFromFilteredItemsCache( const FilteredItem &item )
+{
+    removeFromFilteredItemsCache( 0, item );
+}
+
+void LogFilteredData::removeAllFromFilteredItemsCache( FilteredLineType type )
+{
+    using std::begin;
+    using std::end;
+
+    if ( visibility_ != MarksAndMatches ) {
+        // this is invalidated and will be regenerated when we need it
+        filteredItemsCache_.clear();
+        LOG(logDEBUG) << "cache is invalidated";
+        return;
+    }
+
+    auto erase_begin = std::remove_if( begin( filteredItemsCache_ ), end( filteredItemsCache_ ), [type]( FilteredItem& item ) { return !item.remove( type ); } );
+    filteredItemsCache_.erase( erase_begin, end( filteredItemsCache_ ) );
+}