diff --git a/src/engine/Bind.cpp b/src/engine/Bind.cpp index dde7e019d9..7ebad19812 100644 --- a/src/engine/Bind.cpp +++ b/src/engine/Bind.cpp @@ -8,6 +8,7 @@ #include "engine/QueryExecutionTree.h" #include "engine/sparqlExpressions/SparqlExpression.h" #include "engine/sparqlExpressions/SparqlExpressionGenerators.h" +#include "util/ChunkedForLoop.h" #include "util/Exception.h" // BIND adds exactly one new column @@ -170,9 +171,11 @@ IdTable Bind::computeExpressionBind( getInternallyVisibleVariableColumns().at(singleResult).columnIndex_; auto inputColumn = idTable.getColumn(columnIndex); AD_CORRECTNESS_CHECK(inputColumn.size() == outputColumn.size()); - std::ranges::copy(inputColumn, outputColumn.begin()); + ad_utility::chunkedCopy(inputColumn, outputColumn.begin(), CHUNK_SIZE, + [this]() { checkCancellation(); }); } else if constexpr (isStrongId) { - std::ranges::fill(outputColumn, singleResult); + ad_utility::chunkedFill(outputColumn, singleResult, CHUNK_SIZE, + [this]() { checkCancellation(); }); } else { constexpr bool isConstant = sparqlExpression::isConstantResult; @@ -187,7 +190,8 @@ IdTable Bind::computeExpressionBind( sparqlExpression::detail::constantExpressionResultToId( std::move(*it), *outputLocalVocab); checkCancellation(); - std::ranges::fill(outputColumn, constantId); + ad_utility::chunkedFill(outputColumn, constantId, CHUNK_SIZE, + [this]() { checkCancellation(); }); } } else { size_t i = 0; diff --git a/src/engine/Union.cpp b/src/engine/Union.cpp index 2661bf5515..07ef2a401e 100644 --- a/src/engine/Union.cpp +++ b/src/engine/Union.cpp @@ -6,6 +6,7 @@ #include "Union.h" #include "engine/CallFixedSize.h" +#include "util/ChunkedForLoop.h" #include "util/TransparentFunctors.h" const size_t Union::NO_COLUMN = std::numeric_limits::max(); @@ -184,26 +185,6 @@ ProtoResult Union::computeResult(bool requestLaziness) { Result::getMergedLocalVocab(*subRes1, *subRes2)}; } -// _____________________________________________________________________________ -void Union::copyChunked(auto beg, auto end, auto target) const { - size_t total = end - beg; - for (size_t i = 0; i < total; i += chunkSize) { - checkCancellation(); - size_t actualEnd = std::min(i + chunkSize, total); - std::copy(beg + i, beg + actualEnd, target + i); - } -} - -// _____________________________________________________________________________ -void Union::fillChunked(auto beg, auto end, const auto& value) const { - size_t total = end - beg; - for (size_t i = 0; i < total; i += chunkSize) { - checkCancellation(); - size_t actualEnd = std::min(i + chunkSize, total); - std::fill(beg + i, beg + actualEnd, value); - } -}; - // _____________________________________________________________________________ IdTable Union::computeUnion( const IdTable& left, const IdTable& right, @@ -220,11 +201,14 @@ IdTable Union::computeUnion( size_t inputColumnIndex, size_t offset) { if (inputColumnIndex != NO_COLUMN) { decltype(auto) input = inputTable.getColumn(inputColumnIndex); - copyChunked(input.begin(), input.end(), targetColumn.begin() + offset); + ad_utility::chunkedCopy(input, targetColumn.begin() + offset, chunkSize, + [this]() { checkCancellation(); }); } else { - fillChunked(targetColumn.begin() + offset, - targetColumn.begin() + offset + inputTable.size(), - Id::makeUndefined()); + ad_utility::chunkedFill( + std::ranges::subrange{ + targetColumn.begin() + offset, + targetColumn.begin() + offset + inputTable.size()}, + Id::makeUndefined(), chunkSize, [this]() { checkCancellation(); }); } }; @@ -263,8 +247,9 @@ IdTable Union::transformToCorrectColumnFormat( IdTable idTable, const std::vector& permutation) const { while (idTable.numColumns() < getResultWidth()) { idTable.addEmptyColumn(); - auto column = idTable.getColumn(idTable.numColumns() - 1); - fillChunked(column.begin(), column.end(), Id::makeUndefined()); + ad_utility::chunkedFill(idTable.getColumn(idTable.numColumns() - 1), + Id::makeUndefined(), chunkSize, + [this]() { checkCancellation(); }); } idTable.setColumnSubset(permutation); diff --git a/src/engine/Union.h b/src/engine/Union.h index 6932b946d4..782c94e469 100644 --- a/src/engine/Union.h +++ b/src/engine/Union.h @@ -63,13 +63,6 @@ class Union : public Operation { } private: - // A drop-in replacement for `std::copy` that performs the copying in chunks - // of `chunkSize` and checks the timeout after each chunk. - void copyChunked(auto beg, auto end, auto target) const; - - // A similar timeout-checking replacement for `std::fill`. - void fillChunked(auto beg, auto end, const auto& value) const; - ProtoResult computeResult(bool requestLaziness) override; VariableToColumnMap computeVariableToColumnMap() const override; diff --git a/src/util/ChunkedForLoop.h b/src/util/ChunkedForLoop.h index 4896a21470..9569633e24 100644 --- a/src/util/ChunkedForLoop.h +++ b/src/util/ChunkedForLoop.h @@ -63,6 +63,56 @@ inline void chunkedForLoop(std::size_t start, std::size_t end, std::invoke(chunkOperation); } } + +// Helper concept that combines the sized range and input range concepts. +template +concept SizedInputRange = + std::ranges::sized_range && std::ranges::input_range; + +// Similar to `std::ranges::copy`, but invokes `chunkOperation` every +// `chunkSize` elements. (Round up to the next chunk size if the range size is +// not a multiple of `chunkSize`.) +template +inline void chunkedCopy(R&& inputRange, O result, + std::ranges::range_difference_t chunkSize, + const std::invocable auto& chunkOperation) + requires std::indirectly_copyable, O> { + auto begin = std::ranges::begin(inputRange); + auto end = std::ranges::end(inputRange); + auto target = result; + while (std::ranges::distance(begin, end) >= chunkSize) { + auto start = begin; + std::ranges::advance(begin, chunkSize); + target = std::ranges::copy(start, begin, target).out; + chunkOperation(); + } + std::ranges::copy(begin, end, target); + chunkOperation(); +} + +// Helper concept that combines the sized range and output range concepts. +template +concept SizedOutputRange = + std::ranges::sized_range && std::ranges::output_range; + +// Similar to `std::ranges::fill`, but invokes `chunkOperation` every +// `chunkSize` elements. (Round up to the next chunk size if the range size is +// not a multiple of `chunkSize`.) +template R> +inline void chunkedFill(R&& outputRange, const T& value, + std::ranges::range_difference_t chunkSize, + const std::invocable auto& chunkOperation) { + auto begin = std::ranges::begin(outputRange); + auto end = std::ranges::end(outputRange); + while (std::ranges::distance(begin, end) >= chunkSize) { + auto start = begin; + std::ranges::advance(begin, chunkSize); + std::ranges::fill(start, begin, value); + chunkOperation(); + } + std::ranges::fill(begin, end, value); + chunkOperation(); +} } // namespace ad_utility #endif // QLEVER_CHUNKEDFORLOOP_H diff --git a/test/ChunkedForLoopTest.cpp b/test/ChunkedForLoopTest.cpp index ecccdffe40..b04c6ca6c7 100644 --- a/test/ChunkedForLoopTest.cpp +++ b/test/ChunkedForLoopTest.cpp @@ -2,12 +2,14 @@ // Chair of Algorithms and Data Structures. // Author: Robin Textor-Falconi -#include +#include #include #include "util/ChunkedForLoop.h" +using ad_utility::chunkedCopy; +using ad_utility::chunkedFill; using ad_utility::chunkedForLoop; TEST(ChunkedForLoop, testEmptyRange) { @@ -103,3 +105,42 @@ TEST(ChunkedForLoop, verifyBreakWorksAsExpected) { EXPECT_EQ(counter, 4); EXPECT_EQ(chunkCounter, 1); } + +// _____________________________________________________________________________________________________________________ +TEST(ChunkedForLoop, chunkedFillHandlesEmptyRange) { + size_t chunkCounter = 0; + chunkedFill(std::array{}, 0, 10, [&]() { chunkCounter++; }); + + EXPECT_EQ(chunkCounter, 1); +} + +// _____________________________________________________________________________________________________________________ +TEST(ChunkedForLoop, chunkedFillFillsCorrectly) { + size_t chunkCounter = 0; + std::array elements{}; + chunkedFill(elements, 42, 10, [&]() { chunkCounter++; }); + + EXPECT_EQ(chunkCounter, 3); + EXPECT_THAT(elements, ::testing::Each(::testing::Eq(42))); +} + +// _____________________________________________________________________________________________________________________ +TEST(ChunkedForLoop, chunkedCopyHandlesEmptyRange) { + size_t chunkCounter = 0; + std::array output{}; + chunkedCopy(std::array{}, output.begin(), 2, + [&]() { chunkCounter++; }); + + EXPECT_EQ(chunkCounter, 1); +} + +// _____________________________________________________________________________________________________________________ +TEST(ChunkedForLoop, chunkedCopyCopiesCorrectly) { + size_t chunkCounter = 0; + std::array input{5, 4, 3, 2, 1}; + std::array output{}; + chunkedCopy(input, output.begin(), 2, [&]() { chunkCounter++; }); + + EXPECT_EQ(chunkCounter, 3); + EXPECT_EQ(input, output); +}