Invent roundripping between BitStreamer state and byte stream position

src/librawspeed/bitstreams/BitStreamPosition.h

@@ -0,0 +1,77 @@
+/*
+    RawSpeed - RAW file decoder.
+
+    Copyright (C) 2024 Roman Lebedev
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#pragma once
+
+#include "adt/Casts.h"
+#include "adt/Invariant.h"
+#include "bitstreams/BitStreams.h"
+#include "common/Common.h"
+
+namespace rawspeed {
+
+template <BitOrder bo> struct BitStreamTraits;
+
+template <BitOrder bo> struct BitStreamPosition {
+  int pos;
+  int fillLevel;
+};
+
+template <BitOrder bo> struct ByteStreamPosition {
+  int bytePos;
+  int numBitsToSkip;
+};
+
+template <BitOrder bo>
+ByteStreamPosition<bo> getAsByteStreamPosition(BitStreamPosition<bo> state) {
+  const int MinByteStepMultiple = BitStreamTraits<bo>::MinLoadStepByteMultiple;
+
+  invariant(state.pos >= 0);
+  invariant(state.pos % MinByteStepMultiple == 0);
+  invariant(state.fillLevel >= 0);
+
+  auto numBytesRemainingInCache =
+      implicit_cast<int>(roundUpDivision(state.fillLevel, CHAR_BIT));
+  invariant(numBytesRemainingInCache >= 0);
+  invariant(numBytesRemainingInCache <= state.pos);
+
+  auto numBytesToBacktrack = implicit_cast<int>(
+      roundUp(numBytesRemainingInCache, MinByteStepMultiple));
+  invariant(numBytesToBacktrack >= 0);
+  invariant(numBytesToBacktrack <= state.pos);
+  invariant(numBytesToBacktrack % MinByteStepMultiple == 0);
+
+  auto numBitsToBacktrack = CHAR_BIT * numBytesToBacktrack;
+  invariant(numBitsToBacktrack >= 0);
+
+  ByteStreamPosition<bo> res;
+  invariant(state.pos >= numBytesToBacktrack);
+  res.bytePos = state.pos - numBytesToBacktrack;
+  invariant(numBitsToBacktrack >= state.fillLevel);
+  res.numBitsToSkip = numBitsToBacktrack - state.fillLevel;
+
+  invariant(res.bytePos >= 0);
+  invariant(res.bytePos <= state.pos);
+  invariant(res.bytePos % MinByteStepMultiple == 0);
+  invariant(res.numBitsToSkip >= 0);
+  return res;
+}
+
+} // namespace rawspeed

src/librawspeed/bitstreams/CMakeLists.txt

@@ -8,6 +8,7 @@ FILE(GLOB SOURCES
   "BitStreamMSB.h"
   "BitStreamMSB16.h"
   "BitStreamMSB32.h"
+  "BitStreamPosition.h"
   "BitStreamer.cpp"
   "BitStreamer.h"
   "BitStreamerJPEG.h"

test/librawspeed/bitstreams/BitVacuumerLSBTest.cpp

@@ -22,7 +22,9 @@
 #include "adt/Array1DRef.h"
 #include "adt/Casts.h"
 #include "adt/PartitioningOutputIterator.h"
+#include "bitstreams/BitStreamPosition.h"
 #include "bitstreams/BitStreamerLSB.h"
+#include <climits>
 #include <cstdint>
 #include <iterator>
 #include <utility>
@@ -250,6 +252,49 @@ TEST(BitVacuumerLSBTest, LoadPos) {
   }
 }
 
+TEST(BitVacuumerLSBTest, DependencyBreaking) {
+  std::vector<uint8_t> bitstream;
+
+  using BitStreamer = BitStreamerLSB;
+  using BitStreamerTraits = BitStreamer::Traits;
+
+  constexpr int numByteElts = 33;
+
+  {
+    auto bsInserter = PartitioningOutputIterator(std::back_inserter(bitstream));
+    using BitVacuumer = BitVacuumerLSB<decltype(bsInserter)>;
+    auto bv = BitVacuumer(bsInserter);
+
+    for (int e = 0; e != numByteElts + BitStreamerTraits::MaxProcessBytes; ++e)
+      bv.put(e, 8);
+  }
+  constexpr int numBitsTotal = CHAR_BIT * numByteElts;
+
+  auto fullInput =
+      Array1DRef(bitstream.data(), implicit_cast<int>(bitstream.size()));
+
+  for (int numBitsToSkip = 0; numBitsToSkip <= numBitsTotal; ++numBitsToSkip) {
+    const int numBitsRemaining = numBitsTotal - numBitsToSkip;
+    auto bsRef = BitStreamer(fullInput);
+    bsRef.fill();
+    bsRef.skipManyBits(numBitsToSkip);
+
+    BitStreamPosition<BitStreamerTraits::Tag> state;
+    state.pos = bsRef.getInputPosition();
+    state.fillLevel = bsRef.getFillLevel();
+    const auto bsPos = getAsByteStreamPosition(state);
+
+    auto rebasedInput =
+        fullInput.getCrop(bsPos.bytePos, fullInput.size() - bsPos.bytePos)
+            .getAsArray1DRef();
+    auto bsRebased = BitStreamer(rebasedInput);
+    if (bsPos.numBitsToSkip != 0)
+      bsRebased.skipBits(bsPos.numBitsToSkip);
+    for (int i = 0; i != numBitsRemaining; ++i)
+      ASSERT_THAT(bsRebased.getBits(1), bsRef.getBits(1));
+  }
+}
+
 } // namespace
 
 } // namespace rawspeed

test/librawspeed/bitstreams/BitVacuumerMSB16Test.cpp

@@ -22,7 +22,9 @@
 #include "adt/Array1DRef.h"
 #include "adt/Casts.h"
 #include "adt/PartitioningOutputIterator.h"
+#include "bitstreams/BitStreamPosition.h"
 #include "bitstreams/BitStreamerMSB16.h"
+#include <climits>
 #include <cstdint>
 #include <iterator>
 #include <utility>
@@ -247,6 +249,49 @@ TEST(BitVacuumerMSB16Test, LoadPos) {
   }
 }
 
+TEST(BitVacuumerMSB16Test, DependencyBreaking) {
+  std::vector<uint8_t> bitstream;
+
+  using BitStreamer = BitStreamerMSB16;
+  using BitStreamerTraits = BitStreamer::Traits;
+
+  constexpr int numByteElts = 33;
+
+  {
+    auto bsInserter = PartitioningOutputIterator(std::back_inserter(bitstream));
+    using BitVacuumer = BitVacuumerMSB16<decltype(bsInserter)>;
+    auto bv = BitVacuumer(bsInserter);
+
+    for (int e = 0; e != numByteElts + BitStreamerTraits::MaxProcessBytes; ++e)
+      bv.put(e, 8);
+  }
+  constexpr int numBitsTotal = CHAR_BIT * numByteElts;
+
+  auto fullInput =
+      Array1DRef(bitstream.data(), implicit_cast<int>(bitstream.size()));
+
+  for (int numBitsToSkip = 0; numBitsToSkip <= numBitsTotal; ++numBitsToSkip) {
+    const int numBitsRemaining = numBitsTotal - numBitsToSkip;
+    auto bsRef = BitStreamer(fullInput);
+    bsRef.fill();
+    bsRef.skipManyBits(numBitsToSkip);
+
+    BitStreamPosition<BitStreamerTraits::Tag> state;
+    state.pos = bsRef.getInputPosition();
+    state.fillLevel = bsRef.getFillLevel();
+    const auto bsPos = getAsByteStreamPosition(state);
+
+    auto rebasedInput =
+        fullInput.getCrop(bsPos.bytePos, fullInput.size() - bsPos.bytePos)
+            .getAsArray1DRef();
+    auto bsRebased = BitStreamer(rebasedInput);
+    if (bsPos.numBitsToSkip != 0)
+      bsRebased.skipBits(bsPos.numBitsToSkip);
+    for (int i = 0; i != numBitsRemaining; ++i)
+      ASSERT_THAT(bsRebased.getBits(1), bsRef.getBits(1));
+  }
+}
+
 } // namespace
 
 } // namespace rawspeed

test/librawspeed/bitstreams/BitVacuumerMSB32Test.cpp

@@ -22,7 +22,9 @@
 #include "adt/Array1DRef.h"
 #include "adt/Casts.h"
 #include "adt/PartitioningOutputIterator.h"
+#include "bitstreams/BitStreamPosition.h"
 #include "bitstreams/BitStreamerMSB32.h"
+#include <climits>
 #include <cstdint>
 #include <iterator>
 #include <utility>
@@ -250,6 +252,49 @@ TEST(BitVacuumerMSB32Test, LoadPos) {
   }
 }
 
+TEST(BitVacuumerMSB32Test, DependencyBreaking) {
+  std::vector<uint8_t> bitstream;
+
+  using BitStreamer = BitStreamerMSB32;
+  using BitStreamerTraits = BitStreamer::Traits;
+
+  constexpr int numByteElts = 33;
+
+  {
+    auto bsInserter = PartitioningOutputIterator(std::back_inserter(bitstream));
+    using BitVacuumer = BitVacuumerMSB32<decltype(bsInserter)>;
+    auto bv = BitVacuumer(bsInserter);
+
+    for (int e = 0; e != numByteElts + BitStreamerTraits::MaxProcessBytes; ++e)
+      bv.put(e, 8);
+  }
+  constexpr int numBitsTotal = CHAR_BIT * numByteElts;
+
+  auto fullInput =
+      Array1DRef(bitstream.data(), implicit_cast<int>(bitstream.size()));
+
+  for (int numBitsToSkip = 0; numBitsToSkip <= numBitsTotal; ++numBitsToSkip) {
+    const int numBitsRemaining = numBitsTotal - numBitsToSkip;
+    auto bsRef = BitStreamer(fullInput);
+    bsRef.fill();
+    bsRef.skipManyBits(numBitsToSkip);
+
+    BitStreamPosition<BitStreamerTraits::Tag> state;
+    state.pos = bsRef.getInputPosition();
+    state.fillLevel = bsRef.getFillLevel();
+    const auto bsPos = getAsByteStreamPosition(state);
+
+    auto rebasedInput =
+        fullInput.getCrop(bsPos.bytePos, fullInput.size() - bsPos.bytePos)
+            .getAsArray1DRef();
+    auto bsRebased = BitStreamer(rebasedInput);
+    if (bsPos.numBitsToSkip != 0)
+      bsRebased.skipBits(bsPos.numBitsToSkip);
+    for (int i = 0; i != numBitsRemaining; ++i)
+      ASSERT_THAT(bsRebased.getBits(1), bsRef.getBits(1));
+  }
+}
+
 } // namespace
 
 } // namespace rawspeed

test/librawspeed/bitstreams/BitVacuumerMSBTest.cpp

@@ -22,7 +22,9 @@
 #include "adt/Array1DRef.h"
 #include "adt/Casts.h"
 #include "adt/PartitioningOutputIterator.h"
+#include "bitstreams/BitStreamPosition.h"
 #include "bitstreams/BitStreamerMSB.h"
+#include <climits>
 #include <cstdint>
 #include <iterator>
 #include <utility>
@@ -250,6 +252,49 @@ TEST(BitVacuumerMSBTest, LoadPos) {
   }
 }
 
+TEST(BitVacuumerMSBTest, DependencyBreaking) {
+  std::vector<uint8_t> bitstream;
+
+  using BitStreamer = BitStreamerMSB;
+  using BitStreamerTraits = BitStreamer::Traits;
+
+  constexpr int numByteElts = 33;
+
+  {
+    auto bsInserter = PartitioningOutputIterator(std::back_inserter(bitstream));
+    using BitVacuumer = BitVacuumerMSB<decltype(bsInserter)>;
+    auto bv = BitVacuumer(bsInserter);
+
+    for (int e = 0; e != numByteElts + BitStreamerTraits::MaxProcessBytes; ++e)
+      bv.put(e, 8);
+  }
+  constexpr int numBitsTotal = CHAR_BIT * numByteElts;
+
+  auto fullInput =
+      Array1DRef(bitstream.data(), implicit_cast<int>(bitstream.size()));
+
+  for (int numBitsToSkip = 0; numBitsToSkip <= numBitsTotal; ++numBitsToSkip) {
+    const int numBitsRemaining = numBitsTotal - numBitsToSkip;
+    auto bsRef = BitStreamer(fullInput);
+    bsRef.fill();
+    bsRef.skipManyBits(numBitsToSkip);
+
+    BitStreamPosition<BitStreamerTraits::Tag> state;
+    state.pos = bsRef.getInputPosition();
+    state.fillLevel = bsRef.getFillLevel();
+    const auto bsPos = getAsByteStreamPosition(state);
+
+    auto rebasedInput =
+        fullInput.getCrop(bsPos.bytePos, fullInput.size() - bsPos.bytePos)
+            .getAsArray1DRef();
+    auto bsRebased = BitStreamer(rebasedInput);
+    if (bsPos.numBitsToSkip != 0)
+      bsRebased.skipBits(bsPos.numBitsToSkip);
+    for (int i = 0; i != numBitsRemaining; ++i)
+      ASSERT_THAT(bsRebased.getBits(1), bsRef.getBits(1));
+  }
+}
+
 } // namespace
 
 } // namespace rawspeed