diff --git a/build/source_list.bzl b/build/source_list.bzl index a27b05a2b..5b95fcfa0 100644 --- a/build/source_list.bzl +++ b/build/source_list.bzl @@ -58,6 +58,7 @@ quiche_core_hdrs = [ "common/quiche_text_utils.h", "common/simple_buffer_allocator.h", "common/structured_headers.h", + "common/vectorized_io_utils.h", "common/wire_serialization.h", "http2/adapter/chunked_buffer.h", "http2/adapter/data_source.h", @@ -422,6 +423,7 @@ quiche_core_srcs = [ "common/quiche_text_utils.cc", "common/simple_buffer_allocator.cc", "common/structured_headers.cc", + "common/vectorized_io_utils.cc", "http2/adapter/chunked_buffer.cc", "http2/adapter/event_forwarder.cc", "http2/adapter/header_validator.cc", @@ -1102,6 +1104,7 @@ quiche_tests_srcs = [ "common/structured_headers_test.cc", "common/test_tools/mock_streams_test.cc", "common/test_tools/quiche_test_utils_test.cc", + "common/vectorized_io_utils_test.cc", "common/wire_serialization_test.cc", "http2/adapter/chunked_buffer_test.cc", "http2/adapter/event_forwarder_test.cc", diff --git a/build/source_list.gni b/build/source_list.gni index 17315d72c..dc5104f5b 100644 --- a/build/source_list.gni +++ b/build/source_list.gni @@ -58,6 +58,7 @@ quiche_core_hdrs = [ "src/quiche/common/quiche_text_utils.h", "src/quiche/common/simple_buffer_allocator.h", "src/quiche/common/structured_headers.h", + "src/quiche/common/vectorized_io_utils.h", "src/quiche/common/wire_serialization.h", "src/quiche/http2/adapter/chunked_buffer.h", "src/quiche/http2/adapter/data_source.h", @@ -422,6 +423,7 @@ quiche_core_srcs = [ "src/quiche/common/quiche_text_utils.cc", "src/quiche/common/simple_buffer_allocator.cc", "src/quiche/common/structured_headers.cc", + "src/quiche/common/vectorized_io_utils.cc", "src/quiche/http2/adapter/chunked_buffer.cc", "src/quiche/http2/adapter/event_forwarder.cc", "src/quiche/http2/adapter/header_validator.cc", @@ -1103,6 +1105,7 @@ quiche_tests_srcs = [ "src/quiche/common/structured_headers_test.cc", "src/quiche/common/test_tools/mock_streams_test.cc", "src/quiche/common/test_tools/quiche_test_utils_test.cc", + "src/quiche/common/vectorized_io_utils_test.cc", "src/quiche/common/wire_serialization_test.cc", "src/quiche/http2/adapter/chunked_buffer_test.cc", "src/quiche/http2/adapter/event_forwarder_test.cc", diff --git a/build/source_list.json b/build/source_list.json index 5ea661c6c..5b746d0c4 100644 --- a/build/source_list.json +++ b/build/source_list.json @@ -57,6 +57,7 @@ "quiche/common/quiche_text_utils.h", "quiche/common/simple_buffer_allocator.h", "quiche/common/structured_headers.h", + "quiche/common/vectorized_io_utils.h", "quiche/common/wire_serialization.h", "quiche/http2/adapter/chunked_buffer.h", "quiche/http2/adapter/data_source.h", @@ -421,6 +422,7 @@ "quiche/common/quiche_text_utils.cc", "quiche/common/simple_buffer_allocator.cc", "quiche/common/structured_headers.cc", + "quiche/common/vectorized_io_utils.cc", "quiche/http2/adapter/chunked_buffer.cc", "quiche/http2/adapter/event_forwarder.cc", "quiche/http2/adapter/header_validator.cc", @@ -1102,6 +1104,7 @@ "quiche/common/structured_headers_test.cc", "quiche/common/test_tools/mock_streams_test.cc", "quiche/common/test_tools/quiche_test_utils_test.cc", + "quiche/common/vectorized_io_utils_test.cc", "quiche/common/wire_serialization_test.cc", "quiche/http2/adapter/chunked_buffer_test.cc", "quiche/http2/adapter/event_forwarder_test.cc", diff --git a/quiche/common/quiche_buffer_allocator.h b/quiche/common/quiche_buffer_allocator.h index 0f3684299..4fb1bd44d 100644 --- a/quiche/common/quiche_buffer_allocator.h +++ b/quiche/common/quiche_buffer_allocator.h @@ -10,6 +10,7 @@ #include #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "quiche/common/platform/api/quiche_export.h" #include "quiche/common/platform/api/quiche_iovec.h" @@ -109,6 +110,7 @@ class QUICHE_EXPORT QuicheBuffer { absl::string_view AsStringView() const { return absl::string_view(data(), size()); } + absl::Span AsSpan() { return absl::Span(data(), size()); } // Releases the ownership of the underlying buffer. QuicheUniqueBufferPtr Release() { diff --git a/quiche/common/quiche_stream.h b/quiche/common/quiche_stream.h index 187e36217..5a8a1f779 100644 --- a/quiche/common/quiche_stream.h +++ b/quiche/common/quiche_stream.h @@ -199,15 +199,6 @@ inline absl::Status SendFinOnStream(WriteStream& stream) { return stream.Writev(absl::Span(), options); } -inline size_t TotalStringViewSpanSize( - absl::Span span) { - size_t total = 0; - for (absl::string_view view : span) { - total += view.size(); - } - return total; -} - } // namespace quiche #endif // QUICHE_COMMON_QUICHE_STREAM_H_ diff --git a/quiche/common/vectorized_io_utils.cc b/quiche/common/vectorized_io_utils.cc new file mode 100644 index 000000000..1ce421a22 --- /dev/null +++ b/quiche/common/vectorized_io_utils.cc @@ -0,0 +1,56 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "quiche/common/vectorized_io_utils.h" + +#include +#include +#include + +#include "absl/base/optimization.h" +#include "absl/base/prefetch.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace quiche { + +size_t TotalStringViewSpanSize(absl::Span span) { + size_t total = 0; + for (absl::string_view view : span) { + total += view.size(); + } + return total; +} + +size_t GatherStringViewSpan(absl::Span inputs, + absl::Span output) { + size_t bytes_copied = 0; + for (size_t i = 0; i < inputs.size(); ++i) { + if (inputs[i].empty()) { + continue; + } + const size_t bytes_to_copy = std::min(inputs[i].size(), output.size()); + if (bytes_to_copy == 0) { + break; + } + const absl::Span next_output = output.subspan(bytes_to_copy); + + // Prefetch the first two lines of the next input; the hardware prefetcher + // is expected to take care of the rest. + if (!next_output.empty() && (i + 1) < inputs.size() && + !inputs[i + 1].empty()) { + absl::PrefetchToLocalCache(&inputs[i + 1][0]); + if (inputs[i + 1].size() > ABSL_CACHELINE_SIZE) { + absl::PrefetchToLocalCache(&inputs[i + 1][ABSL_CACHELINE_SIZE]); + } + } + + memcpy(output.data(), inputs[i].data(), bytes_to_copy); + bytes_copied += bytes_to_copy; + output = next_output; + } + return bytes_copied; +} + +} // namespace quiche diff --git a/quiche/common/vectorized_io_utils.h b/quiche/common/vectorized_io_utils.h new file mode 100644 index 000000000..2ac000250 --- /dev/null +++ b/quiche/common/vectorized_io_utils.h @@ -0,0 +1,26 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef QUICHE_COMMON_VECTORIZED_IO_UTILS_H_ +#define QUICHE_COMMON_VECTORIZED_IO_UTILS_H_ + +#include + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace quiche { + +// Computes the total size of all strings in the provided span. +size_t TotalStringViewSpanSize(absl::Span span); + +// Copies data contained in `inputs` into `output`, up until either the `output` +// is full or the `inputs` are copied fully; returns the actual number of bytes +// copied. +size_t GatherStringViewSpan(absl::Span inputs, + absl::Span output); + +} // namespace quiche + +#endif // QUICHE_COMMON_VECTORIZED_IO_UTILS_H_ diff --git a/quiche/common/vectorized_io_utils_test.cc b/quiche/common/vectorized_io_utils_test.cc new file mode 100644 index 000000000..dfa5e9cf6 --- /dev/null +++ b/quiche/common/vectorized_io_utils_test.cc @@ -0,0 +1,79 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "quiche/common/vectorized_io_utils.h" + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "quiche/common/platform/api/quiche_test.h" + +namespace quiche { +namespace { + +using ::testing::ElementsAre; + +TEST(VectorizedIoUtils, GatherStringViewSpanEmpty) { + std::vector views = {"a", "b", "c"}; + size_t bytes_copied = GatherStringViewSpan(views, absl::Span()); + EXPECT_EQ(bytes_copied, 0); +} + +TEST(VectorizedIoUtils, GatherStringViewSpanSingle) { + std::vector views = {"test"}; + char buffer_small[2]; + size_t bytes_copied = + GatherStringViewSpan(views, absl::MakeSpan(buffer_small)); + EXPECT_EQ(bytes_copied, 2); + EXPECT_THAT(buffer_small, ElementsAre('t', 'e')); + + char buffer_exact[4]; + bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer_exact)); + EXPECT_EQ(bytes_copied, 4); + EXPECT_THAT(buffer_exact, ElementsAre('t', 'e', 's', 't')); + + char buffer_large[6] = {'\0'}; + bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer_large)); + EXPECT_EQ(bytes_copied, 4); + EXPECT_THAT(buffer_large, ElementsAre('t', 'e', 's', 't', '\0', '\0')); +} + +TEST(VectorizedIoUtils, GatherStringViewSpanMultiple) { + const std::vector views = {"foo", ",", "bar"}; + constexpr absl::string_view kViewsJoined = "foo,bar"; + char buffer[kViewsJoined.size()]; + for (size_t i = 0; i < sizeof(buffer); ++i) { + size_t bytes_copied = + GatherStringViewSpan(views, absl::Span(buffer, i)); + absl::string_view result(buffer, bytes_copied); + EXPECT_EQ(result, kViewsJoined.substr(0, i)); + } +} + +TEST(VectorizedIoUtils, GatherStringViewSpanEmptyElement) { + const std::vector views = {"foo", "", "bar"}; + constexpr absl::string_view kViewsJoined = "foobar"; + char buffer[kViewsJoined.size()]; + size_t bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer)); + absl::string_view result(buffer, bytes_copied); + EXPECT_EQ(result, kViewsJoined); +} + +TEST(VectorizedIoUtils, GatherStringViewSpanLarge) { + const std::string a(8192, 'a'); + const std::string b(8192, 'b'); + const std::vector views = {a, b}; + const std::string joined = a + b; + char buffer[8192 * 2]; + size_t bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer)); + EXPECT_EQ(bytes_copied, 8192 * 2); + absl::string_view result(buffer, bytes_copied); + EXPECT_EQ(result, joined); +} + +} // namespace +} // namespace quiche diff --git a/quiche/quic/core/http/web_transport_stream_adapter.cc b/quiche/quic/core/http/web_transport_stream_adapter.cc index 5f84081dd..0900323a7 100644 --- a/quiche/quic/core/http/web_transport_stream_adapter.cc +++ b/quiche/quic/core/http/web_transport_stream_adapter.cc @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -25,8 +25,11 @@ #include "quiche/quic/platform/api/quic_bug_tracker.h" #include "quiche/quic/platform/api/quic_flags.h" #include "quiche/quic/platform/api/quic_logging.h" -#include "quiche/common/quiche_mem_slice_storage.h" +#include "quiche/common/platform/api/quiche_logging.h" +#include "quiche/common/platform/api/quiche_mem_slice.h" +#include "quiche/common/quiche_buffer_allocator.h" #include "quiche/common/quiche_stream.h" +#include "quiche/common/vectorized_io_utils.h" #include "quiche/web_transport/web_transport.h" namespace quic { @@ -79,21 +82,20 @@ absl::Status WebTransportStreamAdapter::Writev( return initial_check_status; } - std::vector iovecs; - size_t total_size = 0; - iovecs.resize(data.size()); - for (size_t i = 0; i < data.size(); i++) { - // QuicheMemSliceStorage only reads iovec, thus this is safe. - iovecs[i].iov_base = const_cast(data[i].data()); - iovecs[i].iov_len = data[i].size(); - total_size += data[i].size(); + size_t total_size = quiche::TotalStringViewSpanSize(data); + quiche::QuicheMemSlice slice; + if (total_size > 0) { + quiche::QuicheBuffer buffer( + session_->connection()->helper()->GetStreamSendBufferAllocator(), + total_size); + size_t bytes_copied = quiche::GatherStringViewSpan(data, buffer.AsSpan()); + QUICHE_DCHECK_EQ(total_size, bytes_copied); + slice = quiche::QuicheMemSlice(std::move(buffer)); } - quiche::QuicheMemSliceStorage storage( - iovecs.data(), iovecs.size(), - session_->connection()->helper()->GetStreamSendBufferAllocator(), - GetQuicFlag(quic_send_buffer_max_data_slice_size)); QuicConsumedData consumed = stream_->WriteMemSlices( - storage.ToSpan(), /*fin=*/options.send_fin(), + slice.empty() ? absl::Span() + : absl::MakeSpan(&slice, 1), + /*fin=*/options.send_fin(), /*buffer_uncondtionally=*/options.buffer_unconditionally()); if (consumed.bytes_consumed == total_size) { diff --git a/quiche/web_transport/encapsulated/encapsulated_web_transport.cc b/quiche/web_transport/encapsulated/encapsulated_web_transport.cc index b6380e823..30ab1d449 100644 --- a/quiche/web_transport/encapsulated/encapsulated_web_transport.cc +++ b/quiche/web_transport/encapsulated/encapsulated_web_transport.cc @@ -36,6 +36,7 @@ #include "quiche/common/quiche_circular_deque.h" #include "quiche/common/quiche_status_utils.h" #include "quiche/common/quiche_stream.h" +#include "quiche/common/vectorized_io_utils.h" #include "quiche/web_transport/web_transport.h" namespace webtransport {