From c239a897f1fbd42f75169e279bf5d457e89a0171 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Wed, 27 Nov 2024 23:17:03 +0100 Subject: [PATCH] Enable fully automated schema generation --- include/rfl/avro/SchemaImpl.hpp | 7 +- include/rfl/avro/read.hpp | 15 +++++ include/rfl/avro/schema/Type.hpp | 8 ++- include/rfl/avro/to_schema.hpp | 3 +- include/rfl/avro/write.hpp | 9 +++ src/reflectcpp_avro.cpp | 1 + src/rfl/avro/SchemaImpl.cpp | 2 +- src/rfl/avro/Type.cpp | 45 +++++++++++++ src/rfl/avro/to_schema.cpp | 66 +++++++++++++------ ...test_tutorial_example_automated_schema.cpp | 30 +++++++++ 10 files changed, 161 insertions(+), 25 deletions(-) create mode 100644 src/rfl/avro/Type.cpp create mode 100644 tests/avro/test_tutorial_example_automated_schema.cpp diff --git a/include/rfl/avro/SchemaImpl.hpp b/include/rfl/avro/SchemaImpl.hpp index 2f2fa50f..73e00d2c 100644 --- a/include/rfl/avro/SchemaImpl.hpp +++ b/include/rfl/avro/SchemaImpl.hpp @@ -5,7 +5,7 @@ #include -#include "../Box.hpp" +#include "../Ref.hpp" #include "../Result.hpp" namespace rfl::avro { @@ -27,6 +27,9 @@ class SchemaImpl { /// The JSON string used to create this schema. const std::string& json_str() const { return json_str_; } + /// The JSON string used to create this schema. + const std::string& str() const { return json_str_; } + /// The interface used to create new values. avro_value_iface_t* iface() const { return iface_; }; @@ -35,7 +38,7 @@ class SchemaImpl { std::string json_str_; /// The actual schema - Box schema_; + Ref schema_; /// The interface used to create new, generic classes. avro_value_iface_t* iface_; diff --git a/include/rfl/avro/read.hpp b/include/rfl/avro/read.hpp index 20765944..3505ab30 100644 --- a/include/rfl/avro/read.hpp +++ b/include/rfl/avro/read.hpp @@ -6,12 +6,14 @@ #include #include #include +#include #include "../Processors.hpp" #include "../internal/wrap_in_rfl_array_t.hpp" #include "Parser.hpp" #include "Reader.hpp" #include "Schema.hpp" +#include "to_schema.hpp" namespace rfl::avro { @@ -42,12 +44,25 @@ Result> read( return result; } +/// Parses an object from AVRO using reflection. +template +auto read(const char* _bytes, const size_t _size) noexcept { + const auto schema = to_schema, Ps...>(); + return read(_bytes, _size, schema); +} + /// Parses an object from AVRO using reflection. template auto read(const std::vector& _bytes, const Schema& _schema) { return read(_bytes.data(), _bytes.size(), _schema); } +/// Parses an object from AVRO using reflection. +template +auto read(const std::vector& _bytes) { + return read(_bytes.data(), _bytes.size()); +} + /// Parses an object from a stream. template auto read(std::istream& _stream) { diff --git a/include/rfl/avro/schema/Type.hpp b/include/rfl/avro/schema/Type.hpp index bbdbbeef..8e518cc4 100644 --- a/include/rfl/avro/schema/Type.hpp +++ b/include/rfl/avro/schema/Type.hpp @@ -75,12 +75,18 @@ struct Type { Rename<"default", std::map> default_; }; + struct Reference { + std::string type; + }; + using ReflectionType = rfl::Variant>; + Record, Enum, Array, Map, Reference, std::vector>; const auto& reflection() const { return value; } + Type with_name(const std::string& _name) const; + ReflectionType value; }; diff --git a/include/rfl/avro/to_schema.hpp b/include/rfl/avro/to_schema.hpp index 19c979e9..24038055 100644 --- a/include/rfl/avro/to_schema.hpp +++ b/include/rfl/avro/to_schema.hpp @@ -21,7 +21,6 @@ #include "Schema.hpp" #include "Writer.hpp" #include "schema/Type.hpp" -#include "write.hpp" namespace rfl::avro { @@ -34,7 +33,7 @@ Schema to_schema() { const auto internal_schema = parsing::schema::make>(); const auto json_str = to_json_representation(internal_schema); - return Schema::from_json(json_str).value(); + return std::move(Schema::from_json(json_str).value()); } } // namespace rfl::avro diff --git a/include/rfl/avro/write.hpp b/include/rfl/avro/write.hpp index 53c99682..316c6559 100644 --- a/include/rfl/avro/write.hpp +++ b/include/rfl/avro/write.hpp @@ -15,6 +15,7 @@ #include "Parser.hpp" #include "Schema.hpp" #include "Writer.hpp" +#include "to_schema.hpp" namespace rfl::avro { @@ -42,6 +43,14 @@ std::vector write(const auto& _obj, const auto& _schema) noexcept { return buffer; } +/// Returns AVRO bytes. +template +std::vector write(const auto& _obj) noexcept { + using T = std::remove_cvref_t; + const auto schema = to_schema(); + return write(_obj, schema); +} + /// Writes a AVRO into an ostream. template std::ostream& write(const auto& _obj, std::ostream& _stream) noexcept { diff --git a/src/reflectcpp_avro.cpp b/src/reflectcpp_avro.cpp index 31c3422a..d7b64f63 100644 --- a/src/reflectcpp_avro.cpp +++ b/src/reflectcpp_avro.cpp @@ -31,5 +31,6 @@ SOFTWARE. #include "rfl/avro/Reader.cpp" #include "rfl/avro/SchemaImpl.cpp" +#include "rfl/avro/Type.cpp" #include "rfl/avro/Writer.cpp" #include "rfl/avro/to_schema.cpp" diff --git a/src/rfl/avro/SchemaImpl.cpp b/src/rfl/avro/SchemaImpl.cpp index 21786bc7..dbf62cff 100644 --- a/src/rfl/avro/SchemaImpl.cpp +++ b/src/rfl/avro/SchemaImpl.cpp @@ -4,7 +4,7 @@ namespace rfl::avro { SchemaImpl::SchemaImpl(const std::string& _json_str) : json_str_(_json_str), - schema_(Box::make()), + schema_(Ref::make()), iface_(nullptr) { const auto err = avro_schema_from_json_length( _json_str.c_str(), _json_str.size(), schema_.get()); diff --git a/src/rfl/avro/Type.cpp b/src/rfl/avro/Type.cpp new file mode 100644 index 00000000..b4f850fb --- /dev/null +++ b/src/rfl/avro/Type.cpp @@ -0,0 +1,45 @@ +/* + +MIT License + +Copyright (c) 2023-2024 Code17 GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +*/ + +#include "rfl/avro/schema/Type.hpp" + +namespace rfl::avro::schema { + +Type Type::with_name(const std::string& _name) const { + const auto set_name = [&](const auto& _v) -> ReflectionType { + using T = std::remove_cvref_t; + if constexpr (std::is_same() || std::is_same()) { + auto v_with_name = _v; + v_with_name.name = _name; + return v_with_name; + } else { + return _v; + } + }; + return Type{.value = value.visit(set_name)}; +} + +} // namespace rfl::avro::schema diff --git a/src/rfl/avro/to_schema.cpp b/src/rfl/avro/to_schema.cpp index 933cacf9..6f5cec46 100644 --- a/src/rfl/avro/to_schema.cpp +++ b/src/rfl/avro/to_schema.cpp @@ -31,8 +31,11 @@ SOFTWARE. namespace rfl::avro { -schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) { - auto handle_variant = [](const auto& _t) -> schema::Type { +schema::Type type_to_avro_schema_type( + const parsing::schema::Type& _type, + const std::map& _definitions, + std::set* _already_known) { + auto handle_variant = [&](const auto& _t) -> schema::Type { using T = std::remove_cvref_t; using Type = parsing::schema::Type; if constexpr (std::is_same()) { @@ -62,18 +65,20 @@ schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) { } else if constexpr (std::is_same()) { auto any_of = std::vector(); for (const auto& t : _t.types_) { - any_of.emplace_back(type_to_avro_schema_type(t)); + any_of.emplace_back( + type_to_avro_schema_type(t, _definitions, _already_known)); } return schema::Type{.value = any_of}; } else if constexpr (std::is_same()) { // TODO: Return descriptions - return type_to_avro_schema_type(*_t.type_); + return type_to_avro_schema_type(*_t.type_, _definitions, _already_known); } else if constexpr (std::is_same()) { - return schema::Type{.value = schema::Type::Array{ - .items = Ref::make( - type_to_avro_schema_type(*_t.type_))}}; + return schema::Type{ + .value = schema::Type::Array{ + .items = Ref::make(type_to_avro_schema_type( + *_t.type_, _definitions, _already_known))}}; } else if constexpr (std::is_same()) { return schema::Type{.value = schema::Type::Enum{.symbols = _t.values_}}; @@ -83,35 +88,46 @@ schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) { for (const auto& [k, v] : _t.types_) { record.fields.push_back(schema::Type::RecordField{ .name = k, - .type = Ref::make(type_to_avro_schema_type(v))}); + .type = Ref::make( + type_to_avro_schema_type(v, _definitions, _already_known))}); } return schema::Type{.value = record}; } else if constexpr (std::is_same()) { return schema::Type{.value = std::vector( - {type_to_avro_schema_type(*_t.type_), + {type_to_avro_schema_type(*_t.type_, _definitions, + _already_known), schema::Type{schema::Type::Null{}}})}; } else if constexpr (std::is_same()) { - return schema::Type{.value = _t.name_}; + if (!_already_known || + _already_known->find(_t.name_) != _already_known->end() || + _definitions.find(_t.name_) == _definitions.end()) { + return schema::Type{.value = schema::Type::Reference{.type = _t.name_}}; + } else { + _already_known->insert(_t.name_); + return _definitions.at(_t.name_); + } } else if constexpr (std::is_same()) { - return schema::Type{.value = schema::Type::Map{ - .values = Ref::make( - type_to_avro_schema_type(*_t.value_type_))}}; + return schema::Type{ + .value = schema::Type::Map{ + .values = Ref::make(type_to_avro_schema_type( + *_t.value_type_, _definitions, _already_known))}}; } else if constexpr (std::is_same()) { // TODO: Handle tuples. return schema::Type{.value = schema::Type::Record{}}; } else if constexpr (std::is_same()) { - return schema::Type{.value = schema::Type::Array{ - .items = Ref::make( - type_to_avro_schema_type(*_t.type_))}}; + return schema::Type{ + .value = schema::Type::Array{ + .items = Ref::make(type_to_avro_schema_type( + *_t.type_, _definitions, _already_known))}}; } else if constexpr (std::is_same()) { // Avro knows no validation. - return type_to_avro_schema_type(*_t.type_); + return type_to_avro_schema_type(*_t.type_, _definitions, _already_known); } else { static_assert(rfl::always_false_v, "Not all cases were covered."); @@ -121,9 +137,21 @@ schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) { return rfl::visit(handle_variant, _type.variant_); } +std::map transform_definitions( + const std::map& _definitions) { + std::map definitions; + for (const auto& [k, v] : _definitions) { + definitions[k] = type_to_avro_schema_type(v, {}, nullptr).with_name(k); + } + return definitions; +} + std::string to_json_representation( - const parsing::schema::Definition& internal_schema) { - const auto avro_schema = type_to_avro_schema_type(internal_schema.root_); + const parsing::schema::Definition& _internal_schema) { + const auto definitions = transform_definitions(_internal_schema.definitions_); + std::set already_known; + const auto avro_schema = type_to_avro_schema_type( + _internal_schema.root_, definitions, &already_known); return rfl::json::write(avro_schema); } diff --git a/tests/avro/test_tutorial_example_automated_schema.cpp b/tests/avro/test_tutorial_example_automated_schema.cpp new file mode 100644 index 00000000..7d2af0f3 --- /dev/null +++ b/tests/avro/test_tutorial_example_automated_schema.cpp @@ -0,0 +1,30 @@ +#include + +#include +#include +#include +#include +#include + +#include "write_and_read.hpp" + +/// The basic example from the Avro C tutorial. +namespace test_tutorial_example_automated_schema { + +struct Person { + size_t ID; + std::string First; + std::string Last; + std::string Phone; + int Age; +}; + +TEST(avro, test_tutorial_example_automated_schema) { + const auto person = Person{.ID = 1, + .First = "Randal", + .Last = "Graves", + .Phone = "(555) 123-5678", + .Age = 30}; + write_and_read(person); +} +} // namespace test_tutorial_example_automated_schema