Enable fully automated schema generation

getml · Nov 27, 2024 · c239a89 · c239a89
1 parent d8a7236
commit c239a89
Show file tree

Hide file tree

Showing 10 changed files with 161 additions and 25 deletions.
diff --git a/include/rfl/avro/SchemaImpl.hpp b/include/rfl/avro/SchemaImpl.hpp
@@ -5,7 +5,7 @@
 
 #include <string>
 
-#include "../Box.hpp"
+#include "../Ref.hpp"
 #include "../Result.hpp"
 
 namespace rfl::avro {
@@ -27,6 +27,9 @@ class SchemaImpl {
   /// The JSON string used to create this schema.
   const std::string& json_str() const { return json_str_; }
 
+  /// The JSON string used to create this schema.
+  const std::string& str() const { return json_str_; }
+
   /// The interface used to create new values.
   avro_value_iface_t* iface() const { return iface_; };
 
@@ -35,7 +38,7 @@ class SchemaImpl {
   std::string json_str_;
 
   /// The actual schema
-  Box<avro_schema_t> schema_;
+  Ref<avro_schema_t> schema_;
 
   /// The interface used to create new, generic classes.
   avro_value_iface_t* iface_;

diff --git a/include/rfl/avro/read.hpp b/include/rfl/avro/read.hpp
@@ -6,12 +6,14 @@
 #include <bit>
 #include <istream>
 #include <string>
+#include <type_traits>
 
 #include "../Processors.hpp"
 #include "../internal/wrap_in_rfl_array_t.hpp"
 #include "Parser.hpp"
 #include "Reader.hpp"
 #include "Schema.hpp"
+#include "to_schema.hpp"
 
 namespace rfl::avro {
 
@@ -42,12 +44,25 @@ Result<internal::wrap_in_rfl_array_t<T>> read(
   return result;
 }
 
+/// Parses an object from AVRO using reflection.
+template <class T, class... Ps>
+auto read(const char* _bytes, const size_t _size) noexcept {
+  const auto schema = to_schema<std::remove_cvref_t<T>, Ps...>();
+  return read(_bytes, _size, schema);
+}
+
 /// Parses an object from AVRO using reflection.
 template <class T, class... Ps>
 auto read(const std::vector<char>& _bytes, const Schema<T>& _schema) {
   return read<T, Ps...>(_bytes.data(), _bytes.size(), _schema);
 }
 
+/// Parses an object from AVRO using reflection.
+template <class T, class... Ps>
+auto read(const std::vector<char>& _bytes) {
+  return read<T, Ps...>(_bytes.data(), _bytes.size());
+}
+
 /// Parses an object from a stream.
 template <class T, class... Ps>
 auto read(std::istream& _stream) {

diff --git a/include/rfl/avro/schema/Type.hpp b/include/rfl/avro/schema/Type.hpp
@@ -75,12 +75,18 @@ struct Type {
     Rename<"default", std::map<std::string, std::string>> default_;
   };
 
+  struct Reference {
+    std::string type;
+  };
+
   using ReflectionType =
       rfl::Variant<Null, Boolean, Int, Long, Float, Double, Bytes, String,
-                   Record, Enum, Array, Map, std::string, std::vector<Type>>;
+                   Record, Enum, Array, Map, Reference, std::vector<Type>>;
 
   const auto& reflection() const { return value; }
 
+  Type with_name(const std::string& _name) const;
+
   ReflectionType value;
 };
 

diff --git a/include/rfl/avro/to_schema.hpp b/include/rfl/avro/to_schema.hpp
@@ -21,7 +21,6 @@
 #include "Schema.hpp"
 #include "Writer.hpp"
 #include "schema/Type.hpp"
-#include "write.hpp"
 
 namespace rfl::avro {
 
@@ -34,7 +33,7 @@ Schema<T> to_schema() {
   const auto internal_schema =
       parsing::schema::make<Reader, Writer, T, Processors<Ps...>>();
   const auto json_str = to_json_representation(internal_schema);
-  return Schema<T>::from_json(json_str).value();
+  return std::move(Schema<T>::from_json(json_str).value());
 }
 }  // namespace rfl::avro
 

diff --git a/include/rfl/avro/write.hpp b/include/rfl/avro/write.hpp
@@ -15,6 +15,7 @@
 #include "Parser.hpp"
 #include "Schema.hpp"
 #include "Writer.hpp"
+#include "to_schema.hpp"
 
 namespace rfl::avro {
 
@@ -42,6 +43,14 @@ std::vector<char> write(const auto& _obj, const auto& _schema) noexcept {
   return buffer;
 }
 
+/// Returns AVRO bytes.
+template <class... Ps>
+std::vector<char> write(const auto& _obj) noexcept {
+  using T = std::remove_cvref_t<decltype(_obj)>;
+  const auto schema = to_schema<T, Ps...>();
+  return write(_obj, schema);
+}
+
 /// Writes a AVRO into an ostream.
 template <class... Ps>
 std::ostream& write(const auto& _obj, std::ostream& _stream) noexcept {

diff --git a/src/reflectcpp_avro.cpp b/src/reflectcpp_avro.cpp
@@ -31,5 +31,6 @@ SOFTWARE.
 
 #include "rfl/avro/Reader.cpp"
 #include "rfl/avro/SchemaImpl.cpp"
+#include "rfl/avro/Type.cpp"
 #include "rfl/avro/Writer.cpp"
 #include "rfl/avro/to_schema.cpp"
diff --git a/src/rfl/avro/SchemaImpl.cpp b/src/rfl/avro/SchemaImpl.cpp
@@ -4,7 +4,7 @@ namespace rfl::avro {
 
 SchemaImpl::SchemaImpl(const std::string& _json_str)
     : json_str_(_json_str),
-      schema_(Box<avro_schema_t>::make()),
+      schema_(Ref<avro_schema_t>::make()),
       iface_(nullptr) {
   const auto err = avro_schema_from_json_length(
       _json_str.c_str(), _json_str.size(), schema_.get());

diff --git a/src/rfl/avro/Type.cpp b/src/rfl/avro/Type.cpp
@@ -0,0 +1,45 @@
+/*
+
+MIT License
+
+Copyright (c) 2023-2024 Code17 GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "rfl/avro/schema/Type.hpp"
+
+namespace rfl::avro::schema {
+
+Type Type::with_name(const std::string& _name) const {
+  const auto set_name = [&](const auto& _v) -> ReflectionType {
+    using T = std::remove_cvref_t<decltype(_v)>;
+    if constexpr (std::is_same<T, Record>() || std::is_same<T, Enum>()) {
+      auto v_with_name = _v;
+      v_with_name.name = _name;
+      return v_with_name;
+    } else {
+      return _v;
+    }
+  };
+  return Type{.value = value.visit(set_name)};
+}
+
+}  // namespace rfl::avro::schema
diff --git a/src/rfl/avro/to_schema.cpp b/src/rfl/avro/to_schema.cpp
@@ -31,8 +31,11 @@ SOFTWARE.
 
 namespace rfl::avro {
 
-schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) {
-  auto handle_variant = [](const auto& _t) -> schema::Type {
+schema::Type type_to_avro_schema_type(
+    const parsing::schema::Type& _type,
+    const std::map<std::string, schema::Type>& _definitions,
+    std::set<std::string>* _already_known) {
+  auto handle_variant = [&](const auto& _t) -> schema::Type {
     using T = std::remove_cvref_t<decltype(_t)>;
     using Type = parsing::schema::Type;
     if constexpr (std::is_same<T, Type::Boolean>()) {
@@ -62,18 +65,20 @@ schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) {
     } else if constexpr (std::is_same<T, Type::AnyOf>()) {
       auto any_of = std::vector<schema::Type>();
       for (const auto& t : _t.types_) {
-        any_of.emplace_back(type_to_avro_schema_type(t));
+        any_of.emplace_back(
+            type_to_avro_schema_type(t, _definitions, _already_known));
       }
       return schema::Type{.value = any_of};
 
     } else if constexpr (std::is_same<T, Type::Description>()) {
       // TODO: Return descriptions
-      return type_to_avro_schema_type(*_t.type_);
+      return type_to_avro_schema_type(*_t.type_, _definitions, _already_known);
 
     } else if constexpr (std::is_same<T, Type::FixedSizeTypedArray>()) {
-      return schema::Type{.value = schema::Type::Array{
-                              .items = Ref<schema::Type>::make(
-                                  type_to_avro_schema_type(*_t.type_))}};
+      return schema::Type{
+          .value = schema::Type::Array{
+              .items = Ref<schema::Type>::make(type_to_avro_schema_type(
+                  *_t.type_, _definitions, _already_known))}};
 
     } else if constexpr (std::is_same<T, Type::Literal>()) {
       return schema::Type{.value = schema::Type::Enum{.symbols = _t.values_}};
@@ -83,35 +88,46 @@ schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) {
       for (const auto& [k, v] : _t.types_) {
         record.fields.push_back(schema::Type::RecordField{
             .name = k,
-            .type = Ref<schema::Type>::make(type_to_avro_schema_type(v))});
+            .type = Ref<schema::Type>::make(
+                type_to_avro_schema_type(v, _definitions, _already_known))});
       }
       return schema::Type{.value = record};
 
     } else if constexpr (std::is_same<T, Type::Optional>()) {
       return schema::Type{.value = std::vector<schema::Type>(
-                              {type_to_avro_schema_type(*_t.type_),
+                              {type_to_avro_schema_type(*_t.type_, _definitions,
+                                                        _already_known),
                                schema::Type{schema::Type::Null{}}})};
 
     } else if constexpr (std::is_same<T, Type::Reference>()) {
-      return schema::Type{.value = _t.name_};
+      if (!_already_known ||
+          _already_known->find(_t.name_) != _already_known->end() ||
+          _definitions.find(_t.name_) == _definitions.end()) {
+        return schema::Type{.value = schema::Type::Reference{.type = _t.name_}};
+      } else {
+        _already_known->insert(_t.name_);
+        return _definitions.at(_t.name_);
+      }
 
     } else if constexpr (std::is_same<T, Type::StringMap>()) {
-      return schema::Type{.value = schema::Type::Map{
-                              .values = Ref<schema::Type>::make(
-                                  type_to_avro_schema_type(*_t.value_type_))}};
+      return schema::Type{
+          .value = schema::Type::Map{
+              .values = Ref<schema::Type>::make(type_to_avro_schema_type(
+                  *_t.value_type_, _definitions, _already_known))}};
 
     } else if constexpr (std::is_same<T, Type::Tuple>()) {
       // TODO: Handle tuples.
       return schema::Type{.value = schema::Type::Record{}};
 
     } else if constexpr (std::is_same<T, Type::TypedArray>()) {
-      return schema::Type{.value = schema::Type::Array{
-                              .items = Ref<schema::Type>::make(
-                                  type_to_avro_schema_type(*_t.type_))}};
+      return schema::Type{
+          .value = schema::Type::Array{
+              .items = Ref<schema::Type>::make(type_to_avro_schema_type(
+                  *_t.type_, _definitions, _already_known))}};
 
     } else if constexpr (std::is_same<T, Type::Validated>()) {
       // Avro knows no validation.
-      return type_to_avro_schema_type(*_t.type_);
+      return type_to_avro_schema_type(*_t.type_, _definitions, _already_known);
 
     } else {
       static_assert(rfl::always_false_v<T>, "Not all cases were covered.");
@@ -121,9 +137,21 @@ schema::Type type_to_avro_schema_type(const parsing::schema::Type& _type) {
   return rfl::visit(handle_variant, _type.variant_);
 }
 
+std::map<std::string, schema::Type> transform_definitions(
+    const std::map<std::string, parsing::schema::Type>& _definitions) {
+  std::map<std::string, schema::Type> definitions;
+  for (const auto& [k, v] : _definitions) {
+    definitions[k] = type_to_avro_schema_type(v, {}, nullptr).with_name(k);
+  }
+  return definitions;
+}
+
 std::string to_json_representation(
-    const parsing::schema::Definition& internal_schema) {
-  const auto avro_schema = type_to_avro_schema_type(internal_schema.root_);
+    const parsing::schema::Definition& _internal_schema) {
+  const auto definitions = transform_definitions(_internal_schema.definitions_);
+  std::set<std::string> already_known;
+  const auto avro_schema = type_to_avro_schema_type(
+      _internal_schema.root_, definitions, &already_known);
   return rfl::json::write(avro_schema);
 }
 

diff --git a/tests/avro/test_tutorial_example_automated_schema.cpp b/tests/avro/test_tutorial_example_automated_schema.cpp
@@ -0,0 +1,30 @@
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <rfl.hpp>
+#include <rfl/avro.hpp>
+#include <string>
+#include <vector>
+
+#include "write_and_read.hpp"
+
+/// The basic example from the Avro C tutorial.
+namespace test_tutorial_example_automated_schema {
+
+struct Person {
+  size_t ID;
+  std::string First;
+  std::string Last;
+  std::string Phone;
+  int Age;
+};
+
+TEST(avro, test_tutorial_example_automated_schema) {
+  const auto person = Person{.ID = 1,
+                             .First = "Randal",
+                             .Last = "Graves",
+                             .Phone = "(555) 123-5678",
+                             .Age = 30};
+  write_and_read(person);
+}
+}  // namespace test_tutorial_example_automated_schema