Guard against self-referencing schemas, stabilize scan order with sort

Signed-off-by: Michael Edgar <[email protected]>

core/src/main/java/io/smallrye/openapi/runtime/scanner/OpenApiAnnotationScanner.java

@@ -242,6 +242,7 @@ private void processClassSchemas(final AnnotationScannerContext context) {
                 .stream()
                 .filter(this::annotatedClasses)
                 .map(annotation -> Type.create(annotation.target().asClass().name(), Type.Kind.CLASS))
+                .sorted(Comparator.comparing(Type::name)) // Process annotation classes in predictable order
                 .forEach(type -> SchemaFactory.typeToSchema(context, type, null, context.getExtensions()));
     }
 

core/src/main/java/io/smallrye/openapi/runtime/scanner/OpenApiDataObjectScanner.java

@@ -236,10 +236,11 @@ private void depthFirstGraphSearch() {
              * reference a registered schema.
              */
             Schema entrySchema = currentPathEntry.getSchema();
+            SchemaRegistry registry = context.getSchemaRegistry();
 
-            if (context.getSchemaRegistry().hasSchema(currentType, context.getJsonViews(), null)) {
+            if (registry.hasSchema(currentType, context.getJsonViews(), null)) {
                 // This type has already been scanned and registered, don't do it again!
-                entrySchema.setRef(context.getSchemaRegistry().lookupRef(currentType, context.getJsonViews()).getRef());
+                entrySchema.setRef(registry.lookupRef(currentType, context.getJsonViews()).getRef());
                 continue;
             }
 
@@ -260,12 +261,17 @@ private void depthFirstGraphSearch() {
                 /*
                  * Ignore the returned ref when:
                  *
+                 * - the ref is the currentSchema, i.e. registration did not occur
                  * - the currentSchema is an object type and will be further modified with added properties
-                 * - this is the root object entry that should never be set to a ref here. That may be done
-                 * by the process that is invoking this instance of OpenApiDataObjectScanner.
+                 * - the target of the ref is the schema currently being processed and using the ref would
+                 * result in a self-referencing schema.
                  */
-                if (currentSchema.getType() != Schema.SchemaType.OBJECT && entrySchema != rootSchema) {
-                    entrySchema.setRef(ref.getRef());
+                if (ref != currentSchema && currentSchema.getType() != Schema.SchemaType.OBJECT) {
+                    Schema refTarget = registry.lookupSchema(currentType, context.getJsonViews());
+
+                    if (refTarget != entrySchema) {
+                        entrySchema.setRef(ref.getRef());
+                    }
                 }
             }
 

core/src/test/java/io/smallrye/openapi/runtime/scanner/StandaloneSchemaScanTest.java

@@ -738,4 +738,34 @@ class Bean {
         printToConsole(result);
         assertJsonEquals("components.schemas.iterator-stream-map-types.json", result);
     }
+
+    /*
+     * https://github.com/smallrye/smallrye-open-api/issues/1565
+     *
+     * Verify that registered schemas are not set to a self reference.
+     * Previously, a schema of an object property may have been set
+     * in components as a ref to itself when the property schema was
+     * discovered as one of the parent object's fields. Here, Class2
+     * would have been a self-ref if Class1 were scanned first. If Class2
+     * were scanned first, the issue would not occur.
+     */
+    @Test
+    void testNoSelfRefToSchemaOfAnnotatedObjectProperty() throws IOException, JSONException {
+        @Schema(type = SchemaType.STRING, name = "MyValueClass")
+        class Class2 {
+        }
+
+        @Schema(description = "some description", name = "MyClass")
+        class Class1 {
+            @SuppressWarnings("unused")
+            Class2 value;
+        }
+
+        Index index = indexOf(Class1.class, Class2.class);
+        OpenApiAnnotationScanner scanner = new OpenApiAnnotationScanner(emptyConfig(), index);
+        OpenAPI result = scanner.scan();
+        printToConsole(result);
+        assertJsonEquals("components.schemas.no-self-ref-for-property-schema.json", result);
+    }
+
 }

core/src/test/resources/io/smallrye/openapi/runtime/scanner/components.schemas.no-self-ref-for-property-schema.json

@@ -0,0 +1,19 @@
+{
+  "openapi" : "3.0.3",
+  "components" : {
+    "schemas" : {
+      "MyClass" : {
+        "description" : "some description",
+        "type" : "object",
+        "properties" : {
+          "value" : {
+            "$ref" : "#/components/schemas/MyValueClass"
+          }
+        }
+      },
+      "MyValueClass" : {
+        "type" : "string"
+      }
+    }
+  }
+}