Updated option, updated test

data-avro-generator/build.gradle

@@ -2,6 +2,7 @@ dependencies {
   compile project(':data')
   compile project(':data-avro')
   compile externalDependency.avro
+  compile externalDependency.skyScreamer
   testCompile project(path: ':data', configuration: 'testArtifacts')
   testCompile externalDependency.testng
 }

data-avro-generator/src/main/java/com/linkedin/data/avro/generator/AvroSchemaGenerator.java

@@ -44,10 +44,17 @@
 import java.util.Set;
 
 import java.util.stream.Collectors;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.skyscreamer.jsonassert.Customization;
+import org.skyscreamer.jsonassert.JSONAssert;
+import org.skyscreamer.jsonassert.JSONCompareMode;
+import org.skyscreamer.jsonassert.JSONParser;
+import org.skyscreamer.jsonassert.comparator.CustomComparator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static com.linkedin.data.avro.SchemaTranslator.AVRO_PREFIX;
+import static com.linkedin.data.avro.SchemaTranslator.*;
 
 
 /**
@@ -162,7 +169,11 @@ public static void run(String resolverPath,
     {
       targetDirectoryPath += "/" + AVRO_PREFIX;
     }
-    generator.generate(targetDirectoryPath, sources);
+    try {
+      generator.generate(targetDirectoryPath, sources);
+    } catch (JSONException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   /**
@@ -180,8 +191,7 @@ public DataToAvroSchemaTranslationOptions getDataToAvroSchemaTranslationOptions(
    * @param targetDirectoryPath path to target root java source directory
    * @throws IOException if there are problems opening or deleting files.
    */
-  private void generate(String targetDirectoryPath, String[] sources) throws IOException
-  {
+  private void generate(String targetDirectoryPath, String[] sources) throws IOException, JSONException {
     initSchemaResolver();
 
     _fileToAvroSchemaMap.clear();
@@ -242,8 +252,7 @@ protected void outputAvroSchemas(File targetDirectory) throws IOException
     }
   }
 
-  protected List<File> targetFiles(File targetDirectory)
-  {
+  protected List<File> targetFiles(File targetDirectory) throws JSONException {
     ArrayList<File> generatedFiles = new ArrayList<>();
 
     DataSchemaResolver resolver = getSchemaResolver();
@@ -267,7 +276,14 @@ protected List<File> targetFiles(File targetDirectory)
           String avroSchemaText = SchemaTranslator.dataToAvroSchemaJson(recordDataSchema, _options);
           _fileToAvroSchemaMap.put(generatedFile, avroSchemaText);
           String postTranslateSchemaText = recordDataSchema.toString();
-          assert(preTranslateSchemaText.equals(postTranslateSchemaText));
+
+          // JSON compare except TRANSLATED_FROM_SOURCE_OPTION in root
+          JSONAssert.assertEquals(preTranslateSchemaText, postTranslateSchemaText,
+              new CustomComparator(JSONCompareMode.LENIENT,
+                  new Customization(TRANSLATED_FROM_SOURCE_OPTION, (o1, o2) -> true)));
+
+          assert (((JSONObject) JSONParser.parseJSON(postTranslateSchemaText)).get(TRANSLATED_FROM_SOURCE_OPTION)
+              != null);
         }
       }
     }

data-avro-generator/src/test/java/com/linkedin/data/avro/generator/TestAvroSchemaGenerator.java

@@ -34,14 +34,19 @@
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Parser;
 import org.apache.commons.compress.utils.IOUtils;
+import org.json.JSONException;
+import org.skyscreamer.jsonassert.Customization;
+import org.skyscreamer.jsonassert.JSONAssert;
+import org.skyscreamer.jsonassert.JSONCompareMode;
+import org.skyscreamer.jsonassert.comparator.CustomComparator;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import static com.linkedin.data.TestUtil.*;
+import static com.linkedin.data.avro.SchemaTranslator.*;
 import static com.linkedin.data.avro.generator.AvroSchemaGenerator.GENERATOR_AVRO_NAMESPACE_OVERRIDE;
-import static com.linkedin.data.avro.SchemaTranslator.AVRO_PREFIX;
 import static com.linkedin.data.schema.generator.AbstractGenerator.GENERATOR_RESOLVER_PATH;
 import static com.linkedin.util.FileUtil.buildSystemIndependentPath;
 import static org.testng.Assert.assertEquals;
@@ -175,8 +180,8 @@ public Object[][] toAvroSchemaData()
   }
 
   @Test(dataProvider = "toAvroSchemaData")
-  public void testFileNameAsArgs(Map<String, String> testSchemas, Map<String, String> expectedAvroSchemas, List<String> paths, boolean override) throws IOException
-  {
+  public void testFileNameAsArgs(Map<String, String> testSchemas, Map<String, String> expectedAvroSchemas, List<String> paths, boolean override)
+      throws IOException, JSONException {
     Map<File, Map.Entry<String,String>>  files = TestUtil.createSchemaFiles(_testDir, testSchemas, _debug);
     // directory in path
     Collection<String> testPaths = computePathFromRelativePaths(_testDir, paths);
@@ -193,8 +198,8 @@ public void testFileNameAsArgs(Map<String, String> testSchemas, Map<String, Stri
   }
 
   @Test(dataProvider = "toAvroSchemaData")
-  public void testFullNameAsArgsWithJarInPath(Map<String, String> testSchemas, Map<String, String> expectedAvroSchemas, List<String> paths, boolean override) throws IOException
-  {
+  public void testFullNameAsArgsWithJarInPath(Map<String, String> testSchemas, Map<String, String> expectedAvroSchemas, List<String> paths, boolean override)
+      throws IOException, JSONException {
     Map<File, Map.Entry<String,String>>  files = TestUtil.createSchemaFiles(_testDir, testSchemas, _debug);
     // jar files in path, create jar files
     Collection<String> testPaths = createJarsFromRelativePaths(_testDir, testSchemas, paths, _debug);
@@ -324,8 +329,8 @@ private File setup(Collection<String> paths, boolean override) throws IOExceptio
     return targetDir;
   }
 
-  private void run(String[] args, Map.Entry<File, Map.Entry<String, String>> entry, File targetDir, Map<String, String> expectedAvroSchemas) throws IOException
-  {
+  private void run(String[] args, Map.Entry<File, Map.Entry<String, String>> entry, File targetDir, Map<String, String> expectedAvroSchemas)
+      throws IOException, JSONException {
     Exception exc = null;
     try
     {
@@ -359,7 +364,11 @@ private void run(String[] args, Map.Entry<File, Map.Entry<String, String>> entry
       assertFalse(avroSchema.isError());
       String avroSchemaText = avroSchema.toString();
       if (_debug) out.println(avroSchemaText);
-      assertEquals(avroSchemaText, expectedAvroSchemas.get(pdscFileName));
+
+      // JSON compare except TRANSLATED_FROM_SOURCE_OPTION in root
+      JSONAssert.assertEquals(expectedAvroSchemas.get(pdscFileName), avroSchemaText,
+          new CustomComparator(JSONCompareMode.LENIENT,
+              new Customization(TRANSLATED_FROM_SOURCE_OPTION, (o1, o2) -> true)));
     }
   }
 }

data-avro/src/main/java/com/linkedin/data/avro/SchemaTranslator.java

@@ -58,7 +58,7 @@ public class SchemaTranslator
   private static final Logger log = LoggerFactory.getLogger(SchemaTranslator.class);
 
   public static final String DATA_PROPERTY = "com.linkedin.data";
-  public static final String TRANSLATED_FROM_SOURCE_OPTION = "li.data.translated.from";
+  public static final String TRANSLATED_FROM_SOURCE_OPTION = "schema.translated.from.src";
   public static final String SCHEMA_PROPERTY = "schema";
   public static final String OPTIONAL_DEFAULT_MODE_PROPERTY = "optionalDefaultMode";
   public static final String AVRO_FILE_EXTENSION = ".avsc";

data-avro/src/test/java/com/linkedin/data/avro/TestFilteredSchemaDataTranslation.java

@@ -137,7 +137,7 @@ public void testFilteredDataSchemaDataTranslation() throws IOException
         "{ " +
         "  \"type\" : \"record\", " +
         "  \"name\" : \"Foo\", " +
-        "  \"li.data.translated.from\" : \"Foo\", " +
+        "  \"schema.translated.from.src\" : \"Foo\", " +
         "  \"fields\" : [ " +
         "    { \"name\" : \"a\", \"type\" : \"int\" }, " +
         "    { \"name\" : \"b\", \"type\" : \"int\", \"optional\" : true } " +