update test script - change label checking to an implicit empty value…

… checking, change indo_nli to indonli, add jsonlines requirements

nusantara/nusa_datasets/indo_nli/indo_nli.py → nusantara/nusa_datasets/indonli/indonli.py

@@ -53,7 +53,7 @@
 }
 """
 
-_DATASETNAME = "indo_nli"
+_DATASETNAME = "indonli"
 
 _DESCRIPTION = """\
 This dataset is designed for Natural Language Inference NLP task.  It is designed to provide a challenging test-bed
@@ -100,22 +100,22 @@ class IndoNli(datasets.GeneratorBasedBuilder):
 
     BUILDER_CONFIGS = [
         NusantaraConfig(
-            name="indo_nli_source",
+            name="indonli_source",
             version=SOURCE_VERSION,
-            description="indo_nli source schema",
+            description="indonli source schema",
             schema="source",
-            subset_id="indo_nli",
+            subset_id="indonli",
         ),
         NusantaraConfig(
-            name="indo_nli_nusantara_pairs",
+            name="indonli_nusantara_pairs",
             version=NUSANTARA_VERSION,
-            description="indo_nli Nusantara schema",
+            description="indonli Nusantara schema",
             schema="nusantara_pairs",
-            subset_id="indo_nli",
+            subset_id="indonli",
         ),
     ]
 
-    DEFAULT_CONFIG_NAME = "indo_nli_source"
+    DEFAULT_CONFIG_NAME = "indonli_source"
     labels = ["c", "e", "n"]
 
     def _info(self) -> datasets.DatasetInfo:

requirements.txt

@@ -11,3 +11,4 @@ aiohttp==3.8.1
 pre-commit==2.19.0
 soundfile
 librosa
+jsonlines>=3.1.0

tests/test_nusantara.py

@@ -224,7 +224,7 @@ def get_feature_statistics(self, features: Features, schema: str) -> Dict:
                 for feature_name, feature in features.items():
                     if example.get(feature_name, None) is not None:
                         if isinstance(feature, datasets.ClassLabel) or isinstance(feature, datasets.Value):
-                            if example[feature_name]:
+                            if example[feature_name] is not None:
                                 counter[feature_name] += 1
                         else:
                             counter[feature_name] += len(example[feature_name])