fixes required for deeper pymathics modules (#1031)

This PR seems to fix the issue with Pymathics Natlang documentation
(Mathics3/pymathics-natlang#19 (comment))

mathics/core/load_builtin.py

@@ -141,6 +141,41 @@ def get_module_names(builtin_path: str, exclude_files: set) -> list:
     return [f for f in py_files if f not in exclude_files]
 
 
+def get_submodule_names(obj) -> list:
+    """Many builtins are organized into modules which, from a documentation
+    standpoint, are like Mathematica Online Guide Docs.
+
+    "List Functions", "Colors", or "Distance and Similarity Measures"
+    are some examples Guide Documents group group various Builtin Functions,
+    under submodules relate to that general classification.
+
+    Here, we want to return a list of the Python modules under a "Guide Doc"
+    module.
+
+    As an example of a "Guide Doc" and its submodules, consider the
+    module named mathics.builtin.colors. It collects code and documentation pertaining
+    to the builtin functions that would be found in the Guide documentation for "Colors".
+
+    The `mathics.builtin.colors` module has a submodule
+    `mathics.builtin.colors.named_colors`.
+
+    The builtin functions defined in `named_colors` then are those found in the
+    "Named Colors" group of the "Colors" Guide Doc.
+
+    So in this example then, in the list the modules returned for
+    Python module `mathics.builtin.colors` would be the
+    `mathics.builtin.colors.named_colors` module which contains the
+    definition and docs for the "Named Colors" Mathics Bultin
+    Functions.
+    """
+    modpkgs = []
+    if hasattr(obj, "__path__"):
+        for _, modname, __ in pkgutil.iter_modules(obj.__path__):
+            modpkgs.append(modname)
+        modpkgs.sort()
+    return modpkgs
+
+
 def import_and_load_builtins():
     """
     Imports Builtin modules in mathics.builtin and add rules, and definitions from that.
@@ -306,6 +341,17 @@ def name_is_builtin_symbol(module: ModuleType, name: str) -> Optional[type]:
     return module_object
 
 
+def submodules(package):
+    """Generator of the submodules in a package"""
+    package_folder = package.__file__[: -len("__init__.py")]
+    for _, module_name, __ in pkgutil.iter_modules([package_folder]):
+        try:
+            module = importlib.import_module(package.__name__ + "." + module_name)
+        except Exception:
+            continue
+        yield module
+
+
 def update_display_operators_set(builtin_instance):
     """
     If builtin_instance is an operator of some kind, add that

mathics/doc/gather.py

@@ -14,6 +14,7 @@
 from typing import Tuple, Union
 
 from mathics.core.builtin import Builtin, check_requires_list
+from mathics.core.load_builtin import get_submodule_names, submodules
 from mathics.core.util import IS_PYPY
 from mathics.doc.doc_entries import DocumentationEntry
 from mathics.doc.structure import DocChapter, DocGuideSection, DocSection, DocSubsection
@@ -138,7 +139,7 @@ def gather_sections(chapter, module, builtins_by_module, section_class=None) ->
     # converting the entries into `set`s.
     #
     visited = set()
-    for symbol_instance in builtins_by_module[module.__name__]:
+    for symbol_instance in builtins_by_module.get(module.__name__, []):
         if skip_doc(symbol_instance, module):
             continue
         default_contexts = ("System`", "Pymathics`")
@@ -361,14 +362,3 @@ def sorted_modules(modules) -> list:
         if hasattr(module, "sort_order")
         else module.__name__,
     )
-
-
-def submodules(package):
-    """Generator of the submodules in a package"""
-    package_folder = package.__file__[: -len("__init__.py")]
-    for _, module_name, __ in pkgutil.iter_modules([package_folder]):
-        try:
-            module = importlib.import_module(package.__name__ + "." + module_name)
-        except Exception:
-            continue
-        yield module

mathics/eval/pymathics.py

@@ -69,10 +69,11 @@ def load_pymathics_module(definitions, module_name: str):
         if var is not None:
             instance = var(expression=False)
             if isinstance(instance, Builtin):
+                submodule_name = var.__module__
                 if not var.context:
                     var.context = "Pymathics`"
                 symbol_name = instance.get_name()
-                builtins_by_module[loaded_module.__name__].append(instance)
+                builtins_by_module.setdefault(submodule_name, []).append(instance)
                 newsymbols[symbol_name] = instance
 
     for name in newsymbols: