Move spec org/group/status info to a config file (#2908)

* wip

* WIP rewrite the kdl format, start creating the manager class

* Put statuses.kdl in the boilerplate folder, as it'll need to be modified alongside boilerplate changes/additions

* Finish the basic formating/importing of data.

* wip

* wip

* Finish updating all usage sites of the old status code.

* Fix file location

* Add test group and bs-test org. Fix nodes to all be org.

* Allow type to be omitted

* Grab from the right nodes

* wip fixing errors

* Move org/group/status from MetadataManager to a new Doctype object on Spec.

* Swap the org/group/status order around to be consistent at all callsites.

* Some bugs, and move/copy some statuses into the w3c org. Fix the test boilerplate to not assume it's a w3c file.

* rebase tests

* Uppercase org/group/status. Fold type/groupTypes into the general Group/Status classes; kill GroupW3C/StatusW3C.

* A few more little fixes, and rebase tests

* Add the few groups that are in boilerplates project but not present here.

* Export Doctype in the module

* Refresh spec data, so doctypes.kdl will be included by default

* Make the sources list a little less stupid (no Nones or strings containing None)

* refresh data again

* Rebase manual tests

* Rebase non-manual tests. Not reviewed; they're just data changes. This PR's actual changes were tested on these files earlier.

bikeshed/Spec.py

@@ -18,6 +18,7 @@
     constants,
     datablocks,
     dfns,
+    doctypes,
     extensions,
     fingerprinting,
     h,
@@ -108,6 +109,7 @@ def initializeState(self) -> bool:
         self.widl: widlparser.Parser = idl.getParser()
 
         self.languages: dict[str, language.Language] = fetchLanguages(self.dataFile)
+        self.doctypes: doctypes.DoctypeManager = fetchDoctypes(self.dataFile)
 
         self.extraJC = stylescript.JCManager()
         self.extraJC.addColors()
@@ -141,21 +143,27 @@ def initMetadata(self, inputContent: InputSource.InputContent) -> None:
         #       in a markdown code span or an <xmp> to show off.
         _, self.mdDocument = metadata.parse(lines=inputContent.lines)
 
-        # Combine the data so far...
+        # Combine the data so far, and compute the doctype
+        # (the other md sources need the doctype in order to be found)
         self.md = metadata.join(self.mdBaseline, self.mdDocument, self.mdCommandLine)
+        rawDoctype = (self.md.rawOrg, self.md.rawGroup, self.md.rawStatus)
+        self.doctype = self.doctypes.getDoctype(self.md.rawOrg, self.md.rawGroup, self.md.rawStatus)
 
-        # Using that to determine the Group and Status, load the correct defaults.include boilerplate
         self.mdDefaults = metadata.fromJson(
-            data=retrieve.retrieveBoilerplateFile(self, "defaults", error=True),
+            data=retrieve.retrieveBoilerplateFile(self, "defaults"),
             source="defaults",
         )
         self.md = metadata.join(self.mdBaseline, self.mdDefaults, self.mdDocument, self.mdCommandLine)
+        if rawDoctype != (self.md.rawOrg, self.md.rawGroup, self.md.rawStatus):
+            # recompute doctype
+            self.doctype = self.doctypes.getDoctype(self.md.rawOrg, self.md.rawGroup, self.md.rawStatus)
 
         # Using all of that, load up the text macros so I can sub them into the computed-metadata file.
         self.md.fillTextMacros(self.macros, doc=self)
         jsonEscapedMacros = {k: json.dumps(v)[1:-1] for k, v in self.macros.items()}
+
         computedMdText = h.replaceMacrosTextly(
-            retrieve.retrieveBoilerplateFile(self, "computed-metadata", error=True),
+            retrieve.retrieveBoilerplateFile(self, "computed-metadata"),
             macros=jsonEscapedMacros,
             context="? of computed-metadata.include",
         )
@@ -173,7 +181,7 @@ def initMetadata(self, inputContent: InputSource.InputContent) -> None:
         # And compute macros again, in case the preceding steps changed them.
         self.md.fillTextMacros(self.macros, doc=self)
 
-        self.md.validate()
+        self.md.validate(doc=self)
         m.retroactivelyCheckErrorLevel()
 
     def earlyParse(self, inputContent: InputSource.InputContent) -> list[l.Line]:
@@ -231,7 +239,7 @@ def assembleDocument(self) -> Spec:
             features=markdownFeatures,
         )
 
-        self.refs.setSpecData(self.md)
+        self.refs.setSpecData(self)
 
         # Convert to a single string of html now, for convenience.
         self.html = "".join(x.text for x in self.lines)
@@ -554,6 +562,10 @@ def fetchLanguages(dataFile: retrieve.DataFileRequester) -> dict[str, language.L
     }
 
 
+def fetchDoctypes(dataFile: retrieve.DataFileRequester) -> doctypes.DoctypeManager:
+    return doctypes.DoctypeManager.fromKdlStr(dataFile.fetch("boilerplate", "doctypes.kdl", str=True))
+
+
 def addDomintroStyles(doc: Spec) -> None:
     # Adds common WHATWG styles for domintro blocks.
 

bikeshed/boilerplate.py

@@ -157,7 +157,7 @@ def getFillContainer(tag: str, doc: t.SpecT, default: bool = False) -> t.Element
 
     # Otherwise, append to the end of the document,
     # unless you're in the byos group
-    if doc.md.group == "byos":
+    if doc.doctype.group.name == "BYOS":
         return None
     if default:
         return doc.body
@@ -240,7 +240,7 @@ def removeUnwantedBoilerplate(doc: t.SpecT) -> None:
 
 
 def w3cStylesheetInUse(doc: t.SpecT) -> bool:
-    return doc.md.prepTR or doc.md.status in config.snapshotStatuses
+    return doc.md.prepTR or doc.doctype.group.name == "W3C"
 
 
 def addBikeshedBoilerplate(doc: t.SpecT) -> None:
@@ -933,7 +933,7 @@ def printPreviousVersion(v: dict[str, str]) -> t.ElementT | None:
         md.setdefault("This version", []).append(h.E.a({"href": mac["version"], "class": "u-url"}, mac["version"]))
     if doc.md.TR:
         md.setdefault("Latest published version", []).append(h.E.a({"href": doc.md.TR}, doc.md.TR))
-    if doc.md.ED and doc.md.status in config.snapshotStatuses:
+    if doc.md.ED and "TR" in doc.doctype.status.requires:
         md.setdefault("Editor's Draft", []).append(h.E.a({"href": doc.md.ED}, doc.md.ED))
     if doc.md.previousVersions:
         md["Previous Versions"] = [printPreviousVersion(ver) for ver in doc.md.previousVersions]

bikeshed/conditional.py

@@ -3,7 +3,7 @@
 import dataclasses
 import re
 
-from . import config, h, t
+from . import h, t
 from . import messages as m
 
 # Any element can have an include-if or exclude-if attribute,
@@ -61,7 +61,7 @@ def processConditionals(doc: t.SpecT, container: t.ElementT | None = None) -> No
 def evalConditions(doc: t.SpecT, el: t.ElementT, conditionString: str) -> t.Generator[bool, None, None]:
     for cond in parseConditions(conditionString, el):
         if cond.type == "status":
-            yield config.looselyMatch(cond.value, doc.md.status)
+            yield doc.doctype.status.looselyMatch(cond.value)
         elif cond.type == "text macro":
             for k in doc.macros:
                 if k.upper() == cond.value:

bikeshed/config/__init__.py

@@ -35,23 +35,3 @@
     simplifyText,
     splitForValues,
 )
-from .status import (
-    canonicalizeStatus,
-    datedStatuses,
-    deadlineStatuses,
-    implementationStatuses,
-    looselyMatch,
-    megaGroups,
-    noEDStatuses,
-    shortToLongStatus,
-    snapshotStatuses,
-    splitStatus,
-    unlevelledStatuses,
-    w3cCgs,
-    w3cCommunityStatuses,
-    w3cIgs,
-    w3cIGStatuses,
-    w3cProcessDocumentStatuses,
-    w3cTAGStatuses,
-    w3cWGStatuses,
-)

bikeshed/config/main.py

@@ -12,6 +12,7 @@
 def englishFromList(items: t.Iterable[str], conjunction: str = "or") -> str:
     # Format a list of strings into an English list.
     items = list(items)
+    assert len(items) > 0
     if len(items) == 1:
         return items[0]
     if len(items) == 2: