envoy.base.utils: Add protoc worker protocol

README.md

@@ -124,15 +124,16 @@ pypi: https://pypi.org/project/aio.run.runner
 
 #### [envoy.base.utils](envoy.base.utils)
 
-version: 0.3.7.dev0
+version: 0.3.7
 
 pypi: https://pypi.org/project/envoy.base.utils
 
 ##### requirements:
 
 - [abstracts](https://pypi.org/project/abstracts) >=0.0.12
+- [aio.api.bazel](https://pypi.org/project/aio.api.bazel) >=0.0.2
 - [aio.api.github](https://pypi.org/project/aio.api.github) >=0.1.2
-- [aio.core](https://pypi.org/project/aio.core) >=0.8.6
+- [aio.core](https://pypi.org/project/aio.core) >=0.8.9
 - [aio.run.runner](https://pypi.org/project/aio.run.runner) >=0.3.3
 - [aiohttp](https://pypi.org/project/aiohttp) >=3.8.1
 - [frozendict](https://pypi.org/project/frozendict)

envoy.base.utils/VERSION

@@ -1 +1 @@
-0.3.7-dev
+0.3.7

envoy.base.utils/envoy/base/utils/BUILD

@@ -3,8 +3,9 @@ pytooling_library(
     "envoy.base.utils",
     dependencies=[
         "//deps:abstracts",
-        "//deps:aio.core",
+        "//deps:aio.api.bazel",
         "//deps:aio.api.github",
+        "//deps:aio.core",
         "//deps:aio.run.runner",
         "//deps:aiohttp",
         "//deps:frozendict",

envoy.base.utils/envoy/base/utils/__init__.py

@@ -21,13 +21,23 @@
     is_sha,
     tar_mode,
     last_n_bytes_of)
-from . import interface, typing
+from . import abstract, interface, typing
+from .abstract import (
+    AChangelog,
+    AChangelogEntry,
+    AChangelogs,
+    AInventories,
+    AProject,
+    AProtobufSet,
+    AProtobufValidator,
+    AProtocProtocol)
+
 from .parallel_cmd import parallel_cmd
 from .parallel_runner import ParallelRunner
 from .project import Changelog, ChangelogEntry, Changelogs, Project
 from .project_cmd import project_cmd
 from .project_runner import ProjectRunner
-from .protobuf import ProtobufSet, ProtobufValidator
+from .protobuf import ProtobufSet, ProtobufValidator, ProtocProtocol
 from .interface import IProject
 from .data_env import DataEnvironment
 from .data_env_cmd import data_env_cmd
@@ -36,7 +46,17 @@
 
 
 __all__ = (
+    "abstract",
+    "AChangelog",
+    "AChangelogEntry",
+    "AChangelogs",
+    "AInventories",
+    "AProject",
+    "AProtobufSet",
+    "AProtobufValidator",
+    "AProtocProtocol",
     "async_list",
+    "bazel_worker_cmd",
     "cd_and_return",
     "Changelog",
     "ChangelogEntry",
@@ -67,6 +87,7 @@
     "project_cmd",
     "ProtobufSet",
     "ProtobufValidator",
+    "ProtocProtocol",
     "typed",
     "untar",
     "TAR_EXTS",

envoy.base.utils/envoy/base/utils/abstract/__init__.py

@@ -5,7 +5,10 @@
     AChangelogs,
     AInventories,
     AProject)
-from .protobuf import AProtobufSet, AProtobufValidator
+from .protobuf import (
+    AProtobufSet,
+    AProtobufValidator,
+    AProtocProtocol)
 
 __all__ = (
     "AChangelog",
@@ -14,4 +17,5 @@
     "AInventories",
     "AProject",
     "AProtobufSet",
-    "AProtobufValidator")
+    "AProtobufValidator",
+    "AProtocProtocol")

envoy.base.utils/envoy/base/utils/abstract/protobuf.py

@@ -1,19 +1,24 @@
 
+import argparse
 import importlib
 import json
 import pathlib
 from functools import cached_property, lru_cache
-from typing import Callable, Type
+from typing import Callable, Dict, Type
 
 from google.protobuf import descriptor, descriptor_pb2
 from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import json_format
+from google.protobuf import json_format, text_format
 from google.protobuf import (
     message as _message,
     message_factory as _message_factory)
+from google.protobuf.compiler import plugin_pb2
 
 import abstracts
 
+from aio.api import bazel
+from aio.core.utils import dottedname_resolve
+
 from envoy.base.utils import interface
 
 
@@ -32,29 +37,57 @@ def _yaml():
     return _envoy_yaml
 
 
+@abstracts.implementer(interface.IProtobufSet)
 class AProtobufSet(metaclass=abstracts.Abstraction):
 
     def __init__(self, descriptor_path: str | pathlib.Path) -> None:
         self._descriptor_path = descriptor_path
 
+    def __getitem__(self, name: str) -> descriptor_pb2.FileDescriptorProto:
+        return self.source_files[name]
+
     @cached_property
     def descriptor_path(self) -> pathlib.Path:
         return pathlib.Path(self._descriptor_path)
 
     @cached_property
     def descriptor_pool(self) -> _descriptor_pool.DescriptorPool:
         pool = _descriptor_pool.DescriptorPool()
-        for f in self.descriptor_set.file:
+        for f in self.source_files.values():
             pool.Add(f)
         return pool
 
     @cached_property
     def descriptor_set(self) -> descriptor_pb2.FileDescriptorSet:
+        # from tools.protoxform.utils import load_protos
+
+        # load_protos()
+
         descriptor = descriptor_pb2.FileDescriptorSet()
+
+        # text_format.Parse(
+        #    self.descriptor_path.read_text(),
+        #    descriptor)
+            # allow_unknown_extension=True)
+
         descriptor.ParseFromString(self.descriptor_path.read_bytes())
         return descriptor
 
+    @cached_property
+    def source_files(self) -> Dict[str, descriptor_pb2.FileDescriptorProto]:
+        return {
+            f.name: f
+            for f
+            in self.descriptor_set.file}
+
+    def find_file(self, type_name: str) -> descriptor.Descriptor:
+        return self.descriptor_pool.FindFileByName(type_name)
 
+    def find_message(self, type_name: str) -> descriptor.Descriptor:
+        return self.descriptor_pool.FindMessageTypeByName(type_name)
+
+
+@abstracts.implementer(interface.IProtobufValidator)
 class AProtobufValidator(metaclass=abstracts.Abstraction):
 
     def __init__(self, descriptor_path: str | pathlib.Path) -> None:
@@ -81,17 +114,15 @@ def protobuf_set_class(self) -> Type[interface.IProtobufSet]:
     def yaml(self):
         return _yaml()
 
-    def find_message(self, type_name: str) -> descriptor.Descriptor:
-        return self.descriptor_pool.FindMessageTypeByName(type_name)
-
     @lru_cache
     def message(self, type_name: str) -> _message.Message:
         return self.message_prototype(type_name)()
 
     def message_prototype(
             self,
             type_name: str) -> Callable[[], _message.Message]:
-        return self.message_factory.GetPrototype(self.find_message(type_name))
+        return self.message_factory.GetPrototype(
+            self.protobuf_set.find_message(type_name))
 
     def validate_fragment(
             self,
@@ -113,3 +144,67 @@ def validate_yaml(
             fragment: str,
             type_name: str = BOOTSTRAP_PROTO) -> None:
         self.validate_fragment(self.yaml.safe_load(fragment), type_name)
+
+
+@abstracts.implementer(bazel.IBazelProcessProtocol)
+class AProtocProtocol(
+        bazel.ABazelProcessProtocol,
+        metaclass=abstracts.Abstraction):
+
+    @classmethod
+    def add_protocol_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("descriptor_set")
+        parser.add_argument("plugin")
+
+    @property
+    def descriptor_set(self) -> str:
+        return self.args.descriptor_set
+
+    @cached_property
+    def plugin(self) -> Callable:
+        return dottedname_resolve(f"{self.args.plugin}.main")
+
+    @cached_property
+    def proto_set(self) -> interface.IProtobufSet:
+        return self.proto_set_class(
+            pathlib.Path(self.descriptor_set).absolute())
+
+    @property  # type:ignore
+    @abstracts.interfacemethod
+    def proto_set_class(self) -> Type[interface.IProtobufSet]:
+        raise NotImplementedError
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("--in")
+        parser.add_argument("--out")
+        parser.add_argument("--suffixes")
+        parser.add_argument("--type_db_path")
+        parser.add_argument("--extra_args")
+
+    def parse_request_files(
+            self,
+            request: argparse.Namespace) -> Dict[str, pathlib.Path]:
+        return dict(
+            zip(vars(request)["in"].split(","),
+                (pathlib.Path(p)
+                 for p
+                 in request.out.split(","))))
+
+    async def process(self, request: argparse.Namespace) -> None:
+        for infile, outfile in self.parse_request_files(request).items():
+            self._process_item(self._generate_request(request, infile), outfile)
+
+    def _generate_request(self, request, infile):
+        proto_file = self.proto_set[infile]
+        code_request = plugin_pb2.CodeGeneratorRequest(
+            file_to_generate=[proto_file.name],
+            parameter=f"type_db_path={request.type_db_path}")
+        code_request.proto_file.append(proto_file)
+        return code_request
+
+    def _output(self, request) -> str:
+        return self.plugin(request).file[0].content
+
+    def _process_item(self, request, outfile: pathlib.Path) -> None:
+        outfile.parent.mkdir(exist_ok=True, parents=True)
+        outfile.write_text(self._output(request))

envoy.base.utils/envoy/base/utils/interface.py

@@ -1,32 +1,58 @@
 
 import pathlib
 from typing import (
-    AsyncGenerator, ItemsView, Iterator, KeysView, List,
+    AsyncGenerator, Dict, ItemsView, Iterator, KeysView, List,
     Optional, Set, Tuple, Type, Union, ValuesView)
 
 import aiohttp
-from google.protobuf import descriptor_pool
+from google.protobuf import (
+    descriptor,
+    descriptor_pool,
+    descriptor_pb2)
 from packaging import version as _version
 
 import abstracts
 
-from aio.api import github as _github
+from aio.api import bazel, github as _github
 from aio.core import directory as _directory, event
 
 from envoy.base.utils import typing
 
 
+class IProtocProtocol(
+        bazel.IBazelProcessProtocol,
+        metaclass=abstracts.Interface):
+    pass
+
+
 class IProtobufSet(metaclass=abstracts.Interface):
 
     @abstracts.interfacemethod
     def __init__(self, descriptor_path: str | pathlib.Path) -> None:
         raise NotImplementedError
 
+    @abstracts.interfacemethod
+    def __getitem__(self, name: str) -> descriptor_pb2.FileDescriptorProto:
+        raise NotImplementedError
+
     @property  # type:ignore
     @abstracts.interfacemethod
     def descriptor_pool(self) -> descriptor_pool.DescriptorPool:
         raise NotImplementedError
 
+    @property  # type:ignore
+    @abstracts.interfacemethod
+    def source_files(self) -> Dict[str, descriptor_pb2.FileDescriptorProto]:
+        raise NotImplementedError
+
+    @abstracts.interfacemethod
+    def find_file(self, type_name: str) -> descriptor.Descriptor:
+        raise NotImplementedError
+
+    @abstracts.interfacemethod
+    def find_message(self, type_name: str) -> descriptor.Descriptor:
+        raise NotImplementedError
+
 
 class IProtobufValidator(metaclass=abstracts.Interface):
 

envoy.base.utils/envoy/base/utils/protobuf.py

@@ -1,11 +1,15 @@
 
+import sys
 from typing import Type
 
 import abstracts
 
 from envoy.base.utils import abstract, interface
 
 
+sys.path = [p for p in sys.path if not p.endswith('bazel_tools')]
+
+
 @abstracts.implementer(interface.IProtobufSet)
 class ProtobufSet(abstract.AProtobufSet):
     pass
@@ -17,3 +21,11 @@ class ProtobufValidator(abstract.AProtobufValidator):
     @property
     def protobuf_set_class(self) -> Type[interface.IProtobufSet]:
         return ProtobufSet
+
+
+@abstracts.implementer(interface.IProtocProtocol)
+class ProtocProtocol(abstract.AProtocProtocol):
+
+    @property
+    def proto_set_class(self):
+        return ProtobufSet

envoy.base.utils/setup.cfg

@@ -29,7 +29,8 @@ py_modules = envoy.base.utils
 packages = find_namespace:
 install_requires =
     abstracts>=0.0.12
-    aio.core>=0.8.6
+    aio.core>=0.8.9
+    aio.api.bazel>=0.0.2
     aio.api.github>=0.1.2
     aio.run.runner>=0.3.3
     aiohttp>=3.8.1
@@ -44,6 +45,8 @@ install_requires =
     trycast>=0.7.3
     yarl>=1.7.2
 
+    remote_pdb
+
 [options.extras_require]
 test =
     pytest

envoy.base.utils/tests/BUILD

@@ -4,6 +4,7 @@ pytooling_tests(
     dependencies=[
         ":data",
         "//deps:abstracts",
+        "//deps:aio.api.bazel",
         "//deps:aio.core",
         "//deps:aio.run.runner",
         "//deps:pyyaml",