From 49183534f72f46ca5ace3d0f818616873b57c136 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 13:37:59 -0800 Subject: [PATCH 01/72] add in memory queue option --- baseplate/lib/events.py | 13 ++-- baseplate/lib/message_queue.py | 84 ++++++++++++++++++++++-- baseplate/observers/tracing.py | 22 ++++--- baseplate/sidecars/event_publisher.py | 23 +++++-- baseplate/sidecars/trace_publisher.py | 23 +++++-- docker-compose.yml | 13 ++++ docs/api/baseplate/lib/message_queue.rst | 17 +++-- requirements.txt | 1 + tests/integration/message_queue_tests.py | 71 +++++++++++++++++--- tests/unit/lib/events/queue_tests.py | 36 ++++++++-- 10 files changed, 254 insertions(+), 49 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 838bd6fbe..5aa6cb0cd 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,7 +22,7 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -93,10 +93,13 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): """ - def __init__(self, name: str, event_serializer: Callable[[T], bytes]): - self.queue = MessageQueue( - "/events-" + name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_EVENT_SIZE - ) + def __init__(self, name: str, event_serializer: Callable[[T], bytes], use_in_memory_queue: bool = False): + if use_in_memory_queue: + self.queue = InMemoryMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) + else: + self.queue = PosixMessageQueue( + "/events-" + name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_EVENT_SIZE + ) self.serialize_event = event_serializer def put(self, event: T) -> None: diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index e521772cf..bd957f5dd 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,4 +1,6 @@ """A Gevent-friendly POSIX message queue.""" +from abc import abstractmethod +import queue import select from typing import Optional @@ -34,8 +36,78 @@ class MessageQueueOSError(OSError): def __init__(self, inner: Exception): super().__init__(f"{inner} (check `ulimit -q`?)") - class MessageQueue: + @abstractmethod + def __init__(self, name: str, max_messages: int, max_message_size: int): + pass + + @abstractmethod + def get(self, timeout: Optional[float] = None) -> bytes: + """Read a message from the queue. + + :param timeout: If the queue is empty, the call will block up to + ``timeout`` seconds or forever if ``None``. + :raises: :py:exc:`TimedOutError` The queue was empty for the allowed + duration of the call. + + """ + pass + + @abstractmethod + def put(self, message: bytes, timeout: Optional[float] = None) -> None: + """Add a message to the queue. + + :param timeout: If the queue is full, the call will block up to + ``timeout`` seconds or forever if ``None``. + :raises: :py:exc:`TimedOutError` The queue was full for the allowed + duration of the call. + + """ + pass + +class InMemoryMessageQueue(MessageQueue): + def __init__(self, name: str, max_messages: int): + self.queue = queue.Queue(max_messages) + self.max_messages = max_messages + self.name = name + + def get(self, timeout: Optional[float] = None) -> bytes: + """Read a message from the queue. + + :param timeout: If the queue is empty, the call will block up to + ``timeout`` seconds or forever if ``None``. + :raises: :py:exc:`TimedOutError` The queue was empty for the allowed + duration of the call. + + """ + try: + message = self.queue.get(timeout=timeout) + self.queue.task_done() + return message + except queue.Empty: + raise TimedOutError + + def put(self, message: bytes, timeout: Optional[float] = None) -> None: + """Add a message to the queue. + + :param timeout: If the queue is full, the call will block up to + ``timeout`` seconds or forever if ``None``. + :raises: :py:exc:`TimedOutError` The queue was full for the allowed + duration of the call. + + """ + try: + return self.queue.put(message, timeout=timeout) + except queue.Full: + raise TimedOutError + + def unlink(self) -> None: + pass + + def close(self) -> None: + pass + +class PosixMessageQueue(MessageQueue): """A Gevent-friendly (but not required) inter process message queue. ``name`` should be a string of up to 255 characters consisting of an @@ -47,7 +119,6 @@ class MessageQueue: support this. """ - def __init__(self, name: str, max_messages: int, max_message_size: int): try: self.queue = posix_ipc.MessageQueue( @@ -101,7 +172,7 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: select.select([], [self.queue.mqd], [], time_remaining) raise TimedOutError - + def unlink(self) -> None: """Remove the queue from the system. @@ -119,6 +190,7 @@ def close(self) -> None: """ self.queue.close() + def queue_tool() -> None: @@ -140,6 +212,7 @@ def queue_tool() -> None: help="if creating the queue, what to set the maximum message size to", ) parser.add_argument("queue_name", help="the name of the queue to consume") + parser.add_argument("use_in_memory_queue", default=False, help="whether to use an in-memory queue or a posix queue") group = parser.add_mutually_exclusive_group(required=True) group.add_argument( @@ -166,7 +239,10 @@ def queue_tool() -> None: args = parser.parse_args() - queue = MessageQueue(args.queue_name, args.max_messages, args.max_message_size) + if args.use_in_memory_queue: + queue = InMemoryMessageQueue(args.queue_name, args.max_messages) + else: + queue = PosixMessageQueue(args.queue_name, args.max_messages, args.max_message_size) if args.mode == "read": while True: diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index a593ee1e8..2d9206790 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,7 +29,7 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout @@ -539,18 +539,24 @@ class TraceQueueFullError(Exception): class SidecarRecorder(Recorder): - """Interface for recording spans to a POSIX message queue. + """Interface for recording spans to a message queue. The SidecarRecorder serializes spans to a string representation before adding them to the queue. """ - def __init__(self, queue_name: str): - self.queue = MessageQueue( - "/traces-" + queue_name, - max_messages=MAX_QUEUE_SIZE, - max_message_size=MAX_SPAN_SIZE, - ) + def __init__(self, queue_name: str, use_in_memory_queue: bool = False): + if use_in_memory_queue: + self.queue = InMemoryMessageQueue( + "/traces-" + queue_name, + max_messages=MAX_QUEUE_SIZE, + ) + else: + self.queue = PosixMessageQueue( + "/traces-" + queue_name, + max_messages=MAX_QUEUE_SIZE, + max_message_size=MAX_SPAN_SIZE, + ) def send(self, span: TraceSpanObserver) -> None: # Don't raise exceptions from here. This is called in the diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 9c3dcb2eb..afdec9864 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -17,7 +17,7 @@ from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy @@ -174,6 +174,11 @@ def publish_events() -> None: default="main", help="name of event queue / publisher config (default: main)", ) + arg_parser.add_argument( + "--use-in-memory-queue", + default=False, + help="use an in memory queue instead of a posix queue", + ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" ) @@ -203,11 +208,17 @@ def publish_events() -> None: metrics_client = metrics_client_from_config(raw_config) - event_queue = MessageQueue( - "/events-" + args.queue_name, - max_messages=cfg.max_queue_size, - max_message_size=MAX_EVENT_SIZE, - ) + if args.use_in_memory_queue: + event_queue = InMemoryMessageQueue( + "/events-" + args.queue_name, + max_messages=cfg.max_queue_size, + ) + else: + event_queue = PosixMessageQueue( + "/events-" + args.queue_name, + max_messages=cfg.max_queue_size, + max_message_size=MAX_EVENT_SIZE, + ) # pylint: disable=maybe-no-member serializer = SERIALIZER_BY_VERSION[cfg.collector.version]() diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index b7d91859c..83ae42f89 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,7 +10,7 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy @@ -128,6 +128,11 @@ def publish_traces() -> None: default="main", help="name of trace queue / publisher config (default: main)", ) + arg_parser.add_argument( + "--use-in-memory-queue", + default=False, + help="use in memory queue instead of a posix queue", + ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" ) @@ -160,11 +165,17 @@ def publish_traces() -> None: }, ) - trace_queue = MessageQueue( - "/traces-" + args.queue_name, - max_messages=publisher_cfg.max_queue_size, - max_message_size=MAX_SPAN_SIZE, - ) + if args.use_in_memory_queue: + trace_queue = InMemoryMessageQueue( + "/traces-" + args.queue_name, + max_messages=publisher_cfg.max_queue_size, + ) + else: + trace_queue = PosixMessageQueue( + "/traces-" + args.queue_name, + max_messages=publisher_cfg.max_queue_size, + max_message_size=MAX_SPAN_SIZE, + ) # pylint: disable=maybe-no-member inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size) diff --git a/docker-compose.yml b/docker-compose.yml index bf32bb41d..e2e49eb4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,19 @@ services: - "redis" - "zookeeper" - "redis-cluster-node" + test-baseplate: + build: + context: "." + dockerfile: "Dockerfile" + command: "pytest -v tests/" + volumes: + - ".:/src" + # links: + # - "cassandra" + # - "memcached" + # - "redis" + # - "zookeeper" + # - "redis-cluster-node" cassandra: image: "cassandra:3.11" environment: diff --git a/docs/api/baseplate/lib/message_queue.rst b/docs/api/baseplate/lib/message_queue.rst index 474c12bc3..d5c3758fb 100644 --- a/docs/api/baseplate/lib/message_queue.rst +++ b/docs/api/baseplate/lib/message_queue.rst @@ -27,10 +27,10 @@ process pair (run the producer then the consumer): .. testcode:: # producer.py - from baseplate.lib.message_queue import MessageQueue + from baseplate.lib.message_queue import PosixMessageQueue # If the queue doesn't already exist, we'll create it. - mq = MessageQueue( + mq = PosixMessageQueue( "/baseplate-testing", max_messages=1, max_message_size=1) message = "1" mq.put(message) @@ -48,9 +48,9 @@ POSIX message queue. Next up, run the consumer: .. testcode:: # consumer.py - from baseplate.lib.message_queue import MessageQueue + from baseplate.lib.message_queue import PosixMessageQueue - mq = MessageQueue( + mq = PosixMessageQueue( "/baseplate-testing", max_messages=1, max_message_size=1) # Unless a `timeout` kwarg is passed, this will block until # we can pop a message from the queue. @@ -97,8 +97,8 @@ up seeing a vague ``ValueError`` exception. Here's an example: .. code-block:: pycon - >>> from baseplate.lib.message_queue import MessageQueue - >>> mq = MessageQueue( + >>> from baseplate.lib.message_queue import PosixMessageQueue + >>> mq = PosixMessageQueue( "/over-the-limit", max_messages=11, max_message_size=8096) Traceback (most recent call last): File "", line 2, in @@ -138,7 +138,10 @@ See ``--help`` for more info. .. automodule:: baseplate.lib.message_queue -.. autoclass:: MessageQueue +.. autoclass:: PosixMessageQueue + :members: + +.. autoclass:: InMemoryMessageQueue :members: diff --git a/requirements.txt b/requirements.txt index cf169c397..982955b06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ lxml==4.6.5 mypy==0.910 prometheus-client==0.14.1 pydocstyle==5.1.1 + pylint==2.11.1 pytest==6.2.2 pytest-cov==2.11.1 diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 59938c71f..df26cc869 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -4,11 +4,11 @@ import posix_ipc -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import PosixMessageQueue, InMemoryMessageQueue from baseplate.lib.message_queue import TimedOutError -class TestMessageQueueCreation(unittest.TestCase): +class TestPosixMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" def setUp(self): @@ -21,14 +21,14 @@ def setUp(self): queue.close() def test_create_queue(self): - message_queue = MessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) with contextlib.closing(message_queue) as mq: self.assertEqual(mq.queue.max_messages, 1) self.assertEqual(mq.queue.max_message_size, 1) def test_put_get(self): - message_queue = MessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) with contextlib.closing(message_queue) as mq: mq.put(b"x") @@ -36,7 +36,7 @@ def test_put_get(self): self.assertEqual(message, b"x") def test_get_timeout(self): - message_queue = MessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) with contextlib.closing(message_queue) as mq: start = time.time() @@ -46,7 +46,7 @@ def test_get_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=2) def test_put_timeout(self): - message_queue = MessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) with contextlib.closing(message_queue) as mq: mq.put(b"x") @@ -57,7 +57,7 @@ def test_put_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=1) def test_put_zero_timeout(self): - message_queue = MessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) with contextlib.closing(message_queue) as mq: mq.put(b"x", timeout=0) @@ -65,7 +65,7 @@ def test_put_zero_timeout(self): self.assertEqual(message, b"x") def test_put_full_zero_timeout(self): - message_queue = MessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) with contextlib.closing(message_queue) as mq: mq.put(b"1", timeout=0) @@ -81,3 +81,58 @@ def tearDown(self): else: queue.unlink() queue.close() + +class TestInMemoryMessageQueueCreation(unittest.TestCase): + qname = "/baseplate-test-queue" + + def test_create_queue(self): + message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + self.assertEqual(mq.queue.maxsize, 1) + + def test_put_get(self): + message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x") + message = mq.get() + self.assertEqual(message, b"x") + + def test_get_timeout(self): + message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + start = time.time() + with self.assertRaises(TimedOutError): + mq.get(timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=2) + + def test_put_timeout(self): + message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x") + start = time.time() + with self.assertRaises(TimedOutError): + mq.put(b"x", timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=1) + + def test_put_zero_timeout(self): + message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x", timeout=0) + message = mq.get() + self.assertEqual(message, b"x") + + def test_put_full_zero_timeout(self): + message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.put(b"1", timeout=0) + + with self.assertRaises(TimedOutError): + mq.put(b"2", timeout=0) diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 429ace542..5435fcc5e 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -6,14 +6,14 @@ from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import PosixMessageQueue, InMemoryMessageQueue from baseplate.lib.message_queue import TimedOutError -class EventQueueTests(unittest.TestCase): - @mock.patch("baseplate.lib.events.MessageQueue", autospec=MessageQueue) - def setUp(self, MessageQueue): - self.message_queue = MessageQueue.return_value +class PosixEventQueueTests(unittest.TestCase): + @mock.patch("baseplate.lib.events.PosixMessageQueue", autospec=PosixMessageQueue) + def setUp(self, PosixMessageQueue): + self.message_queue = PosixMessageQueue.return_value self.mock_serializer = mock.Mock() self.queue = EventQueue("test", event_serializer=self.mock_serializer) @@ -41,3 +41,29 @@ def test_event_queue_full(self): with self.assertRaises(EventQueueFullError): self.queue.put(object()) + +class InMemoryEventQueueTests(unittest.TestCase): + @mock.patch("baseplate.lib.events.InMemoryMessageQueue", autospec=InMemoryMessageQueue) + def setUp(self, InMemoryMessageQueue): + self.message_queue = InMemoryMessageQueue.return_value + self.mock_serializer = mock.Mock() + self.queue = EventQueue("test", event_serializer=self.mock_serializer, use_in_memory_queue=True) + + def test_send_event(self): + self.mock_serializer.return_value = "i_am_serialized" + event = object() + + self.queue.put(event) + + self.assertEqual(self.message_queue.put.call_count, 1) + self.mock_serializer.assert_called_with(event) + args, kwargs = self.message_queue.put.call_args + self.assertEqual(args[0], self.mock_serializer.return_value) + + def test_event_queue_full(self): + self.mock_serializer.return_value = "" + + self.message_queue.put.side_effect = TimedOutError + + with self.assertRaises(EventQueueFullError): + self.queue.put(object()) From 7558da79b0f6694c8cad384e87358276f3085738 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 14:26:47 -0800 Subject: [PATCH 02/72] cleanup --- baseplate/lib/message_queue.py | 18 ++++++++++++++++-- baseplate/observers/tracing.py | 4 ++-- docker-compose.yml | 6 ------ tests/integration/message_queue_tests.py | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index bd957f5dd..4cae5e44f 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -65,7 +65,18 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: """ pass -class InMemoryMessageQueue(MessageQueue): +class InMemoryMessageQueue(MessageQueue): + """An in-memory inter process message queue. + + ``name`` should be a string of up to 255 characters consisting of an + initial slash, followed by one or more characters, none of which are + slashes. + + Note: This relies on POSIX message queues being available and + select(2)-able like other file descriptors. Not all operating systems + support this. + + """ def __init__(self, name: str, max_messages: int): self.queue = queue.Queue(max_messages) self.max_messages = max_messages @@ -102,9 +113,11 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: raise TimedOutError def unlink(self) -> None: + """Not implemented for in-memory queue""" pass - def close(self) -> None: + def close(self) -> None: + """Not implemented for in-memory queue""" pass class PosixMessageQueue(MessageQueue): @@ -133,6 +146,7 @@ def __init__(self, name: str, max_messages: int, max_message_size: int): except OSError as exc: raise MessageQueueOSError(exc) self.queue.block = False + self.name = name def get(self, timeout: Optional[float] = None) -> bytes: """Read a message from the queue. diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 2d9206790..66b3be5fb 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -567,7 +567,7 @@ def send(self, span: TraceSpanObserver) -> None: "Trace too big. Traces published to %s are not allowed to be larger " "than %d bytes. Received trace is %d bytes. This can be caused by " "an excess amount of tags or a large amount of child spans.", - self.queue.queue.name, + self.queue.name, MAX_SPAN_SIZE, len(serialized_str), ) @@ -576,7 +576,7 @@ def send(self, span: TraceSpanObserver) -> None: self.queue.put(serialized_str, timeout=0) except TimedOutError: logger.warning( - "Trace queue %s is full. Is trace sidecar healthy?", self.queue.queue.name + "Trace queue %s is full. Is trace sidecar healthy?", self.queue.name ) diff --git a/docker-compose.yml b/docker-compose.yml index e2e49eb4b..faa382bb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,12 +21,6 @@ services: command: "pytest -v tests/" volumes: - ".:/src" - # links: - # - "cassandra" - # - "memcached" - # - "redis" - # - "zookeeper" - # - "redis-cluster-node" cassandra: image: "cassandra:3.11" environment: diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index df26cc869..e5dad1ba1 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -84,7 +84,7 @@ def tearDown(self): class TestInMemoryMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" - + def test_create_queue(self): message_queue = InMemoryMessageQueue(self.qname, max_messages=1) From b71fac2bd57c830954c389731f8dede02ba7c02f Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 14:54:36 -0800 Subject: [PATCH 03/72] lint, cleanup --- baseplate/lib/events.py | 6 +- baseplate/lib/message_queue.py | 130 +++++++++-------------- baseplate/observers/tracing.py | 4 +- baseplate/sidecars/event_publisher.py | 2 +- requirements.txt | 1 - tests/integration/message_queue_tests.py | 3 +- tests/unit/lib/events/queue_tests.py | 5 +- 7 files changed, 65 insertions(+), 86 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 5aa6cb0cd..6b3f96fe7 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -93,8 +93,10 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): """ - def __init__(self, name: str, event_serializer: Callable[[T], bytes], use_in_memory_queue: bool = False): - if use_in_memory_queue: + def __init__( + self, name: str, event_serializer: Callable[[T], bytes], use_in_memory_queue: bool = False + ): + if use_in_memory_queue: self.queue = InMemoryMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) else: self.queue = PosixMessageQueue( diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 4cae5e44f..072f5e50b 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -36,7 +36,10 @@ class MessageQueueOSError(OSError): def __init__(self, inner: Exception): super().__init__(f"{inner} (check `ulimit -q`?)") + class MessageQueue: + """Abstract class for an inter-process message queue.""" + @abstractmethod def __init__(self, name: str, max_messages: int, max_message_size: int): pass @@ -65,60 +68,26 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: """ pass -class InMemoryMessageQueue(MessageQueue): - """An in-memory inter process message queue. - - ``name`` should be a string of up to 255 characters consisting of an - initial slash, followed by one or more characters, none of which are - slashes. - - Note: This relies on POSIX message queues being available and - select(2)-able like other file descriptors. Not all operating systems - support this. - - """ - def __init__(self, name: str, max_messages: int): - self.queue = queue.Queue(max_messages) - self.max_messages = max_messages - self.name = name - - def get(self, timeout: Optional[float] = None) -> bytes: - """Read a message from the queue. + @abstractmethod + def unlink(self) -> None: + """Remove the queue from the system. - :param timeout: If the queue is empty, the call will block up to - ``timeout`` seconds or forever if ``None``. - :raises: :py:exc:`TimedOutError` The queue was empty for the allowed - duration of the call. + The queue will not leave until the last active user closes it. """ - try: - message = self.queue.get(timeout=timeout) - self.queue.task_done() - return message - except queue.Empty: - raise TimedOutError - - def put(self, message: bytes, timeout: Optional[float] = None) -> None: - """Add a message to the queue. + pass - :param timeout: If the queue is full, the call will block up to - ``timeout`` seconds or forever if ``None``. - :raises: :py:exc:`TimedOutError` The queue was full for the allowed - duration of the call. + @abstractmethod + def close(self) -> None: + """Close the queue, freeing related resources. - """ - try: - return self.queue.put(message, timeout=timeout) - except queue.Full: - raise TimedOutError + This must be called explicitly if queues are created/destroyed on the + fly. It is not automatically called when the object is reclaimed by + Python. - def unlink(self) -> None: - """Not implemented for in-memory queue""" + """ pass - def close(self) -> None: - """Not implemented for in-memory queue""" - pass class PosixMessageQueue(MessageQueue): """A Gevent-friendly (but not required) inter process message queue. @@ -132,6 +101,7 @@ class PosixMessageQueue(MessageQueue): support this. """ + def __init__(self, name: str, max_messages: int, max_message_size: int): try: self.queue = posix_ipc.MessageQueue( @@ -149,14 +119,6 @@ def __init__(self, name: str, max_messages: int, max_message_size: int): self.name = name def get(self, timeout: Optional[float] = None) -> bytes: - """Read a message from the queue. - - :param timeout: If the queue is empty, the call will block up to - ``timeout`` seconds or forever if ``None``. - :raises: :py:exc:`TimedOutError` The queue was empty for the allowed - duration of the call. - - """ for time_remaining in RetryPolicy.new(budget=timeout): try: message, _ = self.queue.receive() @@ -169,14 +131,6 @@ def get(self, timeout: Optional[float] = None) -> bytes: raise TimedOutError def put(self, message: bytes, timeout: Optional[float] = None) -> None: - """Add a message to the queue. - - :param timeout: If the queue is full, the call will block up to - ``timeout`` seconds or forever if ``None``. - :raises: :py:exc:`TimedOutError` The queue was full for the allowed - duration of the call. - - """ for time_remaining in RetryPolicy.new(budget=timeout): try: return self.queue.send(message=message) @@ -186,25 +140,43 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: select.select([], [self.queue.mqd], [], time_remaining) raise TimedOutError - - def unlink(self) -> None: - """Remove the queue from the system. - - The queue will not leave until the last active user closes it. - """ + def unlink(self) -> None: self.queue.unlink() def close(self) -> None: - """Close the queue, freeing related resources. + self.queue.close() - This must be called explicitly if queues are created/destroyed on the - fly. It is not automatically called when the object is reclaimed by - Python. - """ - self.queue.close() - +class InMemoryMessageQueue(MessageQueue): + """An in-memory inter process message queue.""" + + def __init__(self, name: str, max_messages: int): + self.queue = queue.Queue(max_messages) + self.max_messages = max_messages + self.name = name + + def get(self, timeout: Optional[float] = None) -> bytes: + try: + message = self.queue.get(timeout=timeout) + self.queue.task_done() + return message + except queue.Empty: + raise TimedOutError + + def put(self, message: bytes, timeout: Optional[float] = None) -> None: + try: + return self.queue.put(message, timeout=timeout) + except queue.Full: + raise TimedOutError + + def unlink(self) -> None: + """Not implemented for in-memory queue""" + pass + + def close(self) -> None: + """Not implemented for in-memory queue""" + pass def queue_tool() -> None: @@ -226,7 +198,11 @@ def queue_tool() -> None: help="if creating the queue, what to set the maximum message size to", ) parser.add_argument("queue_name", help="the name of the queue to consume") - parser.add_argument("use_in_memory_queue", default=False, help="whether to use an in-memory queue or a posix queue") + parser.add_argument( + "use_in_memory_queue", + default=False, + help="whether to use an in-memory queue or a posix queue", + ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( @@ -255,7 +231,7 @@ def queue_tool() -> None: if args.use_in_memory_queue: queue = InMemoryMessageQueue(args.queue_name, args.max_messages) - else: + else: queue = PosixMessageQueue(args.queue_name, args.max_messages, args.max_message_size) if args.mode == "read": diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 66b3be5fb..1c5c6fe44 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -575,9 +575,7 @@ def send(self, span: TraceSpanObserver) -> None: try: self.queue.put(serialized_str, timeout=0) except TimedOutError: - logger.warning( - "Trace queue %s is full. Is trace sidecar healthy?", self.queue.name - ) + logger.warning("Trace queue %s is full. Is trace sidecar healthy?", self.queue.name) def tracing_client_from_config( diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index afdec9864..96b96b2ec 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -213,7 +213,7 @@ def publish_events() -> None: "/events-" + args.queue_name, max_messages=cfg.max_queue_size, ) - else: + else: event_queue = PosixMessageQueue( "/events-" + args.queue_name, max_messages=cfg.max_queue_size, diff --git a/requirements.txt b/requirements.txt index 982955b06..cf169c397 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ lxml==4.6.5 mypy==0.910 prometheus-client==0.14.1 pydocstyle==5.1.1 - pylint==2.11.1 pytest==6.2.2 pytest-cov==2.11.1 diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index e5dad1ba1..330a79981 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -82,9 +82,10 @@ def tearDown(self): queue.unlink() queue.close() + class TestInMemoryMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" - + def test_create_queue(self): message_queue = InMemoryMessageQueue(self.qname, max_messages=1) diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 5435fcc5e..6b1d8447c 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -42,12 +42,15 @@ def test_event_queue_full(self): with self.assertRaises(EventQueueFullError): self.queue.put(object()) + class InMemoryEventQueueTests(unittest.TestCase): @mock.patch("baseplate.lib.events.InMemoryMessageQueue", autospec=InMemoryMessageQueue) def setUp(self, InMemoryMessageQueue): self.message_queue = InMemoryMessageQueue.return_value self.mock_serializer = mock.Mock() - self.queue = EventQueue("test", event_serializer=self.mock_serializer, use_in_memory_queue=True) + self.queue = EventQueue( + "test", event_serializer=self.mock_serializer, use_in_memory_queue=True + ) def test_send_event(self): self.mock_serializer.return_value = "i_am_serialized" From 84c50c9c912db531c121e1e17e46f456be300710 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 15:02:19 -0800 Subject: [PATCH 04/72] linting --- baseplate/lib/events.py | 3 ++- baseplate/lib/message_queue.py | 2 +- baseplate/observers/tracing.py | 3 ++- baseplate/sidecars/event_publisher.py | 3 ++- baseplate/sidecars/trace_publisher.py | 3 ++- tests/integration/message_queue_tests.py | 3 ++- tests/unit/lib/events/queue_tests.py | 3 ++- 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 6b3f96fe7..44c507e8a 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,7 +22,8 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 072f5e50b..111cbdbaa 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,8 +1,8 @@ """A Gevent-friendly POSIX message queue.""" -from abc import abstractmethod import queue import select +from abc import abstractmethod from typing import Optional import posix_ipc diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 1c5c6fe44..a41bedc92 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,7 +29,8 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 96b96b2ec..4d4b899de 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -17,7 +17,8 @@ from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 83ae42f89..a067565cf 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,7 +10,8 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import InMemoryMessageQueue, PosixMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 330a79981..ad2d3b234 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -4,7 +4,8 @@ import posix_ipc -from baseplate.lib.message_queue import PosixMessageQueue, InMemoryMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 6b1d8447c..805fc36ca 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -6,7 +6,8 @@ from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE -from baseplate.lib.message_queue import PosixMessageQueue, InMemoryMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError From 8cd02e94f0c23d696379f5a57c80866b8b5cf85f Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 15:22:23 -0800 Subject: [PATCH 05/72] linting --- baseplate/lib/message_queue.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 111cbdbaa..2da0f9152 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,8 +1,8 @@ """A Gevent-friendly POSIX message queue.""" -import queue +import abc +import queue as q import select -from abc import abstractmethod from typing import Optional import posix_ipc @@ -37,14 +37,10 @@ def __init__(self, inner: Exception): super().__init__(f"{inner} (check `ulimit -q`?)") -class MessageQueue: +class MessageQueue(abc.ABC): """Abstract class for an inter-process message queue.""" - @abstractmethod - def __init__(self, name: str, max_messages: int, max_message_size: int): - pass - - @abstractmethod + @abc.abstractmethod def get(self, timeout: Optional[float] = None) -> bytes: """Read a message from the queue. @@ -54,9 +50,8 @@ def get(self, timeout: Optional[float] = None) -> bytes: duration of the call. """ - pass - @abstractmethod + @abc.abstractmethod def put(self, message: bytes, timeout: Optional[float] = None) -> None: """Add a message to the queue. @@ -66,18 +61,16 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: duration of the call. """ - pass - @abstractmethod + @abc.abstractmethod def unlink(self) -> None: """Remove the queue from the system. The queue will not leave until the last active user closes it. """ - pass - @abstractmethod + @abc.abstractmethod def close(self) -> None: """Close the queue, freeing related resources. @@ -86,7 +79,6 @@ def close(self) -> None: Python. """ - pass class PosixMessageQueue(MessageQueue): @@ -152,7 +144,7 @@ class InMemoryMessageQueue(MessageQueue): """An in-memory inter process message queue.""" def __init__(self, name: str, max_messages: int): - self.queue = queue.Queue(max_messages) + self.queue = q.Queue(max_messages) self.max_messages = max_messages self.name = name @@ -161,13 +153,13 @@ def get(self, timeout: Optional[float] = None) -> bytes: message = self.queue.get(timeout=timeout) self.queue.task_done() return message - except queue.Empty: + except q.Empty: raise TimedOutError def put(self, message: bytes, timeout: Optional[float] = None) -> None: try: return self.queue.put(message, timeout=timeout) - except queue.Full: + except q.Full: raise TimedOutError def unlink(self) -> None: From 719749ad57df7f130a216b3c69b6182557e9ca14 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 15:26:27 -0800 Subject: [PATCH 06/72] linting :sob: --- baseplate/lib/message_queue.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 2da0f9152..893e3ccf4 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -164,11 +164,9 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: def unlink(self) -> None: """Not implemented for in-memory queue""" - pass def close(self) -> None: """Not implemented for in-memory queue""" - pass def queue_tool() -> None: From 7e849989188424c397eb4654f793ad35d88f8094 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 7 Dec 2022 16:36:00 -0800 Subject: [PATCH 07/72] linting --- baseplate/lib/events.py | 2 +- baseplate/lib/message_queue.py | 6 ++++-- baseplate/observers/tracing.py | 2 +- baseplate/sidecars/event_publisher.py | 2 +- baseplate/sidecars/trace_publisher.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 44c507e8a..79ac8678d 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -100,7 +100,7 @@ def __init__( if use_in_memory_queue: self.queue = InMemoryMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) else: - self.queue = PosixMessageQueue( + self.queue = PosixMessageQueue( # type: ignore "/events-" + name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_EVENT_SIZE ) self.serialize_event = event_serializer diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 893e3ccf4..c75869144 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -144,7 +144,7 @@ class InMemoryMessageQueue(MessageQueue): """An in-memory inter process message queue.""" def __init__(self, name: str, max_messages: int): - self.queue = q.Queue(max_messages) + self.queue: q.Queue = q.Queue(max_messages) self.max_messages = max_messages self.name = name @@ -222,7 +222,9 @@ def queue_tool() -> None: if args.use_in_memory_queue: queue = InMemoryMessageQueue(args.queue_name, args.max_messages) else: - queue = PosixMessageQueue(args.queue_name, args.max_messages, args.max_message_size) + queue = PosixMessageQueue( # type: ignore + args.queue_name, args.max_messages, args.max_message_size + ) if args.mode == "read": while True: diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index a41bedc92..eb1735f8d 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -553,7 +553,7 @@ def __init__(self, queue_name: str, use_in_memory_queue: bool = False): max_messages=MAX_QUEUE_SIZE, ) else: - self.queue = PosixMessageQueue( + self.queue = PosixMessageQueue( # type: ignore "/traces-" + queue_name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_SPAN_SIZE, diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 4d4b899de..2b5ecb900 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -215,7 +215,7 @@ def publish_events() -> None: max_messages=cfg.max_queue_size, ) else: - event_queue = PosixMessageQueue( + event_queue = PosixMessageQueue( # type: ignore "/events-" + args.queue_name, max_messages=cfg.max_queue_size, max_message_size=MAX_EVENT_SIZE, diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index a067565cf..f080dae46 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -172,7 +172,7 @@ def publish_traces() -> None: max_messages=publisher_cfg.max_queue_size, ) else: - trace_queue = PosixMessageQueue( + trace_queue = PosixMessageQueue( # type: ignore "/traces-" + args.queue_name, max_messages=publisher_cfg.max_queue_size, max_message_size=MAX_SPAN_SIZE, From 057f2952ee3d1513f560629161f52b939cf9c683 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 14 Dec 2022 07:56:39 -0800 Subject: [PATCH 08/72] wip --- baseplate/lib/events.py | 4 +-- baseplate/lib/message_queue.py | 36 ++++++++++++++++++++++-- baseplate/observers/tracing.py | 4 +-- baseplate/sidecars/event_publisher.py | 40 +++++++++++++++++++++++---- baseplate/sidecars/trace_publisher.py | 4 +-- baseplate/thrift/message_queue.thrift | 35 +++++++++++++++++++++++ 6 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 baseplate/thrift/message_queue.thrift diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 79ac8678d..e16611dd8 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,7 +22,7 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -98,7 +98,7 @@ def __init__( self, name: str, event_serializer: Callable[[T], bytes], use_in_memory_queue: bool = False ): if use_in_memory_queue: - self.queue = InMemoryMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) + self.queue = RemoteMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) else: self.queue = PosixMessageQueue( # type: ignore "/events-" + name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_EVENT_SIZE diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index c75869144..ce6e840a7 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -4,11 +4,16 @@ import select from typing import Optional +from baseplate.thrift import RemoteMessageQueueService import posix_ipc from baseplate.lib.retry import RetryPolicy +from thrift.Thrift import TBinaryProtocol +from thrift.Thrift import TSocket +from thrift.transport import TTransport + class MessageQueueError(Exception): """Base exception for message queue related errors.""" @@ -141,7 +146,7 @@ def close(self) -> None: class InMemoryMessageQueue(MessageQueue): - """An in-memory inter process message queue.""" + """An in-memory inter process message queue.""" # Used by the sidecar def __init__(self, name: str, max_messages: int): self.queue: q.Queue = q.Queue(max_messages) @@ -158,7 +163,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: def put(self, message: bytes, timeout: Optional[float] = None) -> None: try: - return self.queue.put(message, timeout=timeout) + self.queue.put(message, timeout=timeout) except q.Full: raise TimedOutError @@ -169,6 +174,31 @@ def close(self) -> None: """Not implemented for in-memory queue""" +class RemoteMessageQueue(MessageQueue): + def __init__(self, max_messages: int): # Connect to the sidecar and instantiate a remote queue + transport = TSocket.TSocket( "localhost" , 9090) # todo: how do I connect to the sidecar? I dont think this is right + transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + + self.client = RemoteMessageQueueService.Client(protocol) + # todo: create queue? + + # Connect to server + self.transport.open() + + def get(self, timeout: Optional[float] = None) -> bytes: + return self.client.get(timeout) + + def put(self, message: bytes, timeout: Optional[float] = None) -> None: + return self.client.put(message, timeout) + + def unlink(self) -> None: + """Not implemented for remote queue""" + + def close(self) -> None: + self.transport.close() + + def queue_tool() -> None: import argparse import sys @@ -220,7 +250,7 @@ def queue_tool() -> None: args = parser.parse_args() if args.use_in_memory_queue: - queue = InMemoryMessageQueue(args.queue_name, args.max_messages) + queue = RemoteMessageQueue(args.queue_name, args.max_messages) else: queue = PosixMessageQueue( # type: ignore args.queue_name, args.max_messages, args.max_message_size diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index eb1735f8d..949c10000 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,7 +29,7 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout @@ -548,7 +548,7 @@ class SidecarRecorder(Recorder): def __init__(self, queue_name: str, use_in_memory_queue: bool = False): if use_in_memory_queue: - self.queue = InMemoryMessageQueue( + self.queue = RemoteMessageQueue( "/traces-" + queue_name, max_messages=MAX_QUEUE_SIZE, ) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 2b5ecb900..26186ff22 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -12,7 +12,7 @@ import requests -from baseplate import __version__ as baseplate_version +from baseplate import RequestContext, __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE @@ -23,11 +23,17 @@ from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy from baseplate.server import EnvironmentInterpolation +from baseplate.thrift import RemoteMessageQueueService from baseplate.sidecars import Batch from baseplate.sidecars import BatchFull from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch +from thrift.Thrift import TServer +from thrift.Thrift import TBinaryProtocol +from thrift.Thrift import TSocket +from thrift.transport import TTransport + logger = logging.getLogger(__name__) @@ -162,6 +168,21 @@ def publish(self, payload: SerializedBatch) -> None: raise MaxRetriesError("could not sent batch") + +class RemoteMessageQueueHandler: # From the sidecar, create the queue and define get/put using the InMemoryQueue implementation + def is_healthy(self, context: RequestContext) -> bool: + return True + + def __init__(self, name: str, max_messages: int): + self.queue = InMemoryMessageQueue(name, max_messages) + + def get(self, timeout: Optional[float] = None) -> bytes: + return self.queue.get(timeout) + + def put(self, message: bytes, timeout: Optional[float] = None) -> None: + self.queue.put(message, timeout) + + SERIALIZER_BY_VERSION = {"2": V2Batch, "2j": V2JBatch} @@ -210,10 +231,19 @@ def publish_events() -> None: metrics_client = metrics_client_from_config(raw_config) if args.use_in_memory_queue: - event_queue = InMemoryMessageQueue( - "/events-" + args.queue_name, - max_messages=cfg.max_queue_size, - ) + # start a thrift server that will be used to communicate with the main baseplate app + # this should not happen here, where are sidecars running? + processor = RemoteMessageQueueService.processor(RemoteMessageQueueHandler()) + transport = TSocket.TServerSocket(host='127.0.0.1', port=9090) + tfactory = TTransport.TBufferedTransportFactory() + pfactory = TBinaryProtocol.TBinaryProtocolFactory() + + server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) + print('Starting the Message Queue server...') + server.serve() + print('done.') + + else: event_queue = PosixMessageQueue( # type: ignore "/events-" + args.queue_name, diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index f080dae46..2d8a2369c 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,7 +10,7 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config @@ -167,7 +167,7 @@ def publish_traces() -> None: ) if args.use_in_memory_queue: - trace_queue = InMemoryMessageQueue( + trace_queue = RemoteMessageQueue( "/traces-" + args.queue_name, max_messages=publisher_cfg.max_queue_size, ) diff --git a/baseplate/thrift/message_queue.thrift b/baseplate/thrift/message_queue.thrift new file mode 100644 index 000000000..67f1b2091 --- /dev/null +++ b/baseplate/thrift/message_queue.thrift @@ -0,0 +1,35 @@ +namespace go baseplate.remote_message_queue +namespace py baseplate_remote_message_queue +namespace java com.reddit.baseplate.remote_message_queue + +struct PutRequest { + 1: binary data; +} + +struct PutResponse {} + + +struct GetRequest { + 1: binary key; +} + +struct GetResponse { + 1: binary value; +} + +exception PutFailedError {} + +exception GetFailedError {} + +service RemoteMessageQueueService { + PutResponse put( + 1: PutRequest request + ) throws ( + 1: PutFailedError put_failed_error + ); + GetResponse get( + 1: GetRequest request + ) throws ( + 1: GetFailedError get_failed_error + ); +} From 3740cc633aecd4e28a51d1c02846df8bcfc5d440 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 20 Dec 2022 15:47:39 -0800 Subject: [PATCH 09/72] wip --- Dockerfile | 1 + baseplate/lib/events.py | 7 +- baseplate/lib/message_queue.py | 34 +- baseplate/observers/tracing.py | 4 +- baseplate/sidecars/event_publisher.py | 95 ++-- baseplate/sidecars/trace_publisher.py | 6 +- .../RemoteMessageQueueService-remote | 124 +++++ .../RemoteMessageQueueService.py | 471 ++++++++++++++++++ baseplate/thrift/message_queue/__init__.py | 1 + baseplate/thrift/message_queue/constants.py | 14 + .../{ => message_queue}/message_queue.thrift | 0 baseplate/thrift/message_queue/ttypes.py | 361 ++++++++++++++ tests/integration/message_queue_tests.py | 10 + tests/unit/lib/events/publisher_tests.py | 55 ++ tests/unit/lib/events/queue_tests.py | 12 +- 15 files changed, 1134 insertions(+), 61 deletions(-) create mode 100755 baseplate/thrift/message_queue/RemoteMessageQueueService-remote create mode 100644 baseplate/thrift/message_queue/RemoteMessageQueueService.py create mode 100644 baseplate/thrift/message_queue/__init__.py create mode 100644 baseplate/thrift/message_queue/constants.py rename baseplate/thrift/{ => message_queue}/message_queue.thrift (100%) create mode 100644 baseplate/thrift/message_queue/ttypes.py diff --git a/Dockerfile b/Dockerfile index da54f0961..775db7208 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,3 +17,4 @@ RUN pip install -r requirements.txt RUN touch /baseplate-py-dev-docker-image CMD ["/bin/bash"] +# ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index e16611dd8..871223e6d 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -95,9 +95,9 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): """ def __init__( - self, name: str, event_serializer: Callable[[T], bytes], use_in_memory_queue: bool = False + self, name: str, event_serializer: Callable[[T], bytes], queue_type: str = "posix" ): - if use_in_memory_queue: + if queue_type == "in_memory": self.queue = RemoteMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) else: self.queue = PosixMessageQueue( # type: ignore @@ -128,6 +128,9 @@ def put(self, event: T) -> None: except TimedOutError: raise EventQueueFullError + def get(self): + return self.queue.get() + def make_object_for_context(self, name: str, span: Span) -> "EventQueue[T]": return self diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index ce6e840a7..73565d18d 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -4,14 +4,14 @@ import select from typing import Optional -from baseplate.thrift import RemoteMessageQueueService +from baseplate.thrift.message_queue import RemoteMessageQueueService import posix_ipc from baseplate.lib.retry import RetryPolicy -from thrift.Thrift import TBinaryProtocol -from thrift.Thrift import TSocket +from thrift.protocol import TBinaryProtocol +from thrift.transport import TSocket from thrift.transport import TTransport @@ -175,22 +175,30 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): - def __init__(self, max_messages: int): # Connect to the sidecar and instantiate a remote queue - transport = TSocket.TSocket( "localhost" , 9090) # todo: how do I connect to the sidecar? I dont think this is right - transport = TTransport.TBufferedTransport(transport) - protocol = TBinaryProtocol.TBinaryProtocol(transport) + def __init__(self, name: str, max_messages: int): + # Establish a connection with the queue server and create a new queue + self.queue_name = name + + transport = TSocket.TSocket( "127.0.0.1" , 9090) # TODO: I dont think this is right + self.transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(self.transport) self.client = RemoteMessageQueueService.Client(protocol) - # todo: create queue? # Connect to server self.transport.open() + # If the server doesnt have this queue yet, create it + if not self.client.queue_exists(name): + self.queue = self.client.new_queue(name, max_messages) + def get(self, timeout: Optional[float] = None) -> bytes: - return self.client.get(timeout) + # Call the remote server and get an element for the correct queue + return self.client.get(self.queue_name, timeout) def put(self, message: bytes, timeout: Optional[float] = None) -> None: - return self.client.put(message, timeout) + # Call the remote server and put an element on the correct queue + return self.client.put(self.queue_name, message, timeout) def unlink(self) -> None: """Not implemented for remote queue""" @@ -219,8 +227,8 @@ def queue_tool() -> None: ) parser.add_argument("queue_name", help="the name of the queue to consume") parser.add_argument( - "use_in_memory_queue", - default=False, + "queue_type", + default="posix", help="whether to use an in-memory queue or a posix queue", ) @@ -249,7 +257,7 @@ def queue_tool() -> None: args = parser.parse_args() - if args.use_in_memory_queue: + if args.queue_type == "in_memory": queue = RemoteMessageQueue(args.queue_name, args.max_messages) else: queue = PosixMessageQueue( # type: ignore diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 949c10000..5b25c33a6 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -546,8 +546,8 @@ class SidecarRecorder(Recorder): adding them to the queue. """ - def __init__(self, queue_name: str, use_in_memory_queue: bool = False): - if use_in_memory_queue: + def __init__(self, queue_name: str, queue_type: str = "posix"): + if queue_type == "in_memory": self.queue = RemoteMessageQueue( "/traces-" + queue_name, max_messages=MAX_QUEUE_SIZE, diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 26186ff22..b65a6629c 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -18,20 +18,22 @@ from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy from baseplate.server import EnvironmentInterpolation -from baseplate.thrift import RemoteMessageQueueService +from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.sidecars import Batch from baseplate.sidecars import BatchFull from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch -from thrift.Thrift import TServer -from thrift.Thrift import TBinaryProtocol -from thrift.Thrift import TSocket +from thrift.server import TServer +from thrift.protocol import TBinaryProtocol +from thrift.transport import TSocket from thrift.transport import TTransport @@ -169,22 +171,68 @@ def publish(self, payload: SerializedBatch) -> None: -class RemoteMessageQueueHandler: # From the sidecar, create the queue and define get/put using the InMemoryQueue implementation +class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation def is_healthy(self, context: RequestContext) -> bool: return True def __init__(self, name: str, max_messages: int): - self.queue = InMemoryMessageQueue(name, max_messages) + self.queues = {} - def get(self, timeout: Optional[float] = None) -> bytes: - return self.queue.get(timeout) + def new_queue(self, name: str, max_messages: int) -> InMemoryMessageQueue: + queue = InMemoryMessageQueue(name, max_messages) + self.queues[name] = InMemoryMessageQueue(name, max_messages) + return queue - def put(self, message: bytes, timeout: Optional[float] = None) -> None: - self.queue.put(message, timeout) + def queue_exists(self, queue_name: str) -> bool: + return self.queues.get(queue_name) is not None + + def get(self, queue_name: str, timeout: Optional[float] = None) -> bytes: + # If the queue has not been created yet, return None + if not self.queue_exists(queue_name): + return None + + queue: InMemoryMessageQueue = self.queues.get(queue_name) + return queue.get(timeout) + + def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) -> None: + queue: InMemoryMessageQueue = self.queues.get(queue_name) + queue.put(message, timeout) SERIALIZER_BY_VERSION = {"2": V2Batch, "2j": V2JBatch} +def start_queue_server(host: str, port: int) -> None: + # start a thrift server that will be used to communicate with the main baseplate app + # this should not happen here, where are sidecars running? + processor = RemoteMessageQueueService.processor(RemoteMessageQueueHandler()) + transport = TSocket.TServerSocket(host='127.0.0.1', port=9090) + tfactory = TTransport.TBufferedTransportFactory() + pfactory = TBinaryProtocol.TBinaryProtocolFactory() + + server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) + print('Starting the Message Queue server...') + server.serve() + + +def create_queue(queue_type: str, queue_name: str, max_queue_size: int) -> MessageQueue: + if queue_type == "in_memory": + start_queue_server(host='127.0.0.1', port=9090) + print('server started.') + + event_queue = RemoteMessageQueue( # type: ignore + "/events-" + queue_name, + max_queue_size + ) + + else: + event_queue = PosixMessageQueue( # type: ignore + "/events-" + queue_name, + max_messages=max_queue_size, + max_message_size=MAX_EVENT_SIZE, + ) + + return event_queue + def publish_events() -> None: arg_parser = argparse.ArgumentParser() @@ -196,11 +244,6 @@ def publish_events() -> None: default="main", help="name of event queue / publisher config (default: main)", ) - arg_parser.add_argument( - "--use-in-memory-queue", - default=False, - help="use an in memory queue instead of a posix queue", - ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" ) @@ -225,31 +268,13 @@ def publish_events() -> None: }, "key": {"name": config.String, "secret": config.Base64}, "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), + "queue_type": config.Optional(config.String, default="in_memory") }, ) metrics_client = metrics_client_from_config(raw_config) - if args.use_in_memory_queue: - # start a thrift server that will be used to communicate with the main baseplate app - # this should not happen here, where are sidecars running? - processor = RemoteMessageQueueService.processor(RemoteMessageQueueHandler()) - transport = TSocket.TServerSocket(host='127.0.0.1', port=9090) - tfactory = TTransport.TBufferedTransportFactory() - pfactory = TBinaryProtocol.TBinaryProtocolFactory() - - server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) - print('Starting the Message Queue server...') - server.serve() - print('done.') - - - else: - event_queue = PosixMessageQueue( # type: ignore - "/events-" + args.queue_name, - max_messages=cfg.max_queue_size, - max_message_size=MAX_EVENT_SIZE, - ) + event_queue: MessageQueue = create_queue(cfg.queue_type, args.queue_name, cfg.max_queue_size) # pylint: disable=maybe-no-member serializer = SERIALIZER_BY_VERSION[cfg.collector.version]() diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 2d8a2369c..fb28ce1e0 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -130,9 +130,9 @@ def publish_traces() -> None: help="name of trace queue / publisher config (default: main)", ) arg_parser.add_argument( - "--use-in-memory-queue", + "--queue-type", default=False, - help="use in memory queue instead of a posix queue", + help="Options: `in_memory` or `posix`. Default is posix", ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" @@ -166,7 +166,7 @@ def publish_traces() -> None: }, ) - if args.use_in_memory_queue: + if args.queue_type == "in_memory": trace_queue = RemoteMessageQueue( "/traces-" + args.queue_name, max_messages=publisher_cfg.max_queue_size, diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService-remote b/baseplate/thrift/message_queue/RemoteMessageQueueService-remote new file mode 100755 index 000000000..a3abdd1f7 --- /dev/null +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService-remote @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# +# Autogenerated by Thrift Compiler (0.17.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +import sys +import pprint +if sys.version_info[0] > 2: + from urllib.parse import urlparse +else: + from urlparse import urlparse +from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient +from thrift.protocol.TBinaryProtocol import TBinaryProtocol + +from baseplate_remote_message_queue import RemoteMessageQueueService +from baseplate_remote_message_queue.ttypes import * + +if len(sys.argv) <= 1 or sys.argv[1] == '--help': + print('') + print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') + print('') + print('Functions:') + print(' PutResponse put(PutRequest request)') + print(' GetResponse get(GetRequest request)') + print('') + sys.exit(0) + +pp = pprint.PrettyPrinter(indent=2) +host = 'localhost' +port = 9090 +uri = '' +framed = False +ssl = False +validate = True +ca_certs = None +keyfile = None +certfile = None +http = False +argi = 1 + +if sys.argv[argi] == '-h': + parts = sys.argv[argi + 1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + argi += 2 + +if sys.argv[argi] == '-u': + url = urlparse(sys.argv[argi + 1]) + parts = url[1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 80 + uri = url[2] + if url[4]: + uri += '?%s' % url[4] + http = True + argi += 2 + +if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': + framed = True + argi += 1 + +if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': + ssl = True + argi += 1 + +if sys.argv[argi] == '-novalidate': + validate = False + argi += 1 + +if sys.argv[argi] == '-ca_certs': + ca_certs = sys.argv[argi+1] + argi += 2 + +if sys.argv[argi] == '-keyfile': + keyfile = sys.argv[argi+1] + argi += 2 + +if sys.argv[argi] == '-certfile': + certfile = sys.argv[argi+1] + argi += 2 + +cmd = sys.argv[argi] +args = sys.argv[argi + 1:] + +if http: + transport = THttpClient.THttpClient(host, port, uri) +else: + if ssl: + socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) + else: + socket = TSocket.TSocket(host, port) + if framed: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) +protocol = TBinaryProtocol(transport) +client = RemoteMessageQueueService.Client(protocol) +transport.open() + +if cmd == 'put': + if len(args) != 1: + print('put requires 1 args') + sys.exit(1) + pp.pprint(client.put(eval(args[0]),)) + +elif cmd == 'get': + if len(args) != 1: + print('get requires 1 args') + sys.exit(1) + pp.pprint(client.get(eval(args[0]),)) + +else: + print('Unrecognized method %s' % cmd) + sys.exit(1) + +transport.close() diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py new file mode 100644 index 000000000..ff184c32e --- /dev/null +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -0,0 +1,471 @@ +# +# Autogenerated by Thrift Compiler (0.17.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +import logging +from .ttypes import * +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +all_structs = [] + + +class Iface(object): + def put(self, request): + """ + Parameters: + - request + + """ + pass + + def get(self, request): + """ + Parameters: + - request + + """ + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def put(self, request): + """ + Parameters: + - request + + """ + self.send_put(request) + return self.recv_put() + + def send_put(self, request): + self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) + args = put_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_put(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = put_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.put_failed_error is not None: + raise result.put_failed_error + raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") + + def get(self, request): + """ + Parameters: + - request + + """ + self.send_get(request) + return self.recv_get() + + def send_get(self, request): + self._oprot.writeMessageBegin('get', TMessageType.CALL, self._seqid) + args = get_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.get_failed_error is not None: + raise result.get_failed_error + raise TApplicationException(TApplicationException.MISSING_RESULT, "get failed: unknown result") + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["put"] = Processor.process_put + self._processMap["get"] = Processor.process_get + self._on_message_begin = None + + def on_message_begin(self, func): + self._on_message_begin = func + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if self._on_message_begin: + self._on_message_begin(name, type, seqid) + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_put(self, seqid, iprot, oprot): + args = put_args() + args.read(iprot) + iprot.readMessageEnd() + result = put_result() + try: + result.success = self._handler.put(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except PutFailedError as put_failed_error: + msg_type = TMessageType.REPLY + result.put_failed_error = put_failed_error + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("put", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get(self, seqid, iprot, oprot): + args = get_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_result() + try: + result.success = self._handler.get(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except GetFailedError as get_failed_error: + msg_type = TMessageType.REPLY + result.get_failed_error = get_failed_error + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("get", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + +# HELPER FUNCTIONS AND STRUCTURES + + +class put_args(object): + """ + Attributes: + - request + + """ + + + def __init__(self, request=None,): + self.request = request + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = PutRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('put_args') + if self.request is not None: + oprot.writeFieldBegin('request', TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(put_args) +put_args.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'request', [PutRequest, None], None, ), # 1 +) + + +class put_result(object): + """ + Attributes: + - success + - put_failed_error + + """ + + + def __init__(self, success=None, put_failed_error=None,): + self.success = success + self.put_failed_error = put_failed_error + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PutResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.put_failed_error = PutFailedError.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('put_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.put_failed_error is not None: + oprot.writeFieldBegin('put_failed_error', TType.STRUCT, 1) + self.put_failed_error.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(put_result) +put_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 + (1, TType.STRUCT, 'put_failed_error', [PutFailedError, None], None, ), # 1 +) + + +class get_args(object): + """ + Attributes: + - request + + """ + + + def __init__(self, request=None,): + self.request = request + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('get_args') + if self.request is not None: + oprot.writeFieldBegin('request', TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(get_args) +get_args.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'request', [GetRequest, None], None, ), # 1 +) + + +class get_result(object): + """ + Attributes: + - success + - get_failed_error + + """ + + + def __init__(self, success=None, get_failed_error=None,): + self.success = success + self.get_failed_error = get_failed_error + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.get_failed_error = GetFailedError.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('get_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.get_failed_error is not None: + oprot.writeFieldBegin('get_failed_error', TType.STRUCT, 1) + self.get_failed_error.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(get_result) +get_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [GetResponse, None], None, ), # 0 + (1, TType.STRUCT, 'get_failed_error', [GetFailedError, None], None, ), # 1 +) +fix_spec(all_structs) +del all_structs diff --git a/baseplate/thrift/message_queue/__init__.py b/baseplate/thrift/message_queue/__init__.py new file mode 100644 index 000000000..947e972be --- /dev/null +++ b/baseplate/thrift/message_queue/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'RemoteMessageQueueService'] diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py new file mode 100644 index 000000000..511b9cad4 --- /dev/null +++ b/baseplate/thrift/message_queue/constants.py @@ -0,0 +1,14 @@ +# +# Autogenerated by Thrift Compiler (0.17.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +from .ttypes import * diff --git a/baseplate/thrift/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift similarity index 100% rename from baseplate/thrift/message_queue.thrift rename to baseplate/thrift/message_queue/message_queue.thrift diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py new file mode 100644 index 000000000..4e883490c --- /dev/null +++ b/baseplate/thrift/message_queue/ttypes.py @@ -0,0 +1,361 @@ +# +# Autogenerated by Thrift Compiler (0.17.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys + +from thrift.transport import TTransport +all_structs = [] + + +class PutRequest(object): + """ + Attributes: + - data + + """ + + + def __init__(self, data=None,): + self.data = data + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.data = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('PutRequest') + if self.data is not None: + oprot.writeFieldBegin('data', TType.STRING, 1) + oprot.writeBinary(self.data) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PutResponse(object): + + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('PutResponse') + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetRequest(object): + """ + Attributes: + - key + + """ + + + def __init__(self, key=None,): + self.key = key + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('GetRequest') + if self.key is not None: + oprot.writeFieldBegin('key', TType.STRING, 1) + oprot.writeBinary(self.key) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetResponse(object): + """ + Attributes: + - value + + """ + + + def __init__(self, value=None,): + self.value = value + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.value = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('GetResponse') + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRING, 1) + oprot.writeBinary(self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PutFailedError(TException): + + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash(()) + + @classmethod + def read(cls, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('PutFailedError') + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFailedError(TException): + + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash(()) + + @classmethod + def read(cls, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('GetFailedError') + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(PutRequest) +PutRequest.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'data', 'BINARY', None, ), # 1 +) +all_structs.append(PutResponse) +PutResponse.thrift_spec = ( +) +all_structs.append(GetRequest) +GetRequest.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'key', 'BINARY', None, ), # 1 +) +all_structs.append(GetResponse) +GetResponse.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'value', 'BINARY', None, ), # 1 +) +all_structs.append(PutFailedError) +PutFailedError.thrift_spec = ( +) +all_structs.append(GetFailedError) +GetFailedError.thrift_spec = ( +) +fix_spec(all_structs) +del all_structs diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index ad2d3b234..50c21cdba 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -6,6 +6,7 @@ from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -138,3 +139,12 @@ def test_put_full_zero_timeout(self): with self.assertRaises(TimedOutError): mq.put(b"2", timeout=0) + +class TestRemoteMessageQueueCreation(unittest.TestCase): + qname = "/baseplate-test-queue" + + def test_create_queue_remote(self): + message_queue = RemoteMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + message_queue.put("yero") \ No newline at end of file diff --git a/tests/unit/lib/events/publisher_tests.py b/tests/unit/lib/events/publisher_tests.py index fc1d84f8b..4eec4d48a 100644 --- a/tests/unit/lib/events/publisher_tests.py +++ b/tests/unit/lib/events/publisher_tests.py @@ -86,6 +86,61 @@ def setUp(self, Session): self.config.key = config.ConfigNamespace() self.config.key.name = "TestKey" self.config.key.secret = b"hunter2" + # self.config.queue.type = "posix" + + self.session = Session.return_value + self.session.headers = {} + + self.metrics_client = mock.MagicMock(autospec=metrics.Client) + + self.publisher = event_publisher.BatchPublisher(self.metrics_client, self.config) + + def test_empty_batch(self): + self.publisher.publish(SerializedBatch(item_count=0, serialized=b"")) + self.assertEqual(self.session.post.call_count, 0) + + def test_publish_success(self): + events = b'[{"example": "value"}]' + + self.publisher.publish(SerializedBatch(item_count=1, serialized=events)) + + self.assertEqual(self.session.post.call_count, 1) + + args, kwargs = self.session.post.call_args + headers = kwargs.get("headers", {}) + self.assertEqual(args[0], "https://test.local/v1") + self.assertEqual(headers["Content-Type"], "application/json") + self.assertEqual( + headers["X-Signature"], + "key=TestKey, mac=7c46d56b99cd4cb05e08238c1d4c10a2f330795e9d7327f17cc66fd206bf1179", + ) + + @mock.patch("time.sleep") + def test_fail_on_client_error(self, mock_sleep): + self.session.post.side_effect = [ + requests.HTTPError(400, response=mock.Mock(status_code=400)) + ] + events = b'[{"example": "value"}]' + + with self.assertRaises(requests.HTTPError): + self.publisher.publish(SerializedBatch(item_count=1, serialized=events)) + + self.assertEqual(mock_sleep.call_count, 0) + self.assertEqual(self.session.post.call_count, 1) + +class RemoteQueuePublisherTests(unittest.TestCase): + @mock.patch("requests.Session", autospec=True) + def setUp(self, Session): + self.config = config.ConfigNamespace() + self.config.collector = config.ConfigNamespace() + self.config.collector.hostname = "test.local" + self.config.collector.version = 1 + self.config.collector.scheme = "https" + self.config.key = config.ConfigNamespace() + self.config.key.name = "TestKey" + self.config.key.secret = b"hunter2" + self.config.queue = config.ConfigNamespace() + self.config.queue.type = "in_memory" self.session = Session.return_value self.session.headers = {} diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 805fc36ca..5b7089484 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -6,7 +6,7 @@ from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE -from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -44,13 +44,13 @@ def test_event_queue_full(self): self.queue.put(object()) -class InMemoryEventQueueTests(unittest.TestCase): - @mock.patch("baseplate.lib.events.InMemoryMessageQueue", autospec=InMemoryMessageQueue) - def setUp(self, InMemoryMessageQueue): - self.message_queue = InMemoryMessageQueue.return_value +class RemoteMessageQueueTests(unittest.TestCase): + @mock.patch("baseplate.lib.events.RemoteMessageQueue", autospec=RemoteMessageQueue) + def setUp(self, RemoteMessageQueue): + self.message_queue = RemoteMessageQueue.return_value self.mock_serializer = mock.Mock() self.queue = EventQueue( - "test", event_serializer=self.mock_serializer, use_in_memory_queue=True + "test", event_serializer=self.mock_serializer, queue_type="in_memory" ) def test_send_event(self): From 1b1d5cf112a4aef2bdba9f17e821a4c3101dee2c Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 22 Dec 2022 16:25:07 -0800 Subject: [PATCH 10/72] thrift updates --- Makefile | 9 +- .../RemoteMessageQueueService-remote | 124 ----------- .../RemoteMessageQueueService.py | 193 ++++++++++++------ baseplate/thrift/message_queue/constants.py | 2 +- .../thrift/message_queue/message_queue.thrift | 21 +- baseplate/thrift/message_queue/ttypes.py | 191 +++++------------ 6 files changed, 197 insertions(+), 343 deletions(-) delete mode 100755 baseplate/thrift/message_queue/RemoteMessageQueueService-remote diff --git a/Makefile b/Makefile index dacabeb6a..3f3d521ef 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ all: thrift THRIFT=thrift THRIFT_OPTS=-strict -gen py:slots THRIFT_BUILDDIR=build/thrift -THRIFT_SOURCE=baseplate/thrift/baseplate.thrift tests/integration/test.thrift +THRIFT_SOURCE=baseplate/thrift/baseplate.thrift baseplate/thrift/message_queue/message_queue.thrift tests/integration/test.thrift THRIFT_BUILDSTAMPS=$(patsubst %,$(THRIFT_BUILDDIR)/%_buildstamp,$(THRIFT_SOURCE)) thrift: $(THRIFT_BUILDSTAMPS) @@ -22,6 +22,13 @@ $(THRIFT_BUILDDIR)/baseplate/thrift/baseplate.thrift_buildstamp: baseplate/thrif rm -f baseplate/thrift/BaseplateServiceV2-remote touch $@ +$(THRIFT_BUILDDIR)/baseplate/thrift/message_queue/message_queue.thrift_buildstamp: baseplate/thrift/message_queue/message_queue.thrift + mkdir -p $(THRIFT_BUILDDIR)/$< + $(THRIFT) $(THRIFT_OPTS) -out $(THRIFT_BUILDDIR)/$< $< + cp -r $(THRIFT_BUILDDIR)/$ 2: - from urllib.parse import urlparse -else: - from urlparse import urlparse -from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient -from thrift.protocol.TBinaryProtocol import TBinaryProtocol - -from baseplate_remote_message_queue import RemoteMessageQueueService -from baseplate_remote_message_queue.ttypes import * - -if len(sys.argv) <= 1 or sys.argv[1] == '--help': - print('') - print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') - print('') - print('Functions:') - print(' PutResponse put(PutRequest request)') - print(' GetResponse get(GetRequest request)') - print('') - sys.exit(0) - -pp = pprint.PrettyPrinter(indent=2) -host = 'localhost' -port = 9090 -uri = '' -framed = False -ssl = False -validate = True -ca_certs = None -keyfile = None -certfile = None -http = False -argi = 1 - -if sys.argv[argi] == '-h': - parts = sys.argv[argi + 1].split(':') - host = parts[0] - if len(parts) > 1: - port = int(parts[1]) - argi += 2 - -if sys.argv[argi] == '-u': - url = urlparse(sys.argv[argi + 1]) - parts = url[1].split(':') - host = parts[0] - if len(parts) > 1: - port = int(parts[1]) - else: - port = 80 - uri = url[2] - if url[4]: - uri += '?%s' % url[4] - http = True - argi += 2 - -if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': - framed = True - argi += 1 - -if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': - ssl = True - argi += 1 - -if sys.argv[argi] == '-novalidate': - validate = False - argi += 1 - -if sys.argv[argi] == '-ca_certs': - ca_certs = sys.argv[argi+1] - argi += 2 - -if sys.argv[argi] == '-keyfile': - keyfile = sys.argv[argi+1] - argi += 2 - -if sys.argv[argi] == '-certfile': - certfile = sys.argv[argi+1] - argi += 2 - -cmd = sys.argv[argi] -args = sys.argv[argi + 1:] - -if http: - transport = THttpClient.THttpClient(host, port, uri) -else: - if ssl: - socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) - else: - socket = TSocket.TSocket(host, port) - if framed: - transport = TTransport.TFramedTransport(socket) - else: - transport = TTransport.TBufferedTransport(socket) -protocol = TBinaryProtocol(transport) -client = RemoteMessageQueueService.Client(protocol) -transport.open() - -if cmd == 'put': - if len(args) != 1: - print('put requires 1 args') - sys.exit(1) - pp.pprint(client.put(eval(args[0]),)) - -elif cmd == 'get': - if len(args) != 1: - print('get requires 1 args') - sys.exit(1) - pp.pprint(client.get(eval(args[0]),)) - -else: - print('Unrecognized method %s' % cmd) - sys.exit(1) - -transport.close() diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index ff184c32e..b7d5e5494 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -3,7 +3,7 @@ # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # -# options string: py +# options string: py:slots # from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException @@ -19,20 +19,18 @@ class Iface(object): - def put(self, request): + def put(self, queue_name, max_messages, message, timeout): """ Parameters: - - request + - queue_name + - max_messages + - message + - timeout """ pass - def get(self, request): - """ - Parameters: - - request - - """ + def get(self): pass @@ -43,19 +41,25 @@ def __init__(self, iprot, oprot=None): self._oprot = oprot self._seqid = 0 - def put(self, request): + def put(self, queue_name, max_messages, message, timeout): """ Parameters: - - request + - queue_name + - max_messages + - message + - timeout """ - self.send_put(request) + self.send_put(queue_name, max_messages, message, timeout) return self.recv_put() - def send_put(self, request): + def send_put(self, queue_name, max_messages, message, timeout): self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) args = put_args() - args.request = request + args.queue_name = queue_name + args.max_messages = max_messages + args.message = message + args.timeout = timeout args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -77,19 +81,13 @@ def recv_put(self): raise result.put_failed_error raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") - def get(self, request): - """ - Parameters: - - request - - """ - self.send_get(request) + def get(self): + self.send_get() return self.recv_get() - def send_get(self, request): + def send_get(self): self._oprot.writeMessageBegin('get', TMessageType.CALL, self._seqid) args = get_args() - args.request = request args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -146,7 +144,7 @@ def process_put(self, seqid, iprot, oprot): iprot.readMessageEnd() result = put_result() try: - result.success = self._handler.put(args.request) + result.success = self._handler.put(args.queue_name, args.max_messages, args.message, args.timeout) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -172,7 +170,7 @@ def process_get(self, seqid, iprot, oprot): iprot.readMessageEnd() result = get_result() try: - result.success = self._handler.get(args.request) + result.success = self._handler.get() msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -198,13 +196,26 @@ def process_get(self, seqid, iprot, oprot): class put_args(object): """ Attributes: - - request + - queue_name + - max_messages + - message + - timeout """ + __slots__ = ( + 'queue_name', + 'max_messages', + 'message', + 'timeout', + ) + - def __init__(self, request=None,): - self.request = request + def __init__(self, queue_name=None, max_messages=None, message=None, timeout=None,): + self.queue_name = queue_name + self.max_messages = max_messages + self.message = message + self.timeout = timeout def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -216,9 +227,23 @@ def read(self, iprot): if ftype == TType.STOP: break if fid == 1: - if ftype == TType.STRUCT: - self.request = PutRequest() - self.request.read(iprot) + if ftype == TType.STRING: + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.max_messages = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.message = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.DOUBLE: + self.timeout = iprot.readDouble() else: iprot.skip(ftype) else: @@ -231,9 +256,21 @@ def write(self, oprot): oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return oprot.writeStructBegin('put_args') - if self.request is not None: - oprot.writeFieldBegin('request', TType.STRUCT, 1) - self.request.write(oprot) + if self.queue_name is not None: + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldEnd() + if self.max_messages is not None: + oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeI64(self.max_messages) + oprot.writeFieldEnd() + if self.message is not None: + oprot.writeFieldBegin('message', TType.STRING, 3) + oprot.writeBinary(self.message) + oprot.writeFieldEnd() + if self.timeout is not None: + oprot.writeFieldBegin('timeout', TType.DOUBLE, 4) + oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -242,19 +279,29 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) all_structs.append(put_args) put_args.thrift_spec = ( None, # 0 - (1, TType.STRUCT, 'request', [PutRequest, None], None, ), # 1 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + (2, TType.I64, 'max_messages', None, None, ), # 2 + (3, TType.STRING, 'message', 'BINARY', None, ), # 3 + (4, TType.DOUBLE, 'timeout', None, None, ), # 4 ) @@ -266,6 +313,11 @@ class put_result(object): """ + __slots__ = ( + 'success', + 'put_failed_error', + ) + def __init__(self, success=None, put_failed_error=None,): self.success = success @@ -316,12 +368,19 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) @@ -333,15 +392,10 @@ def __ne__(self, other): class get_args(object): - """ - Attributes: - - request - - """ + __slots__ = ( + ) - def __init__(self, request=None,): - self.request = request def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -352,12 +406,6 @@ def read(self, iprot): (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break - if fid == 1: - if ftype == TType.STRUCT: - self.request = GetRequest() - self.request.read(iprot) - else: - iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -368,10 +416,6 @@ def write(self, oprot): oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return oprot.writeStructBegin('get_args') - if self.request is not None: - oprot.writeFieldBegin('request', TType.STRUCT, 1) - self.request.write(oprot) - oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -379,19 +423,24 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) all_structs.append(get_args) get_args.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'request', [GetRequest, None], None, ), # 1 ) @@ -403,6 +452,11 @@ class get_result(object): """ + __slots__ = ( + 'success', + 'get_failed_error', + ) + def __init__(self, success=None, get_failed_error=None,): self.success = success @@ -453,12 +507,19 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py index 511b9cad4..964098e62 100644 --- a/baseplate/thrift/message_queue/constants.py +++ b/baseplate/thrift/message_queue/constants.py @@ -3,7 +3,7 @@ # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # -# options string: py +# options string: py:slots # from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index 67f1b2091..01ab8801c 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -1,18 +1,9 @@ -namespace go baseplate.remote_message_queue -namespace py baseplate_remote_message_queue -namespace java com.reddit.baseplate.remote_message_queue - -struct PutRequest { - 1: binary data; -} +namespace go reddit.message_queue +namespace py message_queue.thrift +namespace java com.reddit.baseplate.message_queue struct PutResponse {} - -struct GetRequest { - 1: binary key; -} - struct GetResponse { 1: binary value; } @@ -23,12 +14,14 @@ exception GetFailedError {} service RemoteMessageQueueService { PutResponse put( - 1: PutRequest request + 1: string queue_name + 2: i64 max_messages + 3: binary message + 4: double timeout ) throws ( 1: PutFailedError put_failed_error ); GetResponse get( - 1: GetRequest request ) throws ( 1: GetFailedError get_failed_error ); diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index 4e883490c..de611e70d 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -3,7 +3,7 @@ # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # -# options string: py +# options string: py:slots # from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException @@ -16,113 +16,11 @@ all_structs = [] -class PutRequest(object): - """ - Attributes: - - data - - """ - - - def __init__(self, data=None,): - self.data = data - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.data = iprot.readBinary() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('PutRequest') - if self.data is not None: - oprot.writeFieldBegin('data', TType.STRING, 1) - oprot.writeBinary(self.data) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - class PutResponse(object): + __slots__ = ( + ) - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('PutResponse') - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class GetRequest(object): - """ - Attributes: - - key - - """ - - - def __init__(self, key=None,): - self.key = key def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -133,11 +31,6 @@ def read(self, iprot): (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break - if fid == 1: - if ftype == TType.STRING: - self.key = iprot.readBinary() - else: - iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -147,11 +40,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('GetRequest') - if self.key is not None: - oprot.writeFieldBegin('key', TType.STRING, 1) - oprot.writeBinary(self.key) - oprot.writeFieldEnd() + oprot.writeStructBegin('PutResponse') oprot.writeFieldStop() oprot.writeStructEnd() @@ -159,12 +48,19 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) @@ -177,6 +73,10 @@ class GetResponse(object): """ + __slots__ = ( + 'value', + ) + def __init__(self, value=None,): self.value = value @@ -216,12 +116,19 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) @@ -229,6 +136,9 @@ def __ne__(self, other): class PutFailedError(TException): + __slots__ = ( + ) + def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -270,12 +180,19 @@ def __str__(self): return repr(self) def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) @@ -283,6 +200,9 @@ def __ne__(self, other): class GetFailedError(TException): + __slots__ = ( + ) + def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -324,28 +244,25 @@ def __str__(self): return repr(self) def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True def __ne__(self, other): return not (self == other) -all_structs.append(PutRequest) -PutRequest.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'data', 'BINARY', None, ), # 1 -) all_structs.append(PutResponse) PutResponse.thrift_spec = ( ) -all_structs.append(GetRequest) -GetRequest.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'key', 'BINARY', None, ), # 1 -) all_structs.append(GetResponse) GetResponse.thrift_spec = ( None, # 0 From 5fd17f5211790f130f1afcb3a748a19c58e10337 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 3 Jan 2023 15:03:17 -0800 Subject: [PATCH 11/72] working --- baseplate/lib/message_queue.py | 40 ++++-- baseplate/sidecars/event_publisher.py | 130 ++++++------------ baseplate/sidecars/publisher_queue_utils.py | 98 +++++++++++++ baseplate/sidecars/trace_publisher.py | 36 ++--- .../RemoteMessageQueueService.py | 125 ++++++++++++----- .../thrift/message_queue/message_queue.thrift | 11 +- baseplate/thrift/message_queue/ttypes.py | 75 +--------- tests/integration/message_queue_tests.py | 70 +++++++++- .../sidecars/publisher_queue_utils_tests.py | 64 +++++++++ tests/unit/lib/events/publisher_tests.py | 52 ------- 10 files changed, 416 insertions(+), 285 deletions(-) create mode 100644 baseplate/sidecars/publisher_queue_utils.py create mode 100644 tests/integration/sidecars/publisher_queue_utils_tests.py diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 73565d18d..a49d3d994 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -2,9 +2,11 @@ import abc import queue as q import select +import gevent from typing import Optional from baseplate.thrift.message_queue import RemoteMessageQueueService +from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError import posix_ipc @@ -104,7 +106,7 @@ def __init__(self, name: str, max_messages: int, max_message_size: int): self.queue = posix_ipc.MessageQueue( name, flags=posix_ipc.O_CREAT, - mode=0o0644, + # mode=0o0644, max_messages=max_messages, max_message_size=max_message_size, ) @@ -175,30 +177,38 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): - def __init__(self, name: str, max_messages: int): - # Establish a connection with the queue server and create a new queue + def __init__(self, name: str, max_messages: int, host: str="127.0.0.1", port: int=9090): self.queue_name = name - - transport = TSocket.TSocket( "127.0.0.1" , 9090) # TODO: I dont think this is right + self.max_messages = max_messages + self.host = host + self.port = port + + def connect(self): + # Establish a connection with the queue server + transport = TSocket.TSocket(self.host, self.port) self.transport = TTransport.TBufferedTransport(transport) protocol = TBinaryProtocol.TBinaryProtocol(self.transport) - self.client = RemoteMessageQueueService.Client(protocol) - - # Connect to server self.transport.open() - # If the server doesnt have this queue yet, create it - if not self.client.queue_exists(name): - self.queue = self.client.new_queue(name, max_messages) - def get(self, timeout: Optional[float] = None) -> bytes: - # Call the remote server and get an element for the correct queue - return self.client.get(self.queue_name, timeout) + # Call the remote server and get an element for the correct queue + try: + self.connect() + print("trying get...") + return self.client.get(self.queue_name, self.max_messages, timeout).value + except ThriftTimedOutError: + raise TimedOutError def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Call the remote server and put an element on the correct queue - return self.client.put(self.queue_name, message, timeout) + # Will create the queue if it doesnt exist + try: + self.connect() + print("trying put:") + self.client.put(self.queue_name, self.max_messages, message, timeout) + except ThriftTimedOutError: + raise TimedOutError def unlink(self) -> None: """Not implemented for remote queue""" diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index b65a6629c..33b7f3a71 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -1,12 +1,16 @@ import argparse import configparser +import contextlib import email.utils +import sys +from baseplate.thrift.message_queue.ttypes import GetResponse, PutResponse +import gevent import gzip import hashlib import hmac import logging -from typing import Any +from typing import Any, Tuple from typing import List from typing import Optional @@ -21,13 +25,17 @@ from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue -from baseplate.lib.message_queue import TimedOutError +from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy from baseplate.server import EnvironmentInterpolation +from baseplate.server import make_listener +from baseplate.server.thrift import make_server from baseplate.thrift.message_queue import RemoteMessageQueueService +from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError from baseplate.sidecars import Batch from baseplate.sidecars import BatchFull +from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch @@ -106,7 +114,6 @@ def serialize(self) -> SerializedBatch: serialized = self._header.encode() + b",".join(self._items) + self._end return SerializedBatch(item_count=len(self._items), serialized=serialized) - class BatchPublisher: def __init__(self, metrics_client: metrics.Client, cfg: Any): self.metrics = metrics_client @@ -170,70 +177,22 @@ def publish(self, payload: SerializedBatch) -> None: raise MaxRetriesError("could not sent batch") +def build_batch(queue: MessageQueue, batcher: TimeLimitedBatch) -> bytes: + while True: + message: Optional[bytes] + try: + message = queue.get(timeout=0.2) + except MessageQueueTimedOutError: + message = None -class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation - def is_healthy(self, context: RequestContext) -> bool: - return True - - def __init__(self, name: str, max_messages: int): - self.queues = {} - - def new_queue(self, name: str, max_messages: int) -> InMemoryMessageQueue: - queue = InMemoryMessageQueue(name, max_messages) - self.queues[name] = InMemoryMessageQueue(name, max_messages) - return queue - - def queue_exists(self, queue_name: str) -> bool: - return self.queues.get(queue_name) is not None - - def get(self, queue_name: str, timeout: Optional[float] = None) -> bytes: - # If the queue has not been created yet, return None - if not self.queue_exists(queue_name): - return None - - queue: InMemoryMessageQueue = self.queues.get(queue_name) - return queue.get(timeout) - - def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) -> None: - queue: InMemoryMessageQueue = self.queues.get(queue_name) - queue.put(message, timeout) - + try: + batcher.add(message) + continue + except BatchFull: + return message SERIALIZER_BY_VERSION = {"2": V2Batch, "2j": V2JBatch} -def start_queue_server(host: str, port: int) -> None: - # start a thrift server that will be used to communicate with the main baseplate app - # this should not happen here, where are sidecars running? - processor = RemoteMessageQueueService.processor(RemoteMessageQueueHandler()) - transport = TSocket.TServerSocket(host='127.0.0.1', port=9090) - tfactory = TTransport.TBufferedTransportFactory() - pfactory = TBinaryProtocol.TBinaryProtocolFactory() - - server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) - print('Starting the Message Queue server...') - server.serve() - - -def create_queue(queue_type: str, queue_name: str, max_queue_size: int) -> MessageQueue: - if queue_type == "in_memory": - start_queue_server(host='127.0.0.1', port=9090) - print('server started.') - - event_queue = RemoteMessageQueue( # type: ignore - "/events-" + queue_name, - max_queue_size - ) - - else: - event_queue = PosixMessageQueue( # type: ignore - "/events-" + queue_name, - max_messages=max_queue_size, - max_message_size=MAX_EVENT_SIZE, - ) - - return event_queue - - def publish_events() -> None: arg_parser = argparse.ArgumentParser() arg_parser.add_argument( @@ -268,13 +227,14 @@ def publish_events() -> None: }, "key": {"name": config.String, "secret": config.Base64}, "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), - "queue_type": config.Optional(config.String, default="in_memory") + "max_element_size": config.Optional(config.Integer, MAX_EVENT_SIZE), + "queue_type": config.Optional(config.String, default="posix") }, ) metrics_client = metrics_client_from_config(raw_config) - event_queue: MessageQueue = create_queue(cfg.queue_type, args.queue_name, cfg.max_queue_size) + event_queue: MessageQueue = publisher_utils.create_queue(cfg.queue_type, args.queue_name, cfg.max_queue_size, cfg.max_element_size) # pylint: disable=maybe-no-member serializer = SERIALIZER_BY_VERSION[cfg.collector.version]() @@ -282,27 +242,25 @@ def publish_events() -> None: publisher = BatchPublisher(metrics_client, cfg) while True: - message: Optional[bytes] - - try: - message = event_queue.get(timeout=0.2) - except TimedOutError: - message = None - - try: - batcher.add(message) - continue - except BatchFull: - pass - - serialized = batcher.serialize() - try: - publisher.publish(serialized) - except Exception: - logger.exception("Events publishing failed.") - batcher.reset() - batcher.add(message) - + if cfg.queue_type == "in_memory": + with publisher_queue_utils.start_queue_server(host='127.0.0.1', port=9090) as server: + last_message = publisher_queue_utils.build_batch(event_queue, batcher) + serialized = batcher.serialize() + try: + publisher.publish(serialized) + except Exception: + logger.exception("Events publishing failed.") + batcher.reset() + batcher.add(last_message) + else: + last_message = publisher_queue_utils.build_batch(event_queue, batcher) + serialized = batcher.serialize() + try: + publisher.publish(serialized) + except Exception: + logger.exception("Events publishing failed.") + batcher.reset() + batcher.add(last_message) if __name__ == "__main__": publish_events() diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py new file mode 100644 index 000000000..840a3976f --- /dev/null +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -0,0 +1,98 @@ +import contextlib +import gevent +from typing import Optional +from baseplate.lib import config +from baseplate.server import make_listener, make_server +from baseplate.thrift.message_queue import RemoteMessageQueueService +from baseplate.thrift.message_queue.ttypes import GetResponse, PutResponse +from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError +from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError + +from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue + + +class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation + def is_healthy(self) -> bool: + return True + + def __init__(self): + self.queue_list = {} + print("initting") + + def get(self, queue_name: str, max_messages: int, timeout: Optional[float] = None) -> GetResponse: + print("in get") + try: + # Create queue if doesnt exist + # We need to create the queue on both get & put - if get() is called on a queue before it exists, we + # still want to wait the appropriate timeout in case anyone puts elements in it + print("in get") + queue = self.queue_list.get(queue_name) + if not queue: + queue = InMemoryMessageQueue(queue_name, max_messages) + self.queue_list[queue_name] = queue + # Get element from list, waiting if necessary + result = queue.get(timeout) + except MessageQueueTimedOutError: + raise ThriftTimedOutError() + + return GetResponse(result) + + def put(self, queue_name: str, max_messages: int, message: bytes, timeout: Optional[float] = None) -> PutResponse: + # Create queue if it does not exist yet + print("in put") + try: + queue = self.queue_list.get(queue_name) + if not self.queue_list.get(queue_name): + print("didnt find queue") + queue = InMemoryMessageQueue(queue_name, max_messages) + self.queue_list[queue_name] = queue + print("putting") + queue.put(message, timeout) + except MessageQueueTimedOutError: + raise ThriftTimedOutError() + print("responding") + return PutResponse() + + +@contextlib.contextmanager +def start_queue_server(host: str, port: int) -> None: + # Start a thrift server that will house the remote queue data + print("starting server") + processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler()) + server_bind_endpoint = config.Endpoint(f"{host}:{port}") + listener = make_listener(server_bind_endpoint) + server = make_server(server_config={}, listener=listener, app=processor) + + # figure out what port the server ended up on + server_address = listener.getsockname() + server.endpoint = config.Endpoint(f"{server_address[0]}:{server_address[1]}") + print("finished starting server") + print(server_address) # localhost 9090 + # run the server until our caller is done with it + server_greenlet = gevent.spawn(server.serve_forever) + try: + yield server + finally: + server_greenlet.kill() + + +def create_queue(queue_type: str, queue_name: str, max_queue_size: int, max_element_size: int) -> MessageQueue: + if queue_type == "in_memory": + with start_queue_server(host='127.0.0.1', port=9090) as server: + print('server started.') + + event_queue = RemoteMessageQueue( # type: ignore + "/events-" + queue_name, + max_queue_size + ) + print("heyo") + + else: + event_queue = PosixMessageQueue( # type: ignore + "/events-" + queue_name, + max_messages=max_queue_size, + max_message_size=max_element_size, + ) + + return event_queue + diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index fb28ce1e0..4871c034d 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,7 +10,7 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import RemoteMessageQueue +from baseplate.lib.message_queue import MessageQueue, RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config @@ -18,7 +18,7 @@ from baseplate.observers.tracing import MAX_QUEUE_SIZE from baseplate.observers.tracing import MAX_SPAN_SIZE from baseplate.server import EnvironmentInterpolation -from baseplate.sidecars import BatchFull +from baseplate.sidecars import BatchFull, publisher_queue_utils from baseplate.sidecars import RawJSONBatch from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch @@ -163,20 +163,13 @@ def publish_traces() -> None: "max_batch_size": config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT), "retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT), "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), + "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), + "max_element_size": config.Optional(config.Integer, MAX_SPAN_SIZE), + "queue_type": config.Optional(config.String, default="posix") }, ) - if args.queue_type == "in_memory": - trace_queue = RemoteMessageQueue( - "/traces-" + args.queue_name, - max_messages=publisher_cfg.max_queue_size, - ) - else: - trace_queue = PosixMessageQueue( # type: ignore - "/traces-" + args.queue_name, - max_messages=publisher_cfg.max_queue_size, - max_message_size=MAX_SPAN_SIZE, - ) + trace_queue: MessageQueue = publisher_queue_utils.create_queue(publisher_cfg.queue_type, args.queue_name, publisher_cfg.max_queue_size, publisher_cfg.max_element_size) # pylint: disable=maybe-no-member inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size) @@ -191,11 +184,18 @@ def publish_traces() -> None: while True: message: Optional[bytes] - try: - message = trace_queue.get(timeout=0.2) - except TimedOutError: - message = None - + if publisher_cfg.queue_type == "in_memory": + with publisher_queue_utils.start_queue_server(host='127.0.0.1', port=9090) as server: + try: + message = trace_queue.get(timeout=0.2) + except TimedOutError: + message = None + else: + try: + message = trace_queue.get(timeout=0.2) + except TimedOutError: + message = None + try: batcher.add(message) except BatchFull: diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index b7d5e5494..b2ee83272 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -30,7 +30,14 @@ def put(self, queue_name, max_messages, message, timeout): """ pass - def get(self): + def get(self, queue_name, max_messages, timeout): + """ + Parameters: + - queue_name + - max_messages + - timeout + + """ pass @@ -77,17 +84,27 @@ def recv_put(self): iprot.readMessageEnd() if result.success is not None: return result.success - if result.put_failed_error is not None: - raise result.put_failed_error + if result.timed_out_error is not None: + raise result.timed_out_error raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") - def get(self): - self.send_get() + def get(self, queue_name, max_messages, timeout): + """ + Parameters: + - queue_name + - max_messages + - timeout + + """ + self.send_get(queue_name, max_messages, timeout) return self.recv_get() - def send_get(self): + def send_get(self, queue_name, max_messages, timeout): self._oprot.writeMessageBegin('get', TMessageType.CALL, self._seqid) args = get_args() + args.queue_name = queue_name + args.max_messages = max_messages + args.timeout = timeout args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -105,8 +122,8 @@ def recv_get(self): iprot.readMessageEnd() if result.success is not None: return result.success - if result.get_failed_error is not None: - raise result.get_failed_error + if result.timed_out_error is not None: + raise result.timed_out_error raise TApplicationException(TApplicationException.MISSING_RESULT, "get failed: unknown result") @@ -148,9 +165,9 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY except TTransport.TTransportException: raise - except PutFailedError as put_failed_error: + except TimedOutError as timed_out_error: msg_type = TMessageType.REPLY - result.put_failed_error = put_failed_error + result.timed_out_error = timed_out_error except TApplicationException as ex: logging.exception('TApplication exception in handler') msg_type = TMessageType.EXCEPTION @@ -170,13 +187,13 @@ def process_get(self, seqid, iprot, oprot): iprot.readMessageEnd() result = get_result() try: - result.success = self._handler.get() + result.success = self._handler.get(args.queue_name, args.max_messages, args.timeout) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise - except GetFailedError as get_failed_error: + except TimedOutError as timed_out_error: msg_type = TMessageType.REPLY - result.get_failed_error = get_failed_error + result.timed_out_error = timed_out_error except TApplicationException as ex: logging.exception('TApplication exception in handler') msg_type = TMessageType.EXCEPTION @@ -309,19 +326,19 @@ class put_result(object): """ Attributes: - success - - put_failed_error + - timed_out_error """ __slots__ = ( 'success', - 'put_failed_error', + 'timed_out_error', ) - def __init__(self, success=None, put_failed_error=None,): + def __init__(self, success=None, timed_out_error=None,): self.success = success - self.put_failed_error = put_failed_error + self.timed_out_error = timed_out_error def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -340,7 +357,7 @@ def read(self, iprot): iprot.skip(ftype) elif fid == 1: if ftype == TType.STRUCT: - self.put_failed_error = PutFailedError.read(iprot) + self.timed_out_error = TimedOutError.read(iprot) else: iprot.skip(ftype) else: @@ -357,9 +374,9 @@ def write(self, oprot): oprot.writeFieldBegin('success', TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() - if self.put_failed_error is not None: - oprot.writeFieldBegin('put_failed_error', TType.STRUCT, 1) - self.put_failed_error.write(oprot) + if self.timed_out_error is not None: + oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -387,16 +404,31 @@ def __ne__(self, other): all_structs.append(put_result) put_result.thrift_spec = ( (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 - (1, TType.STRUCT, 'put_failed_error', [PutFailedError, None], None, ), # 1 + (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 ) class get_args(object): + """ + Attributes: + - queue_name + - max_messages + - timeout + + """ __slots__ = ( + 'queue_name', + 'max_messages', + 'timeout', ) + def __init__(self, queue_name=None, max_messages=None, timeout=None,): + self.queue_name = queue_name + self.max_messages = max_messages + self.timeout = timeout + def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) @@ -406,6 +438,21 @@ def read(self, iprot): (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break + if fid == 1: + if ftype == TType.STRING: + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.max_messages = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.DOUBLE: + self.timeout = iprot.readDouble() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -416,6 +463,18 @@ def write(self, oprot): oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return oprot.writeStructBegin('get_args') + if self.queue_name is not None: + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldEnd() + if self.max_messages is not None: + oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeI64(self.max_messages) + oprot.writeFieldEnd() + if self.timeout is not None: + oprot.writeFieldBegin('timeout', TType.DOUBLE, 3) + oprot.writeDouble(self.timeout) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -441,6 +500,10 @@ def __ne__(self, other): return not (self == other) all_structs.append(get_args) get_args.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + (2, TType.I64, 'max_messages', None, None, ), # 2 + (3, TType.DOUBLE, 'timeout', None, None, ), # 3 ) @@ -448,19 +511,19 @@ class get_result(object): """ Attributes: - success - - get_failed_error + - timed_out_error """ __slots__ = ( 'success', - 'get_failed_error', + 'timed_out_error', ) - def __init__(self, success=None, get_failed_error=None,): + def __init__(self, success=None, timed_out_error=None,): self.success = success - self.get_failed_error = get_failed_error + self.timed_out_error = timed_out_error def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -479,7 +542,7 @@ def read(self, iprot): iprot.skip(ftype) elif fid == 1: if ftype == TType.STRUCT: - self.get_failed_error = GetFailedError.read(iprot) + self.timed_out_error = TimedOutError.read(iprot) else: iprot.skip(ftype) else: @@ -496,9 +559,9 @@ def write(self, oprot): oprot.writeFieldBegin('success', TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() - if self.get_failed_error is not None: - oprot.writeFieldBegin('get_failed_error', TType.STRUCT, 1) - self.get_failed_error.write(oprot) + if self.timed_out_error is not None: + oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -526,7 +589,7 @@ def __ne__(self, other): all_structs.append(get_result) get_result.thrift_spec = ( (0, TType.STRUCT, 'success', [GetResponse, None], None, ), # 0 - (1, TType.STRUCT, 'get_failed_error', [GetFailedError, None], None, ), # 1 + (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index 01ab8801c..b75b82b21 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -8,9 +8,7 @@ struct GetResponse { 1: binary value; } -exception PutFailedError {} - -exception GetFailedError {} +exception TimedOutError {} service RemoteMessageQueueService { PutResponse put( @@ -19,10 +17,13 @@ service RemoteMessageQueueService { 3: binary message 4: double timeout ) throws ( - 1: PutFailedError put_failed_error + 1: TimedOutError timed_out_error ); GetResponse get( + 1: string queue_name + 2: i64 max_messages + 3: double timeout ) throws ( - 1: GetFailedError get_failed_error + 1: TimedOutError timed_out_error ); } diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index de611e70d..6033d6d19 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -134,7 +134,7 @@ def __ne__(self, other): return not (self == other) -class PutFailedError(TException): +class TimedOutError(TException): __slots__ = ( ) @@ -169,71 +169,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('PutFailedError') - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __str__(self): - return repr(self) - - def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - -class GetFailedError(TException): - - __slots__ = ( - ) - - - def __setattr__(self, *args): - raise TypeError("can't modify immutable instance") - - def __delattr__(self, *args): - raise TypeError("can't modify immutable instance") - - def __hash__(self): - return hash(self.__class__) ^ hash(()) - - @classmethod - def read(cls, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: - return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - return cls( - ) - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetFailedError') + oprot.writeStructBegin('TimedOutError') oprot.writeFieldStop() oprot.writeStructEnd() @@ -268,11 +204,8 @@ def __ne__(self, other): None, # 0 (1, TType.STRING, 'value', 'BINARY', None, ), # 1 ) -all_structs.append(PutFailedError) -PutFailedError.thrift_spec = ( -) -all_structs.append(GetFailedError) -GetFailedError.thrift_spec = ( +all_structs.append(TimedOutError) +TimedOutError.thrift_spec = ( ) fix_spec(all_structs) del all_structs diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 50c21cdba..eec6dacf3 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -1,13 +1,18 @@ +from importlib import reload +import pytest import contextlib +import logging import time import unittest +import gevent import posix_ipc -from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError +from baseplate.sidecars import event_publisher class TestPosixMessageQueueCreation(unittest.TestCase): @@ -84,7 +89,6 @@ def tearDown(self): queue.unlink() queue.close() - class TestInMemoryMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" @@ -140,11 +144,63 @@ def test_put_full_zero_timeout(self): with self.assertRaises(TimedOutError): mq.put(b"2", timeout=0) -class TestRemoteMessageQueueCreation(unittest.TestCase): +class GeventPatchedTestCase(unittest.TestCase): + def setUp(self): + gevent.monkey.patch_socket() + + def tearDown(self): + import socket + + reload(socket) + gevent.monkey.saved.clear() + + +class TestRemoteMessageQueueCreation(GeventPatchedTestCase): qname = "/baseplate-test-queue" - def test_create_queue_remote(self): - message_queue = RemoteMessageQueue(self.qname, max_messages=1) + def test_put_get(self): + with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + message_queue = RemoteMessageQueue(self.qname, max_messages=10) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x", timeout=0) + message = mq.get() + self.assertEqual(message, b"x") + + def test_multiple_queues(self): + with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + mq1 = RemoteMessageQueue(self.qname, max_messages=10) + mq2 = RemoteMessageQueue(self.qname+"2", max_messages=10) + + mq1.put(b"x", timeout=0) + mq2.put(b"a", timeout=0) + + # Check the queues in reverse order + self.assertEqual(mq2.get(), b"a") + self.assertEqual(mq1.get(), b"x") + + mq1.close() + mq2.close() - with contextlib.closing(message_queue) as mq: - message_queue.put("yero") \ No newline at end of file + def test_get_timeout(self): + with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + message_queue = RemoteMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + start = time.time() + with self.assertRaises(TimedOutError): + mq.get(timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=2) + + def test_put_timeout(self): + with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + message_queue = RemoteMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x") + start = time.time() + with self.assertRaises(TimedOutError): + mq.put(b"x", timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=1) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py new file mode 100644 index 000000000..4a4a4eaab --- /dev/null +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -0,0 +1,64 @@ +import contextlib +from importlib import reload +import unittest +from baseplate.thrift.message_queue.ttypes import GetResponse +import gevent.monkey +from unittest import mock +# from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue, PosixMessageQueue +from baseplate.lib.events import MAX_QUEUE_SIZE + +import posix_ipc + +from baseplate.lib import config +from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue +# from baseplate.lib.message_queue import TimedOutError +from baseplate.sidecars import publisher_queue_utils + + +class GeventPatchedTestCase(unittest.TestCase): + def setUp(self): + gevent.monkey.patch_socket() + + def tearDown(self): + import socket + + reload(socket) + gevent.monkey.saved.clear() + + +class PublisherQueueTests(GeventPatchedTestCase): + def test_posix_queue(self): + queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) + assert isinstance(queue, PosixMessageQueue) + + def test_posix_queue_get_put(self): + queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) + + test_message = bytes("message", "utf-8") + test_message_2 = bytes("2nd message", "utf-8") + queue.put(test_message) + queue.put(test_message_2) + output = queue.get() + assert output == test_message + output = queue.get() + assert output == test_message_2 + + def test_in_memory_create_queue(self): + queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) + assert isinstance(queue, RemoteMessageQueue) + + def test_in_memory_queue_get_put(self): + queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) + + with publisher_queue_utils.start_queue_server(host='127.0.0.1', port=9090) as server: + queue.connect() + test_message = bytes("message", "utf-8") + test_message_2 = bytes("2nd message", "utf-8") + queue.put(test_message) + queue.put(test_message_2) + output = queue.get() + assert output == test_message + output = queue.get() + assert output == test_message_2 diff --git a/tests/unit/lib/events/publisher_tests.py b/tests/unit/lib/events/publisher_tests.py index 4eec4d48a..fa6ddca3c 100644 --- a/tests/unit/lib/events/publisher_tests.py +++ b/tests/unit/lib/events/publisher_tests.py @@ -128,56 +128,4 @@ def test_fail_on_client_error(self, mock_sleep): self.assertEqual(mock_sleep.call_count, 0) self.assertEqual(self.session.post.call_count, 1) -class RemoteQueuePublisherTests(unittest.TestCase): - @mock.patch("requests.Session", autospec=True) - def setUp(self, Session): - self.config = config.ConfigNamespace() - self.config.collector = config.ConfigNamespace() - self.config.collector.hostname = "test.local" - self.config.collector.version = 1 - self.config.collector.scheme = "https" - self.config.key = config.ConfigNamespace() - self.config.key.name = "TestKey" - self.config.key.secret = b"hunter2" - self.config.queue = config.ConfigNamespace() - self.config.queue.type = "in_memory" - - self.session = Session.return_value - self.session.headers = {} - self.metrics_client = mock.MagicMock(autospec=metrics.Client) - - self.publisher = event_publisher.BatchPublisher(self.metrics_client, self.config) - - def test_empty_batch(self): - self.publisher.publish(SerializedBatch(item_count=0, serialized=b"")) - self.assertEqual(self.session.post.call_count, 0) - - def test_publish_success(self): - events = b'[{"example": "value"}]' - - self.publisher.publish(SerializedBatch(item_count=1, serialized=events)) - - self.assertEqual(self.session.post.call_count, 1) - - args, kwargs = self.session.post.call_args - headers = kwargs.get("headers", {}) - self.assertEqual(args[0], "https://test.local/v1") - self.assertEqual(headers["Content-Type"], "application/json") - self.assertEqual( - headers["X-Signature"], - "key=TestKey, mac=7c46d56b99cd4cb05e08238c1d4c10a2f330795e9d7327f17cc66fd206bf1179", - ) - - @mock.patch("time.sleep") - def test_fail_on_client_error(self, mock_sleep): - self.session.post.side_effect = [ - requests.HTTPError(400, response=mock.Mock(status_code=400)) - ] - events = b'[{"example": "value"}]' - - with self.assertRaises(requests.HTTPError): - self.publisher.publish(SerializedBatch(item_count=1, serialized=events)) - - self.assertEqual(mock_sleep.call_count, 0) - self.assertEqual(self.session.post.call_count, 1) From 1d9510bee158f8da4bec470f74ea4424e968593a Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 3 Jan 2023 15:05:17 -0800 Subject: [PATCH 12/72] cleanup, wipg --- baseplate/sidecars/event_publisher.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 33b7f3a71..a396f5fb8 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -16,35 +16,22 @@ import requests -from baseplate import RequestContext, __version__ as baseplate_version +from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import MessageQueue -from baseplate.lib.message_queue import PosixMessageQueue -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy from baseplate.server import EnvironmentInterpolation -from baseplate.server import make_listener -from baseplate.server.thrift import make_server -from baseplate.thrift.message_queue import RemoteMessageQueueService -from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError from baseplate.sidecars import Batch from baseplate.sidecars import BatchFull from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch -from thrift.server import TServer -from thrift.protocol import TBinaryProtocol -from thrift.transport import TSocket -from thrift.transport import TTransport - - logger = logging.getLogger(__name__) From 6628c1ccc0620203625ad3d3904d3b45e18212d0 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 3 Jan 2023 15:56:22 -0800 Subject: [PATCH 13/72] linting --- baseplate/lib/events.py | 2 +- baseplate/lib/message_queue.py | 47 +-- baseplate/sidecars/event_publisher.py | 22 +- baseplate/sidecars/publisher_queue_utils.py | 55 ++-- baseplate/sidecars/trace_publisher.py | 27 +- .../RemoteMessageQueueService.py | 277 +++++++++++++----- baseplate/thrift/message_queue/__init__.py | 2 +- baseplate/thrift/message_queue/ttypes.py | 79 ++--- tests/integration/message_queue_tests.py | 18 +- .../sidecars/publisher_queue_utils_tests.py | 10 +- tests/unit/lib/events/publisher_tests.py | 3 - 11 files changed, 332 insertions(+), 210 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 871223e6d..685892553 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -128,7 +128,7 @@ def put(self, event: T) -> None: except TimedOutError: raise EventQueueFullError - def get(self): + def get(self): return self.queue.get() def make_object_for_context(self, name: str, span: Span) -> "EventQueue[T]": diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index a49d3d994..177b681ab 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -2,20 +2,19 @@ import abc import queue as q import select -import gevent - from typing import Optional -from baseplate.thrift.message_queue import RemoteMessageQueueService -from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError +import gevent import posix_ipc -from baseplate.lib.retry import RetryPolicy - from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport +from baseplate.lib.retry import RetryPolicy +from baseplate.thrift.message_queue import RemoteMessageQueueService +from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError + class MessageQueueError(Exception): """Base exception for message queue related errors.""" @@ -69,14 +68,6 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: """ - @abc.abstractmethod - def unlink(self) -> None: - """Remove the queue from the system. - - The queue will not leave until the last active user closes it. - - """ - @abc.abstractmethod def close(self) -> None: """Close the queue, freeing related resources. @@ -102,11 +93,12 @@ class PosixMessageQueue(MessageQueue): """ def __init__(self, name: str, max_messages: int, max_message_size: int): + super().__init__() try: self.queue = posix_ipc.MessageQueue( name, flags=posix_ipc.O_CREAT, - # mode=0o0644, + mode=0o0644, max_messages=max_messages, max_message_size=max_message_size, ) @@ -148,7 +140,7 @@ def close(self) -> None: class InMemoryMessageQueue(MessageQueue): - """An in-memory inter process message queue.""" # Used by the sidecar + """An in-memory inter process message queue.""" # Used by the sidecar def __init__(self, name: str, max_messages: int): self.queue: q.Queue = q.Queue(max_messages) @@ -169,51 +161,43 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: except q.Full: raise TimedOutError - def unlink(self) -> None: - """Not implemented for in-memory queue""" - def close(self) -> None: """Not implemented for in-memory queue""" -class RemoteMessageQueue(MessageQueue): - def __init__(self, name: str, max_messages: int, host: str="127.0.0.1", port: int=9090): +class RemoteMessageQueue(MessageQueue): + def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): self.queue_name = name self.max_messages = max_messages self.host = host self.port = port def connect(self): - # Establish a connection with the queue server + # Establish a connection with the queue server transport = TSocket.TSocket(self.host, self.port) self.transport = TTransport.TBufferedTransport(transport) protocol = TBinaryProtocol.TBinaryProtocol(self.transport) self.client = RemoteMessageQueueService.Client(protocol) self.transport.open() - def get(self, timeout: Optional[float] = None) -> bytes: - # Call the remote server and get an element for the correct queue + def get(self, timeout: Optional[float] = None) -> bytes: + # Call the remote server and get an element for the correct queue try: self.connect() - print("trying get...") return self.client.get(self.queue_name, self.max_messages, timeout).value except ThriftTimedOutError: raise TimedOutError - def put(self, message: bytes, timeout: Optional[float] = None) -> None: + def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Call the remote server and put an element on the correct queue # Will create the queue if it doesnt exist try: self.connect() - print("trying put:") self.client.put(self.queue_name, self.max_messages, message, timeout) except ThriftTimedOutError: raise TimedOutError - def unlink(self) -> None: - """Not implemented for remote queue""" - - def close(self) -> None: + def close(self) -> None: self.transport.close() @@ -239,6 +223,7 @@ def queue_tool() -> None: parser.add_argument( "queue_type", default="posix", + choices=["posix", "in_memory"], help="whether to use an in-memory queue or a posix queue", ) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index a396f5fb8..ba3eb9eef 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -1,16 +1,12 @@ import argparse import configparser -import contextlib import email.utils -import sys -from baseplate.thrift.message_queue.ttypes import GetResponse, PutResponse -import gevent import gzip import hashlib import hmac import logging -from typing import Any, Tuple +from typing import Any from typing import List from typing import Optional @@ -101,6 +97,7 @@ def serialize(self) -> SerializedBatch: serialized = self._header.encode() + b",".join(self._items) + self._end return SerializedBatch(item_count=len(self._items), serialized=serialized) + class BatchPublisher: def __init__(self, metrics_client: metrics.Client, cfg: Any): self.metrics = metrics_client @@ -178,8 +175,10 @@ def build_batch(queue: MessageQueue, batcher: TimeLimitedBatch) -> bytes: except BatchFull: return message + SERIALIZER_BY_VERSION = {"2": V2Batch, "2j": V2JBatch} + def publish_events() -> None: arg_parser = argparse.ArgumentParser() arg_parser.add_argument( @@ -215,13 +214,15 @@ def publish_events() -> None: "key": {"name": config.String, "secret": config.Base64}, "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), "max_element_size": config.Optional(config.Integer, MAX_EVENT_SIZE), - "queue_type": config.Optional(config.String, default="posix") + "queue_type": config.Optional(config.String, default="posix"), }, ) metrics_client = metrics_client_from_config(raw_config) - event_queue: MessageQueue = publisher_utils.create_queue(cfg.queue_type, args.queue_name, cfg.max_queue_size, cfg.max_element_size) + event_queue: MessageQueue = publisher_queue_utils.create_queue( + cfg.queue_type, args.queue_name, cfg.max_queue_size, cfg.max_element_size + ) # pylint: disable=maybe-no-member serializer = SERIALIZER_BY_VERSION[cfg.collector.version]() @@ -229,8 +230,8 @@ def publish_events() -> None: publisher = BatchPublisher(metrics_client, cfg) while True: - if cfg.queue_type == "in_memory": - with publisher_queue_utils.start_queue_server(host='127.0.0.1', port=9090) as server: + if cfg.queue_type == "in_memory": + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): last_message = publisher_queue_utils.build_batch(event_queue, batcher) serialized = batcher.serialize() try: @@ -239,7 +240,7 @@ def publish_events() -> None: logger.exception("Events publishing failed.") batcher.reset() batcher.add(last_message) - else: + else: last_message = publisher_queue_utils.build_batch(event_queue, batcher) serialized = batcher.serialize() try: @@ -249,5 +250,6 @@ def publish_events() -> None: batcher.reset() batcher.add(last_message) + if __name__ == "__main__": publish_events() diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 840a3976f..ba8dd5a7e 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,63 +1,62 @@ import contextlib -import gevent from typing import Optional + +import gevent from baseplate.lib import config from baseplate.server import make_listener, make_server from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.thrift.message_queue.ttypes import GetResponse, PutResponse +from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue -class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation +class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation def is_healthy(self) -> bool: - return True - + pass + def __init__(self): self.queue_list = {} - print("initting") - def get(self, queue_name: str, max_messages: int, timeout: Optional[float] = None) -> GetResponse: - print("in get") + def get( + self, queue_name: str, max_messages: int, timeout: Optional[float] = None + ) -> GetResponse: try: # Create queue if doesnt exist - # We need to create the queue on both get & put - if get() is called on a queue before it exists, we + # We need to create the queue on both get & put - if get() is called on a queue before it exists, we # still want to wait the appropriate timeout in case anyone puts elements in it - print("in get") queue = self.queue_list.get(queue_name) - if not queue: + if not queue: queue = InMemoryMessageQueue(queue_name, max_messages) self.queue_list[queue_name] = queue # Get element from list, waiting if necessary result = queue.get(timeout) - except MessageQueueTimedOutError: + except MessageQueueTimedOutError: raise ThriftTimedOutError() return GetResponse(result) - def put(self, queue_name: str, max_messages: int, message: bytes, timeout: Optional[float] = None) -> PutResponse: + def put( + self, queue_name: str, max_messages: int, message: bytes, timeout: Optional[float] = None + ) -> PutResponse: # Create queue if it does not exist yet - print("in put") try: queue = self.queue_list.get(queue_name) if not self.queue_list.get(queue_name): - print("didnt find queue") queue = InMemoryMessageQueue(queue_name, max_messages) self.queue_list[queue_name] = queue - print("putting") queue.put(message, timeout) - except MessageQueueTimedOutError: + except MessageQueueTimedOutError: raise ThriftTimedOutError() - print("responding") return PutResponse() @contextlib.contextmanager -def start_queue_server(host: str, port: int) -> None: +def start_queue_server(host: str, port: int) -> None: # Start a thrift server that will house the remote queue data - print("starting server") processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler()) server_bind_endpoint = config.Endpoint(f"{host}:{port}") listener = make_listener(server_bind_endpoint) @@ -66,8 +65,7 @@ def start_queue_server(host: str, port: int) -> None: # figure out what port the server ended up on server_address = listener.getsockname() server.endpoint = config.Endpoint(f"{server_address[0]}:{server_address[1]}") - print("finished starting server") - print(server_address) # localhost 9090 + print(server_address) # localhost 9090 # run the server until our caller is done with it server_greenlet = gevent.spawn(server.serve_forever) try: @@ -76,16 +74,14 @@ def start_queue_server(host: str, port: int) -> None: server_greenlet.kill() -def create_queue(queue_type: str, queue_name: str, max_queue_size: int, max_element_size: int) -> MessageQueue: +def create_queue( + queue_type: str, queue_name: str, max_queue_size: int, max_element_size: int +) -> MessageQueue: if queue_type == "in_memory": - with start_queue_server(host='127.0.0.1', port=9090) as server: - print('server started.') - + with start_queue_server(host="127.0.0.1", port=9090): event_queue = RemoteMessageQueue( # type: ignore - "/events-" + queue_name, - max_queue_size + "/events-" + queue_name, max_queue_size ) - print("heyo") else: event_queue = PosixMessageQueue( # type: ignore @@ -93,6 +89,5 @@ def create_queue(queue_type: str, queue_name: str, max_queue_size: int, max_elem max_messages=max_queue_size, max_message_size=max_element_size, ) - - return event_queue + return event_queue diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 4871c034d..56d35e308 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,8 +10,7 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import MessageQueue, RemoteMessageQueue -from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy @@ -131,8 +130,9 @@ def publish_traces() -> None: ) arg_parser.add_argument( "--queue-type", - default=False, - help="Options: `in_memory` or `posix`. Default is posix", + default="posix", + choices=["posix", "in_memory"], + help="whether to use an in-memory queue or a posix queue", ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" @@ -143,6 +143,7 @@ def publish_traces() -> None: metavar="NAME", help="name of app to load from config_file (default: main)", ) + args = arg_parser.parse_args() if args.debug: @@ -163,13 +164,17 @@ def publish_traces() -> None: "max_batch_size": config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT), "retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT), "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), - "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), "max_element_size": config.Optional(config.Integer, MAX_SPAN_SIZE), - "queue_type": config.Optional(config.String, default="posix") + "queue_type": config.Optional(config.String, default="posix"), }, ) - trace_queue: MessageQueue = publisher_queue_utils.create_queue(publisher_cfg.queue_type, args.queue_name, publisher_cfg.max_queue_size, publisher_cfg.max_element_size) + trace_queue: MessageQueue = publisher_queue_utils.create_queue( + publisher_cfg.queue_type, + args.queue_name, + publisher_cfg.max_queue_size, + publisher_cfg.max_element_size, + ) # pylint: disable=maybe-no-member inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size) @@ -184,18 +189,18 @@ def publish_traces() -> None: while True: message: Optional[bytes] - if publisher_cfg.queue_type == "in_memory": - with publisher_queue_utils.start_queue_server(host='127.0.0.1', port=9090) as server: + if publisher_cfg.queue_type == "in_memory": + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): try: message = trace_queue.get(timeout=0.2) except TimedOutError: message = None - else: + else: try: message = trace_queue.get(timeout=0.2) except TimedOutError: message = None - + try: batcher.add(message) except BatchFull: diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index b2ee83272..1886b405d 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -15,6 +15,7 @@ from .ttypes import * from thrift.Thrift import TProcessor from thrift.transport import TTransport + all_structs = [] @@ -61,7 +62,7 @@ def put(self, queue_name, max_messages, message, timeout): return self.recv_put() def send_put(self, queue_name, max_messages, message, timeout): - self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("put", TMessageType.CALL, self._seqid) args = put_args() args.queue_name = queue_name args.max_messages = max_messages @@ -86,7 +87,9 @@ def recv_put(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "put failed: unknown result" + ) def get(self, queue_name, max_messages, timeout): """ @@ -100,7 +103,7 @@ def get(self, queue_name, max_messages, timeout): return self.recv_get() def send_get(self, queue_name, max_messages, timeout): - self._oprot.writeMessageBegin('get', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("get", TMessageType.CALL, self._seqid) args = get_args() args.queue_name = queue_name args.max_messages = max_messages @@ -124,7 +127,9 @@ def recv_get(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException(TApplicationException.MISSING_RESULT, "get failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get failed: unknown result" + ) class Processor(Iface, TProcessor): @@ -145,7 +150,9 @@ def process(self, iprot, oprot): if name not in self._processMap: iprot.skip(TType.STRUCT) iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + x = TApplicationException( + TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name) + ) oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) x.write(oprot) oprot.writeMessageEnd() @@ -161,7 +168,9 @@ def process_put(self, seqid, iprot, oprot): iprot.readMessageEnd() result = put_result() try: - result.success = self._handler.put(args.queue_name, args.max_messages, args.message, args.timeout) + result.success = self._handler.put( + args.queue_name, args.max_messages, args.message, args.timeout + ) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -169,13 +178,13 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("put", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -195,18 +204,19 @@ def process_get(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("get", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() oprot.trans.flush() + # HELPER FUNCTIONS AND STRUCTURES @@ -221,21 +231,30 @@ class put_args(object): """ __slots__ = ( - 'queue_name', - 'max_messages', - 'message', - 'timeout', + "queue_name", + "max_messages", + "message", + "timeout", ) - - def __init__(self, queue_name=None, max_messages=None, message=None, timeout=None,): + def __init__( + self, + queue_name=None, + max_messages=None, + message=None, + timeout=None, + ): self.queue_name = queue_name self.max_messages = max_messages self.message = message self.timeout = timeout def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -245,7 +264,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 2: @@ -272,21 +295,23 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('put_args') + oprot.writeStructBegin("put_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.max_messages is not None: - oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeFieldBegin("max_messages", TType.I64, 2) oprot.writeI64(self.max_messages) oprot.writeFieldEnd() if self.message is not None: - oprot.writeFieldBegin('message', TType.STRING, 3) + oprot.writeFieldBegin("message", TType.STRING, 3) oprot.writeBinary(self.message) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin('timeout', TType.DOUBLE, 4) + oprot.writeFieldBegin("timeout", TType.DOUBLE, 4) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -296,9 +321,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -312,13 +336,39 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(put_args) put_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 - (2, TType.I64, 'max_messages', None, None, ), # 2 - (3, TType.STRING, 'message', 'BINARY', None, ), # 3 - (4, TType.DOUBLE, 'timeout', None, None, ), # 4 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "max_messages", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "message", + "BINARY", + None, + ), # 3 + ( + 4, + TType.DOUBLE, + "timeout", + None, + None, + ), # 4 ) @@ -331,17 +381,24 @@ class put_result(object): """ __slots__ = ( - 'success', - 'timed_out_error', + "success", + "timed_out_error", ) - - def __init__(self, success=None, timed_out_error=None,): + def __init__( + self, + success=None, + timed_out_error=None, + ): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -369,13 +426,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('put_result') + oprot.writeStructBegin("put_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -385,9 +442,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -401,10 +457,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(put_result) put_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 + ( + 0, + TType.STRUCT, + "success", + [PutResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "timed_out_error", + [TimedOutError, None], + None, + ), # 1 ) @@ -418,19 +488,27 @@ class get_args(object): """ __slots__ = ( - 'queue_name', - 'max_messages', - 'timeout', + "queue_name", + "max_messages", + "timeout", ) - - def __init__(self, queue_name=None, max_messages=None, timeout=None,): + def __init__( + self, + queue_name=None, + max_messages=None, + timeout=None, + ): self.queue_name = queue_name self.max_messages = max_messages self.timeout = timeout def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -440,7 +518,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 2: @@ -462,17 +544,19 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('get_args') + oprot.writeStructBegin("get_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.max_messages is not None: - oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeFieldBegin("max_messages", TType.I64, 2) oprot.writeI64(self.max_messages) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin('timeout', TType.DOUBLE, 3) + oprot.writeFieldBegin("timeout", TType.DOUBLE, 3) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -482,9 +566,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -498,12 +581,32 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(get_args) get_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 - (2, TType.I64, 'max_messages', None, None, ), # 2 - (3, TType.DOUBLE, 'timeout', None, None, ), # 3 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "max_messages", + None, + None, + ), # 2 + ( + 3, + TType.DOUBLE, + "timeout", + None, + None, + ), # 3 ) @@ -516,17 +619,24 @@ class get_result(object): """ __slots__ = ( - 'success', - 'timed_out_error', + "success", + "timed_out_error", ) - - def __init__(self, success=None, timed_out_error=None,): + def __init__( + self, + success=None, + timed_out_error=None, + ): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -554,13 +664,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('get_result') + oprot.writeStructBegin("get_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -570,9 +680,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -586,10 +695,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(get_result) get_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [GetResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 + ( + 0, + TType.STRUCT, + "success", + [GetResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "timed_out_error", + [TimedOutError, None], + None, + ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/__init__.py b/baseplate/thrift/message_queue/__init__.py index 947e972be..f37fa9583 100644 --- a/baseplate/thrift/message_queue/__init__.py +++ b/baseplate/thrift/message_queue/__init__.py @@ -1 +1 @@ -__all__ = ['ttypes', 'constants', 'RemoteMessageQueueService'] +__all__ = ["ttypes", "constants", "RemoteMessageQueueService"] diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index 6033d6d19..71ba20046 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -13,17 +13,20 @@ import sys from thrift.transport import TTransport + all_structs = [] class PutResponse(object): - __slots__ = ( - ) - + __slots__ = () def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -40,7 +43,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('PutResponse') + oprot.writeStructBegin("PutResponse") oprot.writeFieldStop() oprot.writeStructEnd() @@ -48,9 +51,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -73,16 +75,20 @@ class GetResponse(object): """ - __slots__ = ( - 'value', - ) + __slots__ = ("value",) - - def __init__(self, value=None,): + def __init__( + self, + value=None, + ): self.value = value def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -104,9 +110,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('GetResponse') + oprot.writeStructBegin("GetResponse") if self.value is not None: - oprot.writeFieldBegin('value', TType.STRING, 1) + oprot.writeFieldBegin("value", TType.STRING, 1) oprot.writeBinary(self.value) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -116,9 +122,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -136,9 +141,7 @@ def __ne__(self, other): class TimedOutError(TException): - __slots__ = ( - ) - + __slots__ = () def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -151,7 +154,11 @@ def __hash__(self): @classmethod def read(cls, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) iprot.readStructBegin() while True: @@ -162,14 +169,13 @@ def read(cls, iprot): iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() - return cls( - ) + return cls() def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('TimedOutError') + oprot.writeStructBegin("TimedOutError") oprot.writeFieldStop() oprot.writeStructEnd() @@ -180,9 +186,8 @@ def __str__(self): return repr(self) def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -196,16 +201,22 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(PutResponse) -PutResponse.thrift_spec = ( -) +PutResponse.thrift_spec = () all_structs.append(GetResponse) GetResponse.thrift_spec = ( None, # 0 - (1, TType.STRING, 'value', 'BINARY', None, ), # 1 + ( + 1, + TType.STRING, + "value", + "BINARY", + None, + ), # 1 ) all_structs.append(TimedOutError) -TimedOutError.thrift_spec = ( -) +TimedOutError.thrift_spec = () fix_spec(all_structs) del all_structs diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index eec6dacf3..d95df0f1c 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -89,6 +89,7 @@ def tearDown(self): queue.unlink() queue.close() + class TestInMemoryMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" @@ -144,6 +145,7 @@ def test_put_full_zero_timeout(self): with self.assertRaises(TimedOutError): mq.put(b"2", timeout=0) + class GeventPatchedTestCase(unittest.TestCase): def setUp(self): gevent.monkey.patch_socket() @@ -159,22 +161,22 @@ class TestRemoteMessageQueueCreation(GeventPatchedTestCase): qname = "/baseplate-test-queue" def test_put_get(self): - with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=10) - + with contextlib.closing(message_queue) as mq: mq.put(b"x", timeout=0) message = mq.get() self.assertEqual(message, b"x") def test_multiple_queues(self): - with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: mq1 = RemoteMessageQueue(self.qname, max_messages=10) - mq2 = RemoteMessageQueue(self.qname+"2", max_messages=10) - + mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10) + mq1.put(b"x", timeout=0) mq2.put(b"a", timeout=0) - + # Check the queues in reverse order self.assertEqual(mq2.get(), b"a") self.assertEqual(mq1.get(), b"x") @@ -183,7 +185,7 @@ def test_multiple_queues(self): mq2.close() def test_get_timeout(self): - with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: @@ -194,7 +196,7 @@ def test_get_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=2) def test_put_timeout(self): - with event_publisher.start_queue_server(host='127.0.0.1', port=9090) as server: + with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 4a4a4eaab..9ea03d049 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -4,6 +4,7 @@ from baseplate.thrift.message_queue.ttypes import GetResponse import gevent.monkey from unittest import mock + # from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue, PosixMessageQueue from baseplate.lib.events import MAX_QUEUE_SIZE @@ -13,8 +14,9 @@ from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue + # from baseplate.lib.message_queue import TimedOutError -from baseplate.sidecars import publisher_queue_utils +from baseplate.sidecars import publisher_queue_utils class GeventPatchedTestCase(unittest.TestCase): @@ -33,7 +35,7 @@ def test_posix_queue(self): queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) assert isinstance(queue, PosixMessageQueue) - def test_posix_queue_get_put(self): + def test_posix_queue_get_put(self): queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) test_message = bytes("message", "utf-8") @@ -51,8 +53,8 @@ def test_in_memory_create_queue(self): def test_in_memory_queue_get_put(self): queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) - - with publisher_queue_utils.start_queue_server(host='127.0.0.1', port=9090) as server: + + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): queue.connect() test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") diff --git a/tests/unit/lib/events/publisher_tests.py b/tests/unit/lib/events/publisher_tests.py index fa6ddca3c..fc1d84f8b 100644 --- a/tests/unit/lib/events/publisher_tests.py +++ b/tests/unit/lib/events/publisher_tests.py @@ -86,7 +86,6 @@ def setUp(self, Session): self.config.key = config.ConfigNamespace() self.config.key.name = "TestKey" self.config.key.secret = b"hunter2" - # self.config.queue.type = "posix" self.session = Session.return_value self.session.headers = {} @@ -127,5 +126,3 @@ def test_fail_on_client_error(self, mock_sleep): self.assertEqual(mock_sleep.call_count, 0) self.assertEqual(self.session.post.call_count, 1) - - From 6834daaac8fb7c3a89906dd56f4277d4303b49ba Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 3 Jan 2023 16:08:30 -0800 Subject: [PATCH 14/72] linting --- baseplate/lib/events.py | 2 +- baseplate/lib/message_queue.py | 1 + baseplate/observers/tracing.py | 2 +- baseplate/sidecars/publisher_queue_utils.py | 15 +++++++++------ baseplate/sidecars/trace_publisher.py | 3 ++- .../message_queue/RemoteMessageQueueService.py | 14 +++++++++----- baseplate/thrift/message_queue/constants.py | 10 ++++++---- baseplate/thrift/message_queue/ttypes.py | 12 +++++++----- tests/integration/message_queue_tests.py | 12 ++++++------ .../sidecars/publisher_queue_utils_tests.py | 14 +++++--------- tests/unit/lib/events/queue_tests.py | 2 +- 11 files changed, 48 insertions(+), 39 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 685892553..cd7ce05c8 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,8 +22,8 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 177b681ab..b0e2347e0 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -2,6 +2,7 @@ import abc import queue as q import select + from typing import Optional import gevent diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 5b25c33a6..8cb445d38 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,8 +29,8 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index ba8dd5a7e..442d1e0b8 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,19 +1,22 @@ import contextlib + from typing import Optional import gevent + from baseplate.lib import config -from baseplate.server import make_listener, make_server -from baseplate.thrift.message_queue import RemoteMessageQueueService -from baseplate.thrift.message_queue.ttypes import GetResponse, PutResponse +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError +from baseplate.server import make_listener +from baseplate.server import make_server +from baseplate.thrift.message_queue import RemoteMessageQueueService +from baseplate.thrift.message_queue.ttypes import GetResponse +from baseplate.thrift.message_queue.ttypes import PutResponse from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError -from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue - - class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation def is_healthy(self) -> bool: pass diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 56d35e308..f45522914 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -17,7 +17,8 @@ from baseplate.observers.tracing import MAX_QUEUE_SIZE from baseplate.observers.tracing import MAX_SPAN_SIZE from baseplate.server import EnvironmentInterpolation -from baseplate.sidecars import BatchFull, publisher_queue_utils +from baseplate.sidecars import BatchFull +from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import RawJSONBatch from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index 1886b405d..99d47e4f1 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -5,16 +5,20 @@ # # options string: py:slots # +import logging +import sys -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import TApplicationException +from thrift.Thrift import TException +from thrift.Thrift import TFrozenDict +from thrift.Thrift import TMessageType +from thrift.Thrift import TProcessor +from thrift.Thrift import TType +from thrift.transport import TTransport from thrift.TRecursive import fix_spec -import sys -import logging from .ttypes import * -from thrift.Thrift import TProcessor -from thrift.transport import TTransport all_structs = [] diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py index 964098e62..05ed8518e 100644 --- a/baseplate/thrift/message_queue/constants.py +++ b/baseplate/thrift/message_queue/constants.py @@ -5,10 +5,12 @@ # # options string: py:slots # +import sys -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import TApplicationException +from thrift.Thrift import TException +from thrift.Thrift import TFrozenDict +from thrift.Thrift import TMessageType +from thrift.Thrift import TType from thrift.TRecursive import fix_spec - -import sys -from .ttypes import * diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index 71ba20046..f0fa66c4d 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -5,14 +5,16 @@ # # options string: py:slots # - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - import sys +from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import TApplicationException +from thrift.Thrift import TException +from thrift.Thrift import TFrozenDict +from thrift.Thrift import TMessageType +from thrift.Thrift import TType from thrift.transport import TTransport +from thrift.TRecursive import fix_spec all_structs = [] diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index d95df0f1c..4fb0486b9 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -1,19 +1,19 @@ -from importlib import reload -import pytest import contextlib import logging import time import unittest -import gevent +from importlib import reload + +import gevent import posix_ipc +import pytest -from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError -from baseplate.sidecars import event_publisher - class TestPosixMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 9ea03d049..e7aa715ee 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -1,23 +1,19 @@ import contextlib -from importlib import reload import unittest -from baseplate.thrift.message_queue.ttypes import GetResponse -import gevent.monkey -from unittest import mock -# from baseplate.lib.message_queue import InMemoryMessageQueue, MessageQueue, PosixMessageQueue -from baseplate.lib.events import MAX_QUEUE_SIZE +from importlib import reload +from unittest import mock +import gevent.monkey import posix_ipc from baseplate.lib import config +from baseplate.lib.events import MAX_QUEUE_SIZE from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue - -# from baseplate.lib.message_queue import TimedOutError from baseplate.sidecars import publisher_queue_utils - +from baseplate.thrift.message_queue.ttypes import GetResponse class GeventPatchedTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 5b7089484..3b5096885 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -6,8 +6,8 @@ from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError From efffbcb5304e4b9f84d3c9f9de2a17ef8ea328ae Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 3 Jan 2023 16:33:36 -0800 Subject: [PATCH 15/72] linting, remove docker-compose change --- baseplate/lib/message_queue.py | 2 +- baseplate/sidecars/publisher_queue_utils.py | 1 + docker-compose.yml | 7 ------- tests/integration/message_queue_tests.py | 1 + tests/integration/sidecars/publisher_queue_utils_tests.py | 1 + 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index b0e2347e0..eef76e68d 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,4 +1,4 @@ -"""A Gevent-friendly POSIX message queue.""" +"""A message queue, with two implementations: POSIX-based, or in-memory using a Thrift server.""" import abc import queue as q import select diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 442d1e0b8..2fae69047 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -17,6 +17,7 @@ from baseplate.thrift.message_queue.ttypes import PutResponse from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError + class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation def is_healthy(self) -> bool: pass diff --git a/docker-compose.yml b/docker-compose.yml index faa382bb0..bf32bb41d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,13 +14,6 @@ services: - "redis" - "zookeeper" - "redis-cluster-node" - test-baseplate: - build: - context: "." - dockerfile: "Dockerfile" - command: "pytest -v tests/" - volumes: - - ".:/src" cassandra: image: "cassandra:3.11" environment: diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 4fb0486b9..dab118c54 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -15,6 +15,7 @@ from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError + class TestPosixMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index e7aa715ee..74caed40a 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -15,6 +15,7 @@ from baseplate.sidecars import publisher_queue_utils from baseplate.thrift.message_queue.ttypes import GetResponse + class GeventPatchedTestCase(unittest.TestCase): def setUp(self): gevent.monkey.patch_socket() From afcbd9f9c31075f471915cdbba5bbfd301aefa9f Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 3 Jan 2023 17:19:56 -0800 Subject: [PATCH 16/72] dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 775db7208..da54f0961 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,4 +17,3 @@ RUN pip install -r requirements.txt RUN touch /baseplate-py-dev-docker-image CMD ["/bin/bash"] -# ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file From 05a10a79ba92a538c800edb28a9243d9fdcf6ea0 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 4 Jan 2023 12:44:56 -0800 Subject: [PATCH 17/72] typo --- baseplate/lib/message_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index eef76e68d..c8d0bdb9f 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -222,7 +222,7 @@ def queue_tool() -> None: ) parser.add_argument("queue_name", help="the name of the queue to consume") parser.add_argument( - "queue_type", + "--queue_type", default="posix", choices=["posix", "in_memory"], help="whether to use an in-memory queue or a posix queue", From 257c0bbf54b476d4fbcad58eb03e072b31c09135 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 9 Jan 2023 13:03:26 -0800 Subject: [PATCH 18/72] wip --- baseplate/lib/message_queue.py | 19 +- baseplate/sidecars/event_publisher.py | 44 +- baseplate/sidecars/publisher_queue_utils.py | 38 +- baseplate/sidecars/trace_publisher.py | 44 +- .../RemoteMessageQueueService.py | 549 ++++++++++-------- baseplate/thrift/message_queue/__init__.py | 2 +- baseplate/thrift/message_queue/constants.py | 10 +- .../thrift/message_queue/message_queue.thrift | 8 +- baseplate/thrift/message_queue/ttypes.py | 142 +++-- tests/integration/message_queue_tests.py | 46 +- .../sidecars/publisher_queue_utils_tests.py | 26 +- 11 files changed, 548 insertions(+), 380 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index c8d0bdb9f..666e2053f 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -168,10 +168,13 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): + # Connect to the remote queue server, and creeate the new queue self.queue_name = name self.max_messages = max_messages self.host = host self.port = port + self.connect() + self.client.create_queue(name, max_messages) def connect(self): # Establish a connection with the queue server @@ -184,8 +187,13 @@ def connect(self): def get(self, timeout: Optional[float] = None) -> bytes: # Call the remote server and get an element for the correct queue try: - self.connect() - return self.client.get(self.queue_name, self.max_messages, timeout).value + try: + return self.client.get(self.queue_name, timeout).value + except TSocket.TTransportException: + # Try reconnecting once, we dont want this as another top-level except because + # we may get a timeout after re-connecting, and we want to catch that + self.connect() + return self.client.get(self.queue_name, timeout).value except ThriftTimedOutError: raise TimedOutError @@ -193,8 +201,11 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Call the remote server and put an element on the correct queue # Will create the queue if it doesnt exist try: - self.connect() - self.client.put(self.queue_name, self.max_messages, message, timeout) + try: + self.client.put(self.queue_name, message, timeout) + except TSocket.TTransportException: # Try reconnecting once + self.connect() + self.client.put(self.queue_name, self.max_messages, message, timeout) except ThriftTimedOutError: raise TimedOutError diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index ba3eb9eef..4baef0a51 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -161,19 +161,28 @@ def publish(self, payload: SerializedBatch) -> None: raise MaxRetriesError("could not sent batch") -def build_batch(queue: MessageQueue, batcher: TimeLimitedBatch) -> bytes: +def build_batch_and_publish(event_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: BatchPublisher) -> bytes: while True: message: Optional[bytes] + try: - message = queue.get(timeout=0.2) - except MessageQueueTimedOutError: + message = event_queue.get(timeout=0.2) + except TimedOutError: message = None try: batcher.add(message) continue except BatchFull: - return message + pass + + serialized = batcher.serialize() + try: + publisher.publish(serialized) + except Exception: + logger.exception("Events publishing failed.") + batcher.reset() + batcher.add(message) SERIALIZER_BY_VERSION = {"2": V2Batch, "2j": V2JBatch} @@ -229,26 +238,13 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - while True: - if cfg.queue_type == "in_memory": - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - last_message = publisher_queue_utils.build_batch(event_queue, batcher) - serialized = batcher.serialize() - try: - publisher.publish(serialized) - except Exception: - logger.exception("Events publishing failed.") - batcher.reset() - batcher.add(last_message) - else: - last_message = publisher_queue_utils.build_batch(event_queue, batcher) - serialized = batcher.serialize() - try: - publisher.publish(serialized) - except Exception: - logger.exception("Events publishing failed.") - batcher.reset() - batcher.add(last_message) + if cfg.queue_type == "in_memory": + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + build_batch_and_publish(event_queue, batcher, publisher) + + else: + build_batch_and_publish(event_queue, batcher, publisher) + if __name__ == "__main__": diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 2fae69047..8f122255b 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -11,8 +11,9 @@ from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError from baseplate.server import make_listener -from baseplate.server import make_server +from baseplate.server.thrift import make_server from baseplate.thrift.message_queue import RemoteMessageQueueService +from baseplate.thrift.message_queue.ttypes import CreateResponse from baseplate.thrift.message_queue.ttypes import GetResponse from baseplate.thrift.message_queue.ttypes import PutResponse from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError @@ -23,19 +24,26 @@ def is_healthy(self) -> bool: pass def __init__(self): - self.queue_list = {} + self.queues = {} + + def create_queue(self, queue_name: str, max_messages: int) -> CreateResponse: + queue = InMemoryMessageQueue(queue_name, max_messages) + self.queues[queue_name] = (queue, max_messages) + + return CreateResponse() def get( - self, queue_name: str, max_messages: int, timeout: Optional[float] = None + self, queue_name: str, timeout: Optional[float] = None ) -> GetResponse: + # Raises TimedOutError try: # Create queue if doesnt exist - # We need to create the queue on both get & put - if get() is called on a queue before it exists, we + # We may need to create the queue on both get & put - if get() is called on a queue before it exists, we # still want to wait the appropriate timeout in case anyone puts elements in it - queue = self.queue_list.get(queue_name) + queue, max_messages = self.queues.get(queue_name) if not queue: queue = InMemoryMessageQueue(queue_name, max_messages) - self.queue_list[queue_name] = queue + self.queues[queue_name] = queue # Get element from list, waiting if necessary result = queue.get(timeout) except MessageQueueTimedOutError: @@ -44,14 +52,10 @@ def get( return GetResponse(result) def put( - self, queue_name: str, max_messages: int, message: bytes, timeout: Optional[float] = None + self, queue_name: str, message: bytes, timeout: Optional[float] = None ) -> PutResponse: - # Create queue if it does not exist yet try: - queue = self.queue_list.get(queue_name) - if not self.queue_list.get(queue_name): - queue = InMemoryMessageQueue(queue_name, max_messages) - self.queue_list[queue_name] = queue + queue, _ = self.queues.get(queue_name) queue.put(message, timeout) except MessageQueueTimedOutError: raise ThriftTimedOutError() @@ -69,7 +73,6 @@ def start_queue_server(host: str, port: int) -> None: # figure out what port the server ended up on server_address = listener.getsockname() server.endpoint = config.Endpoint(f"{server_address[0]}:{server_address[1]}") - print(server_address) # localhost 9090 # run the server until our caller is done with it server_greenlet = gevent.spawn(server.serve_forever) try: @@ -79,13 +82,12 @@ def start_queue_server(host: str, port: int) -> None: def create_queue( - queue_type: str, queue_name: str, max_queue_size: int, max_element_size: int + queue_type: str, queue_name: str, max_queue_size: int, max_element_size: int, host: str = "127.0.0.1", port: int = 9090 ) -> MessageQueue: if queue_type == "in_memory": - with start_queue_server(host="127.0.0.1", port=9090): - event_queue = RemoteMessageQueue( # type: ignore - "/events-" + queue_name, max_queue_size - ) + event_queue = RemoteMessageQueue( # type: ignore + "/events-" + queue_name, max_queue_size, host, port + ) else: event_queue = PosixMessageQueue( # type: ignore diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index f45522914..b308bd78a 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -118,6 +118,23 @@ def publish(self, payload: SerializedBatch) -> None: f"ZipkinPublisher exhausted allowance of {self.retry_limit:d} retries." ) +def build_and_publish_batch(trace_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: ZipkinPublisher) -> None: + while True: + message: Optional[bytes] + + try: + message = trace_queue.get(timeout=0.2) + except TimedOutError: + message = None + + try: + batcher.add(message) + except BatchFull: + serialized = batcher.serialize() + publisher.publish(serialized) + batcher.reset() + batcher.add(message) + def publish_traces() -> None: arg_parser = argparse.ArgumentParser() @@ -187,28 +204,11 @@ def publish_traces() -> None: post_timeout=publisher_cfg.post_timeout, ) - while True: - message: Optional[bytes] - - if publisher_cfg.queue_type == "in_memory": - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - try: - message = trace_queue.get(timeout=0.2) - except TimedOutError: - message = None - else: - try: - message = trace_queue.get(timeout=0.2) - except TimedOutError: - message = None - - try: - batcher.add(message) - except BatchFull: - serialized = batcher.serialize() - publisher.publish(serialized) - batcher.reset() - batcher.add(message) + if publisher_cfg.queue_type == "in_memory": + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + build_and_publish_batch(trace_queue, batcher, publisher) + else: + build_and_publish_batch(trace_queue, batcher, publisher) if __name__ == "__main__": diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index 99d47e4f1..ea6a741f6 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -5,41 +5,43 @@ # # options string: py:slots # -import logging -import sys +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TFrozenDict -from thrift.Thrift import TMessageType -from thrift.Thrift import TProcessor -from thrift.Thrift import TType -from thrift.transport import TTransport from thrift.TRecursive import fix_spec +import sys +import logging from .ttypes import * - +from thrift.Thrift import TProcessor +from thrift.transport import TTransport all_structs = [] class Iface(object): - def put(self, queue_name, max_messages, message, timeout): + def create_queue(self, queue_name, max_messages): """ Parameters: - queue_name - max_messages + + """ + pass + + def put(self, queue_name, message, timeout): + """ + Parameters: + - queue_name - message - timeout """ pass - def get(self, queue_name, max_messages, timeout): + def get(self, queue_name, timeout): """ Parameters: - queue_name - - max_messages - timeout """ @@ -53,23 +55,55 @@ def __init__(self, iprot, oprot=None): self._oprot = oprot self._seqid = 0 - def put(self, queue_name, max_messages, message, timeout): + def create_queue(self, queue_name, max_messages): """ Parameters: - queue_name - max_messages + + """ + self.send_create_queue(queue_name, max_messages) + return self.recv_create_queue() + + def send_create_queue(self, queue_name, max_messages): + self._oprot.writeMessageBegin('create_queue', TMessageType.CALL, self._seqid) + args = create_queue_args() + args.queue_name = queue_name + args.max_messages = max_messages + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_queue(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_queue_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_queue failed: unknown result") + + def put(self, queue_name, message, timeout): + """ + Parameters: + - queue_name - message - timeout """ - self.send_put(queue_name, max_messages, message, timeout) + self.send_put(queue_name, message, timeout) return self.recv_put() - def send_put(self, queue_name, max_messages, message, timeout): - self._oprot.writeMessageBegin("put", TMessageType.CALL, self._seqid) + def send_put(self, queue_name, message, timeout): + self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) args = put_args() args.queue_name = queue_name - args.max_messages = max_messages args.message = message args.timeout = timeout args.write(self._oprot) @@ -91,26 +125,22 @@ def recv_put(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException( - TApplicationException.MISSING_RESULT, "put failed: unknown result" - ) + raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") - def get(self, queue_name, max_messages, timeout): + def get(self, queue_name, timeout): """ Parameters: - queue_name - - max_messages - timeout """ - self.send_get(queue_name, max_messages, timeout) + self.send_get(queue_name, timeout) return self.recv_get() - def send_get(self, queue_name, max_messages, timeout): - self._oprot.writeMessageBegin("get", TMessageType.CALL, self._seqid) + def send_get(self, queue_name, timeout): + self._oprot.writeMessageBegin('get', TMessageType.CALL, self._seqid) args = get_args() args.queue_name = queue_name - args.max_messages = max_messages args.timeout = timeout args.write(self._oprot) self._oprot.writeMessageEnd() @@ -131,15 +161,14 @@ def recv_get(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException( - TApplicationException.MISSING_RESULT, "get failed: unknown result" - ) + raise TApplicationException(TApplicationException.MISSING_RESULT, "get failed: unknown result") class Processor(Iface, TProcessor): def __init__(self, handler): self._handler = handler self._processMap = {} + self._processMap["create_queue"] = Processor.process_create_queue self._processMap["put"] = Processor.process_put self._processMap["get"] = Processor.process_get self._on_message_begin = None @@ -154,9 +183,7 @@ def process(self, iprot, oprot): if name not in self._processMap: iprot.skip(TType.STRUCT) iprot.readMessageEnd() - x = TApplicationException( - TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name) - ) + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) x.write(oprot) oprot.writeMessageEnd() @@ -166,15 +193,36 @@ def process(self, iprot, oprot): self._processMap[name](self, seqid, iprot, oprot) return True + def process_create_queue(self, seqid, iprot, oprot): + args = create_queue_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_queue_result() + try: + result.success = self._handler.create_queue(args.queue_name, args.max_messages) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("create_queue", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_put(self, seqid, iprot, oprot): args = put_args() args.read(iprot) iprot.readMessageEnd() result = put_result() try: - result.success = self._handler.put( - args.queue_name, args.max_messages, args.message, args.timeout - ) + result.success = self._handler.put(args.queue_name, args.message, args.timeout) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -182,13 +230,13 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception("TApplication exception in handler") + logging.exception('TApplication exception in handler') msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception("Unexpected exception in handler") + logging.exception('Unexpected exception in handler') msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') oprot.writeMessageBegin("put", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -200,7 +248,7 @@ def process_get(self, seqid, iprot, oprot): iprot.readMessageEnd() result = get_result() try: - result.success = self._handler.get(args.queue_name, args.max_messages, args.timeout) + result.success = self._handler.get(args.queue_name, args.timeout) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -208,57 +256,41 @@ def process_get(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception("TApplication exception in handler") + logging.exception('TApplication exception in handler') msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception("Unexpected exception in handler") + logging.exception('Unexpected exception in handler') msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') oprot.writeMessageBegin("get", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() oprot.trans.flush() - # HELPER FUNCTIONS AND STRUCTURES -class put_args(object): +class create_queue_args(object): """ Attributes: - queue_name - max_messages - - message - - timeout """ __slots__ = ( - "queue_name", - "max_messages", - "message", - "timeout", + 'queue_name', + 'max_messages', ) - def __init__( - self, - queue_name=None, - max_messages=None, - message=None, - timeout=None, - ): + + def __init__(self, queue_name=None, max_messages=None,): self.queue_name = queue_name self.max_messages = max_messages - self.message = message - self.timeout = timeout def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -268,11 +300,7 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) elif fid == 2: @@ -280,6 +308,163 @@ def read(self, iprot): self.max_messages = iprot.readI64() else: iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('create_queue_args') + if self.queue_name is not None: + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldEnd() + if self.max_messages is not None: + oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeI64(self.max_messages) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True + + def __ne__(self, other): + return not (self == other) +all_structs.append(create_queue_args) +create_queue_args.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + (2, TType.I64, 'max_messages', None, None, ), # 2 +) + + +class create_queue_result(object): + """ + Attributes: + - success + + """ + + __slots__ = ( + 'success', + ) + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = CreateResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('create_queue_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True + + def __ne__(self, other): + return not (self == other) +all_structs.append(create_queue_result) +create_queue_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [CreateResponse, None], None, ), # 0 +) + + +class put_args(object): + """ + Attributes: + - queue_name + - message + - timeout + + """ + + __slots__ = ( + 'queue_name', + 'message', + 'timeout', + ) + + + def __init__(self, queue_name=None, message=None, timeout=None,): + self.queue_name = queue_name + self.message = message + self.timeout = timeout + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3: if ftype == TType.STRING: self.message = iprot.readBinary() @@ -299,23 +484,17 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("put_args") + oprot.writeStructBegin('put_args') if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) - oprot.writeFieldEnd() - if self.max_messages is not None: - oprot.writeFieldBegin("max_messages", TType.I64, 2) - oprot.writeI64(self.max_messages) + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) oprot.writeFieldEnd() if self.message is not None: - oprot.writeFieldBegin("message", TType.STRING, 3) + oprot.writeFieldBegin('message', TType.STRING, 3) oprot.writeBinary(self.message) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin("timeout", TType.DOUBLE, 4) + oprot.writeFieldBegin('timeout', TType.DOUBLE, 4) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -325,8 +504,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -340,39 +520,13 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(put_args) put_args.thrift_spec = ( None, # 0 - ( - 1, - TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - ( - 2, - TType.I64, - "max_messages", - None, - None, - ), # 2 - ( - 3, - TType.STRING, - "message", - "BINARY", - None, - ), # 3 - ( - 4, - TType.DOUBLE, - "timeout", - None, - None, - ), # 4 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + None, # 2 + (3, TType.STRING, 'message', 'BINARY', None, ), # 3 + (4, TType.DOUBLE, 'timeout', None, None, ), # 4 ) @@ -385,24 +539,17 @@ class put_result(object): """ __slots__ = ( - "success", - "timed_out_error", + 'success', + 'timed_out_error', ) - def __init__( - self, - success=None, - timed_out_error=None, - ): + + def __init__(self, success=None, timed_out_error=None,): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -430,13 +577,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("put_result") + oprot.writeStructBegin('put_result') if self.success is not None: - oprot.writeFieldBegin("success", TType.STRUCT, 0) + oprot.writeFieldBegin('success', TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) + oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -446,8 +593,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -461,24 +609,10 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(put_result) put_result.thrift_spec = ( - ( - 0, - TType.STRUCT, - "success", - [PutResponse, None], - None, - ), # 0 - ( - 1, - TType.STRUCT, - "timed_out_error", - [TimedOutError, None], - None, - ), # 1 + (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 + (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 ) @@ -486,33 +620,22 @@ class get_args(object): """ Attributes: - queue_name - - max_messages - timeout """ __slots__ = ( - "queue_name", - "max_messages", - "timeout", + 'queue_name', + 'timeout', ) - def __init__( - self, - queue_name=None, - max_messages=None, - timeout=None, - ): + + def __init__(self, queue_name=None, timeout=None,): self.queue_name = queue_name - self.max_messages = max_messages self.timeout = timeout def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -522,16 +645,7 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I64: - self.max_messages = iprot.readI64() + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) elif fid == 3: @@ -548,19 +662,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("get_args") + oprot.writeStructBegin('get_args') if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) - oprot.writeFieldEnd() - if self.max_messages is not None: - oprot.writeFieldBegin("max_messages", TType.I64, 2) - oprot.writeI64(self.max_messages) + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin("timeout", TType.DOUBLE, 3) + oprot.writeFieldBegin('timeout', TType.DOUBLE, 3) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -570,8 +678,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -585,32 +694,12 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(get_args) get_args.thrift_spec = ( None, # 0 - ( - 1, - TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - ( - 2, - TType.I64, - "max_messages", - None, - None, - ), # 2 - ( - 3, - TType.DOUBLE, - "timeout", - None, - None, - ), # 3 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + None, # 2 + (3, TType.DOUBLE, 'timeout', None, None, ), # 3 ) @@ -623,24 +712,17 @@ class get_result(object): """ __slots__ = ( - "success", - "timed_out_error", + 'success', + 'timed_out_error', ) - def __init__( - self, - success=None, - timed_out_error=None, - ): + + def __init__(self, success=None, timed_out_error=None,): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -668,13 +750,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("get_result") + oprot.writeStructBegin('get_result') if self.success is not None: - oprot.writeFieldBegin("success", TType.STRUCT, 0) + oprot.writeFieldBegin('success', TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) + oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -684,8 +766,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -699,24 +782,10 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(get_result) get_result.thrift_spec = ( - ( - 0, - TType.STRUCT, - "success", - [GetResponse, None], - None, - ), # 0 - ( - 1, - TType.STRUCT, - "timed_out_error", - [TimedOutError, None], - None, - ), # 1 + (0, TType.STRUCT, 'success', [GetResponse, None], None, ), # 0 + (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/__init__.py b/baseplate/thrift/message_queue/__init__.py index f37fa9583..947e972be 100644 --- a/baseplate/thrift/message_queue/__init__.py +++ b/baseplate/thrift/message_queue/__init__.py @@ -1 +1 @@ -__all__ = ["ttypes", "constants", "RemoteMessageQueueService"] +__all__ = ['ttypes', 'constants', 'RemoteMessageQueueService'] diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py index 05ed8518e..964098e62 100644 --- a/baseplate/thrift/message_queue/constants.py +++ b/baseplate/thrift/message_queue/constants.py @@ -5,12 +5,10 @@ # # options string: py:slots # -import sys +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TFrozenDict -from thrift.Thrift import TMessageType -from thrift.Thrift import TType from thrift.TRecursive import fix_spec + +import sys +from .ttypes import * diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index b75b82b21..61a0ad79b 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -2,6 +2,8 @@ namespace go reddit.message_queue namespace py message_queue.thrift namespace java com.reddit.baseplate.message_queue +struct CreateResponse {} + struct PutResponse {} struct GetResponse { @@ -11,9 +13,12 @@ struct GetResponse { exception TimedOutError {} service RemoteMessageQueueService { - PutResponse put( + CreateResponse create_queue( 1: string queue_name 2: i64 max_messages + ); + PutResponse put( + 1: string queue_name 3: binary message 4: double timeout ) throws ( @@ -21,7 +26,6 @@ service RemoteMessageQueueService { ); GetResponse get( 1: string queue_name - 2: i64 max_messages 3: double timeout ) throws ( 1: TimedOutError timed_out_error diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index f0fa66c4d..404516b6b 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -5,30 +5,75 @@ # # options string: py:slots # -import sys +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TFrozenDict -from thrift.Thrift import TMessageType -from thrift.Thrift import TType -from thrift.transport import TTransport from thrift.TRecursive import fix_spec +import sys + +from thrift.transport import TTransport all_structs = [] +class CreateResponse(object): + + __slots__ = ( + ) + + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('CreateResponse') + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + for attr in self.__slots__: + my_val = getattr(self, attr) + other_val = getattr(other, attr) + if my_val != other_val: + return False + return True + + def __ne__(self, other): + return not (self == other) + + class PutResponse(object): - __slots__ = () + __slots__ = ( + ) + def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -45,7 +90,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("PutResponse") + oprot.writeStructBegin('PutResponse') oprot.writeFieldStop() oprot.writeStructEnd() @@ -53,8 +98,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -77,20 +123,16 @@ class GetResponse(object): """ - __slots__ = ("value",) + __slots__ = ( + 'value', + ) + - def __init__( - self, - value=None, - ): + def __init__(self, value=None,): self.value = value def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -112,9 +154,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("GetResponse") + oprot.writeStructBegin('GetResponse') if self.value is not None: - oprot.writeFieldBegin("value", TType.STRING, 1) + oprot.writeFieldBegin('value', TType.STRING, 1) oprot.writeBinary(self.value) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -124,8 +166,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -143,7 +186,9 @@ def __ne__(self, other): class TimedOutError(TException): - __slots__ = () + __slots__ = ( + ) + def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -156,11 +201,7 @@ def __hash__(self): @classmethod def read(cls, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and cls.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) iprot.readStructBegin() while True: @@ -171,13 +212,14 @@ def read(cls, iprot): iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() - return cls() + return cls( + ) def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("TimedOutError") + oprot.writeStructBegin('TimedOutError') oprot.writeFieldStop() oprot.writeStructEnd() @@ -188,8 +230,9 @@ def __str__(self): return repr(self) def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -203,22 +246,19 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - +all_structs.append(CreateResponse) +CreateResponse.thrift_spec = ( +) all_structs.append(PutResponse) -PutResponse.thrift_spec = () +PutResponse.thrift_spec = ( +) all_structs.append(GetResponse) GetResponse.thrift_spec = ( None, # 0 - ( - 1, - TType.STRING, - "value", - "BINARY", - None, - ), # 1 + (1, TType.STRING, 'value', 'BINARY', None, ), # 1 ) all_structs.append(TimedOutError) -TimedOutError.thrift_spec = () +TimedOutError.thrift_spec = ( +) fix_spec(all_structs) del all_structs diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index dab118c54..856775e62 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -4,6 +4,7 @@ import unittest from importlib import reload +from baseplate.sidecars import publisher_queue_utils import gevent import posix_ipc @@ -162,7 +163,8 @@ class TestRemoteMessageQueueCreation(GeventPatchedTestCase): qname = "/baseplate-test-queue" def test_put_get(self): - with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: + # start the server that would ordinarily be running on the sidecar + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=10) with contextlib.closing(message_queue) as mq: @@ -171,7 +173,7 @@ def test_put_get(self): self.assertEqual(message, b"x") def test_multiple_queues(self): - with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: mq1 = RemoteMessageQueue(self.qname, max_messages=10) mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10) @@ -185,8 +187,16 @@ def test_multiple_queues(self): mq1.close() mq2.close() + def test_queues_alternate_port(self): + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: + message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x", timeout=0) + self.assertEqual(mq.get(), b"x") + def test_get_timeout(self): - with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: @@ -194,10 +204,10 @@ def test_get_timeout(self): with self.assertRaises(TimedOutError): mq.get(timeout=0.1) elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=2) + self.assertAlmostEqual(elapsed, 0.1, places=1) # TODO: this routinely takes 0.105-0.11 seconds def test_put_timeout(self): - with event_publisher.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: @@ -206,4 +216,28 @@ def test_put_timeout(self): with self.assertRaises(TimedOutError): mq.put(b"x", timeout=0.1) elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=1) + self.assertAlmostEqual(elapsed, 0.1, places=2) + + def test_thrift_retry(self): + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + message_queue = RemoteMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.put(b"x") + # close the connection manually + mq.close() + # this should still pass, as it catches the thrift error and re-connects + self.assertEqual(mq.get(), b"x") + + def test_get_thrift_retry_and_timeout(self): + # Check that we still catch a timeout error even if we have to reconnect + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + message_queue = RemoteMessageQueue(self.qname, max_messages=1) + + with contextlib.closing(message_queue) as mq: + mq.close() + start = time.time() + with self.assertRaises(TimedOutError): + mq.get(timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=1) # TODO: this routinely takes 0.105-0.11 seconds diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 74caed40a..1ee19b8f4 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -27,7 +27,7 @@ def tearDown(self): gevent.monkey.saved.clear() -class PublisherQueueTests(GeventPatchedTestCase): +class PublisherQueueUtilTests(GeventPatchedTestCase): def test_posix_queue(self): queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) assert isinstance(queue, PosixMessageQueue) @@ -45,14 +45,14 @@ def test_posix_queue_get_put(self): assert output == test_message_2 def test_in_memory_create_queue(self): - queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) - assert isinstance(queue, RemoteMessageQueue) + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) + assert isinstance(queue, RemoteMessageQueue) def test_in_memory_queue_get_put(self): - queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue.connect() test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") queue.put(test_message) @@ -61,3 +61,17 @@ def test_in_memory_queue_get_put(self): assert output == test_message output = queue.get() assert output == test_message_2 + + def test_in_memory_queue_alternate_port(self): + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: + queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000, port=9091) + + test_message = bytes("message", "utf-8") + test_message_2 = bytes("2nd message", "utf-8") + queue.put(test_message) + queue.put(test_message_2) + output = queue.get() + assert output == test_message + output = queue.get() + assert output == test_message_2 + From c31eb31756f437b7ba66a134011100b1f381ab02 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 9 Jan 2023 14:57:36 -0800 Subject: [PATCH 19/72] updates --- baseplate/lib/events.py | 6 +++--- baseplate/lib/message_queue.py | 14 +++++++++++--- baseplate/observers/tracing.py | 6 +++--- baseplate/sidecars/event_publisher.py | 15 +++++++++------ baseplate/sidecars/publisher_queue_utils.py | 15 ++++++++------- baseplate/sidecars/trace_publisher.py | 14 ++++++++------ .../sidecars/publisher_queue_utils_tests.py | 8 ++++---- tests/unit/lib/events/queue_tests.py | 4 ++-- 8 files changed, 48 insertions(+), 34 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index cd7ce05c8..cf9217a9a 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,7 +22,7 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue, QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -95,9 +95,9 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): """ def __init__( - self, name: str, event_serializer: Callable[[T], bytes], queue_type: str = "posix" + self, name: str, event_serializer: Callable[[T], bytes], queue_type: QueueType = QueueType.POSIX ): - if queue_type == "in_memory": + if queue_type == QueueType.IN_MEMORY: self.queue = RemoteMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) else: self.queue = PosixMessageQueue( # type: ignore diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 666e2053f..d1b6f72f7 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,5 +1,6 @@ """A message queue, with two implementations: POSIX-based, or in-memory using a Thrift server.""" import abc +from enum import Enum import queue as q import select @@ -44,6 +45,11 @@ def __init__(self, inner: Exception): super().__init__(f"{inner} (check `ulimit -q`?)") +class QueueType(Enum): + IN_MEMORY = "in_memory" + POSIX = "posix" + + class MessageQueue(abc.ABC): """Abstract class for an inter-process message queue.""" @@ -167,6 +173,7 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): + """A message queue that connects to a remote server.""" def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): # Connect to the remote queue server, and creeate the new queue self.queue_name = name @@ -194,6 +201,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: # we may get a timeout after re-connecting, and we want to catch that self.connect() return self.client.get(self.queue_name, timeout).value + # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue type except ThriftTimedOutError: raise TimedOutError @@ -235,7 +243,7 @@ def queue_tool() -> None: parser.add_argument( "--queue_type", default="posix", - choices=["posix", "in_memory"], + choices=[qt.value for qt in QueueType], help="whether to use an in-memory queue or a posix queue", ) @@ -264,8 +272,8 @@ def queue_tool() -> None: args = parser.parse_args() - if args.queue_type == "in_memory": - queue = RemoteMessageQueue(args.queue_name, args.max_messages) + if args.queue_type == QueueType.IN_MEMORY.value: + queue = RemoteMessageQueue(args.queue_name, args.max_messages) # type: ignore else: queue = PosixMessageQueue( # type: ignore args.queue_name, args.max_messages, args.max_message_size diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 8cb445d38..c1b9be8fe 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,7 +29,7 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue, QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout @@ -546,8 +546,8 @@ class SidecarRecorder(Recorder): adding them to the queue. """ - def __init__(self, queue_name: str, queue_type: str = "posix"): - if queue_type == "in_memory": + def __init__(self, queue_name: str, queue_type: QueueType = QueueType.POSIX): + if queue_type == QueueType.IN_MEMORY: self.queue = RemoteMessageQueue( "/traces-" + queue_name, max_messages=MAX_QUEUE_SIZE, diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 4baef0a51..55af72288 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -17,7 +17,7 @@ from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import MessageQueue, QueueType, TimedOutError from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy @@ -42,6 +42,8 @@ MAX_BATCH_AGE = 1 # maximum size (in bytes) of a batch of events MAX_BATCH_SIZE = 500 * 1024 +# Seconds to wait for get/put operations on the event queue +QUEUE_TIMEOUT = 0.2 class MaxRetriesError(Exception): @@ -161,12 +163,13 @@ def publish(self, payload: SerializedBatch) -> None: raise MaxRetriesError("could not sent batch") -def build_batch_and_publish(event_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: BatchPublisher) -> bytes: +def build_batch_and_publish(event_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: BatchPublisher, timeout: float) -> bytes: + # Helper that continuously polls for messages, then batches and publishes them while True: message: Optional[bytes] try: - message = event_queue.get(timeout=0.2) + message = event_queue.get(timeout) except TimedOutError: message = None @@ -238,12 +241,12 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - if cfg.queue_type == "in_memory": + if cfg.queue_type == QueueType.IN_MEMORY.value: with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - build_batch_and_publish(event_queue, batcher, publisher) + build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) else: - build_batch_and_publish(event_queue, batcher, publisher) + build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 8f122255b..4bbb07576 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -5,7 +5,7 @@ import gevent from baseplate.lib import config -from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import InMemoryMessageQueue, QueueType from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue @@ -46,8 +46,9 @@ def get( self.queues[queue_name] = queue # Get element from list, waiting if necessary result = queue.get(timeout) - except MessageQueueTimedOutError: - raise ThriftTimedOutError() + # If the queue timed out, raise a timeout as the server response + except MessageQueueTimedOutError as e: + raise ThriftTimedOutError from e return GetResponse(result) @@ -57,8 +58,8 @@ def put( try: queue, _ = self.queues.get(queue_name) queue.put(message, timeout) - except MessageQueueTimedOutError: - raise ThriftTimedOutError() + except MessageQueueTimedOutError as e: + raise ThriftTimedOutError from e return PutResponse() @@ -82,9 +83,9 @@ def start_queue_server(host: str, port: int) -> None: def create_queue( - queue_type: str, queue_name: str, max_queue_size: int, max_element_size: int, host: str = "127.0.0.1", port: int = 9090 + queue_type: QueueType, queue_name: str, max_queue_size: int, max_element_size: int, host: str = "127.0.0.1", port: int = 9090 ) -> MessageQueue: - if queue_type == "in_memory": + if queue_type == QueueType.IN_MEMORY: event_queue = RemoteMessageQueue( # type: ignore "/events-" + queue_name, max_queue_size, host, port ) diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index b308bd78a..1439755d7 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,7 +10,7 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import MessageQueue, QueueType from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy @@ -38,6 +38,8 @@ # maximum number of retries when publishing traces RETRY_LIMIT_DEFAULT = 10 +# Seconds to wait for get/put operations on the event queue +QUEUE_TIMEOUT = 0.2 class MaxRetriesError(Exception): pass @@ -118,12 +120,12 @@ def publish(self, payload: SerializedBatch) -> None: f"ZipkinPublisher exhausted allowance of {self.retry_limit:d} retries." ) -def build_and_publish_batch(trace_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: ZipkinPublisher) -> None: +def build_and_publish_batch(trace_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: ZipkinPublisher, timeout: float) -> None: while True: message: Optional[bytes] try: - message = trace_queue.get(timeout=0.2) + message = trace_queue.get(timeout) except TimedOutError: message = None @@ -204,11 +206,11 @@ def publish_traces() -> None: post_timeout=publisher_cfg.post_timeout, ) - if publisher_cfg.queue_type == "in_memory": + if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - build_and_publish_batch(trace_queue, batcher, publisher) + build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) else: - build_and_publish_batch(trace_queue, batcher, publisher) + build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) if __name__ == "__main__": diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 1ee19b8f4..d66d811f2 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -9,7 +9,7 @@ from baseplate.lib import config from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import MessageQueue, QueueType from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.sidecars import publisher_queue_utils @@ -46,12 +46,12 @@ def test_posix_queue_get_put(self): def test_in_memory_create_queue(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: - queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) + queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.IN_MEMORY, "test", 5, 8000) assert isinstance(queue, RemoteMessageQueue) def test_in_memory_queue_get_put(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: - queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000) + queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.IN_MEMORY, "test", 5, 8000) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") @@ -64,7 +64,7 @@ def test_in_memory_queue_get_put(self): def test_in_memory_queue_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: - queue: MessageQueue = publisher_queue_utils.create_queue("in_memory", "test", 5, 8000, port=9091) + queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.IN_MEMORY, "test", 5, 8000, port=9091) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 3b5096885..1caaaf4e5 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -6,7 +6,7 @@ from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE -from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import PosixMessageQueue, QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -50,7 +50,7 @@ def setUp(self, RemoteMessageQueue): self.message_queue = RemoteMessageQueue.return_value self.mock_serializer = mock.Mock() self.queue = EventQueue( - "test", event_serializer=self.mock_serializer, queue_type="in_memory" + "test", event_serializer=self.mock_serializer, queue_type=QueueType.IN_MEMORY ) def test_send_event(self): From 7b93a00ddea653d003e8ddcc81f078a38982b390 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 9 Jan 2023 15:04:55 -0800 Subject: [PATCH 20/72] cleanup exceptions --- baseplate/lib/message_queue.py | 3 ++- baseplate/sidecars/publisher_queue_utils.py | 8 ++++---- .../message_queue/RemoteMessageQueueService.py | 12 ++++++------ baseplate/thrift/message_queue/message_queue.thrift | 6 +++--- baseplate/thrift/message_queue/ttypes.py | 8 ++++---- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index d1b6f72f7..634c06f4a 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -15,7 +15,7 @@ from baseplate.lib.retry import RetryPolicy from baseplate.thrift.message_queue import RemoteMessageQueueService -from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError +from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError class MessageQueueError(Exception): @@ -174,6 +174,7 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): """A message queue that connects to a remote server.""" + def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): # Connect to the remote queue server, and creeate the new queue self.queue_name = name diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 4bbb07576..426eb1e0e 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -9,14 +9,14 @@ from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue -from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError +from baseplate.lib.message_queue import TimedOutError from baseplate.server import make_listener from baseplate.server.thrift import make_server from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.thrift.message_queue.ttypes import CreateResponse from baseplate.thrift.message_queue.ttypes import GetResponse from baseplate.thrift.message_queue.ttypes import PutResponse -from baseplate.thrift.message_queue.ttypes import TimedOutError as ThriftTimedOutError +from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation @@ -47,7 +47,7 @@ def get( # Get element from list, waiting if necessary result = queue.get(timeout) # If the queue timed out, raise a timeout as the server response - except MessageQueueTimedOutError as e: + except TimedOutError as e: raise ThriftTimedOutError from e return GetResponse(result) @@ -58,7 +58,7 @@ def put( try: queue, _ = self.queues.get(queue_name) queue.put(message, timeout) - except MessageQueueTimedOutError as e: + except TimedOutError as e: raise ThriftTimedOutError from e return PutResponse() diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index ea6a741f6..4362cd379 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -226,7 +226,7 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY except TTransport.TTransportException: raise - except TimedOutError as timed_out_error: + except ThriftTimedOutError as timed_out_error: msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: @@ -252,7 +252,7 @@ def process_get(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY except TTransport.TTransportException: raise - except TimedOutError as timed_out_error: + except ThriftTimedOutError as timed_out_error: msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: @@ -565,7 +565,7 @@ def read(self, iprot): iprot.skip(ftype) elif fid == 1: if ftype == TType.STRUCT: - self.timed_out_error = TimedOutError.read(iprot) + self.timed_out_error = ThriftTimedOutError.read(iprot) else: iprot.skip(ftype) else: @@ -612,7 +612,7 @@ def __ne__(self, other): all_structs.append(put_result) put_result.thrift_spec = ( (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 + (1, TType.STRUCT, 'timed_out_error', [ThriftTimedOutError, None], None, ), # 1 ) @@ -738,7 +738,7 @@ def read(self, iprot): iprot.skip(ftype) elif fid == 1: if ftype == TType.STRUCT: - self.timed_out_error = TimedOutError.read(iprot) + self.timed_out_error = ThriftTimedOutError.read(iprot) else: iprot.skip(ftype) else: @@ -785,7 +785,7 @@ def __ne__(self, other): all_structs.append(get_result) get_result.thrift_spec = ( (0, TType.STRUCT, 'success', [GetResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [TimedOutError, None], None, ), # 1 + (1, TType.STRUCT, 'timed_out_error', [ThriftTimedOutError, None], None, ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index 61a0ad79b..546e84442 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -10,7 +10,7 @@ struct GetResponse { 1: binary value; } -exception TimedOutError {} +exception ThriftTimedOutError {} service RemoteMessageQueueService { CreateResponse create_queue( @@ -22,12 +22,12 @@ service RemoteMessageQueueService { 3: binary message 4: double timeout ) throws ( - 1: TimedOutError timed_out_error + 1: ThriftTimedOutError timed_out_error ); GetResponse get( 1: string queue_name 3: double timeout ) throws ( - 1: TimedOutError timed_out_error + 1: ThriftTimedOutError timed_out_error ); } diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index 404516b6b..a6c220e8b 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -184,7 +184,7 @@ def __ne__(self, other): return not (self == other) -class TimedOutError(TException): +class ThriftTimedOutError(TException): __slots__ = ( ) @@ -219,7 +219,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('TimedOutError') + oprot.writeStructBegin('ThriftTimedOutError') oprot.writeFieldStop() oprot.writeStructEnd() @@ -257,8 +257,8 @@ def __ne__(self, other): None, # 0 (1, TType.STRING, 'value', 'BINARY', None, ), # 1 ) -all_structs.append(TimedOutError) -TimedOutError.thrift_spec = ( +all_structs.append(ThriftTimedOutError) +ThriftTimedOutError.thrift_spec = ( ) fix_spec(all_structs) del all_structs From f522fce442eda30c9d9077fc52ecf69513185cad Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 9 Jan 2023 16:06:39 -0800 Subject: [PATCH 21/72] linting --- baseplate/lib/events.py | 10 +- baseplate/lib/message_queue.py | 26 +- baseplate/observers/tracing.py | 3 +- baseplate/sidecars/event_publisher.py | 10 +- baseplate/sidecars/publisher_queue_utils.py | 46 ++- baseplate/sidecars/trace_publisher.py | 11 +- .../RemoteMessageQueueService.py | 364 +++++++++++++----- baseplate/thrift/message_queue/__init__.py | 2 +- baseplate/thrift/message_queue/constants.py | 8 +- baseplate/thrift/message_queue/ttypes.py | 109 +++--- tests/integration/message_queue_tests.py | 14 +- .../sidecars/publisher_queue_utils_tests.py | 16 +- tests/unit/lib/events/queue_tests.py | 3 +- 13 files changed, 405 insertions(+), 217 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index cf9217a9a..2f32164a2 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,7 +22,8 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import PosixMessageQueue, QueueType +from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -95,7 +96,10 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): """ def __init__( - self, name: str, event_serializer: Callable[[T], bytes], queue_type: QueueType = QueueType.POSIX + self, + name: str, + event_serializer: Callable[[T], bytes], + queue_type: QueueType = QueueType.POSIX, ): if queue_type == QueueType.IN_MEMORY: self.queue = RemoteMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) @@ -128,7 +132,7 @@ def put(self, event: T) -> None: except TimedOutError: raise EventQueueFullError - def get(self): + def get(self) -> bytes: return self.queue.get() def make_object_for_context(self, name: str, span: Span) -> "EventQueue[T]": diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 634c06f4a..dd091389e 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,9 +1,9 @@ """A message queue, with two implementations: POSIX-based, or in-memory using a Thrift server.""" import abc -from enum import Enum import queue as q import select +from enum import Enum from typing import Optional import gevent @@ -177,14 +177,14 @@ class RemoteMessageQueue(MessageQueue): def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): # Connect to the remote queue server, and creeate the new queue - self.queue_name = name + self.name = name self.max_messages = max_messages self.host = host self.port = port self.connect() self.client.create_queue(name, max_messages) - def connect(self): + def connect(self) -> None: # Establish a connection with the queue server transport = TSocket.TSocket(self.host, self.port) self.transport = TTransport.TBufferedTransport(transport) @@ -195,13 +195,13 @@ def connect(self): def get(self, timeout: Optional[float] = None) -> bytes: # Call the remote server and get an element for the correct queue try: - try: - return self.client.get(self.queue_name, timeout).value - except TSocket.TTransportException: - # Try reconnecting once, we dont want this as another top-level except because + try: + return self.client.get(self.name, timeout).value + except TSocket.TTransportException: + # Try reconnecting once, we dont want this as another top-level except because # we may get a timeout after re-connecting, and we want to catch that self.connect() - return self.client.get(self.queue_name, timeout).value + return self.client.get(self.name, timeout).value # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue type except ThriftTimedOutError: raise TimedOutError @@ -210,11 +210,11 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Call the remote server and put an element on the correct queue # Will create the queue if it doesnt exist try: - try: - self.client.put(self.queue_name, message, timeout) - except TSocket.TTransportException: # Try reconnecting once + try: + self.client.put(self.name, message, timeout) + except TSocket.TTransportException: # Try reconnecting once self.connect() - self.client.put(self.queue_name, self.max_messages, message, timeout) + self.client.put(self.name, message, timeout) except ThriftTimedOutError: raise TimedOutError @@ -274,7 +274,7 @@ def queue_tool() -> None: args = parser.parse_args() if args.queue_type == QueueType.IN_MEMORY.value: - queue = RemoteMessageQueue(args.queue_name, args.max_messages) # type: ignore + queue = RemoteMessageQueue(args.queue_name, args.max_messages) else: queue = PosixMessageQueue( # type: ignore args.queue_name, args.max_messages, args.max_message_size diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index c1b9be8fe..e67b59217 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,7 +29,8 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import PosixMessageQueue, QueueType +from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 55af72288..6ae55d24e 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -17,8 +17,9 @@ from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import MessageQueue, QueueType, TimedOutError -from baseplate.lib.message_queue import TimedOutError as MessageQueueTimedOutError +from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import QueueType +from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy from baseplate.server import EnvironmentInterpolation @@ -163,7 +164,9 @@ def publish(self, payload: SerializedBatch) -> None: raise MaxRetriesError("could not sent batch") -def build_batch_and_publish(event_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: BatchPublisher, timeout: float) -> bytes: +def build_batch_and_publish( + event_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: BatchPublisher, timeout: float +) -> bytes: # Helper that continuously polls for messages, then batches and publishes them while True: message: Optional[bytes] @@ -249,6 +252,5 @@ def publish_events() -> None: build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) - if __name__ == "__main__": publish_events() diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 426eb1e0e..c4b845a05 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,13 +1,18 @@ import contextlib +from typing import Dict +from typing import Generator from typing import Optional import gevent +from gevent.server import StreamServer + from baseplate.lib import config -from baseplate.lib.message_queue import InMemoryMessageQueue, QueueType +from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.server import make_listener @@ -23,27 +28,19 @@ class RemoteMessageQueueHandler: # On the queue server, create the queue and de def is_healthy(self) -> bool: pass - def __init__(self): - self.queues = {} + def __init__(self) -> None: + # Store the queue by name with its max messages + self.queues: Dict[str, MessageQueue] = {} def create_queue(self, queue_name: str, max_messages: int) -> CreateResponse: queue = InMemoryMessageQueue(queue_name, max_messages) - self.queues[queue_name] = (queue, max_messages) + self.queues[queue_name] = queue return CreateResponse() - def get( - self, queue_name: str, timeout: Optional[float] = None - ) -> GetResponse: - # Raises TimedOutError + def get(self, queue_name: str, timeout: Optional[float] = None) -> GetResponse: try: - # Create queue if doesnt exist - # We may need to create the queue on both get & put - if get() is called on a queue before it exists, we - # still want to wait the appropriate timeout in case anyone puts elements in it - queue, max_messages = self.queues.get(queue_name) - if not queue: - queue = InMemoryMessageQueue(queue_name, max_messages) - self.queues[queue_name] = queue + queue = self.queues[queue_name] # Get element from list, waiting if necessary result = queue.get(timeout) # If the queue timed out, raise a timeout as the server response @@ -52,11 +49,9 @@ def get( return GetResponse(result) - def put( - self, queue_name: str, message: bytes, timeout: Optional[float] = None - ) -> PutResponse: + def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) -> PutResponse: try: - queue, _ = self.queues.get(queue_name) + queue = self.queues[queue_name] queue.put(message, timeout) except TimedOutError as e: raise ThriftTimedOutError from e @@ -64,7 +59,7 @@ def put( @contextlib.contextmanager -def start_queue_server(host: str, port: int) -> None: +def start_queue_server(host: str, port: int) -> Generator[StreamServer, None, None]: # Start a thrift server that will house the remote queue data processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler()) server_bind_endpoint = config.Endpoint(f"{host}:{port}") @@ -83,12 +78,15 @@ def start_queue_server(host: str, port: int) -> None: def create_queue( - queue_type: QueueType, queue_name: str, max_queue_size: int, max_element_size: int, host: str = "127.0.0.1", port: int = 9090 + queue_type: QueueType, + queue_name: str, + max_queue_size: int, + max_element_size: int, + host: str = "127.0.0.1", + port: int = 9090, ) -> MessageQueue: if queue_type == QueueType.IN_MEMORY: - event_queue = RemoteMessageQueue( # type: ignore - "/events-" + queue_name, max_queue_size, host, port - ) + event_queue = RemoteMessageQueue("/events-" + queue_name, max_queue_size, host, port) else: event_queue = PosixMessageQueue( # type: ignore diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 1439755d7..c1137d3a5 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,7 +10,8 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics -from baseplate.lib.message_queue import MessageQueue, QueueType +from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError from baseplate.lib.metrics import metrics_client_from_config from baseplate.lib.retry import RetryPolicy @@ -41,6 +42,7 @@ # Seconds to wait for get/put operations on the event queue QUEUE_TIMEOUT = 0.2 + class MaxRetriesError(Exception): pass @@ -120,7 +122,10 @@ def publish(self, payload: SerializedBatch) -> None: f"ZipkinPublisher exhausted allowance of {self.retry_limit:d} retries." ) -def build_and_publish_batch(trace_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: ZipkinPublisher, timeout: float) -> None: + +def build_and_publish_batch( + trace_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: ZipkinPublisher, timeout: float +) -> None: while True: message: Optional[bytes] @@ -209,7 +214,7 @@ def publish_traces() -> None: if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) - else: + else: build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index 4362cd379..863809328 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -5,16 +5,21 @@ # # options string: py:slots # +import logging +import sys -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import TApplicationException +from thrift.Thrift import TException +from thrift.Thrift import TFrozenDict +from thrift.Thrift import TMessageType +from thrift.Thrift import TProcessor +from thrift.Thrift import TType +from thrift.transport import TTransport from thrift.TRecursive import fix_spec -import sys -import logging from .ttypes import * -from thrift.Thrift import TProcessor -from thrift.transport import TTransport + all_structs = [] @@ -66,7 +71,7 @@ def create_queue(self, queue_name, max_messages): return self.recv_create_queue() def send_create_queue(self, queue_name, max_messages): - self._oprot.writeMessageBegin('create_queue', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("create_queue", TMessageType.CALL, self._seqid) args = create_queue_args() args.queue_name = queue_name args.max_messages = max_messages @@ -87,7 +92,9 @@ def recv_create_queue(self): iprot.readMessageEnd() if result.success is not None: return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "create_queue failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "create_queue failed: unknown result" + ) def put(self, queue_name, message, timeout): """ @@ -101,7 +108,7 @@ def put(self, queue_name, message, timeout): return self.recv_put() def send_put(self, queue_name, message, timeout): - self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("put", TMessageType.CALL, self._seqid) args = put_args() args.queue_name = queue_name args.message = message @@ -125,7 +132,9 @@ def recv_put(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "put failed: unknown result" + ) def get(self, queue_name, timeout): """ @@ -138,7 +147,7 @@ def get(self, queue_name, timeout): return self.recv_get() def send_get(self, queue_name, timeout): - self._oprot.writeMessageBegin('get', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("get", TMessageType.CALL, self._seqid) args = get_args() args.queue_name = queue_name args.timeout = timeout @@ -161,7 +170,9 @@ def recv_get(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException(TApplicationException.MISSING_RESULT, "get failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get failed: unknown result" + ) class Processor(Iface, TProcessor): @@ -183,7 +194,9 @@ def process(self, iprot, oprot): if name not in self._processMap: iprot.skip(TType.STRUCT) iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + x = TApplicationException( + TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name) + ) oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) x.write(oprot) oprot.writeMessageEnd() @@ -204,13 +217,13 @@ def process_create_queue(self, seqid, iprot, oprot): except TTransport.TTransportException: raise except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("create_queue", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -230,13 +243,13 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("put", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -256,18 +269,19 @@ def process_get(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("get", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() oprot.trans.flush() + # HELPER FUNCTIONS AND STRUCTURES @@ -280,17 +294,24 @@ class create_queue_args(object): """ __slots__ = ( - 'queue_name', - 'max_messages', + "queue_name", + "max_messages", ) - - def __init__(self, queue_name=None, max_messages=None,): + def __init__( + self, + queue_name=None, + max_messages=None, + ): self.queue_name = queue_name self.max_messages = max_messages def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -300,7 +321,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 2: @@ -317,13 +342,15 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('create_queue_args') + oprot.writeStructBegin("create_queue_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.max_messages is not None: - oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeFieldBegin("max_messages", TType.I64, 2) oprot.writeI64(self.max_messages) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -333,9 +360,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -349,11 +375,25 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(create_queue_args) create_queue_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 - (2, TType.I64, 'max_messages', None, None, ), # 2 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "max_messages", + None, + None, + ), # 2 ) @@ -364,16 +404,20 @@ class create_queue_result(object): """ - __slots__ = ( - 'success', - ) - + __slots__ = ("success",) - def __init__(self, success=None,): + def __init__( + self, + success=None, + ): self.success = success def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -396,9 +440,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('create_queue_result') + oprot.writeStructBegin("create_queue_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -408,9 +452,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -424,9 +467,17 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(create_queue_result) create_queue_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [CreateResponse, None], None, ), # 0 + ( + 0, + TType.STRUCT, + "success", + [CreateResponse, None], + None, + ), # 0 ) @@ -440,19 +491,27 @@ class put_args(object): """ __slots__ = ( - 'queue_name', - 'message', - 'timeout', + "queue_name", + "message", + "timeout", ) - - def __init__(self, queue_name=None, message=None, timeout=None,): + def __init__( + self, + queue_name=None, + message=None, + timeout=None, + ): self.queue_name = queue_name self.message = message self.timeout = timeout def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -462,7 +521,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 3: @@ -484,17 +547,19 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('put_args') + oprot.writeStructBegin("put_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.message is not None: - oprot.writeFieldBegin('message', TType.STRING, 3) + oprot.writeFieldBegin("message", TType.STRING, 3) oprot.writeBinary(self.message) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin('timeout', TType.DOUBLE, 4) + oprot.writeFieldBegin("timeout", TType.DOUBLE, 4) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -504,9 +569,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -520,13 +584,33 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(put_args) put_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 None, # 2 - (3, TType.STRING, 'message', 'BINARY', None, ), # 3 - (4, TType.DOUBLE, 'timeout', None, None, ), # 4 + ( + 3, + TType.STRING, + "message", + "BINARY", + None, + ), # 3 + ( + 4, + TType.DOUBLE, + "timeout", + None, + None, + ), # 4 ) @@ -539,17 +623,24 @@ class put_result(object): """ __slots__ = ( - 'success', - 'timed_out_error', + "success", + "timed_out_error", ) - - def __init__(self, success=None, timed_out_error=None,): + def __init__( + self, + success=None, + timed_out_error=None, + ): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -577,13 +668,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('put_result') + oprot.writeStructBegin("put_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -593,9 +684,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -609,10 +699,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(put_result) put_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [ThriftTimedOutError, None], None, ), # 1 + ( + 0, + TType.STRUCT, + "success", + [PutResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "timed_out_error", + [ThriftTimedOutError, None], + None, + ), # 1 ) @@ -625,17 +729,24 @@ class get_args(object): """ __slots__ = ( - 'queue_name', - 'timeout', + "queue_name", + "timeout", ) - - def __init__(self, queue_name=None, timeout=None,): + def __init__( + self, + queue_name=None, + timeout=None, + ): self.queue_name = queue_name self.timeout = timeout def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -645,7 +756,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 3: @@ -662,13 +777,15 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('get_args') + oprot.writeStructBegin("get_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin('timeout', TType.DOUBLE, 3) + oprot.writeFieldBegin("timeout", TType.DOUBLE, 3) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -678,9 +795,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -694,12 +810,26 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(get_args) get_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 None, # 2 - (3, TType.DOUBLE, 'timeout', None, None, ), # 3 + ( + 3, + TType.DOUBLE, + "timeout", + None, + None, + ), # 3 ) @@ -712,17 +842,24 @@ class get_result(object): """ __slots__ = ( - 'success', - 'timed_out_error', + "success", + "timed_out_error", ) - - def __init__(self, success=None, timed_out_error=None,): + def __init__( + self, + success=None, + timed_out_error=None, + ): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -750,13 +887,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('get_result') + oprot.writeStructBegin("get_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -766,9 +903,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -782,10 +918,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(get_result) get_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [GetResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [ThriftTimedOutError, None], None, ), # 1 + ( + 0, + TType.STRUCT, + "success", + [GetResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "timed_out_error", + [ThriftTimedOutError, None], + None, + ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/__init__.py b/baseplate/thrift/message_queue/__init__.py index 947e972be..f37fa9583 100644 --- a/baseplate/thrift/message_queue/__init__.py +++ b/baseplate/thrift/message_queue/__init__.py @@ -1 +1 @@ -__all__ = ['ttypes', 'constants', 'RemoteMessageQueueService'] +__all__ = ["ttypes", "constants", "RemoteMessageQueueService"] diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py index 964098e62..a19d895e4 100644 --- a/baseplate/thrift/message_queue/constants.py +++ b/baseplate/thrift/message_queue/constants.py @@ -5,10 +5,14 @@ # # options string: py:slots # +import sys -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import TApplicationException +from thrift.Thrift import TException +from thrift.Thrift import TFrozenDict +from thrift.Thrift import TMessageType +from thrift.Thrift import TType from thrift.TRecursive import fix_spec -import sys from .ttypes import * diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index a6c220e8b..d7b6993cf 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -5,25 +5,30 @@ # # options string: py:slots # +import sys -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import TApplicationException +from thrift.Thrift import TException +from thrift.Thrift import TFrozenDict +from thrift.Thrift import TMessageType +from thrift.Thrift import TType +from thrift.transport import TTransport from thrift.TRecursive import fix_spec -import sys - -from thrift.transport import TTransport all_structs = [] class CreateResponse(object): - __slots__ = ( - ) - + __slots__ = () def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -40,7 +45,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('CreateResponse') + oprot.writeStructBegin("CreateResponse") oprot.writeFieldStop() oprot.writeStructEnd() @@ -48,9 +53,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -68,12 +72,14 @@ def __ne__(self, other): class PutResponse(object): - __slots__ = ( - ) - + __slots__ = () def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -90,7 +96,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('PutResponse') + oprot.writeStructBegin("PutResponse") oprot.writeFieldStop() oprot.writeStructEnd() @@ -98,9 +104,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -123,16 +128,20 @@ class GetResponse(object): """ - __slots__ = ( - 'value', - ) + __slots__ = ("value",) - - def __init__(self, value=None,): + def __init__( + self, + value=None, + ): self.value = value def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -154,9 +163,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('GetResponse') + oprot.writeStructBegin("GetResponse") if self.value is not None: - oprot.writeFieldBegin('value', TType.STRING, 1) + oprot.writeFieldBegin("value", TType.STRING, 1) oprot.writeBinary(self.value) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -166,9 +175,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -186,9 +194,7 @@ def __ne__(self, other): class ThriftTimedOutError(TException): - __slots__ = ( - ) - + __slots__ = () def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -201,7 +207,11 @@ def __hash__(self): @classmethod def read(cls, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) iprot.readStructBegin() while True: @@ -212,14 +222,13 @@ def read(cls, iprot): iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() - return cls( - ) + return cls() def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('ThriftTimedOutError') + oprot.writeStructBegin("ThriftTimedOutError") oprot.writeFieldStop() oprot.writeStructEnd() @@ -230,9 +239,8 @@ def __str__(self): return repr(self) def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -246,19 +254,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(CreateResponse) -CreateResponse.thrift_spec = ( -) +CreateResponse.thrift_spec = () all_structs.append(PutResponse) -PutResponse.thrift_spec = ( -) +PutResponse.thrift_spec = () all_structs.append(GetResponse) GetResponse.thrift_spec = ( None, # 0 - (1, TType.STRING, 'value', 'BINARY', None, ), # 1 + ( + 1, + TType.STRING, + "value", + "BINARY", + None, + ), # 1 ) all_structs.append(ThriftTimedOutError) -ThriftTimedOutError.thrift_spec = ( -) +ThriftTimedOutError.thrift_spec = () fix_spec(all_structs) del all_structs diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 856775e62..f400a3704 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -4,7 +4,6 @@ import unittest from importlib import reload -from baseplate.sidecars import publisher_queue_utils import gevent import posix_ipc @@ -15,6 +14,7 @@ from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError +from baseplate.sidecars import publisher_queue_utils class TestPosixMessageQueueCreation(unittest.TestCase): @@ -187,7 +187,7 @@ def test_multiple_queues(self): mq1.close() mq2.close() - def test_queues_alternate_port(self): + def test_queues_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091) @@ -204,7 +204,9 @@ def test_get_timeout(self): with self.assertRaises(TimedOutError): mq.get(timeout=0.1) elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=1) # TODO: this routinely takes 0.105-0.11 seconds + self.assertAlmostEqual( + elapsed, 0.1, places=1 + ) # TODO: this routinely takes 0.105-0.11 seconds def test_put_timeout(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: @@ -228,7 +230,7 @@ def test_thrift_retry(self): mq.close() # this should still pass, as it catches the thrift error and re-connects self.assertEqual(mq.get(), b"x") - + def test_get_thrift_retry_and_timeout(self): # Check that we still catch a timeout error even if we have to reconnect with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: @@ -240,4 +242,6 @@ def test_get_thrift_retry_and_timeout(self): with self.assertRaises(TimedOutError): mq.get(timeout=0.1) elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=1) # TODO: this routinely takes 0.105-0.11 seconds + self.assertAlmostEqual( + elapsed, 0.1, places=1 + ) # TODO: this routinely takes 0.105-0.11 seconds diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index d66d811f2..50c43ea12 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -9,8 +9,9 @@ from baseplate.lib import config from baseplate.lib.events import MAX_QUEUE_SIZE -from baseplate.lib.message_queue import MessageQueue, QueueType +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.sidecars import publisher_queue_utils from baseplate.thrift.message_queue.ttypes import GetResponse @@ -46,12 +47,16 @@ def test_posix_queue_get_put(self): def test_in_memory_create_queue(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: - queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.IN_MEMORY, "test", 5, 8000) + queue: MessageQueue = publisher_queue_utils.create_queue( + QueueType.IN_MEMORY, "test", 5, 8000 + ) assert isinstance(queue, RemoteMessageQueue) def test_in_memory_queue_get_put(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: - queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.IN_MEMORY, "test", 5, 8000) + queue: MessageQueue = publisher_queue_utils.create_queue( + QueueType.IN_MEMORY, "test", 5, 8000 + ) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") @@ -64,7 +69,9 @@ def test_in_memory_queue_get_put(self): def test_in_memory_queue_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: - queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.IN_MEMORY, "test", 5, 8000, port=9091) + queue: MessageQueue = publisher_queue_utils.create_queue( + QueueType.IN_MEMORY, "test", 5, 8000, port=9091 + ) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") @@ -74,4 +81,3 @@ def test_in_memory_queue_alternate_port(self): assert output == test_message output = queue.get() assert output == test_message_2 - diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 1caaaf4e5..58751289e 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -6,7 +6,8 @@ from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE -from baseplate.lib.message_queue import PosixMessageQueue, QueueType +from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError From 11e498e4418b9a49403b5c4254e49cf7e4ae6715 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 9 Jan 2023 16:53:22 -0800 Subject: [PATCH 22/72] linting --- baseplate/lib/message_queue.py | 1 - tests/integration/message_queue_tests.py | 17 +++++++---------- .../sidecars/publisher_queue_utils_tests.py | 11 +++-------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index dd091389e..a784eaabb 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -6,7 +6,6 @@ from enum import Enum from typing import Optional -import gevent import posix_ipc from thrift.protocol import TBinaryProtocol diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index f400a3704..f5933f3b8 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -1,5 +1,4 @@ import contextlib -import logging import time import unittest @@ -7,10 +6,8 @@ import gevent import posix_ipc -import pytest from baseplate.lib.message_queue import InMemoryMessageQueue -from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -164,7 +161,7 @@ class TestRemoteMessageQueueCreation(GeventPatchedTestCase): def test_put_get(self): # start the server that would ordinarily be running on the sidecar - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=10) with contextlib.closing(message_queue) as mq: @@ -173,7 +170,7 @@ def test_put_get(self): self.assertEqual(message, b"x") def test_multiple_queues(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): mq1 = RemoteMessageQueue(self.qname, max_messages=10) mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10) @@ -188,7 +185,7 @@ def test_multiple_queues(self): mq2.close() def test_queues_alternate_port(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091) with contextlib.closing(message_queue) as mq: @@ -196,7 +193,7 @@ def test_queues_alternate_port(self): self.assertEqual(mq.get(), b"x") def test_get_timeout(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: @@ -209,7 +206,7 @@ def test_get_timeout(self): ) # TODO: this routinely takes 0.105-0.11 seconds def test_put_timeout(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: @@ -221,7 +218,7 @@ def test_put_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=2) def test_thrift_retry(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: @@ -233,7 +230,7 @@ def test_thrift_retry(self): def test_get_thrift_retry_and_timeout(self): # Check that we still catch a timeout error even if we have to reconnect - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 50c43ea12..5e1ffa3ce 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -1,20 +1,15 @@ -import contextlib import unittest from importlib import reload from unittest import mock import gevent.monkey -import posix_ipc -from baseplate.lib import config -from baseplate.lib.events import MAX_QUEUE_SIZE from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.sidecars import publisher_queue_utils -from baseplate.thrift.message_queue.ttypes import GetResponse class GeventPatchedTestCase(unittest.TestCase): @@ -46,14 +41,14 @@ def test_posix_queue_get_put(self): assert output == test_message_2 def test_in_memory_create_queue(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): queue: MessageQueue = publisher_queue_utils.create_queue( QueueType.IN_MEMORY, "test", 5, 8000 ) assert isinstance(queue, RemoteMessageQueue) def test_in_memory_queue_get_put(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): queue: MessageQueue = publisher_queue_utils.create_queue( QueueType.IN_MEMORY, "test", 5, 8000 ) @@ -68,7 +63,7 @@ def test_in_memory_queue_get_put(self): assert output == test_message_2 def test_in_memory_queue_alternate_port(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091) as server: + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): queue: MessageQueue = publisher_queue_utils.create_queue( QueueType.IN_MEMORY, "test", 5, 8000, port=9091 ) From 27d4171adb148cb0e0febb2b98a93c845bd443a8 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 9 Jan 2023 17:01:02 -0800 Subject: [PATCH 23/72] linting --- tests/integration/sidecars/publisher_queue_utils_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 5e1ffa3ce..6407c48fc 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -1,7 +1,6 @@ import unittest from importlib import reload -from unittest import mock import gevent.monkey From 78c3102ab91ab772112347d85bc3cbb4f2726391 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 10 Jan 2023 10:58:09 -0800 Subject: [PATCH 24/72] cleanup --- baseplate/lib/message_queue.py | 32 ++++++++++++++++--- baseplate/observers/tracing.py | 2 ++ baseplate/sidecars/event_publisher.py | 5 ++- baseplate/sidecars/publisher_queue_utils.py | 15 ++++----- baseplate/sidecars/trace_publisher.py | 8 ++--- tests/integration/message_queue_tests.py | 20 ++++++------ .../sidecars/publisher_queue_utils_tests.py | 10 +++--- 7 files changed, 59 insertions(+), 33 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index a784eaabb..32687b874 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -139,6 +139,11 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: raise TimedOutError def unlink(self) -> None: + """Remove the queue from the system. + + The queue will not leave until the last active user closes it. + + """ self.queue.unlink() def close(self) -> None: @@ -146,7 +151,18 @@ def close(self) -> None: class InMemoryMessageQueue(MessageQueue): - """An in-memory inter process message queue.""" # Used by the sidecar + """An in-memory inter process message queue. + + Uses a simple Python Queue to store data. + + Used in conjunction with the RemoteMessageQueue to + provide an alternative to Posix queues, for systems + that don't have Posix available. The client will + instantitate a RemoteMessageQueue, which connect to + a Thrift server. The Thrift server internally uses an + InMemoryMessageQueue to store data. + + """ def __init__(self, name: str, max_messages: int): self.queue: q.Queue = q.Queue(max_messages) @@ -172,7 +188,13 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): - """A message queue that connects to a remote server.""" + """A message queue that uses a remote Thrift server. + + Used in conjunction with the InMemoryMessageQueue to + provide an alternative to Posix queues, for systems + that don't have Posix available. + + """ def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): # Connect to the remote queue server, and creeate the new queue @@ -201,7 +223,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: # we may get a timeout after re-connecting, and we want to catch that self.connect() return self.client.get(self.name, timeout).value - # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue type + # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue except ThriftTimedOutError: raise TimedOutError @@ -242,7 +264,7 @@ def queue_tool() -> None: parser.add_argument("queue_name", help="the name of the queue to consume") parser.add_argument( "--queue_type", - default="posix", + default=QueueType.POSIX.value, choices=[qt.value for qt in QueueType], help="whether to use an in-memory queue or a posix queue", ) @@ -273,6 +295,8 @@ def queue_tool() -> None: args = parser.parse_args() if args.queue_type == QueueType.IN_MEMORY.value: + # Start a remote queue, which connects to a Thrift server + # that manages an in-memory queue queue = RemoteMessageQueue(args.queue_name, args.max_messages) else: queue = PosixMessageQueue( # type: ignore diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index e67b59217..dfecba128 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -549,6 +549,8 @@ class SidecarRecorder(Recorder): def __init__(self, queue_name: str, queue_type: QueueType = QueueType.POSIX): if queue_type == QueueType.IN_MEMORY: + # Start a remote queue, which connects to a Thrift server + # that manages an in-memory queue self.queue = RemoteMessageQueue( "/traces-" + queue_name, max_messages=MAX_QUEUE_SIZE, diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 6ae55d24e..993263a39 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -29,6 +29,7 @@ from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch + logger = logging.getLogger(__name__) @@ -236,7 +237,7 @@ def publish_events() -> None: metrics_client = metrics_client_from_config(raw_config) event_queue: MessageQueue = publisher_queue_utils.create_queue( - cfg.queue_type, args.queue_name, cfg.max_queue_size, cfg.max_element_size + cfg.queue_type, f"/events-{args.queue_name}", cfg.max_queue_size, cfg.max_element_size ) # pylint: disable=maybe-no-member @@ -245,6 +246,8 @@ def publish_events() -> None: publisher = BatchPublisher(metrics_client, cfg) if cfg.queue_type == QueueType.IN_MEMORY.value: + # Start the Thrift server that communicates with RemoteMessageQueues and stores + # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index c4b845a05..f0661dcec 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -24,7 +24,9 @@ from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError -class RemoteMessageQueueHandler: # On the queue server, create the queue and define get/put using the InMemoryQueue implementation +class RemoteMessageQueueHandler: + """Create an InMemoryMessageQueue locally and expose get/put methods.""" + def is_healthy(self) -> bool: pass @@ -60,15 +62,12 @@ def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) @contextlib.contextmanager def start_queue_server(host: str, port: int) -> Generator[StreamServer, None, None]: - # Start a thrift server that will house the remote queue data + # Start a thrift server that will store the queue data in memory processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler()) server_bind_endpoint = config.Endpoint(f"{host}:{port}") listener = make_listener(server_bind_endpoint) server = make_server(server_config={}, listener=listener, app=processor) - # figure out what port the server ended up on - server_address = listener.getsockname() - server.endpoint = config.Endpoint(f"{server_address[0]}:{server_address[1]}") # run the server until our caller is done with it server_greenlet = gevent.spawn(server.serve_forever) try: @@ -79,18 +78,18 @@ def start_queue_server(host: str, port: int) -> Generator[StreamServer, None, No def create_queue( queue_type: QueueType, - queue_name: str, + queue_full_name: str, max_queue_size: int, max_element_size: int, host: str = "127.0.0.1", port: int = 9090, ) -> MessageQueue: if queue_type == QueueType.IN_MEMORY: - event_queue = RemoteMessageQueue("/events-" + queue_name, max_queue_size, host, port) + event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) else: event_queue = PosixMessageQueue( # type: ignore - "/events-" + queue_name, + queue_full_name, max_messages=max_queue_size, max_message_size=max_element_size, ) diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index c1137d3a5..b02917311 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -155,8 +155,8 @@ def publish_traces() -> None: ) arg_parser.add_argument( "--queue-type", - default="posix", - choices=["posix", "in_memory"], + default=QueueType.POSIX.value, + choices=[qt.value for qt in QueueType], help="whether to use an in-memory queue or a posix queue", ) arg_parser.add_argument( @@ -190,13 +190,13 @@ def publish_traces() -> None: "retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT), "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), "max_element_size": config.Optional(config.Integer, MAX_SPAN_SIZE), - "queue_type": config.Optional(config.String, default="posix"), + "queue_type": config.Optional(config.String, default=QueueType.POSIX.value), }, ) trace_queue: MessageQueue = publisher_queue_utils.create_queue( publisher_cfg.queue_type, - args.queue_name, + "/traces-" + args.queue_name, publisher_cfg.max_queue_size, publisher_cfg.max_element_size, ) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index f5933f3b8..142030234 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -27,14 +27,14 @@ def setUp(self): queue.close() def test_create_queue(self): - message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: self.assertEqual(mq.queue.max_messages, 1) - self.assertEqual(mq.queue.max_message_size, 1) + self.assertEqual(mq.queue.max_message_size, 1000) def test_put_get(self): - message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: mq.put(b"x") @@ -42,7 +42,7 @@ def test_put_get(self): self.assertEqual(message, b"x") def test_get_timeout(self): - message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: start = time.time() @@ -52,7 +52,7 @@ def test_get_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=2) def test_put_timeout(self): - message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: mq.put(b"x") @@ -63,7 +63,7 @@ def test_put_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=1) def test_put_zero_timeout(self): - message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: mq.put(b"x", timeout=0) @@ -71,7 +71,7 @@ def test_put_zero_timeout(self): self.assertEqual(message, b"x") def test_put_full_zero_timeout(self): - message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1) + message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: mq.put(b"1", timeout=0) @@ -203,7 +203,7 @@ def test_get_timeout(self): elapsed = time.time() - start self.assertAlmostEqual( elapsed, 0.1, places=1 - ) # TODO: this routinely takes 0.105-0.11 seconds + ) # TODO: this routinely takes 0.105-0.11 seconds, is 1 place ok? def test_put_timeout(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): @@ -239,6 +239,4 @@ def test_get_thrift_retry_and_timeout(self): with self.assertRaises(TimedOutError): mq.get(timeout=0.1) elapsed = time.time() - start - self.assertAlmostEqual( - elapsed, 0.1, places=1 - ) # TODO: this routinely takes 0.105-0.11 seconds + self.assertAlmostEqual(elapsed, 0.1, places=1) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 6407c48fc..b4ae35dfc 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -24,11 +24,11 @@ def tearDown(self): class PublisherQueueUtilTests(GeventPatchedTestCase): def test_posix_queue(self): - queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) + queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.POSIX, "/test", 5, 1000) assert isinstance(queue, PosixMessageQueue) def test_posix_queue_get_put(self): - queue: MessageQueue = publisher_queue_utils.create_queue("posix", "test", 5, 8000) + queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.POSIX, "/test", 5, 1000) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") @@ -42,14 +42,14 @@ def test_posix_queue_get_put(self): def test_in_memory_create_queue(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): queue: MessageQueue = publisher_queue_utils.create_queue( - QueueType.IN_MEMORY, "test", 5, 8000 + QueueType.IN_MEMORY, "/test", 5, 1000 ) assert isinstance(queue, RemoteMessageQueue) def test_in_memory_queue_get_put(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): queue: MessageQueue = publisher_queue_utils.create_queue( - QueueType.IN_MEMORY, "test", 5, 8000 + QueueType.IN_MEMORY, "/test", 5, 1000 ) test_message = bytes("message", "utf-8") @@ -64,7 +64,7 @@ def test_in_memory_queue_get_put(self): def test_in_memory_queue_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): queue: MessageQueue = publisher_queue_utils.create_queue( - QueueType.IN_MEMORY, "test", 5, 8000, port=9091 + QueueType.IN_MEMORY, "/test", 5, 1000, port=9091 ) test_message = bytes("message", "utf-8") From 2e364d1c02184a9e92973f94a66cefd668236e96 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 19 Jan 2023 14:55:04 -0800 Subject: [PATCH 25/72] updates --- baseplate/lib/events.py | 28 ++++-- baseplate/lib/message_queue.py | 102 +++++++++++++------- baseplate/observers/tracing.py | 33 +++---- baseplate/sidecars/event_publisher.py | 8 +- baseplate/sidecars/publisher_queue_utils.py | 19 ++-- baseplate/sidecars/trace_publisher.py | 2 +- tests/integration/message_queue_tests.py | 14 +-- tests/unit/lib/events/queue_tests.py | 4 +- 8 files changed, 132 insertions(+), 78 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 2f32164a2..a085b5195 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -22,9 +22,10 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import create_queue +from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST +from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import QueueType -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -92,6 +93,12 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): :param event_serializer: A callable that takes an event object and returns serialized bytes ready to send on the wire. See below for options. + :param queue_type: A QueueType indicating which type of queue the publisher + should use. + :param queue_host: A string indicating the hostname of the queue server, if using + an in-memory remote queue. + :param queue_port: An int indicating what port should be used for the queue server, + if using an in-memory remote queue. """ @@ -100,13 +107,12 @@ def __init__( name: str, event_serializer: Callable[[T], bytes], queue_type: QueueType = QueueType.POSIX, + queue_host: str = DEFAULT_QUEUE_HOST, + queue_port: int = DEFAULT_QUEUE_PORT, ): - if queue_type == QueueType.IN_MEMORY: - self.queue = RemoteMessageQueue("/events-" + name, max_messages=MAX_QUEUE_SIZE) - else: - self.queue = PosixMessageQueue( # type: ignore - "/events-" + name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_EVENT_SIZE - ) + self.queue = create_queue( + queue_type, "/events-" + name, MAX_QUEUE_SIZE, MAX_EVENT_SIZE, queue_host, queue_port + ) self.serialize_event = event_serializer def put(self, event: T) -> None: @@ -133,6 +139,12 @@ def put(self, event: T) -> None: raise EventQueueFullError def get(self) -> bytes: + """Get an event from the queue. + + :returns bytes: The next event in the queue. + :raises: :py:exc:`TimedOutError` There were no elements in the queue. + + """ return self.queue.get() def make_object_for_context(self, name: str, span: Span) -> "EventQueue[T]": diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 32687b874..f5c631ca2 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -16,6 +16,9 @@ from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError +DEFAULT_QUEUE_HOST = "127.0.0.1" +DEFAULT_QUEUE_PORT = 9090 + class MessageQueueError(Exception): """Base exception for message queue related errors.""" @@ -155,19 +158,11 @@ class InMemoryMessageQueue(MessageQueue): Uses a simple Python Queue to store data. - Used in conjunction with the RemoteMessageQueue to - provide an alternative to Posix queues, for systems - that don't have Posix available. The client will - instantitate a RemoteMessageQueue, which connect to - a Thrift server. The Thrift server internally uses an - InMemoryMessageQueue to store data. - """ - def __init__(self, name: str, max_messages: int): + def __init__(self, max_messages: int): self.queue: q.Queue = q.Queue(max_messages) self.max_messages = max_messages - self.name = name def get(self, timeout: Optional[float] = None) -> bytes: try: @@ -191,12 +186,18 @@ class RemoteMessageQueue(MessageQueue): """A message queue that uses a remote Thrift server. Used in conjunction with the InMemoryMessageQueue to - provide an alternative to Posix queues, for systems - that don't have Posix available. + provide an alternative to Posix queues, for use-cases + where Posix is not appropriate. """ - def __init__(self, name: str, max_messages: int, host: str = "127.0.0.1", port: int = 9090): + def __init__( + self, + name: str, + max_messages: int, + host: str = DEFAULT_QUEUE_HOST, + port: int = DEFAULT_QUEUE_PORT, + ): # Connect to the remote queue server, and creeate the new queue self.name = name self.max_messages = max_messages @@ -213,29 +214,33 @@ def connect(self) -> None: self.client = RemoteMessageQueueService.Client(protocol) self.transport.open() + def _try_to_get(self, timeout: Optional[float]) -> bytes: + try: + return self.client.get(self.name, timeout).value + except TSocket.TTransportException: + self.connect() # Try reconnecting once + return self.client.get(self.name, timeout).value + def get(self, timeout: Optional[float] = None) -> bytes: # Call the remote server and get an element for the correct queue try: - try: - return self.client.get(self.name, timeout).value - except TSocket.TTransportException: - # Try reconnecting once, we dont want this as another top-level except because - # we may get a timeout after re-connecting, and we want to catch that - self.connect() - return self.client.get(self.name, timeout).value + return self._try_to_get(timeout) # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue except ThriftTimedOutError: raise TimedOutError + def _try_to_put(self, message: bytes, timeout: Optional[float]) -> None: + try: + self.client.put(self.name, message, timeout) + except TSocket.TTransportException: + self.connect() # Try reconnecting once + self.client.put(self.name, message, timeout) + def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Call the remote server and put an element on the correct queue # Will create the queue if it doesnt exist try: - try: - self.client.put(self.name, message, timeout) - except TSocket.TTransportException: # Try reconnecting once - self.connect() - self.client.put(self.name, message, timeout) + self._try_to_put(message, timeout) except ThriftTimedOutError: raise TimedOutError @@ -243,6 +248,23 @@ def close(self) -> None: self.transport.close() +def create_queue( + queue_type: QueueType, + queue_name: str, + max_messages: int, + max_message_size: int, + queue_host: str, + queue_port: int, +) -> MessageQueue: + if queue_type == QueueType.IN_MEMORY: + # Start a remote queue, which connects to a Thrift server + # that manages an in-memory queue + queue = RemoteMessageQueue(queue_name, max_messages, queue_host, queue_port) + else: + queue = PosixMessageQueue(queue_name, max_messages, max_message_size) # type: ignore + return queue + + def queue_tool() -> None: import argparse import sys @@ -263,10 +285,22 @@ def queue_tool() -> None: ) parser.add_argument("queue_name", help="the name of the queue to consume") parser.add_argument( - "--queue_type", + "--queue-type", default=QueueType.POSIX.value, choices=[qt.value for qt in QueueType], - help="whether to use an in-memory queue or a posix queue", + help="allows selection of the queue implementation", + ) + parser.add_argument( + "--queue-host", + type=str, + default=DEFAULT_QUEUE_HOST, + help="for a remote queue, what host to use", + ) + parser.add_argument( + "--queue-port", + type=int, + default=DEFAULT_QUEUE_PORT, + help="for a remote queue, what port to use", ) group = parser.add_mutually_exclusive_group(required=True) @@ -294,14 +328,14 @@ def queue_tool() -> None: args = parser.parse_args() - if args.queue_type == QueueType.IN_MEMORY.value: - # Start a remote queue, which connects to a Thrift server - # that manages an in-memory queue - queue = RemoteMessageQueue(args.queue_name, args.max_messages) - else: - queue = PosixMessageQueue( # type: ignore - args.queue_name, args.max_messages, args.max_message_size - ) + queue = create_queue( + args.queue_type, + args.queue_name, + args.max_messages, + args.max_message_size, + args.queue_host, + args.queue_port, + ) if args.mode == "read": while True: diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index dfecba128..a50567e8e 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -29,9 +29,10 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import create_queue +from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST +from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import QueueType -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout @@ -547,20 +548,16 @@ class SidecarRecorder(Recorder): adding them to the queue. """ - def __init__(self, queue_name: str, queue_type: QueueType = QueueType.POSIX): - if queue_type == QueueType.IN_MEMORY: - # Start a remote queue, which connects to a Thrift server - # that manages an in-memory queue - self.queue = RemoteMessageQueue( - "/traces-" + queue_name, - max_messages=MAX_QUEUE_SIZE, - ) - else: - self.queue = PosixMessageQueue( # type: ignore - "/traces-" + queue_name, - max_messages=MAX_QUEUE_SIZE, - max_message_size=MAX_SPAN_SIZE, - ) + def __init__( + self, + queue_name: str, + queue_type: QueueType = QueueType.POSIX, + host: str = DEFAULT_QUEUE_HOST, + port: int = DEFAULT_QUEUE_PORT, + ): + self.queue = create_queue( + queue_type, "/traves-" + queue_name, MAX_QUEUE_SIZE, MAX_SPAN_SIZE, host, port + ) def send(self, span: TraceSpanObserver) -> None: # Don't raise exceptions from here. This is called in the @@ -571,7 +568,7 @@ def send(self, span: TraceSpanObserver) -> None: "Trace too big. Traces published to %s are not allowed to be larger " "than %d bytes. Received trace is %d bytes. This can be caused by " "an excess amount of tags or a large amount of child spans.", - self.queue.name, + self.queue.name, # type: ignore MAX_SPAN_SIZE, len(serialized_str), ) @@ -579,7 +576,7 @@ def send(self, span: TraceSpanObserver) -> None: try: self.queue.put(serialized_str, timeout=0) except TimedOutError: - logger.warning("Trace queue %s is full. Is trace sidecar healthy?", self.queue.name) + logger.warning("Trace queue %s is full. Is trace sidecar healthy?", self.queue.name) # type: ignore def tracing_client_from_config( diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 993263a39..7fd3d7a52 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -17,6 +17,8 @@ from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE +from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST +from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError @@ -230,7 +232,7 @@ def publish_events() -> None: "key": {"name": config.String, "secret": config.Base64}, "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), "max_element_size": config.Optional(config.Integer, MAX_EVENT_SIZE), - "queue_type": config.Optional(config.String, default="posix"), + "queue_type": config.Optional(config.String, default=QueueType.POSIX.value), }, ) @@ -248,7 +250,9 @@ def publish_events() -> None: if cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server( + host=DEFAULT_QUEUE_HOST, port=DEFAULT_QUEUE_PORT + ): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) else: diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index f0661dcec..3960743a3 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,3 +1,4 @@ +"""Shared functions for the event & trace publisher sidecars and message queues.""" import contextlib from typing import Dict @@ -9,6 +10,8 @@ from gevent.server import StreamServer from baseplate.lib import config +from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST +from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue @@ -25,17 +28,19 @@ class RemoteMessageQueueHandler: - """Create an InMemoryMessageQueue locally and expose get/put methods.""" + """Create an InMemoryMessageQueue locally and expose get/put methods. - def is_healthy(self) -> bool: - pass + See this overview for more details: + https://docs.google.com/document/d/1soN0UP9P12u3ByUwH_t47Uw9GwdVZRDuRFT0MvR52Dk/ + + """ def __init__(self) -> None: # Store the queue by name with its max messages self.queues: Dict[str, MessageQueue] = {} def create_queue(self, queue_name: str, max_messages: int) -> CreateResponse: - queue = InMemoryMessageQueue(queue_name, max_messages) + queue = InMemoryMessageQueue(max_messages) self.queues[queue_name] = queue return CreateResponse() @@ -81,9 +86,11 @@ def create_queue( queue_full_name: str, max_queue_size: int, max_element_size: int, - host: str = "127.0.0.1", - port: int = 9090, + host: str = DEFAULT_QUEUE_HOST, + port: int = DEFAULT_QUEUE_PORT, ) -> MessageQueue: + # See this overview for the relationship between InMemoryMessageQueues & RemoteMessageQueues + # https://docs.google.com/document/d/1soN0UP9P12u3ByUwH_t47Uw9GwdVZRDuRFT0MvR52Dk/ if queue_type == QueueType.IN_MEMORY: event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index b02917311..310458eb4 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -157,7 +157,7 @@ def publish_traces() -> None: "--queue-type", default=QueueType.POSIX.value, choices=[qt.value for qt in QueueType], - help="whether to use an in-memory queue or a posix queue", + help="allows selection of the queue implementation", ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 142030234..1cd3952a6 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -93,13 +93,13 @@ class TestInMemoryMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" def test_create_queue(self): - message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: self.assertEqual(mq.queue.maxsize, 1) def test_put_get(self): - message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: mq.put(b"x") @@ -107,7 +107,7 @@ def test_put_get(self): self.assertEqual(message, b"x") def test_get_timeout(self): - message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: start = time.time() @@ -117,7 +117,7 @@ def test_get_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=2) def test_put_timeout(self): - message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: mq.put(b"x") @@ -128,7 +128,7 @@ def test_put_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=1) def test_put_zero_timeout(self): - message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: mq.put(b"x", timeout=0) @@ -136,7 +136,7 @@ def test_put_zero_timeout(self): self.assertEqual(message, b"x") def test_put_full_zero_timeout(self): - message_queue = InMemoryMessageQueue(self.qname, max_messages=1) + message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: mq.put(b"1", timeout=0) @@ -215,7 +215,7 @@ def test_put_timeout(self): with self.assertRaises(TimedOutError): mq.put(b"x", timeout=0.1) elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=2) + self.assertAlmostEqual(elapsed, 0.1, places=1) def test_thrift_retry(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 58751289e..07e359780 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -13,7 +13,7 @@ class PosixEventQueueTests(unittest.TestCase): - @mock.patch("baseplate.lib.events.PosixMessageQueue", autospec=PosixMessageQueue) + @mock.patch("baseplate.lib.message_queue.PosixMessageQueue", autospec=PosixMessageQueue) def setUp(self, PosixMessageQueue): self.message_queue = PosixMessageQueue.return_value self.mock_serializer = mock.Mock() @@ -46,7 +46,7 @@ def test_event_queue_full(self): class RemoteMessageQueueTests(unittest.TestCase): - @mock.patch("baseplate.lib.events.RemoteMessageQueue", autospec=RemoteMessageQueue) + @mock.patch("baseplate.lib.message_queue.RemoteMessageQueue", autospec=RemoteMessageQueue) def setUp(self, RemoteMessageQueue): self.message_queue = RemoteMessageQueue.return_value self.mock_serializer = mock.Mock() From 4c4c93817235d1978399461ca938933297045e5f Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 24 Jan 2023 15:23:06 -0800 Subject: [PATCH 26/72] EventQueue takes n optionala queue --- baseplate/lib/events.py | 18 +++++-------- baseplate/lib/message_queue.py | 27 ++++++++++++------- baseplate/observers/tracing.py | 18 +++++-------- baseplate/sidecars/event_publisher.py | 3 ++- baseplate/sidecars/publisher_queue_utils.py | 23 ---------------- tests/integration/message_queue_tests.py | 20 ++++++++++++-- .../sidecars/publisher_queue_utils_tests.py | 20 +++----------- tests/unit/lib/events/queue_tests.py | 15 ++++++++--- 8 files changed, 66 insertions(+), 78 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index a085b5195..986e24103 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -11,7 +11,7 @@ """ import logging -from typing import Any +from typing import Any, Optional, Union from typing import Callable from typing import Generic from typing import TypeVar @@ -22,10 +22,8 @@ from baseplate import Span from baseplate.clients import ContextFactory from baseplate.lib import config -from baseplate.lib.message_queue import create_queue -from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST -from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT -from baseplate.lib.message_queue import QueueType +from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -106,13 +104,11 @@ def __init__( self, name: str, event_serializer: Callable[[T], bytes], - queue_type: QueueType = QueueType.POSIX, - queue_host: str = DEFAULT_QUEUE_HOST, - queue_port: int = DEFAULT_QUEUE_PORT, + queue: Optional[MessageQueue] = None, ): - self.queue = create_queue( - queue_type, "/events-" + name, MAX_QUEUE_SIZE, MAX_EVENT_SIZE, queue_host, queue_port - ) + self.queue = queue + if not queue: + self.queue = PosixMessageQueue("/events-" + name, MAX_QUEUE_SIZE, MAX_EVENT_SIZE) self.serialize_event = event_serializer def put(self, event: T) -> None: diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index f5c631ca2..730acf700 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -5,6 +5,7 @@ from enum import Enum from typing import Optional +# from baseplate.sidecars.publisher_queue_utils import create_queue import posix_ipc @@ -250,19 +251,25 @@ def close(self) -> None: def create_queue( queue_type: QueueType, - queue_name: str, - max_messages: int, - max_message_size: int, - queue_host: str, - queue_port: int, + queue_full_name: str, + max_queue_size: int, + max_element_size: int, + host: str = DEFAULT_QUEUE_HOST, + port: int = DEFAULT_QUEUE_PORT, ) -> MessageQueue: + # See this overview for the relationship between InMemoryMessageQueues & RemoteMessageQueues + # https://docs.google.com/document/d/1soN0UP9P12u3ByUwH_t47Uw9GwdVZRDuRFT0MvR52Dk/ if queue_type == QueueType.IN_MEMORY: - # Start a remote queue, which connects to a Thrift server - # that manages an in-memory queue - queue = RemoteMessageQueue(queue_name, max_messages, queue_host, queue_port) + event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) + else: - queue = PosixMessageQueue(queue_name, max_messages, max_message_size) # type: ignore - return queue + event_queue = PosixMessageQueue( # type: ignore + queue_full_name, + max_messages=max_queue_size, + max_message_size=max_element_size, + ) + + return event_queue def queue_tool() -> None: diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index a50567e8e..d3f63874a 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -10,7 +10,7 @@ import typing from datetime import datetime -from typing import Any +from typing import Any, Union from typing import DefaultDict from typing import Dict from typing import List @@ -29,10 +29,8 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import create_queue -from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST -from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT -from baseplate.lib.message_queue import QueueType +from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout @@ -551,13 +549,11 @@ class SidecarRecorder(Recorder): def __init__( self, queue_name: str, - queue_type: QueueType = QueueType.POSIX, - host: str = DEFAULT_QUEUE_HOST, - port: int = DEFAULT_QUEUE_PORT, + queue: Union[MessageQueue, None], ): - self.queue = create_queue( - queue_type, "/traves-" + queue_name, MAX_QUEUE_SIZE, MAX_SPAN_SIZE, host, port - ) + self.queue = queue + if not queue: + self.queue = PosixMessageQueue("/traces-" + queue_name, MAX_QUEUE_SIZE, MAX_SPAN_SIZE) def send(self, span: TraceSpanObserver) -> None: # Don't raise exceptions from here. This is called in the diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 7fd3d7a52..ce9b4e331 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -17,6 +17,7 @@ from baseplate.lib import metrics from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE +from baseplate.lib.message_queue import create_queue from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import MessageQueue @@ -238,7 +239,7 @@ def publish_events() -> None: metrics_client = metrics_client_from_config(raw_config) - event_queue: MessageQueue = publisher_queue_utils.create_queue( + event_queue: MessageQueue = create_queue( cfg.queue_type, f"/events-{args.queue_name}", cfg.max_queue_size, cfg.max_element_size ) diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 3960743a3..ec64de2f0 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -79,26 +79,3 @@ def start_queue_server(host: str, port: int) -> Generator[StreamServer, None, No yield server finally: server_greenlet.kill() - - -def create_queue( - queue_type: QueueType, - queue_full_name: str, - max_queue_size: int, - max_element_size: int, - host: str = DEFAULT_QUEUE_HOST, - port: int = DEFAULT_QUEUE_PORT, -) -> MessageQueue: - # See this overview for the relationship between InMemoryMessageQueues & RemoteMessageQueues - # https://docs.google.com/document/d/1soN0UP9P12u3ByUwH_t47Uw9GwdVZRDuRFT0MvR52Dk/ - if queue_type == QueueType.IN_MEMORY: - event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) - - else: - event_queue = PosixMessageQueue( # type: ignore - queue_full_name, - max_messages=max_queue_size, - max_message_size=max_element_size, - ) - - return event_queue diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 1cd3952a6..3df3b0ac9 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -7,8 +7,11 @@ import gevent import posix_ipc +from baseplate.lib.message_queue import create_queue from baseplate.lib.message_queue import InMemoryMessageQueue +from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import PosixMessageQueue +from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.sidecars import publisher_queue_utils @@ -26,7 +29,7 @@ def setUp(self): queue.unlink() queue.close() - def test_create_queue(self): + def test_instantiate_queue(self): message_queue = PosixMessageQueue(self.qname, max_messages=1, max_message_size=1000) with contextlib.closing(message_queue) as mq: @@ -92,7 +95,7 @@ def tearDown(self): class TestInMemoryMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" - def test_create_queue(self): + def test_instantiate_queue(self): message_queue = InMemoryMessageQueue(max_messages=1) with contextlib.closing(message_queue) as mq: @@ -240,3 +243,16 @@ def test_get_thrift_retry_and_timeout(self): mq.get(timeout=0.1) elapsed = time.time() - start self.assertAlmostEqual(elapsed, 0.1, places=1) + + +class TestCreateQueue(GeventPatchedTestCase): + def test_posix_queue(self): + queue: MessageQueue = create_queue(QueueType.POSIX, "/test", 5, 1000) + assert isinstance(queue, PosixMessageQueue) + + def test_in_memory_create_queue(self): + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + queue: MessageQueue = create_queue( + QueueType.IN_MEMORY, "/test", 5, 1000 + ) + assert isinstance(queue, RemoteMessageQueue) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index b4ae35dfc..5eb92d80c 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -4,10 +4,9 @@ import gevent.monkey +from baseplate.lib.message_queue import create_queue from baseplate.lib.message_queue import MessageQueue -from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import QueueType -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.sidecars import publisher_queue_utils @@ -23,12 +22,8 @@ def tearDown(self): class PublisherQueueUtilTests(GeventPatchedTestCase): - def test_posix_queue(self): - queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.POSIX, "/test", 5, 1000) - assert isinstance(queue, PosixMessageQueue) - def test_posix_queue_get_put(self): - queue: MessageQueue = publisher_queue_utils.create_queue(QueueType.POSIX, "/test", 5, 1000) + queue: MessageQueue = create_queue(QueueType.POSIX, "/test", 5, 1000) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") @@ -39,16 +34,9 @@ def test_posix_queue_get_put(self): output = queue.get() assert output == test_message_2 - def test_in_memory_create_queue(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue: MessageQueue = publisher_queue_utils.create_queue( - QueueType.IN_MEMORY, "/test", 5, 1000 - ) - assert isinstance(queue, RemoteMessageQueue) - def test_in_memory_queue_get_put(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue: MessageQueue = publisher_queue_utils.create_queue( + queue: MessageQueue = create_queue( QueueType.IN_MEMORY, "/test", 5, 1000 ) @@ -63,7 +51,7 @@ def test_in_memory_queue_get_put(self): def test_in_memory_queue_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): - queue: MessageQueue = publisher_queue_utils.create_queue( + queue: MessageQueue = create_queue( QueueType.IN_MEMORY, "/test", 5, 1000, port=9091 ) diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 07e359780..04278a14d 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -2,12 +2,11 @@ from unittest import mock -from baseplate.lib.events import EventQueue +from baseplate.lib.events import MAX_QUEUE_SIZE, EventQueue from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.message_queue import PosixMessageQueue -from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -17,7 +16,8 @@ class PosixEventQueueTests(unittest.TestCase): def setUp(self, PosixMessageQueue): self.message_queue = PosixMessageQueue.return_value self.mock_serializer = mock.Mock() - self.queue = EventQueue("test", event_serializer=self.mock_serializer) + queue = PosixMessageQueue("/test", 10, 100) + self.queue = EventQueue("test", event_serializer=self.mock_serializer, queue=queue) def test_send_event(self): self.mock_serializer.return_value = "i_am_serialized" @@ -44,14 +44,21 @@ def test_event_queue_full(self): with self.assertRaises(EventQueueFullError): self.queue.put(object()) + @mock.patch("baseplate.lib.events.MAX_QUEUE_SIZE", 10) + @mock.patch("baseplate.lib.events.MAX_EVENT_SIZE", 100) + def test_default_queue(self): + queue = EventQueue("test", event_serializer=self.mock_serializer) + assert isinstance(queue.queue, PosixMessageQueue) + class RemoteMessageQueueTests(unittest.TestCase): @mock.patch("baseplate.lib.message_queue.RemoteMessageQueue", autospec=RemoteMessageQueue) def setUp(self, RemoteMessageQueue): self.message_queue = RemoteMessageQueue.return_value self.mock_serializer = mock.Mock() + queue = RemoteMessageQueue("test", MAX_QUEUE_SIZE, "127.0.0.1", 9090) self.queue = EventQueue( - "test", event_serializer=self.mock_serializer, queue_type=QueueType.IN_MEMORY + "test", event_serializer=self.mock_serializer, queue=queue ) def test_send_event(self): From 95a225e7f206a5fe07b1ce9808e0fc5f60475ff8 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 13 Feb 2023 11:25:40 -0800 Subject: [PATCH 27/72] clarify comments --- baseplate/lib/events.py | 9 +++------ baseplate/lib/message_queue.py | 14 +++++++++----- baseplate/sidecars/publisher_queue_utils.py | 6 ++++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index 986e24103..f99500e94 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -91,12 +91,9 @@ class EventQueue(ContextFactory, config.Parser, Generic[T]): :param event_serializer: A callable that takes an event object and returns serialized bytes ready to send on the wire. See below for options. - :param queue_type: A QueueType indicating which type of queue the publisher - should use. - :param queue_host: A string indicating the hostname of the queue server, if using - an in-memory remote queue. - :param queue_port: An int indicating what port should be used for the queue server, - if using an in-memory remote queue. + :param queue: An optional MessageQueue that will be used for queueing and + publishing messages. If no queue is provided, a PosixMessageQueue will + be used. """ diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 730acf700..83e4baf50 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -186,9 +186,13 @@ def close(self) -> None: class RemoteMessageQueue(MessageQueue): """A message queue that uses a remote Thrift server. - Used in conjunction with the InMemoryMessageQueue to - provide an alternative to Posix queues, for use-cases - where Posix is not appropriate. + Used in conjunction with the InMemoryMessageQueue to provide an alternative + to Posix queues, for use-cases where Posix is not appropriate. + + This implementation is a temporary compromise and should only be used + under very specific circumstances if the POSIX alternative is unavailable. + Specifically, using Thrift here has significant performance and/or + resource impacts. """ @@ -257,8 +261,8 @@ def create_queue( host: str = DEFAULT_QUEUE_HOST, port: int = DEFAULT_QUEUE_PORT, ) -> MessageQueue: - # See this overview for the relationship between InMemoryMessageQueues & RemoteMessageQueues - # https://docs.google.com/document/d/1soN0UP9P12u3ByUwH_t47Uw9GwdVZRDuRFT0MvR52Dk/ + # The in-memory queue is initialized on the sidecar, so we use a remote queue + # from the main baseplate application to interact with it. if queue_type == QueueType.IN_MEMORY: event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index ec64de2f0..89303ca3f 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -30,8 +30,10 @@ class RemoteMessageQueueHandler: """Create an InMemoryMessageQueue locally and expose get/put methods. - See this overview for more details: - https://docs.google.com/document/d/1soN0UP9P12u3ByUwH_t47Uw9GwdVZRDuRFT0MvR52Dk/ + This implementation is a temporary compromise and should only be used + under very specific circumstances if the POSIX alternative is unavailable. + Specifically, using Thrift here has significant performance and/or + resource impacts. """ From 018134eda939346e97cc86625d2cd438ab55fc3b Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 13 Feb 2023 12:22:34 -0800 Subject: [PATCH 28/72] linting --- baseplate/lib/events.py | 9 ++++++--- baseplate/lib/message_queue.py | 2 +- baseplate/observers/tracing.py | 14 ++++++++------ baseplate/sidecars/trace_publisher.py | 3 ++- tests/integration/message_queue_tests.py | 4 +--- .../sidecars/publisher_queue_utils_tests.py | 8 ++------ tests/unit/lib/events/queue_tests.py | 7 +++---- 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/baseplate/lib/events.py b/baseplate/lib/events.py index f99500e94..c16eb3b97 100644 --- a/baseplate/lib/events.py +++ b/baseplate/lib/events.py @@ -11,9 +11,10 @@ """ import logging -from typing import Any, Optional, Union +from typing import Any from typing import Callable from typing import Generic +from typing import Optional from typing import TypeVar from thrift import TSerialization @@ -103,9 +104,11 @@ def __init__( event_serializer: Callable[[T], bytes], queue: Optional[MessageQueue] = None, ): - self.queue = queue - if not queue: + if queue: + self.queue = queue + else: self.queue = PosixMessageQueue("/events-" + name, MAX_QUEUE_SIZE, MAX_EVENT_SIZE) + self.serialize_event = event_serializer def put(self, event: T) -> None: diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 83e4baf50..2d7430378 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -5,7 +5,6 @@ from enum import Enum from typing import Optional -# from baseplate.sidecars.publisher_queue_utils import create_queue import posix_ipc @@ -55,6 +54,7 @@ class QueueType(Enum): class MessageQueue(abc.ABC): """Abstract class for an inter-process message queue.""" + name: str @abc.abstractmethod def get(self, timeout: Optional[float] = None) -> bytes: diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index d3f63874a..3ab1a8591 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -10,12 +10,13 @@ import typing from datetime import datetime -from typing import Any, Union +from typing import Any from typing import DefaultDict from typing import Dict from typing import List from typing import NamedTuple from typing import Optional +from typing import Union import requests @@ -549,10 +550,11 @@ class SidecarRecorder(Recorder): def __init__( self, queue_name: str, - queue: Union[MessageQueue, None], + queue: Optional[MessageQueue] = None, ): - self.queue = queue - if not queue: + if queue: + self.queue = queue + else: self.queue = PosixMessageQueue("/traces-" + queue_name, MAX_QUEUE_SIZE, MAX_SPAN_SIZE) def send(self, span: TraceSpanObserver) -> None: @@ -564,7 +566,7 @@ def send(self, span: TraceSpanObserver) -> None: "Trace too big. Traces published to %s are not allowed to be larger " "than %d bytes. Received trace is %d bytes. This can be caused by " "an excess amount of tags or a large amount of child spans.", - self.queue.name, # type: ignore + self.queue.name, MAX_SPAN_SIZE, len(serialized_str), ) @@ -572,7 +574,7 @@ def send(self, span: TraceSpanObserver) -> None: try: self.queue.put(serialized_str, timeout=0) except TimedOutError: - logger.warning("Trace queue %s is full. Is trace sidecar healthy?", self.queue.name) # type: ignore + logger.warning("Trace queue %s is full. Is trace sidecar healthy?", self.queue.name) def tracing_client_from_config( diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 310458eb4..363e5cee6 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -10,6 +10,7 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config from baseplate.lib import metrics +from baseplate.lib import message_queue from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError @@ -194,7 +195,7 @@ def publish_traces() -> None: }, ) - trace_queue: MessageQueue = publisher_queue_utils.create_queue( + trace_queue: MessageQueue = message_queue.create_queue( publisher_cfg.queue_type, "/traces-" + args.queue_name, publisher_cfg.max_queue_size, diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 3df3b0ac9..c2d9f9f19 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -252,7 +252,5 @@ def test_posix_queue(self): def test_in_memory_create_queue(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue: MessageQueue = create_queue( - QueueType.IN_MEMORY, "/test", 5, 1000 - ) + queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000) assert isinstance(queue, RemoteMessageQueue) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 5eb92d80c..510429f45 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -36,9 +36,7 @@ def test_posix_queue_get_put(self): def test_in_memory_queue_get_put(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue: MessageQueue = create_queue( - QueueType.IN_MEMORY, "/test", 5, 1000 - ) + queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") @@ -51,9 +49,7 @@ def test_in_memory_queue_get_put(self): def test_in_memory_queue_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): - queue: MessageQueue = create_queue( - QueueType.IN_MEMORY, "/test", 5, 1000, port=9091 - ) + queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000, port=9091) test_message = bytes("message", "utf-8") test_message_2 = bytes("2nd message", "utf-8") diff --git a/tests/unit/lib/events/queue_tests.py b/tests/unit/lib/events/queue_tests.py index 04278a14d..4f8217398 100644 --- a/tests/unit/lib/events/queue_tests.py +++ b/tests/unit/lib/events/queue_tests.py @@ -2,10 +2,11 @@ from unittest import mock -from baseplate.lib.events import MAX_QUEUE_SIZE, EventQueue +from baseplate.lib.events import EventQueue from baseplate.lib.events import EventQueueFullError from baseplate.lib.events import EventTooLargeError from baseplate.lib.events import MAX_EVENT_SIZE +from baseplate.lib.events import MAX_QUEUE_SIZE from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError @@ -57,9 +58,7 @@ def setUp(self, RemoteMessageQueue): self.message_queue = RemoteMessageQueue.return_value self.mock_serializer = mock.Mock() queue = RemoteMessageQueue("test", MAX_QUEUE_SIZE, "127.0.0.1", 9090) - self.queue = EventQueue( - "test", event_serializer=self.mock_serializer, queue=queue - ) + self.queue = EventQueue("test", event_serializer=self.mock_serializer, queue=queue) def test_send_event(self): self.mock_serializer.return_value = "i_am_serialized" From 3ba56487ed38ddd5d89d46cb27ab8fbefddd378a Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 13 Feb 2023 12:40:39 -0800 Subject: [PATCH 29/72] linting --- baseplate/observers/tracing.py | 2 +- baseplate/sidecars/trace_publisher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 3ab1a8591..5ff935495 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -30,8 +30,8 @@ from baseplate import SpanObserver from baseplate.lib import config from baseplate.lib import warn_deprecated -from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import MessageQueue +from baseplate.lib.message_queue import PosixMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.observers.timeout import ServerTimeout diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 363e5cee6..2e23d0014 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -9,8 +9,8 @@ from baseplate import __version__ as baseplate_version from baseplate.lib import config -from baseplate.lib import metrics from baseplate.lib import message_queue +from baseplate.lib import metrics from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError From e1345b02b6f8e0daa84cfa4e80663266a85d5abc Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 13 Feb 2023 12:46:05 -0800 Subject: [PATCH 30/72] linting --- baseplate/lib/message_queue.py | 1 + baseplate/observers/tracing.py | 1 - baseplate/sidecars/publisher_queue_utils.py | 5 ----- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 2d7430378..bf67120d1 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -54,6 +54,7 @@ class QueueType(Enum): class MessageQueue(abc.ABC): """Abstract class for an inter-process message queue.""" + name: str @abc.abstractmethod diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index 5ff935495..d211b2202 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -16,7 +16,6 @@ from typing import List from typing import NamedTuple from typing import Optional -from typing import Union import requests diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 89303ca3f..2b1a1f36d 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -10,13 +10,8 @@ from gevent.server import StreamServer from baseplate.lib import config -from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST -from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import MessageQueue -from baseplate.lib.message_queue import PosixMessageQueue -from baseplate.lib.message_queue import QueueType -from baseplate.lib.message_queue import RemoteMessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.server import make_listener from baseplate.server.thrift import make_server From 276e768904f399af041caa45b250b1bb218fbfbb Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 13 Feb 2023 12:59:26 -0800 Subject: [PATCH 31/72] linting :sob: --- baseplate/observers/tracing.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/baseplate/observers/tracing.py b/baseplate/observers/tracing.py index d211b2202..9c6992856 100644 --- a/baseplate/observers/tracing.py +++ b/baseplate/observers/tracing.py @@ -2,7 +2,6 @@ import collections import json import logging -import queue import random import socket import threading @@ -10,6 +9,8 @@ import typing from datetime import datetime +from queue import Empty +from queue import Queue from typing import Any from typing import DefaultDict from typing import Dict @@ -36,9 +37,9 @@ if typing.TYPE_CHECKING: - SpanQueue = queue.Queue["TraceSpanObserver"] # pylint: disable=unsubscriptable-object + SpanQueue = Queue["TraceSpanObserver"] # pylint: disable=unsubscriptable-object else: - SpanQueue = queue.Queue + SpanQueue = Queue logger = logging.getLogger(__name__) @@ -422,7 +423,7 @@ class BaseBatchRecorder(Recorder): def __init__( self, max_queue_size: int, num_workers: int, max_span_batch: int, batch_wait_interval: float ): - self.span_queue: SpanQueue = queue.Queue(maxsize=max_queue_size) + self.span_queue: SpanQueue = Queue(maxsize=max_queue_size) self.batch_wait_interval = batch_wait_interval self.max_span_batch = max_span_batch self.logger = logging.getLogger(self.__class__.__name__) @@ -445,7 +446,7 @@ def _flush_spans(self) -> None: try: while len(spans) < self.max_span_batch: spans.append(self.span_queue.get_nowait()._serialize()) - except queue.Empty: + except Empty: pass finally: if spans: From 50226db4e7da79eefd91398b997b841a7b2bd50c Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 22 Feb 2023 16:47:35 -0800 Subject: [PATCH 32/72] wip need to test metrics --- baseplate/lib/message_queue.py | 106 ++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 3 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index bf67120d1..068386c15 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -4,21 +4,26 @@ import select from enum import Enum +import time from typing import Optional +from prometheus_client import Gauge +from prometheus_client import Histogram +import gevent import posix_ipc from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport +from baseplate.lib.prometheus_metrics import default_latency_buckets from baseplate.lib.retry import RetryPolicy from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError DEFAULT_QUEUE_HOST = "127.0.0.1" DEFAULT_QUEUE_PORT = 9090 - +PROM_PREFIX = "message_queue" class MessageQueueError(Exception): """Base exception for message queue related errors.""" @@ -197,6 +202,42 @@ class RemoteMessageQueue(MessageQueue): """ + prom_labels = [ + "timeout", + "queue_name", + "queue_host", + "queue_port", + "queue_max_messages", + ] + + remote_queue_gets_queued = Gauge( + f"{PROM_PREFIX}_pending_puts_to_sidecar", + "total number of get requests in flight, being sent to the publishing sidecar.", + prom_labels, + multiprocess_mode="livesum", + ) + + remote_queue_puts_queued = Gauge( + f"{PROM_PREFIX}_pending_gets_to_sidecar", + "total number of put requests in flight, being sent to the publishing sidecar.", + prom_labels, + multiprocess_mode="livesum", + ) + + remote_queue_put_latency = Histogram( + f"{PROM_PREFIX}_sidecar_put_latency", + "latency of message `put` to the publishing sidecar.", + prom_labels, + buckets=default_latency_buckets, + ) + + remote_queue_get_latency = Histogram( + f"{PROM_PREFIX}_sidecar_get_latency", + "latency of message `get` to the publishing sidecar.", + prom_labels, + buckets=default_latency_buckets, + ) + def __init__( self, name: str, @@ -230,7 +271,35 @@ def _try_to_get(self, timeout: Optional[float]) -> bytes: def get(self, timeout: Optional[float] = None) -> bytes: # Call the remote server and get an element for the correct queue try: - return self._try_to_get(timeout) + start_time = time.perf_counter() + self.remote_queue_gets_queued.labels( + timeout=timeout, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages + ).inc() + + greenlet = gevent.spawn(self._try_to_get, timeout) + gevent.joinall([greenlet]) + + self.remote_queue_get_latency.labels( + timeout=timeout, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages + ).observe(time.perf_counter() - start_time) + + self.remote_queue_gets_queued.labels( + timeout=timeout, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages + ).dec() + + return greenlet.get() # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue except ThriftTimedOutError: raise TimedOutError @@ -246,7 +315,38 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Call the remote server and put an element on the correct queue # Will create the queue if it doesnt exist try: - self._try_to_put(message, timeout) + # increment in-flight counter + start_time = time.perf_counter() + self.remote_queue_puts_queued.labels( + timeout=timeout, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages + ).inc() + + greenlet = gevent.spawn(self._try_to_put, message, timeout) + gevent.joinall([greenlet]) + + # report latency and decrement in-flight counter + self.remote_queue_put_latency.labels( + timeout=timeout, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages + ).observe(time.perf_counter() - start_time) + + self.remote_queue_puts_queued.labels( + timeout=timeout, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages + ).dec() + + # reraise any exceptions encountered in processing + greenlet.get() except ThriftTimedOutError: raise TimedOutError From e28ed06453190fa162dc7edab8f544c08e74a59d Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 23 Feb 2023 18:23:49 -0800 Subject: [PATCH 33/72] move timer start --- baseplate/lib/message_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 068386c15..1087c99db 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -271,7 +271,6 @@ def _try_to_get(self, timeout: Optional[float]) -> bytes: def get(self, timeout: Optional[float] = None) -> bytes: # Call the remote server and get an element for the correct queue try: - start_time = time.perf_counter() self.remote_queue_gets_queued.labels( timeout=timeout, queue_name=self.name, @@ -279,6 +278,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: queue_port=self.port, queue_max_messages=self.max_messages ).inc() + start_time = time.perf_counter() greenlet = gevent.spawn(self._try_to_get, timeout) gevent.joinall([greenlet]) @@ -316,7 +316,6 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: # Will create the queue if it doesnt exist try: # increment in-flight counter - start_time = time.perf_counter() self.remote_queue_puts_queued.labels( timeout=timeout, queue_name=self.name, @@ -324,6 +323,7 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: queue_port=self.port, queue_max_messages=self.max_messages ).inc() + start_time = time.perf_counter() greenlet = gevent.spawn(self._try_to_put, message, timeout) gevent.joinall([greenlet]) From 7f6538b7e042a7af58156680c4ee04c7fa5775ee Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 23 Feb 2023 18:26:51 -0800 Subject: [PATCH 34/72] import order --- baseplate/lib/message_queue.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 1087c99db..37f646c18 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -2,16 +2,17 @@ import abc import queue as q import select +import time from enum import Enum -import time from typing import Optional -from prometheus_client import Gauge -from prometheus_client import Histogram import gevent import posix_ipc +from prometheus_client import Gauge +from prometheus_client import Histogram + from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport From 406161f887a4760aa32a8ab13cde06b142a76eb9 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 23 Feb 2023 18:33:56 -0800 Subject: [PATCH 35/72] linting --- baseplate/lib/message_queue.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 37f646c18..6541090bc 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -12,7 +12,6 @@ from prometheus_client import Gauge from prometheus_client import Histogram - from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport @@ -26,6 +25,7 @@ DEFAULT_QUEUE_PORT = 9090 PROM_PREFIX = "message_queue" + class MessageQueueError(Exception): """Base exception for message queue related errors.""" @@ -204,7 +204,7 @@ class RemoteMessageQueue(MessageQueue): """ prom_labels = [ - "timeout", + "timeout", "queue_name", "queue_host", "queue_port", @@ -277,19 +277,19 @@ def get(self, timeout: Optional[float] = None) -> bytes: queue_name=self.name, queue_host=self.host, queue_port=self.port, - queue_max_messages=self.max_messages + queue_max_messages=self.max_messages, ).inc() start_time = time.perf_counter() greenlet = gevent.spawn(self._try_to_get, timeout) gevent.joinall([greenlet]) - + self.remote_queue_get_latency.labels( timeout=timeout, queue_name=self.name, queue_host=self.host, queue_port=self.port, - queue_max_messages=self.max_messages + queue_max_messages=self.max_messages, ).observe(time.perf_counter() - start_time) self.remote_queue_gets_queued.labels( @@ -297,7 +297,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: queue_name=self.name, queue_host=self.host, queue_port=self.port, - queue_max_messages=self.max_messages + queue_max_messages=self.max_messages, ).dec() return greenlet.get() @@ -322,20 +322,20 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: queue_name=self.name, queue_host=self.host, queue_port=self.port, - queue_max_messages=self.max_messages + queue_max_messages=self.max_messages, ).inc() start_time = time.perf_counter() greenlet = gevent.spawn(self._try_to_put, message, timeout) gevent.joinall([greenlet]) - + # report latency and decrement in-flight counter self.remote_queue_put_latency.labels( timeout=timeout, queue_name=self.name, queue_host=self.host, queue_port=self.port, - queue_max_messages=self.max_messages + queue_max_messages=self.max_messages, ).observe(time.perf_counter() - start_time) self.remote_queue_puts_queued.labels( @@ -343,7 +343,7 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: queue_name=self.name, queue_host=self.host, queue_port=self.port, - queue_max_messages=self.max_messages + queue_max_messages=self.max_messages, ).dec() # reraise any exceptions encountered in processing From 5a459a924b04ee548cb28d5397c8214d8a8ba024 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 27 Feb 2023 10:14:06 -0800 Subject: [PATCH 36/72] wip --- baseplate/lib/message_queue.py | 190 ++++++++++++----------- tests/integration/message_queue_tests.py | 30 +++- 2 files changed, 125 insertions(+), 95 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 6541090bc..71ccf87c7 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -5,11 +5,12 @@ import time from enum import Enum -from typing import Optional +from typing import Any, Optional import gevent import posix_ipc +from prometheus_client import Counter from prometheus_client import Gauge from prometheus_client import Histogram from thrift.protocol import TBinaryProtocol @@ -204,37 +205,29 @@ class RemoteMessageQueue(MessageQueue): """ prom_labels = [ - "timeout", + "mode", # put or get "queue_name", "queue_host", "queue_port", "queue_max_messages", ] - remote_queue_gets_queued = Gauge( + remote_queue_requests_queued = Gauge( f"{PROM_PREFIX}_pending_puts_to_sidecar", - "total number of get requests in flight, being sent to the publishing sidecar.", + "total number of queue requests in flight, being sent to the publishing sidecar.", prom_labels, multiprocess_mode="livesum", ) - remote_queue_puts_queued = Gauge( - f"{PROM_PREFIX}_pending_gets_to_sidecar", - "total number of put requests in flight, being sent to the publishing sidecar.", - prom_labels, - multiprocess_mode="livesum", - ) - - remote_queue_put_latency = Histogram( - f"{PROM_PREFIX}_sidecar_put_latency", - "latency of message `put` to the publishing sidecar.", - prom_labels, - buckets=default_latency_buckets, + remote_queue_requests_outcome = Counter( + f"{PROM_PREFIX}_sidecar_requests_outcome", + "outcomes (success/fail) or queue requests sent to the publishing sidecar.", + prom_labels + ["outcome"], ) - remote_queue_get_latency = Histogram( - f"{PROM_PREFIX}_sidecar_get_latency", - "latency of message `get` to the publishing sidecar.", + remote_queue_request_latency = Histogram( + f"{PROM_PREFIX}_sidecar_latency", + "latency of message requests to the publishing sidecar.", prom_labels, buckets=default_latency_buckets, ) @@ -262,95 +255,118 @@ def connect(self) -> None: self.client = RemoteMessageQueueService.Client(protocol) self.transport.open() - def _try_to_get(self, timeout: Optional[float]) -> bytes: - try: - return self.client.get(self.name, timeout).value - except TSocket.TTransportException: - self.connect() # Try reconnecting once - return self.client.get(self.name, timeout).value - - def get(self, timeout: Optional[float] = None) -> bytes: - # Call the remote server and get an element for the correct queue - try: - self.remote_queue_gets_queued.labels( - timeout=timeout, - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).inc() - start_time = time.perf_counter() - - greenlet = gevent.spawn(self._try_to_get, timeout) - gevent.joinall([greenlet]) - - self.remote_queue_get_latency.labels( - timeout=timeout, + def _update_counters(self, request_mode: str, outcome_mode: str): + # This request is no longer queued + self.remote_queue_requests_queued.labels( + mode=request_mode, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages, + ).dec() + # Increment success/failure counters + self.remote_queue_requests_outcome.labels( + mode=request_mode, + outcome=outcome_mode, + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages, + ).inc() + + def _get_success_callback(self, thing: Any): + self._update_counters("get", "success") + + def _put_success_callback(self, thing: Any): + self._update_counters("put", "success") + + def _get_fail_callback(self, thing: Any): + self._update_counters("get", "fail") + + def _put_fail_callback(self, thing: Any): + self._update_counters("put", "fail") + + def _try_to_get(self, timeout: Optional[float], start_time: float) -> bytes: + try: # handle timeouts + try: # retry connection errors once + result = self.client.get(self.name, timeout).value + except TSocket.TTransportException: + self.connect() # Try reconnecting once + result = self.client.get(self.name, timeout).value + + # record latency + self.remote_queue_request_latency.labels( + mode="get", queue_name=self.name, queue_host=self.host, queue_port=self.port, queue_max_messages=self.max_messages, ).observe(time.perf_counter() - start_time) + # update other counters + self._update_counters("get", "success") - self.remote_queue_gets_queued.labels( - timeout=timeout, - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).dec() - - return greenlet.get() + return result # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue except ThriftTimedOutError: + self._update_counters("get", "fail") raise TimedOutError - def _try_to_put(self, message: bytes, timeout: Optional[float]) -> None: - try: - self.client.put(self.name, message, timeout) - except TSocket.TTransportException: - self.connect() # Try reconnecting once - self.client.put(self.name, message, timeout) + def get(self, timeout: Optional[float] = None, block: bool = False) -> bytes: + # Call the remote server and get an element for the correct queue + # increment in-flight counter + self.remote_queue_requests_queued.labels( + mode="get", + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages, + ).inc() + start_time = time.perf_counter() - def put(self, message: bytes, timeout: Optional[float] = None) -> None: - # Call the remote server and put an element on the correct queue - # Will create the queue if it doesnt exist try: - # increment in-flight counter - self.remote_queue_puts_queued.labels( - timeout=timeout, - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).inc() - start_time = time.perf_counter() - - greenlet = gevent.spawn(self._try_to_put, message, timeout) - gevent.joinall([greenlet]) + return self._try_to_get(timeout, start_time) + except ThriftTimedOutError: + raise TimedOutError - # report latency and decrement in-flight counter - self.remote_queue_put_latency.labels( - timeout=timeout, + def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: + try: + try: + # Call the remote server and put an element on the correct queue + # Will create the queue if it doesnt exist + self.client.put(self.name, message, timeout) + except TSocket.TTransportException: + self.connect() # Try reconnecting once + self.client.put(self.name, message, timeout) + + # record latency + self.remote_queue_request_latency.labels( + mode="put", queue_name=self.name, queue_host=self.host, queue_port=self.port, queue_max_messages=self.max_messages, ).observe(time.perf_counter() - start_time) - - self.remote_queue_puts_queued.labels( - timeout=timeout, - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).dec() - - # reraise any exceptions encountered in processing - greenlet.get() + return True # Success + # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue except ThriftTimedOutError: raise TimedOutError + def put(self, message: bytes, timeout: Optional[float] = None, block: bool = False) -> Any: + # increment in-flight counter + self.remote_queue_requests_queued.labels( + mode="put", + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages, + ).inc() + start_time = time.perf_counter() + + greenlet = gevent.spawn(self._try_to_put, message, timeout, start_time) + greenlet.link_value(self._get_success_callback) + greenlet.link_exception(self._get_fail_callback) + return greenlet + def close(self) -> None: self.transport.close() diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index c2d9f9f19..08f3bd103 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -168,17 +168,21 @@ def test_put_get(self): message_queue = RemoteMessageQueue(self.qname, max_messages=10) with contextlib.closing(message_queue) as mq: - mq.put(b"x", timeout=0) + g = mq.put(b"x") + gevent.joinall([g]) message = mq.get() self.assertEqual(message, b"x") + + def test_multiple_queues(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): mq1 = RemoteMessageQueue(self.qname, max_messages=10) mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10) - mq1.put(b"x", timeout=0) - mq2.put(b"a", timeout=0) + g = mq1.put(b"x", timeout=0) + g2 = mq2.put(b"a", timeout=0) + gevent.joinall([g, g2]) # Check the queues in reverse order self.assertEqual(mq2.get(), b"a") @@ -192,7 +196,8 @@ def test_queues_alternate_port(self): message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091) with contextlib.closing(message_queue) as mq: - mq.put(b"x", timeout=0) + g = mq.put(b"x", timeout=0) + gevent.joinall([g]) self.assertEqual(mq.get(), b"x") def test_get_timeout(self): @@ -209,14 +214,22 @@ def test_get_timeout(self): ) # TODO: this routinely takes 0.105-0.11 seconds, is 1 place ok? def test_put_timeout(self): + # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError + # is raised, we dont actually know unless we explicitly check with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: - mq.put(b"x") + # TODO: This fails with socket errors if I remove this first joinall - presumably we can't call put twice without + # waiting for the first one to truly finish? + greenlet = mq.put(b"x") # fill the queue + gevent.joinall([greenlet]) start = time.time() - with self.assertRaises(TimedOutError): - mq.put(b"x", timeout=0.1) + with self.assertRaises(TimedOutError): # queue should be full + # put is non-blocking, so we need to wait for the result + greenlet2 = mq.put(b"x", timeout=0.1) + gevent.joinall([greenlet2]) + greenlet2.get() # this should expose any exceptions encountered, i.e. TimedOutError elapsed = time.time() - start self.assertAlmostEqual(elapsed, 0.1, places=1) @@ -225,7 +238,8 @@ def test_thrift_retry(self): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: - mq.put(b"x") + g = mq.put(b"x") + gevent.joinall([g]) # close the connection manually mq.close() # this should still pass, as it catches the thrift error and re-connects From 3eee3d9f486682f58a94cd2fe4090efffea77b38 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 1 Mar 2023 15:25:48 -0800 Subject: [PATCH 37/72] x --- baseplate/lib/message_queue.py | 133 ++++++++++---------- baseplate/sidecars/publisher_queue_utils.py | 1 + tests/integration/message_queue_tests.py | 78 ++++++------ 3 files changed, 103 insertions(+), 109 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 71ccf87c7..f11c27991 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -5,7 +5,10 @@ import time from enum import Enum +import traceback from typing import Any, Optional +from baseplate.lib import config +from baseplate.lib.thrift_pool import ThriftConnectionPool import gevent import posix_ipc @@ -238,22 +241,24 @@ def __init__( max_messages: int, host: str = DEFAULT_QUEUE_HOST, port: int = DEFAULT_QUEUE_PORT, + pool_size: int = 10, + pool_timeout: int = 1, + pool_conn_max_age: int = 120, ): # Connect to the remote queue server, and creeate the new queue self.name = name self.max_messages = max_messages self.host = host self.port = port - self.connect() - self.client.create_queue(name, max_messages) + self.pool = self.create_connection_pool(pool_size, pool_timeout, pool_conn_max_age) + with self.pool.connection() as protocol: + client = RemoteMessageQueueService.Client(protocol) + client.create_queue(name, max_messages) - def connect(self) -> None: - # Establish a connection with the queue server - transport = TSocket.TSocket(self.host, self.port) - self.transport = TTransport.TBufferedTransport(transport) - protocol = TBinaryProtocol.TBinaryProtocol(self.transport) - self.client = RemoteMessageQueueService.Client(protocol) - self.transport.open() + def create_connection_pool(self, pool_size, pool_timeout, pool_conn_max_age) -> ThriftConnectionPool: + endpoint = config.Endpoint(f"{self.host}:{self.port}") + pool = ThriftConnectionPool(endpoint, size=pool_size, timeout=pool_timeout, max_age=pool_conn_max_age) + return pool def _update_counters(self, request_mode: str, outcome_mode: str): # This request is no longer queued @@ -274,45 +279,45 @@ def _update_counters(self, request_mode: str, outcome_mode: str): queue_max_messages=self.max_messages, ).inc() - def _get_success_callback(self, thing: Any): - self._update_counters("get", "success") - - def _put_success_callback(self, thing: Any): + def _put_success_callback(self, greenlet: Any): self._update_counters("put", "success") + gevent.joinall([greenlet]) - def _get_fail_callback(self, thing: Any): - self._update_counters("get", "fail") - - def _put_fail_callback(self, thing: Any): + def _put_fail_callback(self, greenlet: Any): self._update_counters("put", "fail") + gevent.joinall([greenlet]) + try: + greenlet.get() + except Exception as e: + print("Remote queue `put` failed, exception found: ", e) def _try_to_get(self, timeout: Optional[float], start_time: float) -> bytes: - try: # handle timeouts - try: # retry connection errors once - result = self.client.get(self.name, timeout).value - except TSocket.TTransportException: - self.connect() # Try reconnecting once - result = self.client.get(self.name, timeout).value - - # record latency - self.remote_queue_request_latency.labels( - mode="get", - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).observe(time.perf_counter() - start_time) - # update other counters - self._update_counters("get", "success") - - return result - # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue - except ThriftTimedOutError: - self._update_counters("get", "fail") - raise TimedOutError + # get a connection from the pool + with self.pool.connection() as protocol: + client = RemoteMessageQueueService.Client(protocol) + try: # handle timeouts + result = client.get(self.name, timeout).value + + # record latency + self.remote_queue_request_latency.labels( + mode="get", + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages, + ).observe(time.perf_counter() - start_time) + # update other counters + self._update_counters("get", "success") + + return result + # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue + except ThriftTimedOutError: + self._update_counters("get", "fail") + raise TimedOutError def get(self, timeout: Optional[float] = None, block: bool = False) -> bytes: # Call the remote server and get an element for the correct queue + # increment in-flight counter self.remote_queue_requests_queued.labels( mode="get", @@ -322,36 +327,32 @@ def get(self, timeout: Optional[float] = None, block: bool = False) -> bytes: queue_max_messages=self.max_messages, ).inc() start_time = time.perf_counter() - try: return self._try_to_get(timeout, start_time) except ThriftTimedOutError: raise TimedOutError def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: - try: + # get a connection from the pool + with self.pool.connection() as protocol: + client = RemoteMessageQueueService.Client(protocol) try: - # Call the remote server and put an element on the correct queue - # Will create the queue if it doesnt exist - self.client.put(self.name, message, timeout) - except TSocket.TTransportException: - self.connect() # Try reconnecting once - self.client.put(self.name, message, timeout) - - # record latency - self.remote_queue_request_latency.labels( - mode="put", - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).observe(time.perf_counter() - start_time) - return True # Success - # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue - except ThriftTimedOutError: - raise TimedOutError - - def put(self, message: bytes, timeout: Optional[float] = None, block: bool = False) -> Any: + client.put(self.name, message, timeout) + + # record latency + self.remote_queue_request_latency.labels( + mode="put", + queue_name=self.name, + queue_host=self.host, + queue_port=self.port, + queue_max_messages=self.max_messages, + ).observe(time.perf_counter() - start_time) + return True # Success + # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue + except ThriftTimedOutError: + raise TimedOutError + + def put(self, message: bytes, timeout: Optional[float] = None) -> Any: # increment in-flight counter self.remote_queue_requests_queued.labels( mode="put", @@ -363,12 +364,12 @@ def put(self, message: bytes, timeout: Optional[float] = None, block: bool = Fal start_time = time.perf_counter() greenlet = gevent.spawn(self._try_to_put, message, timeout, start_time) - greenlet.link_value(self._get_success_callback) - greenlet.link_exception(self._get_fail_callback) + greenlet.link_value(self._put_success_callback) + greenlet.link_exception(self._put_fail_callback) return greenlet def close(self) -> None: - self.transport.close() + pass # do we need to close anything?? def create_queue( diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 2b1a1f36d..71a39b037 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,5 +1,6 @@ """Shared functions for the event & trace publisher sidecars and message queues.""" import contextlib +import time from typing import Dict from typing import Generator diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 08f3bd103..28df55030 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -1,8 +1,10 @@ import contextlib +import io import time import unittest from importlib import reload +from baseplate.thrift.message_queue import RemoteMessageQueueService import gevent import posix_ipc @@ -16,6 +18,10 @@ from baseplate.lib.message_queue import TimedOutError from baseplate.sidecars import publisher_queue_utils +from thrift.protocol import TBinaryProtocol +from thrift.transport import TSocket +from thrift.transport import TTransport + class TestPosixMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" @@ -169,36 +175,36 @@ def test_put_get(self): with contextlib.closing(message_queue) as mq: g = mq.put(b"x") - gevent.joinall([g]) - message = mq.get() + # Need to join before calling get: put is non-blocking, but get will start before put + # finishes, and since gevent is not true parallelism, get will actually block and put + # will never finish. We need to call joinall before get in all these tests to ensure + # elements finish enqueueing before we try to get them. + gevent.joinall([g]) + message = mq.get(timeout=1) + print("message: ", str(message)) self.assertEqual(message, b"x") - - def test_multiple_queues(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): mq1 = RemoteMessageQueue(self.qname, max_messages=10) mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10) - g = mq1.put(b"x", timeout=0) - g2 = mq2.put(b"a", timeout=0) - gevent.joinall([g, g2]) + g = mq1.put(b"x", timeout=1) + g2 = mq2.put(b"a", timeout=1) + gevent.joinall([g, g2]) # Check the queues in reverse order - self.assertEqual(mq2.get(), b"a") - self.assertEqual(mq1.get(), b"x") - - mq1.close() - mq2.close() + self.assertEqual(mq2.get(timeout=2), b"a") + self.assertEqual(mq1.get(timeout=2), b"x") def test_queues_alternate_port(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091) with contextlib.closing(message_queue) as mq: - g = mq.put(b"x", timeout=0) + g = mq.put(b"x", timeout=1) gevent.joinall([g]) - self.assertEqual(mq.get(), b"x") + self.assertEqual(mq.get(timeout=2), b"x") def test_get_timeout(self): with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): @@ -220,44 +226,30 @@ def test_put_timeout(self): message_queue = RemoteMessageQueue(self.qname, max_messages=1) with contextlib.closing(message_queue) as mq: - # TODO: This fails with socket errors if I remove this first joinall - presumably we can't call put twice without - # waiting for the first one to truly finish? - greenlet = mq.put(b"x") # fill the queue - gevent.joinall([greenlet]) + g = mq.put(b"x") # fill the queue start = time.time() with self.assertRaises(TimedOutError): # queue should be full # put is non-blocking, so we need to wait for the result - greenlet2 = mq.put(b"x", timeout=0.1) - gevent.joinall([greenlet2]) - greenlet2.get() # this should expose any exceptions encountered, i.e. TimedOutError + g2 = mq.put(b"x", timeout=0.1) + gevent.joinall([g, g2]) + g2.get() # this should expose any exceptions encountered, i.e. TimedOutError elapsed = time.time() - start self.assertAlmostEqual(elapsed, 0.1, places=1) - def test_thrift_retry(self): + def test_pool(self): + # If we try to connect with more slots than the pool has, without joining, we should get an error with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=1) + message_queue = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3) with contextlib.closing(message_queue) as mq: - g = mq.put(b"x") - gevent.joinall([g]) - # close the connection manually - mq.close() - # this should still pass, as it catches the thrift error and re-connects - self.assertEqual(mq.get(), b"x") - - def test_get_thrift_retry_and_timeout(self): - # Check that we still catch a timeout error even if we have to reconnect - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=1) - - with contextlib.closing(message_queue) as mq: - mq.close() - start = time.time() - with self.assertRaises(TimedOutError): - mq.get(timeout=0.1) - elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=1) - + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + g1 = mq.put(b"x", timeout=1) + g2 = mq.put(b"x", timeout=1) + g3 = mq.put(b"x", timeout=1) + g4 = mq.put(b"x", timeout=1) + gevent.joinall([g1, g2, g3, g4]) + assert "timed out waiting for a connection slot" in buf.getvalue() class TestCreateQueue(GeventPatchedTestCase): def test_posix_queue(self): From 2f92bb02c208716feede9f74cb215a433508d235 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 1 Mar 2023 18:54:45 -0800 Subject: [PATCH 38/72] remove thrift get --- baseplate/lib/message_queue.py | 61 +- baseplate/sidecars/event_publisher.py | 5 +- baseplate/sidecars/publisher_queue_utils.py | 31 +- .../RemoteMessageQueueService.py | 548 +++--------------- baseplate/thrift/message_queue/__init__.py | 2 +- baseplate/thrift/message_queue/constants.py | 8 +- .../thrift/message_queue/message_queue.thrift | 14 +- baseplate/thrift/message_queue/ttypes.py | 109 ++-- tests/integration/message_queue_tests.py | 75 ++- .../sidecars/publisher_queue_utils_tests.py | 34 +- 10 files changed, 209 insertions(+), 678 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index f11c27991..e1fcf123e 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -254,11 +254,21 @@ def __init__( with self.pool.connection() as protocol: client = RemoteMessageQueueService.Client(protocol) client.create_queue(name, max_messages) + # self.connect() + # self.client.create_queue(name, max_messages) def create_connection_pool(self, pool_size, pool_timeout, pool_conn_max_age) -> ThriftConnectionPool: endpoint = config.Endpoint(f"{self.host}:{self.port}") pool = ThriftConnectionPool(endpoint, size=pool_size, timeout=pool_timeout, max_age=pool_conn_max_age) return pool + + def connect(self) -> None: + # Establish a connection with the queue server + transport = TSocket.TSocket(self.host, self.port) + self.transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(self.transport) + self.client = RemoteMessageQueueService.Client(protocol) + self.transport.open() def _update_counters(self, request_mode: str, outcome_mode: str): # This request is no longer queued @@ -291,46 +301,8 @@ def _put_fail_callback(self, greenlet: Any): except Exception as e: print("Remote queue `put` failed, exception found: ", e) - def _try_to_get(self, timeout: Optional[float], start_time: float) -> bytes: - # get a connection from the pool - with self.pool.connection() as protocol: - client = RemoteMessageQueueService.Client(protocol) - try: # handle timeouts - result = client.get(self.name, timeout).value - - # record latency - self.remote_queue_request_latency.labels( - mode="get", - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).observe(time.perf_counter() - start_time) - # update other counters - self._update_counters("get", "success") - - return result - # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue - except ThriftTimedOutError: - self._update_counters("get", "fail") - raise TimedOutError - - def get(self, timeout: Optional[float] = None, block: bool = False) -> bytes: - # Call the remote server and get an element for the correct queue - - # increment in-flight counter - self.remote_queue_requests_queued.labels( - mode="get", - queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, - ).inc() - start_time = time.perf_counter() - try: - return self._try_to_get(timeout, start_time) - except ThriftTimedOutError: - raise TimedOutError + def get(self) -> bytes: + raise NotImplementedError def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: # get a connection from the pool @@ -377,13 +349,12 @@ def create_queue( queue_full_name: str, max_queue_size: int, max_element_size: int, - host: str = DEFAULT_QUEUE_HOST, - port: int = DEFAULT_QUEUE_PORT, ) -> MessageQueue: - # The in-memory queue is initialized on the sidecar, so we use a remote queue - # from the main baseplate application to interact with it. + # The in-memory queue is initialized here on the sidecar, and the main baseplate + # application will use a remote queue to interact with it. if queue_type == QueueType.IN_MEMORY: - event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) + event_queue = InMemoryMessageQueue(max_queue_size) + # event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) else: event_queue = PosixMessageQueue( # type: ignore diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index ce9b4e331..260f742ce 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -239,8 +239,9 @@ def publish_events() -> None: metrics_client = metrics_client_from_config(raw_config) + queue_name = f"/events-{args.queue_name}" event_queue: MessageQueue = create_queue( - cfg.queue_type, f"/events-{args.queue_name}", cfg.max_queue_size, cfg.max_element_size + cfg.queue_type, queue_name, cfg.max_queue_size, cfg.max_element_size ) # pylint: disable=maybe-no-member @@ -252,7 +253,7 @@ def publish_events() -> None: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server( - host=DEFAULT_QUEUE_HOST, port=DEFAULT_QUEUE_PORT + event_queue, host=DEFAULT_QUEUE_HOST, port=DEFAULT_QUEUE_PORT ): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 71a39b037..ed0c1efbb 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -33,40 +33,27 @@ class RemoteMessageQueueHandler: """ - def __init__(self) -> None: + def __init__(self, queues: Dict[str, MessageQueue]) -> None: # Store the queue by name with its max messages - self.queues: Dict[str, MessageQueue] = {} + self.queues = queues def create_queue(self, queue_name: str, max_messages: int) -> CreateResponse: - queue = InMemoryMessageQueue(max_messages) - self.queues[queue_name] = queue - + self.queues[queue_name] = InMemoryMessageQueue(max_messages) return CreateResponse() - def get(self, queue_name: str, timeout: Optional[float] = None) -> GetResponse: - try: - queue = self.queues[queue_name] - # Get element from list, waiting if necessary - result = queue.get(timeout) - # If the queue timed out, raise a timeout as the server response - except TimedOutError as e: - raise ThriftTimedOutError from e - - return GetResponse(result) - def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) -> PutResponse: + # May raise TimedOutError try: queue = self.queues[queue_name] queue.put(message, timeout) - except TimedOutError as e: - raise ThriftTimedOutError from e - return PutResponse() - + return PutResponse() + except TimedOutError: + raise ThriftTimedOutError @contextlib.contextmanager -def start_queue_server(host: str, port: int) -> Generator[StreamServer, None, None]: +def start_queue_server(queues: Dict[str, MessageQueue], host: str, port: int) -> Generator[StreamServer, None, None]: # Start a thrift server that will store the queue data in memory - processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler()) + processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler(queues)) server_bind_endpoint = config.Endpoint(f"{host}:{port}") listener = make_listener(server_bind_endpoint) server = make_server(server_config={}, listener=listener, app=processor) diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index 863809328..ad6267a9c 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -5,21 +5,16 @@ # # options string: py:slots # -import logging -import sys +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TFrozenDict -from thrift.Thrift import TMessageType -from thrift.Thrift import TProcessor -from thrift.Thrift import TType -from thrift.transport import TTransport from thrift.TRecursive import fix_spec +import sys +import logging from .ttypes import * - +from thrift.Thrift import TProcessor +from thrift.transport import TTransport all_structs = [] @@ -43,15 +38,6 @@ def put(self, queue_name, message, timeout): """ pass - def get(self, queue_name, timeout): - """ - Parameters: - - queue_name - - timeout - - """ - pass - class Client(Iface): def __init__(self, iprot, oprot=None): @@ -71,7 +57,7 @@ def create_queue(self, queue_name, max_messages): return self.recv_create_queue() def send_create_queue(self, queue_name, max_messages): - self._oprot.writeMessageBegin("create_queue", TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin('create_queue', TMessageType.CALL, self._seqid) args = create_queue_args() args.queue_name = queue_name args.max_messages = max_messages @@ -92,9 +78,7 @@ def recv_create_queue(self): iprot.readMessageEnd() if result.success is not None: return result.success - raise TApplicationException( - TApplicationException.MISSING_RESULT, "create_queue failed: unknown result" - ) + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_queue failed: unknown result") def put(self, queue_name, message, timeout): """ @@ -108,7 +92,7 @@ def put(self, queue_name, message, timeout): return self.recv_put() def send_put(self, queue_name, message, timeout): - self._oprot.writeMessageBegin("put", TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) args = put_args() args.queue_name = queue_name args.message = message @@ -132,47 +116,7 @@ def recv_put(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException( - TApplicationException.MISSING_RESULT, "put failed: unknown result" - ) - - def get(self, queue_name, timeout): - """ - Parameters: - - queue_name - - timeout - - """ - self.send_get(queue_name, timeout) - return self.recv_get() - - def send_get(self, queue_name, timeout): - self._oprot.writeMessageBegin("get", TMessageType.CALL, self._seqid) - args = get_args() - args.queue_name = queue_name - args.timeout = timeout - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_get(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = get_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.timed_out_error is not None: - raise result.timed_out_error - raise TApplicationException( - TApplicationException.MISSING_RESULT, "get failed: unknown result" - ) + raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") class Processor(Iface, TProcessor): @@ -181,7 +125,6 @@ def __init__(self, handler): self._processMap = {} self._processMap["create_queue"] = Processor.process_create_queue self._processMap["put"] = Processor.process_put - self._processMap["get"] = Processor.process_get self._on_message_begin = None def on_message_begin(self, func): @@ -194,9 +137,7 @@ def process(self, iprot, oprot): if name not in self._processMap: iprot.skip(TType.STRUCT) iprot.readMessageEnd() - x = TApplicationException( - TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name) - ) + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) x.write(oprot) oprot.writeMessageEnd() @@ -217,13 +158,13 @@ def process_create_queue(self, seqid, iprot, oprot): except TTransport.TTransportException: raise except TApplicationException as ex: - logging.exception("TApplication exception in handler") + logging.exception('TApplication exception in handler') msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception("Unexpected exception in handler") + logging.exception('Unexpected exception in handler') msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') oprot.writeMessageBegin("create_queue", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -243,45 +184,18 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception("TApplication exception in handler") + logging.exception('TApplication exception in handler') msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception("Unexpected exception in handler") + logging.exception('Unexpected exception in handler') msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') oprot.writeMessageBegin("put", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() oprot.trans.flush() - def process_get(self, seqid, iprot, oprot): - args = get_args() - args.read(iprot) - iprot.readMessageEnd() - result = get_result() - try: - result.success = self._handler.get(args.queue_name, args.timeout) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except ThriftTimedOutError as timed_out_error: - msg_type = TMessageType.REPLY - result.timed_out_error = timed_out_error - except TApplicationException as ex: - logging.exception("TApplication exception in handler") - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception("Unexpected exception in handler") - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") - oprot.writeMessageBegin("get", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - # HELPER FUNCTIONS AND STRUCTURES @@ -294,24 +208,17 @@ class create_queue_args(object): """ __slots__ = ( - "queue_name", - "max_messages", + 'queue_name', + 'max_messages', ) - def __init__( - self, - queue_name=None, - max_messages=None, - ): + + def __init__(self, queue_name=None, max_messages=None,): self.queue_name = queue_name self.max_messages = max_messages def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -321,11 +228,7 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) elif fid == 2: @@ -342,15 +245,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("create_queue_args") + oprot.writeStructBegin('create_queue_args') if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) oprot.writeFieldEnd() if self.max_messages is not None: - oprot.writeFieldBegin("max_messages", TType.I64, 2) + oprot.writeFieldBegin('max_messages', TType.I64, 2) oprot.writeI64(self.max_messages) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -360,8 +261,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -375,25 +277,11 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(create_queue_args) create_queue_args.thrift_spec = ( None, # 0 - ( - 1, - TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - ( - 2, - TType.I64, - "max_messages", - None, - None, - ), # 2 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + (2, TType.I64, 'max_messages', None, None, ), # 2 ) @@ -404,20 +292,16 @@ class create_queue_result(object): """ - __slots__ = ("success",) + __slots__ = ( + 'success', + ) + - def __init__( - self, - success=None, - ): + def __init__(self, success=None,): self.success = success def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -440,9 +324,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("create_queue_result") + oprot.writeStructBegin('create_queue_result') if self.success is not None: - oprot.writeFieldBegin("success", TType.STRUCT, 0) + oprot.writeFieldBegin('success', TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -452,8 +336,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -467,17 +352,9 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(create_queue_result) create_queue_result.thrift_spec = ( - ( - 0, - TType.STRUCT, - "success", - [CreateResponse, None], - None, - ), # 0 + (0, TType.STRUCT, 'success', [CreateResponse, None], None, ), # 0 ) @@ -491,27 +368,19 @@ class put_args(object): """ __slots__ = ( - "queue_name", - "message", - "timeout", + 'queue_name', + 'message', + 'timeout', ) - def __init__( - self, - queue_name=None, - message=None, - timeout=None, - ): + + def __init__(self, queue_name=None, message=None, timeout=None,): self.queue_name = queue_name self.message = message self.timeout = timeout def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -521,19 +390,15 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) + self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) - elif fid == 3: + elif fid == 2: if ftype == TType.STRING: self.message = iprot.readBinary() else: iprot.skip(ftype) - elif fid == 4: + elif fid == 3: if ftype == TType.DOUBLE: self.timeout = iprot.readDouble() else: @@ -547,19 +412,17 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("put_args") + oprot.writeStructBegin('put_args') if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) + oprot.writeFieldBegin('queue_name', TType.STRING, 1) + oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) oprot.writeFieldEnd() if self.message is not None: - oprot.writeFieldBegin("message", TType.STRING, 3) + oprot.writeFieldBegin('message', TType.STRING, 2) oprot.writeBinary(self.message) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin("timeout", TType.DOUBLE, 4) + oprot.writeFieldBegin('timeout', TType.DOUBLE, 3) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -569,8 +432,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -584,33 +448,12 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(put_args) put_args.thrift_spec = ( None, # 0 - ( - 1, - TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - None, # 2 - ( - 3, - TType.STRING, - "message", - "BINARY", - None, - ), # 3 - ( - 4, - TType.DOUBLE, - "timeout", - None, - None, - ), # 4 + (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 + (2, TType.STRING, 'message', 'BINARY', None, ), # 2 + (3, TType.DOUBLE, 'timeout', None, None, ), # 3 ) @@ -623,24 +466,17 @@ class put_result(object): """ __slots__ = ( - "success", - "timed_out_error", + 'success', + 'timed_out_error', ) - def __init__( - self, - success=None, - timed_out_error=None, - ): + + def __init__(self, success=None, timed_out_error=None,): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -668,13 +504,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("put_result") + oprot.writeStructBegin('put_result') if self.success is not None: - oprot.writeFieldBegin("success", TType.STRUCT, 0) + oprot.writeFieldBegin('success', TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) + oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -684,8 +520,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -699,243 +536,10 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(put_result) put_result.thrift_spec = ( - ( - 0, - TType.STRUCT, - "success", - [PutResponse, None], - None, - ), # 0 - ( - 1, - TType.STRUCT, - "timed_out_error", - [ThriftTimedOutError, None], - None, - ), # 1 -) - - -class get_args(object): - """ - Attributes: - - queue_name - - timeout - - """ - - __slots__ = ( - "queue_name", - "timeout", - ) - - def __init__( - self, - queue_name=None, - timeout=None, - ): - self.queue_name = queue_name - self.timeout = timeout - - def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.DOUBLE: - self.timeout = iprot.readDouble() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin("get_args") - if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) - oprot.writeFieldEnd() - if self.timeout is not None: - oprot.writeFieldBegin("timeout", TType.DOUBLE, 3) - oprot.writeDouble(self.timeout) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - -all_structs.append(get_args) -get_args.thrift_spec = ( - None, # 0 - ( - 1, - TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - None, # 2 - ( - 3, - TType.DOUBLE, - "timeout", - None, - None, - ), # 3 -) - - -class get_result(object): - """ - Attributes: - - success - - timed_out_error - - """ - - __slots__ = ( - "success", - "timed_out_error", - ) - - def __init__( - self, - success=None, - timed_out_error=None, - ): - self.success = success - self.timed_out_error = timed_out_error - - def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = GetResponse() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.timed_out_error = ThriftTimedOutError.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin("get_result") - if self.success is not None: - oprot.writeFieldBegin("success", TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.timed_out_error is not None: - oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) - self.timed_out_error.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - -all_structs.append(get_result) -get_result.thrift_spec = ( - ( - 0, - TType.STRUCT, - "success", - [GetResponse, None], - None, - ), # 0 - ( - 1, - TType.STRUCT, - "timed_out_error", - [ThriftTimedOutError, None], - None, - ), # 1 + (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 + (1, TType.STRUCT, 'timed_out_error', [ThriftTimedOutError, None], None, ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/__init__.py b/baseplate/thrift/message_queue/__init__.py index f37fa9583..947e972be 100644 --- a/baseplate/thrift/message_queue/__init__.py +++ b/baseplate/thrift/message_queue/__init__.py @@ -1 +1 @@ -__all__ = ["ttypes", "constants", "RemoteMessageQueueService"] +__all__ = ['ttypes', 'constants', 'RemoteMessageQueueService'] diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py index a19d895e4..964098e62 100644 --- a/baseplate/thrift/message_queue/constants.py +++ b/baseplate/thrift/message_queue/constants.py @@ -5,14 +5,10 @@ # # options string: py:slots # -import sys +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TFrozenDict -from thrift.Thrift import TMessageType -from thrift.Thrift import TType from thrift.TRecursive import fix_spec +import sys from .ttypes import * diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index 546e84442..04055e244 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -19,15 +19,15 @@ service RemoteMessageQueueService { ); PutResponse put( 1: string queue_name - 3: binary message - 4: double timeout - ) throws ( - 1: ThriftTimedOutError timed_out_error - ); - GetResponse get( - 1: string queue_name + 2: binary message 3: double timeout ) throws ( 1: ThriftTimedOutError timed_out_error ); + // GetResponse get( + // 1: string queue_name + // 3: double timeout + // ) throws ( + // 1: ThriftTimedOutError timed_out_error + // ); } diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index d7b6993cf..a6c220e8b 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -5,30 +5,25 @@ # # options string: py:slots # -import sys +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException from thrift.protocol.TProtocol import TProtocolException -from thrift.Thrift import TApplicationException -from thrift.Thrift import TException -from thrift.Thrift import TFrozenDict -from thrift.Thrift import TMessageType -from thrift.Thrift import TType -from thrift.transport import TTransport from thrift.TRecursive import fix_spec +import sys + +from thrift.transport import TTransport all_structs = [] class CreateResponse(object): - __slots__ = () + __slots__ = ( + ) + def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -45,7 +40,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("CreateResponse") + oprot.writeStructBegin('CreateResponse') oprot.writeFieldStop() oprot.writeStructEnd() @@ -53,8 +48,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -72,14 +68,12 @@ def __ne__(self, other): class PutResponse(object): - __slots__ = () + __slots__ = ( + ) + def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -96,7 +90,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("PutResponse") + oprot.writeStructBegin('PutResponse') oprot.writeFieldStop() oprot.writeStructEnd() @@ -104,8 +98,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -128,20 +123,16 @@ class GetResponse(object): """ - __slots__ = ("value",) + __slots__ = ( + 'value', + ) - def __init__( - self, - value=None, - ): + + def __init__(self, value=None,): self.value = value def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -163,9 +154,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("GetResponse") + oprot.writeStructBegin('GetResponse') if self.value is not None: - oprot.writeFieldBegin("value", TType.STRING, 1) + oprot.writeFieldBegin('value', TType.STRING, 1) oprot.writeBinary(self.value) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -175,8 +166,9 @@ def validate(self): return def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -194,7 +186,9 @@ def __ne__(self, other): class ThriftTimedOutError(TException): - __slots__ = () + __slots__ = ( + ) + def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -207,11 +201,7 @@ def __hash__(self): @classmethod def read(cls, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and cls.thrift_spec is not None - ): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) iprot.readStructBegin() while True: @@ -222,13 +212,14 @@ def read(cls, iprot): iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() - return cls() + return cls( + ) def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin("ThriftTimedOutError") + oprot.writeStructBegin('ThriftTimedOutError') oprot.writeFieldStop() oprot.writeStructEnd() @@ -239,8 +230,9 @@ def __str__(self): return repr(self) def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) + L = ['%s=%r' % (key, getattr(self, key)) + for key in self.__slots__] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -254,24 +246,19 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - all_structs.append(CreateResponse) -CreateResponse.thrift_spec = () +CreateResponse.thrift_spec = ( +) all_structs.append(PutResponse) -PutResponse.thrift_spec = () +PutResponse.thrift_spec = ( +) all_structs.append(GetResponse) GetResponse.thrift_spec = ( None, # 0 - ( - 1, - TType.STRING, - "value", - "BINARY", - None, - ), # 1 + (1, TType.STRING, 'value', 'BINARY', None, ), # 1 ) all_structs.append(ThriftTimedOutError) -ThriftTimedOutError.thrift_spec = () +ThriftTimedOutError.thrift_spec = ( +) fix_spec(all_structs) del all_structs diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 28df55030..00638370c 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -99,8 +99,6 @@ def tearDown(self): class TestInMemoryMessageQueueCreation(unittest.TestCase): - qname = "/baseplate-test-queue" - def test_instantiate_queue(self): message_queue = InMemoryMessageQueue(max_messages=1) @@ -166,64 +164,65 @@ def tearDown(self): class TestRemoteMessageQueueCreation(GeventPatchedTestCase): - qname = "/baseplate-test-queue" - + qname = "/baseplate-test-queue" def test_put_get(self): - # start the server that would ordinarily be running on the sidecar - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=10) + # create the queue and start the server that would ordinarily be running on the sidecar + queues = {} + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + # create the corresponding queue that would ordinarily be on the main baseplate application/client-side + message_queue = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) with contextlib.closing(message_queue) as mq: g = mq.put(b"x") - # Need to join before calling get: put is non-blocking, but get will start before put - # finishes, and since gevent is not true parallelism, get will actually block and put - # will never finish. We need to call joinall before get in all these tests to ensure - # elements finish enqueueing before we try to get them. - gevent.joinall([g]) - message = mq.get(timeout=1) + gevent.joinall([g]) + message = queues[self.qname].get(timeout=0.1) print("message: ", str(message)) self.assertEqual(message, b"x") def test_multiple_queues(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - mq1 = RemoteMessageQueue(self.qname, max_messages=10) - mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10) + queues = {} + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + mq1 = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) + mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10, pool_size=1) - g = mq1.put(b"x", timeout=1) - g2 = mq2.put(b"a", timeout=1) + g = mq1.put(b"x", timeout=0.1) + g2 = mq2.put(b"a", timeout=0.1) gevent.joinall([g, g2]) # Check the queues in reverse order - self.assertEqual(mq2.get(timeout=2), b"a") - self.assertEqual(mq1.get(timeout=2), b"x") + self.assertEqual(queues[self.qname + "2"].get(timeout=0.1), b"a") + self.assertEqual(queues[self.qname].get(timeout=0.1), b"x") def test_queues_alternate_port(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): - message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091) + queues = {} + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9091): + message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091, pool_size=1) with contextlib.closing(message_queue) as mq: - g = mq.put(b"x", timeout=1) + g = mq.put(b"x", timeout=0.1) gevent.joinall([g]) - self.assertEqual(mq.get(timeout=2), b"x") + self.assertEqual(queues[self.qname].get(timeout=2), b"x") def test_get_timeout(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=1) + queues = {} + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + message_queue = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) with contextlib.closing(message_queue) as mq: start = time.time() with self.assertRaises(TimedOutError): - mq.get(timeout=0.1) + queues[self.qname].get(timeout=0.1) elapsed = time.time() - start self.assertAlmostEqual( elapsed, 0.1, places=1 ) # TODO: this routinely takes 0.105-0.11 seconds, is 1 place ok? def test_put_timeout(self): + queues = {} # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError # is raised, we dont actually know unless we explicitly check - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=1) + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + message_queue = RemoteMessageQueue(self.qname, max_messages=1, pool_size=2) with contextlib.closing(message_queue) as mq: g = mq.put(b"x") # fill the queue @@ -237,26 +236,26 @@ def test_put_timeout(self): self.assertAlmostEqual(elapsed, 0.1, places=1) def test_pool(self): + queues = {} # If we try to connect with more slots than the pool has, without joining, we should get an error - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3) with contextlib.closing(message_queue) as mq: buf = io.StringIO() with contextlib.redirect_stdout(buf): - g1 = mq.put(b"x", timeout=1) - g2 = mq.put(b"x", timeout=1) - g3 = mq.put(b"x", timeout=1) - g4 = mq.put(b"x", timeout=1) + g1 = mq.put(b"x", timeout=0.1) + g2 = mq.put(b"x", timeout=0.1) + g3 = mq.put(b"x", timeout=0.1) + g4 = mq.put(b"x", timeout=0.1) gevent.joinall([g1, g2, g3, g4]) assert "timed out waiting for a connection slot" in buf.getvalue() -class TestCreateQueue(GeventPatchedTestCase): +class TestCreateQueue(): def test_posix_queue(self): queue: MessageQueue = create_queue(QueueType.POSIX, "/test", 5, 1000) assert isinstance(queue, PosixMessageQueue) def test_in_memory_create_queue(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000) - assert isinstance(queue, RemoteMessageQueue) + queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000) + assert isinstance(queue, InMemoryMessageQueue) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 510429f45..fe395900b 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -35,27 +35,13 @@ def test_posix_queue_get_put(self): assert output == test_message_2 def test_in_memory_queue_get_put(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000) - - test_message = bytes("message", "utf-8") - test_message_2 = bytes("2nd message", "utf-8") - queue.put(test_message) - queue.put(test_message_2) - output = queue.get() - assert output == test_message - output = queue.get() - assert output == test_message_2 - - def test_in_memory_queue_alternate_port(self): - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9091): - queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000, port=9091) - - test_message = bytes("message", "utf-8") - test_message_2 = bytes("2nd message", "utf-8") - queue.put(test_message) - queue.put(test_message_2) - output = queue.get() - assert output == test_message - output = queue.get() - assert output == test_message_2 + queue: MessageQueue = create_queue(QueueType.IN_MEMORY, "/test", 5, 1000) + + test_message = bytes("message", "utf-8") + test_message_2 = bytes("2nd message", "utf-8") + queue.put(test_message) + queue.put(test_message_2) + output = queue.get() + assert output == test_message + output = queue.get() + assert output == test_message_2 From da30d5ef7ac07da2250c1d37096c581c62de3559 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 12:04:16 -0800 Subject: [PATCH 39/72] remove get --- baseplate/lib/message_queue.py | 38 +++++++++---------- baseplate/sidecars/event_publisher.py | 12 +----- baseplate/sidecars/publisher_queue_utils.py | 9 ++--- baseplate/sidecars/trace_publisher.py | 7 +--- .../thrift/message_queue/message_queue.thrift | 6 --- tests/integration/message_queue_tests.py | 20 ++++------ .../sidecars/publisher_queue_utils_tests.py | 2 - 7 files changed, 33 insertions(+), 61 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index e1fcf123e..1cb0a983c 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -3,16 +3,12 @@ import queue as q import select import time - from enum import Enum -import traceback -from typing import Any, Optional -from baseplate.lib import config -from baseplate.lib.thrift_pool import ThriftConnectionPool +from typing import Any +from typing import Optional import gevent import posix_ipc - from prometheus_client import Counter from prometheus_client import Gauge from prometheus_client import Histogram @@ -20,8 +16,10 @@ from thrift.transport import TSocket from thrift.transport import TTransport +from baseplate.lib import config from baseplate.lib.prometheus_metrics import default_latency_buckets from baseplate.lib.retry import RetryPolicy +from baseplate.lib.thrift_pool import ThriftConnectionPool from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError @@ -208,7 +206,7 @@ class RemoteMessageQueue(MessageQueue): """ prom_labels = [ - "mode", # put or get + "mode", # put or get "queue_name", "queue_host", "queue_port", @@ -257,11 +255,15 @@ def __init__( # self.connect() # self.client.create_queue(name, max_messages) - def create_connection_pool(self, pool_size, pool_timeout, pool_conn_max_age) -> ThriftConnectionPool: + def create_connection_pool( + self, pool_size: int, pool_timeout: int, pool_conn_max_age: int + ) -> ThriftConnectionPool: endpoint = config.Endpoint(f"{self.host}:{self.port}") - pool = ThriftConnectionPool(endpoint, size=pool_size, timeout=pool_timeout, max_age=pool_conn_max_age) + pool = ThriftConnectionPool( + endpoint, size=pool_size, timeout=pool_timeout, max_age=pool_conn_max_age + ) return pool - + def connect(self) -> None: # Establish a connection with the queue server transport = TSocket.TSocket(self.host, self.port) @@ -270,7 +272,7 @@ def connect(self) -> None: self.client = RemoteMessageQueueService.Client(protocol) self.transport.open() - def _update_counters(self, request_mode: str, outcome_mode: str): + def _update_counters(self, request_mode: str, outcome_mode: str) -> None: # This request is no longer queued self.remote_queue_requests_queued.labels( mode=request_mode, @@ -289,11 +291,11 @@ def _update_counters(self, request_mode: str, outcome_mode: str): queue_max_messages=self.max_messages, ).inc() - def _put_success_callback(self, greenlet: Any): + def _put_success_callback(self, greenlet: Any) -> None: self._update_counters("put", "success") gevent.joinall([greenlet]) - def _put_fail_callback(self, greenlet: Any): + def _put_fail_callback(self, greenlet: Any) -> None: self._update_counters("put", "fail") gevent.joinall([greenlet]) try: @@ -301,8 +303,8 @@ def _put_fail_callback(self, greenlet: Any): except Exception as e: print("Remote queue `put` failed, exception found: ", e) - def get(self) -> bytes: - raise NotImplementedError + def get(self, _: Optional[float] = None) -> bytes: + raise NotImplementedError # TODO: this is yucky, that this queue type does not implement get def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: # get a connection from the pool @@ -319,7 +321,7 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa queue_port=self.port, queue_max_messages=self.max_messages, ).observe(time.perf_counter() - start_time) - return True # Success + return True # Success # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue except ThriftTimedOutError: raise TimedOutError @@ -341,7 +343,7 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> Any: return greenlet def close(self) -> None: - pass # do we need to close anything?? + pass # do we need to close anything?? def create_queue( @@ -434,8 +436,6 @@ def queue_tool() -> None: args.queue_name, args.max_messages, args.max_message_size, - args.queue_host, - args.queue_port, ) if args.mode == "read": diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 260f742ce..26010ec1e 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -5,7 +5,6 @@ import hashlib import hmac import logging - from typing import Any from typing import List from typing import Optional @@ -249,16 +248,7 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - if cfg.queue_type == QueueType.IN_MEMORY.value: - # Start the Thrift server that communicates with RemoteMessageQueues and stores - # data in a InMemoryMessageQueue - with publisher_queue_utils.start_queue_server( - event_queue, host=DEFAULT_QUEUE_HOST, port=DEFAULT_QUEUE_PORT - ): - build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) - - else: - build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) + build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) if __name__ == "__main__": diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index ed0c1efbb..d3234a84e 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,13 +1,10 @@ """Shared functions for the event & trace publisher sidecars and message queues.""" import contextlib -import time - from typing import Dict from typing import Generator from typing import Optional import gevent - from gevent.server import StreamServer from baseplate.lib import config @@ -18,7 +15,6 @@ from baseplate.server.thrift import make_server from baseplate.thrift.message_queue import RemoteMessageQueueService from baseplate.thrift.message_queue.ttypes import CreateResponse -from baseplate.thrift.message_queue.ttypes import GetResponse from baseplate.thrift.message_queue.ttypes import PutResponse from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError @@ -50,8 +46,11 @@ def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) except TimedOutError: raise ThriftTimedOutError + @contextlib.contextmanager -def start_queue_server(queues: Dict[str, MessageQueue], host: str, port: int) -> Generator[StreamServer, None, None]: +def start_queue_server( + queues: Dict[str, MessageQueue], host: str, port: int +) -> Generator[StreamServer, None, None]: # Start a thrift server that will store the queue data in memory processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler(queues)) server_bind_endpoint = config.Endpoint(f"{host}:{port}") diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 2e23d0014..25348232a 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -2,7 +2,6 @@ import configparser import logging import urllib.parse - from typing import Optional import requests @@ -212,11 +211,7 @@ def publish_traces() -> None: post_timeout=publisher_cfg.post_timeout, ) - if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): - build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) - else: - build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) + build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) if __name__ == "__main__": diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index 04055e244..8ae60dcbd 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -24,10 +24,4 @@ service RemoteMessageQueueService { ) throws ( 1: ThriftTimedOutError timed_out_error ); - // GetResponse get( - // 1: string queue_name - // 3: double timeout - // ) throws ( - // 1: ThriftTimedOutError timed_out_error - // ); } diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 00638370c..69636a3e6 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -2,9 +2,7 @@ import io import time import unittest - from importlib import reload -from baseplate.thrift.message_queue import RemoteMessageQueueService import gevent import posix_ipc @@ -18,10 +16,6 @@ from baseplate.lib.message_queue import TimedOutError from baseplate.sidecars import publisher_queue_utils -from thrift.protocol import TBinaryProtocol -from thrift.transport import TSocket -from thrift.transport import TTransport - class TestPosixMessageQueueCreation(unittest.TestCase): qname = "/baseplate-test-queue" @@ -164,7 +158,8 @@ def tearDown(self): class TestRemoteMessageQueueCreation(GeventPatchedTestCase): - qname = "/baseplate-test-queue" + qname = "/baseplate-test-queue" + def test_put_get(self): # create the queue and start the server that would ordinarily be running on the sidecar queues = {} @@ -219,19 +214,19 @@ def test_get_timeout(self): def test_put_timeout(self): queues = {} - # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError + # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError # is raised, we dont actually know unless we explicitly check with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1, pool_size=2) with contextlib.closing(message_queue) as mq: - g = mq.put(b"x") # fill the queue + g = mq.put(b"x") # fill the queue start = time.time() - with self.assertRaises(TimedOutError): # queue should be full + with self.assertRaises(TimedOutError): # queue should be full # put is non-blocking, so we need to wait for the result g2 = mq.put(b"x", timeout=0.1) gevent.joinall([g, g2]) - g2.get() # this should expose any exceptions encountered, i.e. TimedOutError + g2.get() # this should expose any exceptions encountered, i.e. TimedOutError elapsed = time.time() - start self.assertAlmostEqual(elapsed, 0.1, places=1) @@ -251,7 +246,8 @@ def test_pool(self): gevent.joinall([g1, g2, g3, g4]) assert "timed out waiting for a connection slot" in buf.getvalue() -class TestCreateQueue(): + +class TestCreateQueue: def test_posix_queue(self): queue: MessageQueue = create_queue(QueueType.POSIX, "/test", 5, 1000) assert isinstance(queue, PosixMessageQueue) diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index fe395900b..3b3c3c4d2 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -1,5 +1,4 @@ import unittest - from importlib import reload import gevent.monkey @@ -7,7 +6,6 @@ from baseplate.lib.message_queue import create_queue from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType -from baseplate.sidecars import publisher_queue_utils class GeventPatchedTestCase(unittest.TestCase): From bf367e4d8e1259b390f14109bd9b897d7d029ebe Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 13:07:58 -0800 Subject: [PATCH 40/72] linting --- baseplate/lib/message_queue.py | 2 ++ baseplate/sidecars/event_publisher.py | 1 + baseplate/sidecars/publisher_queue_utils.py | 2 ++ baseplate/sidecars/trace_publisher.py | 1 + .../message_queue/RemoteMessageQueueService.py | 14 ++++++++------ baseplate/thrift/message_queue/constants.py | 6 ------ baseplate/thrift/message_queue/ttypes.py | 9 +++------ tests/integration/message_queue_tests.py | 1 + .../sidecars/publisher_queue_utils_tests.py | 1 + 9 files changed, 19 insertions(+), 18 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 1cb0a983c..9667ef575 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -3,12 +3,14 @@ import queue as q import select import time + from enum import Enum from typing import Any from typing import Optional import gevent import posix_ipc + from prometheus_client import Counter from prometheus_client import Gauge from prometheus_client import Histogram diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 26010ec1e..2d5cbcb7d 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -5,6 +5,7 @@ import hashlib import hmac import logging + from typing import Any from typing import List from typing import Optional diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index d3234a84e..dffa8ff95 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,10 +1,12 @@ """Shared functions for the event & trace publisher sidecars and message queues.""" import contextlib + from typing import Dict from typing import Generator from typing import Optional import gevent + from gevent.server import StreamServer from baseplate.lib import config diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 25348232a..ae1f23cf6 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -2,6 +2,7 @@ import configparser import logging import urllib.parse + from typing import Optional import requests diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index ad6267a9c..a8a9643b8 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -6,15 +6,17 @@ # options string: py:slots # -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys import logging -from .ttypes import * +import sys + +from thrift.Thrift import TApplicationException +from thrift.Thrift import TMessageType from thrift.Thrift import TProcessor +from thrift.Thrift import TType from thrift.transport import TTransport +from thrift.TRecursive import fix_spec + +from .ttypes import * all_structs = [] diff --git a/baseplate/thrift/message_queue/constants.py b/baseplate/thrift/message_queue/constants.py index 964098e62..f25235c85 100644 --- a/baseplate/thrift/message_queue/constants.py +++ b/baseplate/thrift/message_queue/constants.py @@ -5,10 +5,4 @@ # # options string: py:slots # - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys from .ttypes import * diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index a6c220e8b..e5bf45df8 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -6,13 +6,10 @@ # options string: py:slots # -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys - +from thrift.Thrift import TException +from thrift.Thrift import TType from thrift.transport import TTransport +from thrift.TRecursive import fix_spec all_structs = [] diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 69636a3e6..b8d3d8069 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -2,6 +2,7 @@ import io import time import unittest + from importlib import reload import gevent diff --git a/tests/integration/sidecars/publisher_queue_utils_tests.py b/tests/integration/sidecars/publisher_queue_utils_tests.py index 3b3c3c4d2..de06f9906 100644 --- a/tests/integration/sidecars/publisher_queue_utils_tests.py +++ b/tests/integration/sidecars/publisher_queue_utils_tests.py @@ -1,4 +1,5 @@ import unittest + from importlib import reload import gevent.monkey From 2d17807a96c292e77749f651bf2a21dd442b8184 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 13:11:41 -0800 Subject: [PATCH 41/72] linting --- baseplate/lib/message_queue.py | 2 +- .../RemoteMessageQueueService.py | 237 ++++++++++++------ baseplate/thrift/message_queue/__init__.py | 2 +- baseplate/thrift/message_queue/ttypes.py | 102 ++++---- 4 files changed, 225 insertions(+), 118 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 9667ef575..2e0fcc3f6 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -306,7 +306,7 @@ def _put_fail_callback(self, greenlet: Any) -> None: print("Remote queue `put` failed, exception found: ", e) def get(self, _: Optional[float] = None) -> bytes: - raise NotImplementedError # TODO: this is yucky, that this queue type does not implement get + raise NotImplementedError # TODO: this is yucky, that this queue type does not implement get def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: # get a connection from the pool diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index a8a9643b8..f9666014e 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -5,7 +5,6 @@ # # options string: py:slots # - import logging import sys @@ -17,6 +16,7 @@ from thrift.TRecursive import fix_spec from .ttypes import * + all_structs = [] @@ -59,7 +59,7 @@ def create_queue(self, queue_name, max_messages): return self.recv_create_queue() def send_create_queue(self, queue_name, max_messages): - self._oprot.writeMessageBegin('create_queue', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("create_queue", TMessageType.CALL, self._seqid) args = create_queue_args() args.queue_name = queue_name args.max_messages = max_messages @@ -80,7 +80,9 @@ def recv_create_queue(self): iprot.readMessageEnd() if result.success is not None: return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "create_queue failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "create_queue failed: unknown result" + ) def put(self, queue_name, message, timeout): """ @@ -94,7 +96,7 @@ def put(self, queue_name, message, timeout): return self.recv_put() def send_put(self, queue_name, message, timeout): - self._oprot.writeMessageBegin('put', TMessageType.CALL, self._seqid) + self._oprot.writeMessageBegin("put", TMessageType.CALL, self._seqid) args = put_args() args.queue_name = queue_name args.message = message @@ -118,7 +120,9 @@ def recv_put(self): return result.success if result.timed_out_error is not None: raise result.timed_out_error - raise TApplicationException(TApplicationException.MISSING_RESULT, "put failed: unknown result") + raise TApplicationException( + TApplicationException.MISSING_RESULT, "put failed: unknown result" + ) class Processor(Iface, TProcessor): @@ -139,7 +143,9 @@ def process(self, iprot, oprot): if name not in self._processMap: iprot.skip(TType.STRUCT) iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + x = TApplicationException( + TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name) + ) oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) x.write(oprot) oprot.writeMessageEnd() @@ -160,13 +166,13 @@ def process_create_queue(self, seqid, iprot, oprot): except TTransport.TTransportException: raise except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("create_queue", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -186,18 +192,19 @@ def process_put(self, seqid, iprot, oprot): msg_type = TMessageType.REPLY result.timed_out_error = timed_out_error except TApplicationException as ex: - logging.exception('TApplication exception in handler') + logging.exception("TApplication exception in handler") msg_type = TMessageType.EXCEPTION result = ex except Exception: - logging.exception('Unexpected exception in handler') + logging.exception("Unexpected exception in handler") msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") oprot.writeMessageBegin("put", msg_type, seqid) result.write(oprot) oprot.writeMessageEnd() oprot.trans.flush() + # HELPER FUNCTIONS AND STRUCTURES @@ -210,17 +217,24 @@ class create_queue_args(object): """ __slots__ = ( - 'queue_name', - 'max_messages', + "queue_name", + "max_messages", ) - - def __init__(self, queue_name=None, max_messages=None,): + def __init__( + self, + queue_name=None, + max_messages=None, + ): self.queue_name = queue_name self.max_messages = max_messages def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -230,7 +244,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 2: @@ -247,13 +265,15 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('create_queue_args') + oprot.writeStructBegin("create_queue_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.max_messages is not None: - oprot.writeFieldBegin('max_messages', TType.I64, 2) + oprot.writeFieldBegin("max_messages", TType.I64, 2) oprot.writeI64(self.max_messages) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -263,9 +283,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -279,11 +298,25 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(create_queue_args) create_queue_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 - (2, TType.I64, 'max_messages', None, None, ), # 2 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "max_messages", + None, + None, + ), # 2 ) @@ -294,16 +327,20 @@ class create_queue_result(object): """ - __slots__ = ( - 'success', - ) - + __slots__ = ("success",) - def __init__(self, success=None,): + def __init__( + self, + success=None, + ): self.success = success def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -326,9 +363,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('create_queue_result') + oprot.writeStructBegin("create_queue_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -338,9 +375,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -354,9 +390,17 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(create_queue_result) create_queue_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [CreateResponse, None], None, ), # 0 + ( + 0, + TType.STRUCT, + "success", + [CreateResponse, None], + None, + ), # 0 ) @@ -370,19 +414,27 @@ class put_args(object): """ __slots__ = ( - 'queue_name', - 'message', - 'timeout', + "queue_name", + "message", + "timeout", ) - - def __init__(self, queue_name=None, message=None, timeout=None,): + def __init__( + self, + queue_name=None, + message=None, + timeout=None, + ): self.queue_name = queue_name self.message = message self.timeout = timeout def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -392,7 +444,11 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRING: - self.queue_name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.queue_name = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) else: iprot.skip(ftype) elif fid == 2: @@ -414,17 +470,19 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('put_args') + oprot.writeStructBegin("put_args") if self.queue_name is not None: - oprot.writeFieldBegin('queue_name', TType.STRING, 1) - oprot.writeString(self.queue_name.encode('utf-8') if sys.version_info[0] == 2 else self.queue_name) + oprot.writeFieldBegin("queue_name", TType.STRING, 1) + oprot.writeString( + self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name + ) oprot.writeFieldEnd() if self.message is not None: - oprot.writeFieldBegin('message', TType.STRING, 2) + oprot.writeFieldBegin("message", TType.STRING, 2) oprot.writeBinary(self.message) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin('timeout', TType.DOUBLE, 3) + oprot.writeFieldBegin("timeout", TType.DOUBLE, 3) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -434,9 +492,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -450,12 +507,32 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(put_args) put_args.thrift_spec = ( None, # 0 - (1, TType.STRING, 'queue_name', 'UTF8', None, ), # 1 - (2, TType.STRING, 'message', 'BINARY', None, ), # 2 - (3, TType.DOUBLE, 'timeout', None, None, ), # 3 + ( + 1, + TType.STRING, + "queue_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "message", + "BINARY", + None, + ), # 2 + ( + 3, + TType.DOUBLE, + "timeout", + None, + None, + ), # 3 ) @@ -468,17 +545,24 @@ class put_result(object): """ __slots__ = ( - 'success', - 'timed_out_error', + "success", + "timed_out_error", ) - - def __init__(self, success=None, timed_out_error=None,): + def __init__( + self, + success=None, + timed_out_error=None, + ): self.success = success self.timed_out_error = timed_out_error def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -506,13 +590,13 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('put_result') + oprot.writeStructBegin("put_result") if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) + oprot.writeFieldBegin("success", TType.STRUCT, 0) self.success.write(oprot) oprot.writeFieldEnd() if self.timed_out_error is not None: - oprot.writeFieldBegin('timed_out_error', TType.STRUCT, 1) + oprot.writeFieldBegin("timed_out_error", TType.STRUCT, 1) self.timed_out_error.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -522,9 +606,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -538,10 +621,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(put_result) put_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [PutResponse, None], None, ), # 0 - (1, TType.STRUCT, 'timed_out_error', [ThriftTimedOutError, None], None, ), # 1 + ( + 0, + TType.STRUCT, + "success", + [PutResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "timed_out_error", + [ThriftTimedOutError, None], + None, + ), # 1 ) fix_spec(all_structs) del all_structs diff --git a/baseplate/thrift/message_queue/__init__.py b/baseplate/thrift/message_queue/__init__.py index 947e972be..f37fa9583 100644 --- a/baseplate/thrift/message_queue/__init__.py +++ b/baseplate/thrift/message_queue/__init__.py @@ -1 +1 @@ -__all__ = ['ttypes', 'constants', 'RemoteMessageQueueService'] +__all__ = ["ttypes", "constants", "RemoteMessageQueueService"] diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index e5bf45df8..31dc582e7 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -5,22 +5,24 @@ # # options string: py:slots # - from thrift.Thrift import TException -from thrift.Thrift import TType +from thrift.Thrift import TType from thrift.transport import TTransport from thrift.TRecursive import fix_spec + all_structs = [] class CreateResponse(object): - __slots__ = ( - ) - + __slots__ = () def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -37,7 +39,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('CreateResponse') + oprot.writeStructBegin("CreateResponse") oprot.writeFieldStop() oprot.writeStructEnd() @@ -45,9 +47,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -65,12 +66,14 @@ def __ne__(self, other): class PutResponse(object): - __slots__ = ( - ) - + __slots__ = () def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -87,7 +90,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('PutResponse') + oprot.writeStructBegin("PutResponse") oprot.writeFieldStop() oprot.writeStructEnd() @@ -95,9 +98,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -120,16 +122,20 @@ class GetResponse(object): """ - __slots__ = ( - 'value', - ) - + __slots__ = ("value",) - def __init__(self, value=None,): + def __init__( + self, + value=None, + ): self.value = value def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) return iprot.readStructBegin() @@ -151,9 +157,9 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('GetResponse') + oprot.writeStructBegin("GetResponse") if self.value is not None: - oprot.writeFieldBegin('value', TType.STRING, 1) + oprot.writeFieldBegin("value", TType.STRING, 1) oprot.writeBinary(self.value) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -163,9 +169,8 @@ def validate(self): return def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -183,9 +188,7 @@ def __ne__(self, other): class ThriftTimedOutError(TException): - __slots__ = ( - ) - + __slots__ = () def __setattr__(self, *args): raise TypeError("can't modify immutable instance") @@ -198,7 +201,11 @@ def __hash__(self): @classmethod def read(cls, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and cls.thrift_spec is not None: + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) iprot.readStructBegin() while True: @@ -209,14 +216,13 @@ def read(cls, iprot): iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() - return cls( - ) + return cls() def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('ThriftTimedOutError') + oprot.writeStructBegin("ThriftTimedOutError") oprot.writeFieldStop() oprot.writeStructEnd() @@ -227,9 +233,8 @@ def __str__(self): return repr(self) def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] + return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -243,19 +248,24 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + all_structs.append(CreateResponse) -CreateResponse.thrift_spec = ( -) +CreateResponse.thrift_spec = () all_structs.append(PutResponse) -PutResponse.thrift_spec = ( -) +PutResponse.thrift_spec = () all_structs.append(GetResponse) GetResponse.thrift_spec = ( None, # 0 - (1, TType.STRING, 'value', 'BINARY', None, ), # 1 + ( + 1, + TType.STRING, + "value", + "BINARY", + None, + ), # 1 ) all_structs.append(ThriftTimedOutError) -ThriftTimedOutError.thrift_spec = ( -) +ThriftTimedOutError.thrift_spec = () fix_spec(all_structs) del all_structs From 3a62e2d8b1758282075623df555041e7372d32d7 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 13:22:59 -0800 Subject: [PATCH 42/72] linting --- baseplate/sidecars/event_publisher.py | 3 --- baseplate/sidecars/trace_publisher.py | 1 - tests/integration/message_queue_tests.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 2d5cbcb7d..bb656d0d1 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -18,8 +18,6 @@ from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE from baseplate.lib.message_queue import create_queue -from baseplate.lib.message_queue import DEFAULT_QUEUE_HOST -from baseplate.lib.message_queue import DEFAULT_QUEUE_PORT from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError @@ -28,7 +26,6 @@ from baseplate.server import EnvironmentInterpolation from baseplate.sidecars import Batch from baseplate.sidecars import BatchFull -from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index ae1f23cf6..a76bc3e1e 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -20,7 +20,6 @@ from baseplate.observers.tracing import MAX_SPAN_SIZE from baseplate.server import EnvironmentInterpolation from baseplate.sidecars import BatchFull -from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import RawJSONBatch from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index b8d3d8069..ad789fc3b 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -204,7 +204,7 @@ def test_get_timeout(self): with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): message_queue = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) - with contextlib.closing(message_queue) as mq: + with contextlib.closing(message_queue): start = time.time() with self.assertRaises(TimedOutError): queues[self.qname].get(timeout=0.1) From eb71f4c62064ce8640f2c7b11e2a03041d1553c4 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 14:08:19 -0800 Subject: [PATCH 43/72] cleanup --- baseplate/lib/message_queue.py | 88 ++++++------- tests/integration/message_queue_tests.py | 149 ++++++++++------------- 2 files changed, 99 insertions(+), 138 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 2e0fcc3f6..cfc9b61b4 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -89,16 +89,6 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: """ - @abc.abstractmethod - def close(self) -> None: - """Close the queue, freeing related resources. - - This must be called explicitly if queues are created/destroyed on the - fly. It is not automatically called when the object is reclaimed by - Python. - - """ - class PosixMessageQueue(MessageQueue): """A Gevent-friendly (but not required) inter process message queue. @@ -162,6 +152,13 @@ def unlink(self) -> None: self.queue.unlink() def close(self) -> None: + """Close the queue, freeing related resources. + + This must be called explicitly if queues are created/destroyed on the + fly. It is not automatically called when the object is reclaimed by + Python. + + """ self.queue.close() @@ -190,45 +187,44 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> None: except q.Full: raise TimedOutError - def close(self) -> None: - """Not implemented for in-memory queue""" - class RemoteMessageQueue(MessageQueue): - """A message queue that uses a remote Thrift server. - - Used in conjunction with the InMemoryMessageQueue to provide an alternative - to Posix queues, for use-cases where Posix is not appropriate. + """A write-only message queue that uses a remote Thrift server. This implementation is a temporary compromise and should only be used under very specific circumstances if the POSIX alternative is unavailable. - Specifically, using Thrift here has significant performance and/or + Specifically, using Thrift here may have significant performance and/or resource impacts. """ prom_labels = [ - "mode", # put or get "queue_name", "queue_host", "queue_port", "queue_max_messages", ] - remote_queue_requests_queued = Gauge( + remote_queue_put_requests_queued = Gauge( f"{PROM_PREFIX}_pending_puts_to_sidecar", "total number of queue requests in flight, being sent to the publishing sidecar.", prom_labels, multiprocess_mode="livesum", ) - remote_queue_requests_outcome = Counter( - f"{PROM_PREFIX}_sidecar_requests_outcome", - "outcomes (success/fail) or queue requests sent to the publishing sidecar.", - prom_labels + ["outcome"], + remote_queue_put_requests_success = Counter( + f"{PROM_PREFIX}_sidecar_put_success", + "successful queue requests sent to the publishing sidecar.", + prom_labels, ) - remote_queue_request_latency = Histogram( + remote_queue_put_requests_fail = Counter( + f"{PROM_PREFIX}_sidecar_put_fail", + "failed queue requests sent to the publishing sidecar.", + prom_labels, + ) + + remote_queue_put_request_latency = Histogram( f"{PROM_PREFIX}_sidecar_latency", "latency of message requests to the publishing sidecar.", prom_labels, @@ -254,8 +250,6 @@ def __init__( with self.pool.connection() as protocol: client = RemoteMessageQueueService.Client(protocol) client.create_queue(name, max_messages) - # self.connect() - # self.client.create_queue(name, max_messages) def create_connection_pool( self, pool_size: int, pool_timeout: int, pool_conn_max_age: int @@ -266,27 +260,20 @@ def create_connection_pool( ) return pool - def connect(self) -> None: - # Establish a connection with the queue server - transport = TSocket.TSocket(self.host, self.port) - self.transport = TTransport.TBufferedTransport(transport) - protocol = TBinaryProtocol.TBinaryProtocol(self.transport) - self.client = RemoteMessageQueueService.Client(protocol) - self.transport.open() - - def _update_counters(self, request_mode: str, outcome_mode: str) -> None: + def _update_counters(self, outcome: str) -> None: # This request is no longer queued - self.remote_queue_requests_queued.labels( - mode=request_mode, + self.remote_queue_put_requests_queued.labels( queue_name=self.name, queue_host=self.host, queue_port=self.port, queue_max_messages=self.max_messages, ).dec() # Increment success/failure counters - self.remote_queue_requests_outcome.labels( - mode=request_mode, - outcome=outcome_mode, + if outcome == "success": + metric = self.remote_queue_put_requests_success + else: + metric = self.remote_queue_put_requests_fail + metric.labels( queue_name=self.name, queue_host=self.host, queue_port=self.port, @@ -294,11 +281,11 @@ def _update_counters(self, request_mode: str, outcome_mode: str) -> None: ).inc() def _put_success_callback(self, greenlet: Any) -> None: - self._update_counters("put", "success") + self._update_counters("success") gevent.joinall([greenlet]) def _put_fail_callback(self, greenlet: Any) -> None: - self._update_counters("put", "fail") + self._update_counters("fail") gevent.joinall([greenlet]) try: greenlet.get() @@ -306,7 +293,7 @@ def _put_fail_callback(self, greenlet: Any) -> None: print("Remote queue `put` failed, exception found: ", e) def get(self, _: Optional[float] = None) -> bytes: - raise NotImplementedError # TODO: this is yucky, that this queue type does not implement get + raise NotImplementedError # This queue type is write-only def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: # get a connection from the pool @@ -316,8 +303,7 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa client.put(self.name, message, timeout) # record latency - self.remote_queue_request_latency.labels( - mode="put", + self.remote_queue_put_request_latency.labels( queue_name=self.name, queue_host=self.host, queue_port=self.port, @@ -330,8 +316,7 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa def put(self, message: bytes, timeout: Optional[float] = None) -> Any: # increment in-flight counter - self.remote_queue_requests_queued.labels( - mode="put", + self.remote_queue_put_requests_queued.labels( queue_name=self.name, queue_host=self.host, queue_port=self.port, @@ -344,21 +329,16 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> Any: greenlet.link_exception(self._put_fail_callback) return greenlet - def close(self) -> None: - pass # do we need to close anything?? - - def create_queue( queue_type: QueueType, queue_full_name: str, max_queue_size: int, max_element_size: int, ) -> MessageQueue: - # The in-memory queue is initialized here on the sidecar, and the main baseplate + # The in-memory queue is created on the sidecar, and the main baseplate # application will use a remote queue to interact with it. if queue_type == QueueType.IN_MEMORY: event_queue = InMemoryMessageQueue(max_queue_size) - # event_queue = RemoteMessageQueue(queue_full_name, max_queue_size, host, port) else: event_queue = PosixMessageQueue( # type: ignore diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index ad789fc3b..9a53528e3 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -95,56 +95,44 @@ def tearDown(self): class TestInMemoryMessageQueueCreation(unittest.TestCase): def test_instantiate_queue(self): - message_queue = InMemoryMessageQueue(max_messages=1) - - with contextlib.closing(message_queue) as mq: - self.assertEqual(mq.queue.maxsize, 1) + mq = InMemoryMessageQueue(max_messages=1) + self.assertEqual(mq.queue.maxsize, 1) def test_put_get(self): - message_queue = InMemoryMessageQueue(max_messages=1) - - with contextlib.closing(message_queue) as mq: - mq.put(b"x") - message = mq.get() - self.assertEqual(message, b"x") + mq = InMemoryMessageQueue(max_messages=1) + mq.put(b"x") + message = mq.get() + self.assertEqual(message, b"x") def test_get_timeout(self): - message_queue = InMemoryMessageQueue(max_messages=1) - - with contextlib.closing(message_queue) as mq: - start = time.time() - with self.assertRaises(TimedOutError): - mq.get(timeout=0.1) - elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=2) + mq = InMemoryMessageQueue(max_messages=1) + start = time.time() + with self.assertRaises(TimedOutError): + mq.get(timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=2) def test_put_timeout(self): - message_queue = InMemoryMessageQueue(max_messages=1) - - with contextlib.closing(message_queue) as mq: - mq.put(b"x") - start = time.time() - with self.assertRaises(TimedOutError): - mq.put(b"x", timeout=0.1) - elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=1) + mq = InMemoryMessageQueue(max_messages=1) + mq.put(b"x") + start = time.time() + with self.assertRaises(TimedOutError): + mq.put(b"x", timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=1) def test_put_zero_timeout(self): - message_queue = InMemoryMessageQueue(max_messages=1) - - with contextlib.closing(message_queue) as mq: - mq.put(b"x", timeout=0) - message = mq.get() - self.assertEqual(message, b"x") + mq = InMemoryMessageQueue(max_messages=1) + mq.put(b"x", timeout=0) + message = mq.get() + self.assertEqual(message, b"x") def test_put_full_zero_timeout(self): - message_queue = InMemoryMessageQueue(max_messages=1) + mq = InMemoryMessageQueue(max_messages=1) + mq.put(b"1", timeout=0) - with contextlib.closing(message_queue) as mq: - mq.put(b"1", timeout=0) - - with self.assertRaises(TimedOutError): - mq.put(b"2", timeout=0) + with self.assertRaises(TimedOutError): + mq.put(b"2", timeout=0) class GeventPatchedTestCase(unittest.TestCase): @@ -166,14 +154,12 @@ def test_put_get(self): queues = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): # create the corresponding queue that would ordinarily be on the main baseplate application/client-side - message_queue = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) - - with contextlib.closing(message_queue) as mq: - g = mq.put(b"x") - gevent.joinall([g]) - message = queues[self.qname].get(timeout=0.1) - print("message: ", str(message)) - self.assertEqual(message, b"x") + mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) + g = mq.put(b"x") + gevent.joinall([g]) + message = queues[self.qname].get(timeout=0.1) + print("message: ", str(message)) + self.assertEqual(message, b"x") def test_multiple_queues(self): queues = {} @@ -192,60 +178,55 @@ def test_multiple_queues(self): def test_queues_alternate_port(self): queues = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9091): - message_queue = RemoteMessageQueue(self.qname, max_messages=10, port=9091, pool_size=1) + mq = RemoteMessageQueue(self.qname, max_messages=10, port=9091, pool_size=1) - with contextlib.closing(message_queue) as mq: - g = mq.put(b"x", timeout=0.1) - gevent.joinall([g]) - self.assertEqual(queues[self.qname].get(timeout=2), b"x") + g = mq.put(b"x", timeout=0.1) + gevent.joinall([g]) + self.assertEqual(queues[self.qname].get(timeout=2), b"x") def test_get_timeout(self): queues = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) + mq = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) - with contextlib.closing(message_queue): - start = time.time() - with self.assertRaises(TimedOutError): - queues[self.qname].get(timeout=0.1) - elapsed = time.time() - start - self.assertAlmostEqual( - elapsed, 0.1, places=1 - ) # TODO: this routinely takes 0.105-0.11 seconds, is 1 place ok? + start = time.time() + with self.assertRaises(TimedOutError): + queues[self.qname].get(timeout=0.1) + elapsed = time.time() - start + self.assertAlmostEqual( + elapsed, 0.1, places=1 + ) # TODO: this routinely takes 0.105-0.11 seconds, is 1 place ok? def test_put_timeout(self): queues = {} # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError # is raised, we dont actually know unless we explicitly check with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=1, pool_size=2) - - with contextlib.closing(message_queue) as mq: - g = mq.put(b"x") # fill the queue - start = time.time() - with self.assertRaises(TimedOutError): # queue should be full - # put is non-blocking, so we need to wait for the result - g2 = mq.put(b"x", timeout=0.1) - gevent.joinall([g, g2]) - g2.get() # this should expose any exceptions encountered, i.e. TimedOutError - elapsed = time.time() - start - self.assertAlmostEqual(elapsed, 0.1, places=1) + mq = RemoteMessageQueue(self.qname, max_messages=1, pool_size=2) + + g = mq.put(b"x") # fill the queue + start = time.time() + with self.assertRaises(TimedOutError): # queue should be full + # put is non-blocking, so we need to wait for the result + g2 = mq.put(b"x", timeout=0.1) + gevent.joinall([g, g2]) + g2.get() # this should expose any exceptions encountered, i.e. TimedOutError + elapsed = time.time() - start + self.assertAlmostEqual(elapsed, 0.1, places=1) def test_pool(self): queues = {} # If we try to connect with more slots than the pool has, without joining, we should get an error with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - message_queue = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3) - - with contextlib.closing(message_queue) as mq: - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - g1 = mq.put(b"x", timeout=0.1) - g2 = mq.put(b"x", timeout=0.1) - g3 = mq.put(b"x", timeout=0.1) - g4 = mq.put(b"x", timeout=0.1) - gevent.joinall([g1, g2, g3, g4]) - assert "timed out waiting for a connection slot" in buf.getvalue() + mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + g1 = mq.put(b"x", timeout=0.1) + g2 = mq.put(b"x", timeout=0.1) + g3 = mq.put(b"x", timeout=0.1) + g4 = mq.put(b"x", timeout=0.1) + gevent.joinall([g1, g2, g3, g4]) + assert "timed out waiting for a connection slot" in buf.getvalue() class TestCreateQueue: From 1ccd7e26f6daf2f5364db40169d9a19d480be37b Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 14:11:21 -0800 Subject: [PATCH 44/72] lint --- baseplate/lib/message_queue.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index cfc9b61b4..d85c04ce9 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,4 +1,4 @@ -"""A message queue, with two implementations: POSIX-based, or in-memory using a Thrift server.""" +"""A message queue, with three implementations: POSIX-based, in-memory, or remote using a Thrift server.""" import abc import queue as q import select @@ -157,7 +157,7 @@ def close(self) -> None: This must be called explicitly if queues are created/destroyed on the fly. It is not automatically called when the object is reclaimed by Python. - + """ self.queue.close() @@ -271,7 +271,7 @@ def _update_counters(self, outcome: str) -> None: # Increment success/failure counters if outcome == "success": metric = self.remote_queue_put_requests_success - else: + else: metric = self.remote_queue_put_requests_fail metric.labels( queue_name=self.name, @@ -293,7 +293,7 @@ def _put_fail_callback(self, greenlet: Any) -> None: print("Remote queue `put` failed, exception found: ", e) def get(self, _: Optional[float] = None) -> bytes: - raise NotImplementedError # This queue type is write-only + raise NotImplementedError # This queue type is write-only def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: float) -> bool: # get a connection from the pool @@ -329,6 +329,7 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> Any: greenlet.link_exception(self._put_fail_callback) return greenlet + def create_queue( queue_type: QueueType, queue_full_name: str, From 48a7624c0b6928013dc8ab2861d955a2d6486fe5 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 14:16:22 -0800 Subject: [PATCH 45/72] comment --- baseplate/lib/message_queue.py | 3 --- tests/integration/message_queue_tests.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index d85c04ce9..a57506c7d 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -14,9 +14,6 @@ from prometheus_client import Counter from prometheus_client import Gauge from prometheus_client import Histogram -from thrift.protocol import TBinaryProtocol -from thrift.transport import TSocket -from thrift.transport import TTransport from baseplate.lib import config from baseplate.lib.prometheus_metrics import default_latency_buckets diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 9a53528e3..d5d50f1dd 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -187,7 +187,7 @@ def test_queues_alternate_port(self): def test_get_timeout(self): queues = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - mq = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) + _ = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) # create the empty queue start = time.time() with self.assertRaises(TimedOutError): From 53a4df02006391a1d8ad6a96c6678d63a0234441 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 14:18:50 -0800 Subject: [PATCH 46/72] x --- tests/integration/message_queue_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index d5d50f1dd..9bb7a8078 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -187,7 +187,9 @@ def test_queues_alternate_port(self): def test_get_timeout(self): queues = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - _ = RemoteMessageQueue(self.qname, max_messages=1, pool_size=1) # create the empty queue + _ = RemoteMessageQueue( + self.qname, max_messages=1, pool_size=1 + ) # create the empty queue start = time.time() with self.assertRaises(TimedOutError): From 210cbb99339505394c25e03672e82ccf4b8f11f5 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 15:25:07 -0800 Subject: [PATCH 47/72] reduce pool timeout for pool test --- tests/integration/message_queue_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 9bb7a8078..0df864cac 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -220,7 +220,7 @@ def test_pool(self): queues = {} # If we try to connect with more slots than the pool has, without joining, we should get an error with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3) + mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3, pool_timeout=0.1) buf = io.StringIO() with contextlib.redirect_stdout(buf): g1 = mq.put(b"x", timeout=0.1) From 654ad09b69b317c2c87bda9af8f7cbf923ec4ad0 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 15:35:41 -0800 Subject: [PATCH 48/72] newline --- tests/integration/message_queue_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 0df864cac..29785cea5 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -221,6 +221,7 @@ def test_pool(self): # If we try to connect with more slots than the pool has, without joining, we should get an error with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3, pool_timeout=0.1) + buf = io.StringIO() with contextlib.redirect_stdout(buf): g1 = mq.put(b"x", timeout=0.1) From 272f80d1ae7419bfd1c487a7370a9fff1838ac28 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 2 Mar 2023 15:40:55 -0800 Subject: [PATCH 49/72] remove pool test --- tests/integration/message_queue_tests.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 29785cea5..9ce3caf35 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -216,21 +216,6 @@ def test_put_timeout(self): elapsed = time.time() - start self.assertAlmostEqual(elapsed, 0.1, places=1) - def test_pool(self): - queues = {} - # If we try to connect with more slots than the pool has, without joining, we should get an error - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=3, pool_timeout=0.1) - - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - g1 = mq.put(b"x", timeout=0.1) - g2 = mq.put(b"x", timeout=0.1) - g3 = mq.put(b"x", timeout=0.1) - g4 = mq.put(b"x", timeout=0.1) - gevent.joinall([g1, g2, g3, g4]) - assert "timed out waiting for a connection slot" in buf.getvalue() - class TestCreateQueue: def test_posix_queue(self): From ffaf766d30fcc90aaec14287915bcef290f218b4 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Fri, 3 Mar 2023 08:08:19 -0800 Subject: [PATCH 50/72] import --- tests/integration/message_queue_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 9ce3caf35..8d71afafd 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -1,5 +1,4 @@ import contextlib -import io import time import unittest From 19f26cfde817793a636f312a0113481be8ca3e4f Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 11:58:53 -0800 Subject: [PATCH 51/72] cleanup --- baseplate/lib/message_queue.py | 48 ++++++++---------------- baseplate/sidecars/event_publisher.py | 17 ++++++--- baseplate/sidecars/trace_publisher.py | 19 +++++++--- tests/integration/message_queue_tests.py | 1 - 4 files changed, 39 insertions(+), 46 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index a57506c7d..5eec762f8 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -1,5 +1,6 @@ """A message queue, with three implementations: POSIX-based, in-memory, or remote using a Thrift server.""" import abc +import logging import queue as q import select import time @@ -24,8 +25,6 @@ DEFAULT_QUEUE_HOST = "127.0.0.1" DEFAULT_QUEUE_PORT = 9090 -PROM_PREFIX = "message_queue" - class MessageQueueError(Exception): """Base exception for message queue related errors.""" @@ -197,32 +196,29 @@ class RemoteMessageQueue(MessageQueue): prom_labels = [ "queue_name", - "queue_host", - "queue_port", - "queue_max_messages", ] - remote_queue_put_requests_queued = Gauge( - f"{PROM_PREFIX}_pending_puts_to_sidecar", + remote_queue_put_length = Gauge( + "message_queue_length", "total number of queue requests in flight, being sent to the publishing sidecar.", prom_labels, multiprocess_mode="livesum", ) - remote_queue_put_requests_success = Counter( - f"{PROM_PREFIX}_sidecar_put_success", + remote_queue_put_success = Counter( + "message_queue_sidecar_put_success", "successful queue requests sent to the publishing sidecar.", prom_labels, ) - remote_queue_put_requests_fail = Counter( - f"{PROM_PREFIX}_sidecar_put_fail", + remote_queue_put_fail = Counter( + "message_queue_sidecar_put_fail", "failed queue requests sent to the publishing sidecar.", prom_labels, ) - remote_queue_put_request_latency = Histogram( - f"{PROM_PREFIX}_sidecar_latency", + remote_queue_put_latency = Histogram( + "message_queue_sidecar_latency", "latency of message requests to the publishing sidecar.", prom_labels, buckets=default_latency_buckets, @@ -259,35 +255,27 @@ def create_connection_pool( def _update_counters(self, outcome: str) -> None: # This request is no longer queued - self.remote_queue_put_requests_queued.labels( + self.remote_queue_put_length.labels( queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, ).dec() # Increment success/failure counters if outcome == "success": - metric = self.remote_queue_put_requests_success + metric = self.remote_queue_put_success else: - metric = self.remote_queue_put_requests_fail + metric = self.remote_queue_put_fail metric.labels( queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, ).inc() def _put_success_callback(self, greenlet: Any) -> None: self._update_counters("success") - gevent.joinall([greenlet]) def _put_fail_callback(self, greenlet: Any) -> None: self._update_counters("fail") - gevent.joinall([greenlet]) try: greenlet.get() except Exception as e: - print("Remote queue `put` failed, exception found: ", e) + logging.info("Remote queue `put` failed, exception found: ", e) def get(self, _: Optional[float] = None) -> bytes: raise NotImplementedError # This queue type is write-only @@ -300,11 +288,8 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa client.put(self.name, message, timeout) # record latency - self.remote_queue_put_request_latency.labels( + self.remote_queue_put_latency.labels( queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, ).observe(time.perf_counter() - start_time) return True # Success # If the server responded with a timeout, raise our own timeout to be consistent with the posix queue @@ -313,11 +298,8 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa def put(self, message: bytes, timeout: Optional[float] = None) -> Any: # increment in-flight counter - self.remote_queue_put_requests_queued.labels( + self.remote_queue_put_length.labels( queue_name=self.name, - queue_host=self.host, - queue_port=self.port, - queue_max_messages=self.max_messages, ).inc() start_time = time.perf_counter() diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index bb656d0d1..d77342a8c 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -26,6 +26,7 @@ from baseplate.server import EnvironmentInterpolation from baseplate.sidecars import Batch from baseplate.sidecars import BatchFull +from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch @@ -174,14 +175,11 @@ def build_batch_and_publish( try: message = event_queue.get(timeout) - except TimedOutError: - message = None - - try: batcher.add(message) + except TimedOutError: continue except BatchFull: - pass + continue serialized = batcher.serialize() try: @@ -246,7 +244,14 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) + if cfg.queue_type == QueueType.IN_MEMORY.value: + # Start the Thrift server that communicates with RemoteMessageQueues and stores + # data in a InMemoryMessageQueue + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) + + else: + build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) if __name__ == "__main__": diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index a76bc3e1e..b8aec7681 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -20,6 +20,7 @@ from baseplate.observers.tracing import MAX_SPAN_SIZE from baseplate.server import EnvironmentInterpolation from baseplate.sidecars import BatchFull +from baseplate.sidecars import publisher_queue_utils from baseplate.sidecars import RawJSONBatch from baseplate.sidecars import SerializedBatch from baseplate.sidecars import TimeLimitedBatch @@ -123,7 +124,7 @@ def publish(self, payload: SerializedBatch) -> None: ) -def build_and_publish_batch( +def build_batch_and_publish( trace_queue: MessageQueue, batcher: TimeLimitedBatch, publisher: ZipkinPublisher, timeout: float ) -> None: while True: @@ -131,11 +132,9 @@ def build_and_publish_batch( try: message = trace_queue.get(timeout) - except TimedOutError: - message = None - - try: batcher.add(message) + except TimedOutError: + continue except BatchFull: serialized = batcher.serialize() publisher.publish(serialized) @@ -211,7 +210,15 @@ def publish_traces() -> None: post_timeout=publisher_cfg.post_timeout, ) - build_and_publish_batch(trace_queue, batcher, publisher, QUEUE_TIMEOUT) + if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: + # Start the Thrift server that communicates with RemoteMessageQueues and stores + # data in a InMemoryMessageQueue + with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + build_batch_and_publish(trace_queue, batcher, publisher, QUEUE_TIMEOUT) + + else: + build_batch_and_publish(trace_queue, batcher, publisher, QUEUE_TIMEOUT) + if __name__ == "__main__": diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 8d71afafd..9e1454d3a 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -157,7 +157,6 @@ def test_put_get(self): g = mq.put(b"x") gevent.joinall([g]) message = queues[self.qname].get(timeout=0.1) - print("message: ", str(message)) self.assertEqual(message, b"x") def test_multiple_queues(self): From d49b2d8880fc99521c3e5f58369afd2a2b212551 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 12:02:05 -0800 Subject: [PATCH 52/72] black --- baseplate/lib/message_queue.py | 1 + baseplate/sidecars/trace_publisher.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 5eec762f8..e24f0849a 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -26,6 +26,7 @@ DEFAULT_QUEUE_HOST = "127.0.0.1" DEFAULT_QUEUE_PORT = 9090 + class MessageQueueError(Exception): """Base exception for message queue related errors.""" diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index b8aec7681..4f55909ac 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -220,6 +220,5 @@ def publish_traces() -> None: build_batch_and_publish(trace_queue, batcher, publisher, QUEUE_TIMEOUT) - if __name__ == "__main__": publish_traces() From 9e13d344dfb7bab9af1161f5a17f43ac3da25e77 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 12:10:05 -0800 Subject: [PATCH 53/72] linting --- baseplate/lib/message_queue.py | 4 ++-- baseplate/sidecars/event_publisher.py | 3 ++- baseplate/sidecars/trace_publisher.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index e24f0849a..02bfd5461 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -268,7 +268,7 @@ def _update_counters(self, outcome: str) -> None: queue_name=self.name, ).inc() - def _put_success_callback(self, greenlet: Any) -> None: + def _put_success_callback(self, _: Any) -> None: self._update_counters("success") def _put_fail_callback(self, greenlet: Any) -> None: @@ -276,7 +276,7 @@ def _put_fail_callback(self, greenlet: Any) -> None: try: greenlet.get() except Exception as e: - logging.info("Remote queue `put` failed, exception found: ", e) + logging.info(f"Remote queue `put` failed, exception found: {e}") def get(self, _: Optional[float] = None) -> bytes: raise NotImplementedError # This queue type is write-only diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index d77342a8c..528438a52 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -247,7 +247,8 @@ def publish_events() -> None: if cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + queues = {} + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) else: diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 4f55909ac..a041c0ea0 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -213,7 +213,8 @@ def publish_traces() -> None: if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - with publisher_queue_utils.start_queue_server(host="127.0.0.1", port=9090): + queues = {} + with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): build_batch_and_publish(trace_queue, batcher, publisher, QUEUE_TIMEOUT) else: From 6d8d96731b4246d6814fbab87fcef690d1fce511 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 12:15:03 -0800 Subject: [PATCH 54/72] lint --- baseplate/lib/message_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 02bfd5461..901fbe707 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -276,7 +276,7 @@ def _put_fail_callback(self, greenlet: Any) -> None: try: greenlet.get() except Exception as e: - logging.info(f"Remote queue `put` failed, exception found: {e}") + logging.info("Remote queue `put` failed, exception found: %s", e) def get(self, _: Optional[float] = None) -> bytes: raise NotImplementedError # This queue type is write-only From cc9714fcb45d5b29b2b2d9f6f4cbafabfdc28023 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 12:20:46 -0800 Subject: [PATCH 55/72] linting --- baseplate/sidecars/event_publisher.py | 3 ++- baseplate/sidecars/trace_publisher.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 528438a52..daa508e27 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -7,6 +7,7 @@ import logging from typing import Any +from typing import Dict from typing import List from typing import Optional @@ -247,7 +248,7 @@ def publish_events() -> None: if cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - queues = {} + queues: Dict[str, MessageQueue] = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index a041c0ea0..205fdb4fc 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -3,6 +3,7 @@ import logging import urllib.parse +from typing import Dict from typing import Optional import requests @@ -213,7 +214,7 @@ def publish_traces() -> None: if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - queues = {} + queues: Dict[str, MessageQueue] = {} with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): build_batch_and_publish(trace_queue, batcher, publisher, QUEUE_TIMEOUT) From c77c67f907b2b22b809e0520eeee1fea90428d5d Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 13:27:02 -0800 Subject: [PATCH 56/72] remove create_queue --- baseplate/lib/message_queue.py | 5 +- baseplate/sidecars/event_publisher.py | 4 +- baseplate/sidecars/publisher_queue_utils.py | 20 +- baseplate/sidecars/trace_publisher.py | 4 +- .../RemoteMessageQueueService.py | 319 +----------------- .../thrift/message_queue/message_queue.thrift | 15 +- baseplate/thrift/message_queue/ttypes.py | 138 +------- tests/integration/message_queue_tests.py | 36 +- 8 files changed, 36 insertions(+), 505 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 901fbe707..6d1fd6b08 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -241,9 +241,6 @@ def __init__( self.host = host self.port = port self.pool = self.create_connection_pool(pool_size, pool_timeout, pool_conn_max_age) - with self.pool.connection() as protocol: - client = RemoteMessageQueueService.Client(protocol) - client.create_queue(name, max_messages) def create_connection_pool( self, pool_size: int, pool_timeout: int, pool_conn_max_age: int @@ -286,7 +283,7 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa with self.pool.connection() as protocol: client = RemoteMessageQueueService.Client(protocol) try: - client.put(self.name, message, timeout) + client.put(message, timeout) # record latency self.remote_queue_put_latency.labels( diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index daa508e27..24c2d02d7 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -7,7 +7,6 @@ import logging from typing import Any -from typing import Dict from typing import List from typing import Optional @@ -248,8 +247,7 @@ def publish_events() -> None: if cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - queues: Dict[str, MessageQueue] = {} - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server(event_queue, host="127.0.0.1", port=9090): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) else: diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index dffa8ff95..12407378e 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -1,7 +1,6 @@ """Shared functions for the event & trace publisher sidecars and message queues.""" import contextlib -from typing import Dict from typing import Generator from typing import Optional @@ -11,12 +10,10 @@ from baseplate.lib import config from baseplate.lib.message_queue import InMemoryMessageQueue -from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import TimedOutError from baseplate.server import make_listener from baseplate.server.thrift import make_server from baseplate.thrift.message_queue import RemoteMessageQueueService -from baseplate.thrift.message_queue.ttypes import CreateResponse from baseplate.thrift.message_queue.ttypes import PutResponse from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError @@ -31,19 +28,14 @@ class RemoteMessageQueueHandler: """ - def __init__(self, queues: Dict[str, MessageQueue]) -> None: + def __init__(self, queue: InMemoryMessageQueue) -> None: # Store the queue by name with its max messages - self.queues = queues + self.queue = queue - def create_queue(self, queue_name: str, max_messages: int) -> CreateResponse: - self.queues[queue_name] = InMemoryMessageQueue(max_messages) - return CreateResponse() - - def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) -> PutResponse: + def put(self, message: bytes, timeout: Optional[float] = None) -> PutResponse: # May raise TimedOutError try: - queue = self.queues[queue_name] - queue.put(message, timeout) + self.queue.put(message, timeout) return PutResponse() except TimedOutError: raise ThriftTimedOutError @@ -51,10 +43,10 @@ def put(self, queue_name: str, message: bytes, timeout: Optional[float] = None) @contextlib.contextmanager def start_queue_server( - queues: Dict[str, MessageQueue], host: str, port: int + queue: InMemoryMessageQueue, host: str, port: int ) -> Generator[StreamServer, None, None]: # Start a thrift server that will store the queue data in memory - processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler(queues)) + processor = RemoteMessageQueueService.Processor(RemoteMessageQueueHandler(queue)) server_bind_endpoint = config.Endpoint(f"{host}:{port}") listener = make_listener(server_bind_endpoint) server = make_server(server_config={}, listener=listener, app=processor) diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 205fdb4fc..9a8d98a0f 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -3,7 +3,6 @@ import logging import urllib.parse -from typing import Dict from typing import Optional import requests @@ -214,8 +213,7 @@ def publish_traces() -> None: if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - queues: Dict[str, MessageQueue] = {} - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server(trace_queue, host="127.0.0.1", port=9090): build_batch_and_publish(trace_queue, batcher, publisher, QUEUE_TIMEOUT) else: diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index f9666014e..9bda9c5c4 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -6,14 +6,13 @@ # options string: py:slots # import logging -import sys from thrift.Thrift import TApplicationException from thrift.Thrift import TMessageType from thrift.Thrift import TProcessor from thrift.Thrift import TType -from thrift.transport import TTransport from thrift.TRecursive import fix_spec +from thrift.transport import TTransport from .ttypes import * @@ -21,19 +20,9 @@ class Iface(object): - def create_queue(self, queue_name, max_messages): - """ - Parameters: - - queue_name - - max_messages - - """ - pass - - def put(self, queue_name, message, timeout): + def put(self, message, timeout): """ Parameters: - - queue_name - message - timeout @@ -48,57 +37,19 @@ def __init__(self, iprot, oprot=None): self._oprot = oprot self._seqid = 0 - def create_queue(self, queue_name, max_messages): - """ - Parameters: - - queue_name - - max_messages - - """ - self.send_create_queue(queue_name, max_messages) - return self.recv_create_queue() - - def send_create_queue(self, queue_name, max_messages): - self._oprot.writeMessageBegin("create_queue", TMessageType.CALL, self._seqid) - args = create_queue_args() - args.queue_name = queue_name - args.max_messages = max_messages - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_create_queue(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = create_queue_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException( - TApplicationException.MISSING_RESULT, "create_queue failed: unknown result" - ) - - def put(self, queue_name, message, timeout): + def put(self, message, timeout): """ Parameters: - - queue_name - message - timeout """ - self.send_put(queue_name, message, timeout) + self.send_put(message, timeout) return self.recv_put() - def send_put(self, queue_name, message, timeout): + def send_put(self, message, timeout): self._oprot.writeMessageBegin("put", TMessageType.CALL, self._seqid) args = put_args() - args.queue_name = queue_name args.message = message args.timeout = timeout args.write(self._oprot) @@ -129,7 +80,6 @@ class Processor(Iface, TProcessor): def __init__(self, handler): self._handler = handler self._processMap = {} - self._processMap["create_queue"] = Processor.process_create_queue self._processMap["put"] = Processor.process_put self._on_message_begin = None @@ -155,36 +105,13 @@ def process(self, iprot, oprot): self._processMap[name](self, seqid, iprot, oprot) return True - def process_create_queue(self, seqid, iprot, oprot): - args = create_queue_args() - args.read(iprot) - iprot.readMessageEnd() - result = create_queue_result() - try: - result.success = self._handler.create_queue(args.queue_name, args.max_messages) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except TApplicationException as ex: - logging.exception("TApplication exception in handler") - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception("Unexpected exception in handler") - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") - oprot.writeMessageBegin("create_queue", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - def process_put(self, seqid, iprot, oprot): args = put_args() args.read(iprot) iprot.readMessageEnd() result = put_result() try: - result.success = self._handler.put(args.queue_name, args.message, args.timeout) + result.success = self._handler.put(args.message, args.timeout) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -208,224 +135,24 @@ def process_put(self, seqid, iprot, oprot): # HELPER FUNCTIONS AND STRUCTURES -class create_queue_args(object): - """ - Attributes: - - queue_name - - max_messages - - """ - - __slots__ = ( - "queue_name", - "max_messages", - ) - - def __init__( - self, - queue_name=None, - max_messages=None, - ): - self.queue_name = queue_name - self.max_messages = max_messages - - def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I64: - self.max_messages = iprot.readI64() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin("create_queue_args") - if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) - oprot.writeFieldEnd() - if self.max_messages is not None: - oprot.writeFieldBegin("max_messages", TType.I64, 2) - oprot.writeI64(self.max_messages) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - -all_structs.append(create_queue_args) -create_queue_args.thrift_spec = ( - None, # 0 - ( - 1, - TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - ( - 2, - TType.I64, - "max_messages", - None, - None, - ), # 2 -) - - -class create_queue_result(object): - """ - Attributes: - - success - - """ - - __slots__ = ("success",) - - def __init__( - self, - success=None, - ): - self.success = success - - def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = CreateResponse() - self.success.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin("create_queue_result") - if self.success is not None: - oprot.writeFieldBegin("success", TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - -all_structs.append(create_queue_result) -create_queue_result.thrift_spec = ( - ( - 0, - TType.STRUCT, - "success", - [CreateResponse, None], - None, - ), # 0 -) - - class put_args(object): """ Attributes: - - queue_name - message - timeout """ __slots__ = ( - "queue_name", "message", "timeout", ) def __init__( self, - queue_name=None, message=None, timeout=None, ): - self.queue_name = queue_name self.message = message self.timeout = timeout @@ -443,20 +170,11 @@ def read(self, iprot): if ftype == TType.STOP: break if fid == 1: - if ftype == TType.STRING: - self.queue_name = ( - iprot.readString().decode("utf-8", errors="replace") - if sys.version_info[0] == 2 - else iprot.readString() - ) - else: - iprot.skip(ftype) - elif fid == 2: if ftype == TType.STRING: self.message = iprot.readBinary() else: iprot.skip(ftype) - elif fid == 3: + elif fid == 2: if ftype == TType.DOUBLE: self.timeout = iprot.readDouble() else: @@ -471,18 +189,12 @@ def write(self, oprot): oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return oprot.writeStructBegin("put_args") - if self.queue_name is not None: - oprot.writeFieldBegin("queue_name", TType.STRING, 1) - oprot.writeString( - self.queue_name.encode("utf-8") if sys.version_info[0] == 2 else self.queue_name - ) - oprot.writeFieldEnd() if self.message is not None: - oprot.writeFieldBegin("message", TType.STRING, 2) + oprot.writeFieldBegin("message", TType.STRING, 1) oprot.writeBinary(self.message) oprot.writeFieldEnd() if self.timeout is not None: - oprot.writeFieldBegin("timeout", TType.DOUBLE, 3) + oprot.writeFieldBegin("timeout", TType.DOUBLE, 2) oprot.writeDouble(self.timeout) oprot.writeFieldEnd() oprot.writeFieldStop() @@ -515,24 +227,17 @@ def __ne__(self, other): ( 1, TType.STRING, - "queue_name", - "UTF8", - None, - ), # 1 - ( - 2, - TType.STRING, "message", "BINARY", None, - ), # 2 + ), # 1 ( - 3, + 2, TType.DOUBLE, "timeout", None, None, - ), # 3 + ), # 2 ) diff --git a/baseplate/thrift/message_queue/message_queue.thrift b/baseplate/thrift/message_queue/message_queue.thrift index 8ae60dcbd..6e0eef99b 100644 --- a/baseplate/thrift/message_queue/message_queue.thrift +++ b/baseplate/thrift/message_queue/message_queue.thrift @@ -2,25 +2,14 @@ namespace go reddit.message_queue namespace py message_queue.thrift namespace java com.reddit.baseplate.message_queue -struct CreateResponse {} - struct PutResponse {} -struct GetResponse { - 1: binary value; -} - exception ThriftTimedOutError {} service RemoteMessageQueueService { - CreateResponse create_queue( - 1: string queue_name - 2: i64 max_messages - ); PutResponse put( - 1: string queue_name - 2: binary message - 3: double timeout + 1: binary message + 2: double timeout ) throws ( 1: ThriftTimedOutError timed_out_error ); diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index 31dc582e7..5b555d47d 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -7,61 +7,11 @@ # from thrift.Thrift import TException from thrift.Thrift import TType -from thrift.transport import TTransport from thrift.TRecursive import fix_spec -all_structs = [] - - -class CreateResponse(object): - - __slots__ = () - - def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin("CreateResponse") - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True +from thrift.transport import TTransport - def __ne__(self, other): - return not (self == other) +all_structs = [] class PutResponse(object): @@ -115,77 +65,6 @@ def __ne__(self, other): return not (self == other) -class GetResponse(object): - """ - Attributes: - - value - - """ - - __slots__ = ("value",) - - def __init__( - self, - value=None, - ): - self.value = value - - def read(self, iprot): - if ( - iprot._fast_decode is not None - and isinstance(iprot.trans, TTransport.CReadableTransport) - and self.thrift_spec is not None - ): - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.value = iprot.readBinary() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin("GetResponse") - if self.value is not None: - oprot.writeFieldBegin("value", TType.STRING, 1) - oprot.writeBinary(self.value) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key)) for key in self.__slots__] - return "%s(%s)" % (self.__class__.__name__, ", ".join(L)) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - for attr in self.__slots__: - my_val = getattr(self, attr) - other_val = getattr(other, attr) - if my_val != other_val: - return False - return True - - def __ne__(self, other): - return not (self == other) - - class ThriftTimedOutError(TException): __slots__ = () @@ -250,21 +129,8 @@ def __ne__(self, other): return not (self == other) -all_structs.append(CreateResponse) -CreateResponse.thrift_spec = () all_structs.append(PutResponse) PutResponse.thrift_spec = () -all_structs.append(GetResponse) -GetResponse.thrift_spec = ( - None, # 0 - ( - 1, - TType.STRING, - "value", - "BINARY", - None, - ), # 1 -) all_structs.append(ThriftTimedOutError) ThriftTimedOutError.thrift_spec = () fix_spec(all_structs) diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index 9e1454d3a..f75a6031f 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -150,58 +150,44 @@ class TestRemoteMessageQueueCreation(GeventPatchedTestCase): def test_put_get(self): # create the queue and start the server that would ordinarily be running on the sidecar - queues = {} - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + imq = InMemoryMessageQueue(max_messages=10) + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9090): # create the corresponding queue that would ordinarily be on the main baseplate application/client-side mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) g = mq.put(b"x") gevent.joinall([g]) - message = queues[self.qname].get(timeout=0.1) + message = imq.get(timeout=0.1) self.assertEqual(message, b"x") - def test_multiple_queues(self): - queues = {} - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): - mq1 = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) - mq2 = RemoteMessageQueue(self.qname + "2", max_messages=10, pool_size=1) - - g = mq1.put(b"x", timeout=0.1) - g2 = mq2.put(b"a", timeout=0.1) - - gevent.joinall([g, g2]) - # Check the queues in reverse order - self.assertEqual(queues[self.qname + "2"].get(timeout=0.1), b"a") - self.assertEqual(queues[self.qname].get(timeout=0.1), b"x") - def test_queues_alternate_port(self): - queues = {} - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9091): + imq = InMemoryMessageQueue(max_messages=10) + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9091): mq = RemoteMessageQueue(self.qname, max_messages=10, port=9091, pool_size=1) g = mq.put(b"x", timeout=0.1) gevent.joinall([g]) - self.assertEqual(queues[self.qname].get(timeout=2), b"x") + self.assertEqual(imq.get(timeout=2), b"x") def test_get_timeout(self): - queues = {} - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + imq = InMemoryMessageQueue(max_messages=1) + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9090): _ = RemoteMessageQueue( self.qname, max_messages=1, pool_size=1 ) # create the empty queue start = time.time() with self.assertRaises(TimedOutError): - queues[self.qname].get(timeout=0.1) + imq.get(timeout=0.1) elapsed = time.time() - start self.assertAlmostEqual( elapsed, 0.1, places=1 ) # TODO: this routinely takes 0.105-0.11 seconds, is 1 place ok? def test_put_timeout(self): - queues = {} + imq = InMemoryMessageQueue(max_messages=1) # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError # is raised, we dont actually know unless we explicitly check - with publisher_queue_utils.start_queue_server(queues, host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9090): mq = RemoteMessageQueue(self.qname, max_messages=1, pool_size=2) g = mq.put(b"x") # fill the queue From b8cfff8e1e4af598b8390bb3e6270fc27829f4cf Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 13:32:14 -0800 Subject: [PATCH 57/72] l --- baseplate/thrift/message_queue/RemoteMessageQueueService.py | 2 +- baseplate/thrift/message_queue/ttypes.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/baseplate/thrift/message_queue/RemoteMessageQueueService.py b/baseplate/thrift/message_queue/RemoteMessageQueueService.py index 9bda9c5c4..a1de1f86a 100644 --- a/baseplate/thrift/message_queue/RemoteMessageQueueService.py +++ b/baseplate/thrift/message_queue/RemoteMessageQueueService.py @@ -11,8 +11,8 @@ from thrift.Thrift import TMessageType from thrift.Thrift import TProcessor from thrift.Thrift import TType -from thrift.TRecursive import fix_spec from thrift.transport import TTransport +from thrift.TRecursive import fix_spec from .ttypes import * diff --git a/baseplate/thrift/message_queue/ttypes.py b/baseplate/thrift/message_queue/ttypes.py index 5b555d47d..754fa93f2 100644 --- a/baseplate/thrift/message_queue/ttypes.py +++ b/baseplate/thrift/message_queue/ttypes.py @@ -7,9 +7,8 @@ # from thrift.Thrift import TException from thrift.Thrift import TType -from thrift.TRecursive import fix_spec - from thrift.transport import TTransport +from thrift.TRecursive import fix_spec all_structs = [] From 442c20d8ac3d938a7fd500a78bcb375e9699f287 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 16:44:38 -0800 Subject: [PATCH 58/72] mypy --- baseplate/sidecars/event_publisher.py | 3 ++- baseplate/sidecars/trace_publisher.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 24c2d02d7..8a8a0880e 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -18,6 +18,7 @@ from baseplate.lib.events import MAX_EVENT_SIZE from baseplate.lib.events import MAX_QUEUE_SIZE from baseplate.lib.message_queue import create_queue +from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError @@ -244,7 +245,7 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - if cfg.queue_type == QueueType.IN_MEMORY.value: + if cfg.queue_type == QueueType.IN_MEMORY.value and isinstance(event_queue, InMemoryMessageQueue): # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(event_queue, host="127.0.0.1", port=9090): diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 9a8d98a0f..e0f236d1a 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -11,6 +11,7 @@ from baseplate.lib import config from baseplate.lib import message_queue from baseplate.lib import metrics +from baseplate.lib.message_queue import InMemoryMessageQueue from baseplate.lib.message_queue import MessageQueue from baseplate.lib.message_queue import QueueType from baseplate.lib.message_queue import TimedOutError @@ -210,7 +211,7 @@ def publish_traces() -> None: post_timeout=publisher_cfg.post_timeout, ) - if publisher_cfg.queue_type == QueueType.IN_MEMORY.value: + if publisher_cfg.queue_type == QueueType.IN_MEMORY.value and isinstance(trace_queue, InMemoryMessageQueue): # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(trace_queue, host="127.0.0.1", port=9090): From 5ec4da3669b6da559938ba8eeb6f77e20a01b56b Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 6 Mar 2023 16:52:58 -0800 Subject: [PATCH 59/72] black --- baseplate/sidecars/event_publisher.py | 4 +++- baseplate/sidecars/trace_publisher.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 8a8a0880e..09d3c05bb 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -245,7 +245,9 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - if cfg.queue_type == QueueType.IN_MEMORY.value and isinstance(event_queue, InMemoryMessageQueue): + if cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( + event_queue, InMemoryMessageQueue + ): # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(event_queue, host="127.0.0.1", port=9090): diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index e0f236d1a..47266a75d 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -211,7 +211,9 @@ def publish_traces() -> None: post_timeout=publisher_cfg.post_timeout, ) - if publisher_cfg.queue_type == QueueType.IN_MEMORY.value and isinstance(trace_queue, InMemoryMessageQueue): + if publisher_cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( + trace_queue, InMemoryMessageQueue + ): # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(trace_queue, host="127.0.0.1", port=9090): From ed0d3289b9d562bb4c12eb656cda06bbaee8c048 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 9 Mar 2023 14:28:09 -0800 Subject: [PATCH 60/72] cleanup and monkeypatch gevent --- baseplate/lib/message_queue.py | 15 +++++++-------- baseplate/sidecars/__init__.py | 4 ++++ baseplate/sidecars/event_publisher.py | 10 ++++++++-- baseplate/sidecars/publisher_queue_utils.py | 1 - tests/integration/message_queue_tests.py | 18 ++++++++---------- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 6d1fd6b08..52a42dd50 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -24,7 +24,7 @@ from baseplate.thrift.message_queue.ttypes import ThriftTimedOutError DEFAULT_QUEUE_HOST = "127.0.0.1" -DEFAULT_QUEUE_PORT = 9090 +DEFAULT_QUEUE_PORT = 9091 class MessageQueueError(Exception): @@ -54,7 +54,7 @@ def __init__(self, inner: Exception): super().__init__(f"{inner} (check `ulimit -q`?)") -class QueueType(Enum): +class QueueType(str, Enum): IN_MEMORY = "in_memory" POSIX = "posix" @@ -228,16 +228,13 @@ class RemoteMessageQueue(MessageQueue): def __init__( self, name: str, - max_messages: int, host: str = DEFAULT_QUEUE_HOST, port: int = DEFAULT_QUEUE_PORT, pool_size: int = 10, pool_timeout: int = 1, pool_conn_max_age: int = 120, ): - # Connect to the remote queue server, and creeate the new queue self.name = name - self.max_messages = max_messages self.host = host self.port = port self.pool = self.create_connection_pool(pool_size, pool_timeout, pool_conn_max_age) @@ -284,7 +281,6 @@ def _try_to_put(self, message: bytes, timeout: Optional[float], start_time: floa client = RemoteMessageQueueService.Client(protocol) try: client.put(message, timeout) - # record latency self.remote_queue_put_latency.labels( queue_name=self.name, @@ -316,14 +312,17 @@ def create_queue( # The in-memory queue is created on the sidecar, and the main baseplate # application will use a remote queue to interact with it. if queue_type == QueueType.IN_MEMORY: + logging.debug("Using in memory message queue") event_queue = InMemoryMessageQueue(max_queue_size) - - else: + elif queue_type == QueueType.POSIX: + logging.debug("Using posix message queue") event_queue = PosixMessageQueue( # type: ignore queue_full_name, max_messages=max_queue_size, max_message_size=max_element_size, ) + else: + raise f"Unknown queue type, options are [{QueueType.IN_MEMORY.value}, {QueueType.POSIX.value}]" return event_queue diff --git a/baseplate/sidecars/__init__.py b/baseplate/sidecars/__init__.py index 0ff05fc4f..d29267c46 100644 --- a/baseplate/sidecars/__init__.py +++ b/baseplate/sidecars/__init__.py @@ -4,6 +4,10 @@ from typing import NamedTuple from typing import Optional +import gevent + +gevent.monkey.patch_all() + class SerializedBatch(NamedTuple): item_count: int diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 09d3c05bb..487e717fb 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -48,6 +48,9 @@ MAX_BATCH_SIZE = 500 * 1024 # Seconds to wait for get/put operations on the event queue QUEUE_TIMEOUT = 0.2 +# Default address for remote queue server +DEFAULT_HOST = "127.0.0.1" +DEFAULT_PORT = 9091 class MaxRetriesError(Exception): @@ -173,7 +176,6 @@ def build_batch_and_publish( # Helper that continuously polls for messages, then batches and publishes them while True: message: Optional[bytes] - try: message = event_queue.get(timeout) batcher.add(message) @@ -230,6 +232,8 @@ def publish_events() -> None: "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), "max_element_size": config.Optional(config.Integer, MAX_EVENT_SIZE), "queue_type": config.Optional(config.String, default=QueueType.POSIX.value), + "queue_host": config.Optional(config.String, DEFAULT_HOST), + "queue_port": config.Optional(config.Integer, DEFAULT_PORT), }, ) @@ -250,7 +254,9 @@ def publish_events() -> None: ): # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue - with publisher_queue_utils.start_queue_server(event_queue, host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server( + event_queue, host=cfg.queue_host, port=cfg.queue_port + ): build_batch_and_publish(event_queue, batcher, publisher, QUEUE_TIMEOUT) else: diff --git a/baseplate/sidecars/publisher_queue_utils.py b/baseplate/sidecars/publisher_queue_utils.py index 12407378e..07d699ea4 100644 --- a/baseplate/sidecars/publisher_queue_utils.py +++ b/baseplate/sidecars/publisher_queue_utils.py @@ -50,7 +50,6 @@ def start_queue_server( server_bind_endpoint = config.Endpoint(f"{host}:{port}") listener = make_listener(server_bind_endpoint) server = make_server(server_config={}, listener=listener, app=processor) - # run the server until our caller is done with it server_greenlet = gevent.spawn(server.serve_forever) try: diff --git a/tests/integration/message_queue_tests.py b/tests/integration/message_queue_tests.py index f75a6031f..5a9373441 100644 --- a/tests/integration/message_queue_tests.py +++ b/tests/integration/message_queue_tests.py @@ -151,9 +151,9 @@ class TestRemoteMessageQueueCreation(GeventPatchedTestCase): def test_put_get(self): # create the queue and start the server that would ordinarily be running on the sidecar imq = InMemoryMessageQueue(max_messages=10) - with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9090): + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9091): # create the corresponding queue that would ordinarily be on the main baseplate application/client-side - mq = RemoteMessageQueue(self.qname, max_messages=10, pool_size=1) + mq = RemoteMessageQueue(self.qname, pool_size=1) g = mq.put(b"x") gevent.joinall([g]) message = imq.get(timeout=0.1) @@ -161,8 +161,8 @@ def test_put_get(self): def test_queues_alternate_port(self): imq = InMemoryMessageQueue(max_messages=10) - with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9091): - mq = RemoteMessageQueue(self.qname, max_messages=10, port=9091, pool_size=1) + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9092): + mq = RemoteMessageQueue(self.qname, port=9092, pool_size=1) g = mq.put(b"x", timeout=0.1) gevent.joinall([g]) @@ -170,10 +170,8 @@ def test_queues_alternate_port(self): def test_get_timeout(self): imq = InMemoryMessageQueue(max_messages=1) - with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9090): - _ = RemoteMessageQueue( - self.qname, max_messages=1, pool_size=1 - ) # create the empty queue + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9091): + _ = RemoteMessageQueue(self.qname, pool_size=1) # create the empty queue start = time.time() with self.assertRaises(TimedOutError): @@ -187,8 +185,8 @@ def test_put_timeout(self): imq = InMemoryMessageQueue(max_messages=1) # `put` is non-blocking, so if we try to put onto a full queue and a TimeOutError # is raised, we dont actually know unless we explicitly check - with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9090): - mq = RemoteMessageQueue(self.qname, max_messages=1, pool_size=2) + with publisher_queue_utils.start_queue_server(imq, host="127.0.0.1", port=9091): + mq = RemoteMessageQueue(self.qname, pool_size=2) g = mq.put(b"x") # fill the queue start = time.time() From f171e10c587141c7260fbd40d6a4a1e11e96fcba Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 9 Mar 2023 14:50:14 -0800 Subject: [PATCH 61/72] exception --- baseplate/lib/message_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 52a42dd50..8dc2efc5d 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -322,7 +322,7 @@ def create_queue( max_message_size=max_element_size, ) else: - raise f"Unknown queue type, options are [{QueueType.IN_MEMORY.value}, {QueueType.POSIX.value}]" + raise Exception(f"Unknown queue type, options are [{QueueType.IN_MEMORY.value}, {QueueType.POSIX.value}]") return event_queue From 38464cf15a3c78aa64d25a8f7f47cebec3f737ab Mon Sep 17 00:00:00 2001 From: syd ryan Date: Wed, 15 Mar 2023 13:48:17 -0700 Subject: [PATCH 62/72] lint --- baseplate/lib/message_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 8dc2efc5d..2006ec97f 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -322,7 +322,9 @@ def create_queue( max_message_size=max_element_size, ) else: - raise Exception(f"Unknown queue type, options are [{QueueType.IN_MEMORY.value}, {QueueType.POSIX.value}]") + raise Exception( + f"Unknown queue type, options are [{QueueType.IN_MEMORY.value}, {QueueType.POSIX.value}]" + ) return event_queue From a456dc8be0012f252029d1f25a71054c3834d727 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 16 Mar 2023 14:06:07 -0700 Subject: [PATCH 63/72] catch gevent errors --- baseplate/lib/message_queue.py | 10 ++++++++-- baseplate/sidecars/event_publisher.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 2006ec97f..277c9ff5e 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -296,8 +296,14 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> Any: queue_name=self.name, ).inc() start_time = time.perf_counter() - - greenlet = gevent.spawn(self._try_to_put, message, timeout, start_time) + try: + greenlet = gevent.spawn(self._try_to_put, message, timeout, start_time) + except Exception as e: + # If the greenlet failed to spawn for some reason, log it as a failure + self._update_counters("fail") + logging.info("Remote queue `put` failed because gevent.spawn failed: %s", e) + raise e + # Process success/failure when the greenlet completes greenlet.link_value(self._put_success_callback) greenlet.link_exception(self._put_fail_callback) return greenlet diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 487e717fb..44fae2aac 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -10,6 +10,7 @@ from typing import List from typing import Optional +import gevent import requests from baseplate import __version__ as baseplate_version @@ -252,6 +253,7 @@ def publish_events() -> None: if cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( event_queue, InMemoryMessageQueue ): + gevent.monkey.patch_all() # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server( From 18242c729a0bf2deaba86e77c2777e43ad3fc07d Mon Sep 17 00:00:00 2001 From: syd ryan Date: Thu, 16 Mar 2023 14:08:42 -0700 Subject: [PATCH 64/72] lint --- baseplate/lib/message_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 277c9ff5e..7c3ba4657 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -298,7 +298,7 @@ def put(self, message: bytes, timeout: Optional[float] = None) -> Any: start_time = time.perf_counter() try: greenlet = gevent.spawn(self._try_to_put, message, timeout, start_time) - except Exception as e: + except Exception as e: # If the greenlet failed to spawn for some reason, log it as a failure self._update_counters("fail") logging.info("Remote queue `put` failed because gevent.spawn failed: %s", e) From 855de0f97ba0fb55236ca25fbaf04f8f8f77a568 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Fri, 17 Mar 2023 09:50:13 -0700 Subject: [PATCH 65/72] remove from trace publisher and sidecar init --- baseplate/sidecars/__init__.py | 4 ---- baseplate/sidecars/trace_publisher.py | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/baseplate/sidecars/__init__.py b/baseplate/sidecars/__init__.py index d29267c46..0ff05fc4f 100644 --- a/baseplate/sidecars/__init__.py +++ b/baseplate/sidecars/__init__.py @@ -4,10 +4,6 @@ from typing import NamedTuple from typing import Optional -import gevent - -gevent.monkey.patch_all() - class SerializedBatch(NamedTuple): item_count: int diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index 47266a75d..a39ac4b66 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -5,6 +5,7 @@ from typing import Optional +import gevent import requests from baseplate import __version__ as baseplate_version @@ -214,6 +215,7 @@ def publish_traces() -> None: if publisher_cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( trace_queue, InMemoryMessageQueue ): + gevent.monkey.patch_all() # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(trace_queue, host="127.0.0.1", port=9090): From b2c2ff0f0436f4f80e33b1c4dfbb9d7614c116cd Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 20 Mar 2023 09:17:56 -0700 Subject: [PATCH 66/72] lint --- baseplate/lib/message_queue.py | 2 +- baseplate/sidecars/event_publisher.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 9b14955d9..42d254b22 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -179,7 +179,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: def put(self, message: bytes, timeout: Optional[float] = None) -> None: try: - self.queue.put(message, block=False,timeout=timeout) + self.queue.put(message, block=False, timeout=timeout) except q.Full: raise TimedOutError diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 63d99da28..2f756730a 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -196,18 +196,20 @@ def build_and_publish_batch( try: message = event_queue.get(timeout) batcher.add(message) - continue # We will publish on the next loop if batch full/queue empty + continue # We will publish on the next loop if batch full/queue empty except TimedOutError: message = None - pass # We may want to publish if we have other messages in the batch and time is up + pass # We may want to publish if we have other messages in the batch and time is up except BatchFull: batcher.is_full = True - pass # We want to publish bc batch is full + pass # We want to publish bc batch is full - if batcher.is_ready: # Time is up or batch is full + if batcher.is_ready: # Time is up or batch is full serialize_and_publish_batch(publisher, batcher) - if message: # If we published because batch was full, we need to add the straggler we popped + if ( + message + ): # If we published because batch was full, we need to add the straggler we popped batcher.add(message) @@ -264,7 +266,6 @@ def publish_events() -> None: batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) - def flush_queue_signal_handler(_signo: int, _frame: FrameType) -> None: """Signal handler for flushing messages from the queue and publishing them.""" message: Optional[bytes] @@ -283,7 +284,6 @@ def flush_queue_signal_handler(_signo: int, _frame: FrameType) -> None: serialize_and_publish_batch(publisher, batcher) batcher.add(message) sys.exit(0) - if cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( event_queue, InMemoryMessageQueue From 8da60eb665a65c7ddcd208329ee7d86fb76c2d16 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 20 Mar 2023 09:23:58 -0700 Subject: [PATCH 67/72] lint --- baseplate/sidecars/event_publisher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 2f756730a..a3f42ffa6 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -196,13 +196,13 @@ def build_and_publish_batch( try: message = event_queue.get(timeout) batcher.add(message) - continue # We will publish on the next loop if batch full/queue empty + continue # Start loop again - we will publish on the next loop if batch full/queue empty except TimedOutError: message = None - pass # We may want to publish if we have other messages in the batch and time is up + # Keep going - we may want to publish if we have other messages in the batch and time is up except BatchFull: batcher.is_full = True - pass # We want to publish bc batch is full + # Keep going - we want to publish bc batch is full if batcher.is_ready: # Time is up or batch is full serialize_and_publish_batch(publisher, batcher) From 608a4465b0640b7b58ca6853d3cfc811d5618f09 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 20 Mar 2023 12:04:56 -0700 Subject: [PATCH 68/72] fix --- baseplate/lib/message_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseplate/lib/message_queue.py b/baseplate/lib/message_queue.py index 42d254b22..6b4a7a458 100644 --- a/baseplate/lib/message_queue.py +++ b/baseplate/lib/message_queue.py @@ -179,7 +179,7 @@ def get(self, timeout: Optional[float] = None) -> bytes: def put(self, message: bytes, timeout: Optional[float] = None) -> None: try: - self.queue.put(message, block=False, timeout=timeout) + self.queue.put(message, block=True, timeout=timeout) except q.Full: raise TimedOutError From adea15ca6b0a0fdd4067bd40f2534d74fbc6973d Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 20 Mar 2023 16:18:25 -0700 Subject: [PATCH 69/72] add scripts, remove signal handling for remote queue --- baseplate/sidecars/event_publisher.py | 7 ++----- bin/baseplate-sidecar-event | 17 +++++++++++++++++ bin/baseplate-sidecar-trace | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100755 bin/baseplate-sidecar-event create mode 100755 bin/baseplate-sidecar-trace diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index a3f42ffa6..170d8b60e 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -288,16 +288,13 @@ def flush_queue_signal_handler(_signo: int, _frame: FrameType) -> None: if cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( event_queue, InMemoryMessageQueue ): - gevent.monkey.patch_all() # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server( event_queue, host=cfg.queue_host, port=cfg.queue_port ): - for sig in (gevent.signal.SIGINT, gevent.signal.SIGTERM): - gevent.signal.signal(sig, flush_queue_signal_handler) - gevent.signal.siginterrupt(sig, False) - print("patched with gevent signal") + # Note: shutting down gracefully with gevent is complicated, so we are not + # implementing for now. There is the possibility of event loss on shutdown. build_and_publish_batch(event_queue, batcher, publisher, QUEUE_TIMEOUT) else: diff --git a/bin/baseplate-sidecar-event b/bin/baseplate-sidecar-event new file mode 100755 index 000000000..378818264 --- /dev/null +++ b/bin/baseplate-sidecar-event @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# this is here and not just a console_script entry point because we want the +# gevent monkeypatching to happen before any pieces of baseplate are imported +# at all, as would happen if we were to have this code be in the baseplate +# package. + +from gevent.monkey import patch_all + +from baseplate.server.monkey import patch_stdlib_queues + +patch_all() +patch_stdlib_queues() + +from baseplate.sidecars.event_publisher import publish_events + +publish_events() \ No newline at end of file diff --git a/bin/baseplate-sidecar-trace b/bin/baseplate-sidecar-trace new file mode 100755 index 000000000..6a458de75 --- /dev/null +++ b/bin/baseplate-sidecar-trace @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# this is here and not just a console_script entry point because we want the +# gevent monkeypatching to happen before any pieces of baseplate are imported +# at all, as would happen if we were to have this code be in the baseplate +# package. + +from gevent.monkey import patch_all + +from baseplate.server.monkey import patch_stdlib_queues + +patch_all() +patch_stdlib_queues() + +from baseplate.sidecars.trace_publisher import publish_traces + +publish_traces() \ No newline at end of file From 14ecb2e3e3f089c245b93f4b2b1165bcf095dc19 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Mon, 20 Mar 2023 16:45:29 -0700 Subject: [PATCH 70/72] lint --- baseplate/sidecars/event_publisher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/baseplate/sidecars/event_publisher.py b/baseplate/sidecars/event_publisher.py index 170d8b60e..0c1a6de2c 100644 --- a/baseplate/sidecars/event_publisher.py +++ b/baseplate/sidecars/event_publisher.py @@ -13,7 +13,6 @@ from typing import List from typing import Optional -import gevent import requests from baseplate import __version__ as baseplate_version From 86f3eca6935097cff2a5bc6af35bc5a3b8b57efa Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 21 Mar 2023 11:53:48 -0700 Subject: [PATCH 71/72] typos --- baseplate/sidecars/trace_publisher.py | 2 -- bin/baseplate-sidecar-event | 2 +- bin/baseplate-sidecar-trace | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/baseplate/sidecars/trace_publisher.py b/baseplate/sidecars/trace_publisher.py index a39ac4b66..47266a75d 100644 --- a/baseplate/sidecars/trace_publisher.py +++ b/baseplate/sidecars/trace_publisher.py @@ -5,7 +5,6 @@ from typing import Optional -import gevent import requests from baseplate import __version__ as baseplate_version @@ -215,7 +214,6 @@ def publish_traces() -> None: if publisher_cfg.queue_type == QueueType.IN_MEMORY.value and isinstance( trace_queue, InMemoryMessageQueue ): - gevent.monkey.patch_all() # Start the Thrift server that communicates with RemoteMessageQueues and stores # data in a InMemoryMessageQueue with publisher_queue_utils.start_queue_server(trace_queue, host="127.0.0.1", port=9090): diff --git a/bin/baseplate-sidecar-event b/bin/baseplate-sidecar-event index 378818264..f736a8cf3 100755 --- a/bin/baseplate-sidecar-event +++ b/bin/baseplate-sidecar-event @@ -14,4 +14,4 @@ patch_stdlib_queues() from baseplate.sidecars.event_publisher import publish_events -publish_events() \ No newline at end of file +publish_events() diff --git a/bin/baseplate-sidecar-trace b/bin/baseplate-sidecar-trace index 6a458de75..c3425653b 100755 --- a/bin/baseplate-sidecar-trace +++ b/bin/baseplate-sidecar-trace @@ -14,4 +14,4 @@ patch_stdlib_queues() from baseplate.sidecars.trace_publisher import publish_traces -publish_traces() \ No newline at end of file +publish_traces() From 9ba9170224b45f37f5a57ecb39383ccfec664967 Mon Sep 17 00:00:00 2001 From: syd ryan Date: Tue, 21 Mar 2023 16:20:42 -0700 Subject: [PATCH 72/72] x --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index f9778ca56..2fc715583 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,8 @@ "bin/baseplate-shell", "bin/baseplate-tshell", "bin/baseplate-healthcheck", + "bin/baseplate-sidecar-event", + "bin/baseplate-sidecar-trace", ], # the thrift compiler must be able to find baseplate.thrift to build # services which extend BaseplateService.