execlib/execlog/router.py

'''
Router
'''
import time
import asyncio
import logging
import inspect
import traceback
import threading
import concurrent 
from enum import Enum
from pathlib import Path
from typing import Any, Callable
from collections import defaultdict
from colorama import Fore, Back, Style
from functools import partial, update_wrapper
from concurrent.futures import ThreadPoolExecutor, wait, as_completed

from tqdm.auto import tqdm

from execlog.event import Event
from execlog.listener import Listener
from execlog.util.generic import color_text, get_func_name


logger = logging.getLogger(__name__)

class FrameDirective(Enum):
    '''
    Indicates frame-level behavior when a callback fails.
    '''
    CONTINUE_WITHOUT = 1
    CANCEL_FRAME = 2

class CallbackTimeoutError(Exception):
    ...

class CancelledFrameError(Exception):
    ...

class Router[E: Event]:
    '''
    Route events to registered callbacks

    .. note::

        Generalized registration includes an endpoint (the origin of an event), a pattern (to
        filter events at the endpoint), and a callback (to be executed if pattern is matched).

        The Router _routes_ events to affiliated callbacks in a multi-threaded fashion. A
        thread pool handles these jobs as events are submitted, typically by a composing
        Listener. The Listener "hears" an event, and passes it on through to a Router to
        further filter and delegate any matching follow-up jobs.

        This base Router implements most of the registry and filter model. When events are
        submitted for propagation, they are checked for matching routes. Events specify an
        origin endpoint, which is used as the filter for attached routes. The event is then
        subjected to the ``filter`` method, which checks if the event matches the registered
        ``pattern`` under the originated ``endpoint``. If so, the callback is scheduled for
        execution, and the matching event is passed as its sole argument.

        Subclasses are expected to implement (at least) the ``filter`` method. This function is
        responsible for wrapping up the task-specific logic needed to determine if an event,
        originating from a known endpoint, matches the callback-specific pattern. This method
        needn't handle any other filter logic, like checking if the event originates from the
        provided endpoint, as this is already handled by the outer look in ``matching_routes``.

        ``get_listener`` is a convenience method that instantiates and populates an affiliated
        Listener over the register paths found in the Router. Listeners require a Router upon
        instantiation so events can be propagated to available targets when they occur.
        ``get_listener()`` is the recommended way to attain a Listener.

    .. admonition:: on debouncing events

        Previously, debouncing was handled by listeners. This logic has been generalized
        and moved to this class, as it's general enough to be desired across various
        Listener types. We also need unique, identifying info only available with a
        ``(endpoint, callback, pattern)`` triple in order to debounce events in accordance
        with their intended target.

    .. admonition:: tracking events and serializing callback frames

        Although not part of the original implementation, we now track which events have a
        callback chain actively being executed, and prevent the same chain from being
        started concurrently. If the callback chain is actively running for an event, and
        that same event is submitted before this chain finishes, the request is simply
        enqueued. The ``clear_event`` method is attached as a "done callback" to each job
        future, and will re-submit the event once the active chain finishes.

        While this could be interpreted as a harsh design choice, it helps prevent many
        many thread conflicts (race conditions, writing to the same resources, etc) when
        the same function is executed concurrently, many times over. Without waiting
        completely for an event to be fully handled, later jobs may complete before
        earlier ones, or interact with intermediate disk states (raw file writes, DB
        inserts, etc), before the earliest call has had a chance to clean up.

    .. admonition:: Details behind the threaded model and future management

        Routers kick off execution in response to *events*. These events are received via
        ``.submit``, and the following process is kick-started:

        1. Each event is wrapped in its own ``.submit_event`` call and submitted as a task
           to the *primary* thread pool. Let $E$ be the set of events, and $|E|$ be the
           number of events. ``.submit`` exits as soon as these $|E|$ tasks are enqueued,
           not waiting for the completion of the corresponding worker threads. There are
           now $|E|$ tier-I tasks waiting to be started by the router's primary thread
           pool, with states set to "pending."
        2. The primary thread pool begins running the enqueued ``.submit_event`` calls
           concurrently using allocated resources (e.g., four threads). Each
           ``.submit_event`` call matches the associated event $e$ to $c_e$ callbacks,
           according to the registered routes. These callbacks are each individually
           submitted to the *secondary* thread pool as tier-II tasks and waited upon
           *within* the ``.submit_event`` call. This thread pool separation prevents
           deadlocks which would otherwise be an issue if submitting both tier-I and
           tier-II tasks to the same thread pool. Tier-I tasks that have begun this
           process are in the "running" state, and the submitted tier-II futures are now
           "pending."
        3. Once the $c_e$ callbacks for event $e$ are completed (which are tier-II tasks
           being waited upon within a tier-I task), their done-callbacks will be invoked
           within "a thread belonging to the process that added them" (with
           ``wait_on_event_callbacks`` calling through to ``submit_callback``, which
           attaches ``general_task_done``). Where these done callbacks are executed
           varies based on a few conditions. See

           https://stackoverflow.com/a/26021772/4573915

           for a great breakdown on this. The gist is the following: 

           a. If the callback is attached to a future that is already cancelled or
              completed, it will be invoked immediately in the current thread doing the
              attaching.
           b. If the future is queued/pending and successfully cancelled, the thread
              doing the cancelling will immediately invoke all of the future's callbacks.
           c. Otherwise, the thread that executes the future's task (could produce either
              a successful result or an exception) will invoke the callbacks.

           So if a task completes (i.e., is not cancelled, producing either a result or an
           exception), the thread that ran the task will also handle the associated
           callbacks. If the task is successfully cancelled (which means it was never
           running and never allocated a thread), the cancelling context will handle the
           callbacks, and this happens here only in ``.shutdown()`` with the call
           ``thread_pool.shutdown(cancel_futures=True)``.

           The results of these futures are made available in this ``submit_event``
           context. Note that these results are dictated by the logic in
           ``wait_on_futures``. If a future was successfully cancelled or raised and
           exception during execution, it will not have a result to add to this list.

           The *router-level* post-callbacks are then submitted to the secondary thread
           pool and awaited in a similar fashion to the individual $c_e$ callbacks. The
           results obtained from the event callbacks are passed through to these
           "post-callbacks" for possible centralized processing.
        4. Once all post-callbacks have completed (along with *their* attached
           "done-callbacks," which are just ``.general_task_done`` checks handled in the
           executor thread that ran the post-callback), finally the tier-I
           ``.submit_event`` future can be marked completed (either with a successfully
           attached result or an exception), and its attached "done-callbacks" will be
           ran the in the same tier-I thread that handled the task (which is
           ``general_task_done`` and ``clear_event``, in that order).

        - General behaviors and additional remarks:
          * Thread pool futures can only be cancelled prior to "running" or "done" states.
            Both successful cancellation or completion trigger a future's done-callbacks,
            which will be executed in one a few possible contexts depending several
            conditions, as detailed above.
          * Tier-I tasks have ``clear_event`` attached as a done-callback, which tracks
            the task result and resubmits the event if valid (successfully debounced)
            repeat requests were received while the event has been handled.
          * Both tier-I and tier-II callbacks have a ``.general_task_done`` callback, which
            attempts to retrieve the future result if it wasn't cancelled (if it was, this
            retrieval would raise a ``CancelledError``). If it wasn't cancelled but an
            exception was raised during execution, this same exception will be re-raised
            and re-caught, logged as an error, and exit "cleanly" (since job failures
            shouldn't throw off the entire process). A successful result retrieval will
            have no effect.

        - On interrupts, exception handling, and future cancellation:
          * 
    '''
    listener_cls = Listener[E]

    def __init__(self, loop=None, workers=None):
        '''
        Parameters:
            loop:
            workers: number of workers to assign the thread pool when the event loop is
                     started. Defaults to ``None``, which, when passed to
                     ThreadPoolExecutor, will by default use 5x the number of available
                     processors on the machine (which the docs claim is a reasonable
                     assumption given threads are more commonly leveraged for I/O work
                     rather than intense CPU operations). Given the intended context for
                     this class, this assumption aligns appropriately.
        '''
        self.loop         = loop
        self.workers      = workers

        self.routemap : dict[str, list[tuple]] = defaultdict(list)
        self.post_callbacks = []

        # track running jobs by event
        self.running_events = defaultdict(set)

        # debounce tracker
        self.next_allowed_time = defaultdict(int)

        # store prepped (e.g., delayed) callbacks
        self.callback_registry = {}
        self.callback_start_times = {}

        # track event history
        self.event_log = []

        # shutdown flag, mostly for callbacks
        self.should_exit = False
        self._active_futures = set()

        self._thread_pool_1 = None
        self._thread_pool_2 = None
        self._route_lock  = threading.Lock()

    @property
    def primary_thread_pool(self):
        '''Handle tier-I futures.'''
        if self._thread_pool_1 is None:
            self._thread_pool_1 = ThreadPoolExecutor(max_workers=self.workers)
        return self._thread_pool_1
    
    @property
    def secondary_thread_pool(self):
        '''Handle tier-II futures.'''
        if self._thread_pool_2 is None:
            self._thread_pool_2 = ThreadPoolExecutor(max_workers=self.workers)
        return self._thread_pool_2

    def register(
        self,
        endpoint,
        callback: Callable,
        pattern,
        debounce=200,
        delay=10,
        callback_timeout=None,
        condition=FrameDirective.CONTINUE_WITHOUT,
        **listener_kwargs,
    ):
        '''
        Register a route. 

        Note: Listener arguments
            Notice how listener_kwargs are accumulated instead of uniquely assigned to an
            endpoint. This is generally acceptable as some listeners may allow various
            configurations for the same endpoint. Note, however, for something like the
            PathListener, this will have no effect. Registering the same endpoint multiple
            times won't cause any errors, but the configuration options will only remain
            for the last registered group.

            (Update) The above remark about PathListener's is no longer, and likely never
            was. Varying flag sets under the same endpoint do in fact have a cumulative
            effect, and we need to be able disentangle events accordingly through
            submitted event's ``action`` value.

        Parameters:
            endpoint:
            callback: callable accepting an event to be executed if when a matching event
                      is received
            pattern: hashable object to be used when filtering event (passed to inherited
                     ``filter(...)``)
            debounce:
            delay:
            callback_timeout: timeout for waiting 
        '''
        route_tuple = (
            callback,
            pattern,
            debounce,
            delay,
            callback_timeout,
            condition,
            listener_kwargs
        )
        self.routemap[endpoint].append(route_tuple)

    def submit(self, events: E | list[E], callbacks: list[Callable] | None = None):
        '''
        Handle a list of events. Each event is matched against the registered callbacks,
        and those callbacks are ran concurrently (be it via a thread pool or an asyncio
        loop).
        '''
        if type(events) is not list:
            events = [events]

        futures = []
        for event in events:
            future = self.submit_callback(self.submit_event, event, callbacks=callbacks)
            future.add_done_callback(lambda f: self.clear_event(event, f))
            futures.append(future)

        return futures

    def submit_event(
        self,
        event      : E,
        callbacks  : list[Callable] | None       = None,
        timeouts   : list[int|float] | None      = None,
        conditions : list[FrameDirective] | None = None,
    ) -> list:
        '''
        Group up and submit all matching callbacks for ``event``. All callbacks are ran
        concurrently in their own threads, and this method blocks until all are completed.

        In the outer ``submit`` context, this blocking method is itself ran in its own
        thread, and the registered post-callbacks are attached to the completion of this
        function, i.e., the finishing of all callbacks matching provided event.

        Note that an event may not match any routes, in which case the method exits early.
        An empty list is returned, and this shows up as the outer future's result. In this
        case, the event is never considered "running," and the non-result picked up in
        ``clear_event`` will ensure it exits right away (not even attempting to pop the
        event from the running list, and for now not tracking it in the event log).
        '''
        if callbacks is None:
            # ensure same thread gets all matching routes & sets debounce updates; else
            # this may be split across threads mid-check, preventing one thread from
            # handling the blocking of the entire group
            with self._route_lock:
                callbacks, timeouts, conditions = self.matching_routes(event)

        # stop early if no work to do
        if len(callbacks) == 0:
            return []

        # enqueue requested/matched callbacks and exit if running
        event_idx = self.event_index(event) 
        if event_idx in self.running_events:
            self.queue_callbacks(event_idx, callbacks)
            return []

        # TODO: Chesterton's fence
        # callbacks now computed, flush the running event
        # note: a separate thread could queue valid callbacks since the running check;
        # o/w we know the running index is empty
        # self.running_events[event_idx] = self.running_events[event_idx]

        # submit matching callbacks and wait for them to complete
        # *may* raise a FrameCancelledError, which we let propagate upward
        completed_futures = self.wait_on_event_callbacks(
            event,
            callbacks,
            timeouts=timeouts,
            conditions=conditions,
        )

        # finally call post event-group callbacks (only if some event callbacks were
        # submitted), wait for them to complete
        if completed_futures:
            wait([
                self.submit_event_callback(post_callback, event, completed_futures)[0]
                for post_callback in self.post_callbacks
            ])

        return completed_futures

    def _submit_with_thread_pool(self, thread_pool, callback: Callable, *args, **kwargs):
        if inspect.iscoroutinefunction(callback):
            if self.loop is None:
                self.loop = asyncio.new_event_loop()
                asyncio.set_event_loop(self.loop)

            #loop.run_in_executor(executor, loop.create_task, callback(event))
            #future = self.loop.call_soon_threadsafe(
            #    self.loop.create_task,
            future = asyncio.run_coroutine_threadsafe(
                callback(*args, **kwargs),
                self.loop,
            )
        else:
            future = thread_pool.submit(
                callback, *args, **kwargs
            )

        return future

    def submit_callback(self, callback: Callable, *args, **kwargs):
        future = self._submit_with_thread_pool(
            self.primary_thread_pool,
            callback,
            *args, **kwargs
        )
        self._active_futures.add(future)

        return future

    def submit_event_callback(self, callback: Callable, event: E, *args, **kwargs):
        '''
        Note: this method is expected to return a future. Perform any event-based
        filtering before submitting a callback with this method.
        '''
        # exit immediately if exit flag is set
        # if self.should_exit:
        #     return
        submitted_time = time.time()
        callback = self.wrap_timed_callback(callback, submitted_time)

        future = self._submit_with_thread_pool(
            self.secondary_thread_pool,
            callback,
            event,
            *args, **kwargs
        )
        future.add_done_callback(self.general_callback)

        return future, submitted_time

    def matching_routes(
        self,
        event: E,
        event_time = None
    ) -> tuple[list[Callable], list[int|float]]:
        '''
        Return eligible matching routes for the provided event.

        Note that we wait as late as possible before enqueuing matches if the event is in
        fact already active in a frame. If this method were start filtering results while
        the frame is active, and the frame were to finish before all matching callbacks
        were determined, we would be perfectly happy to return all matches, and allow the
        outer ``submit_event`` context to run them right away in a newly constructed frame.
        The _very_ next thing that gets done is adding this event to the active event
        tracker. Otherwise, matching is performed as usual, and eligible callbacks are
        simply enqueued for the next event frame, which will be checked in the "done"
        callback of the active frame. The logic here should mostly "seal up" any real
        opportunities for error, e.g., a frame ending and popping off elements from
        ``running_events`` half-way through their inserting at the end of this method, or
        multiple threads checking for matching routes for the same event, and both coming
        away with a non-empty set of matches to run. That last example highlights
        precisely how the single event-frame model works: many threads might be running
        this method at the same time, for the same event (which has fired rapidly), but
        only one should be able to "secure the frame" and begin running the matching
        callbacks. Making the "active frame check" both as late as possible and as close
        to the event blocking stage in the tracker (in ``submit_event``), we make the
        ambiguity gap as small as possible (and almost certainly smaller than any
        realistic I/O-bound event duplication).

        Note: on event actions
            The debounce reset is now only set if the event is successfully filtered. This
            allows some middle ground when trying to depend on event actions: if the
            action passes through, we block the whole range of actions until the debounce
            window completes. Otherwise, the event remains open, only to be blocked by the
            debounce on the first matching action.
        '''
        matches    = []
        timeouts   = []
        conditions = []
        endpoint   = event.endpoint
        name       = event.name
        #action     = tuple(event.action) # should be more general
        event_time = time.time()*1000 if event_time is None else event_time

        for (callback, pattern, debounce, delay, ctimeout, condition, listen_kwargs) in self.routemap[endpoint]:
            #index = (endpoint, name, action, callback, pattern, debounce, delay)
            index = (endpoint, name, callback, pattern, debounce, delay)

            if event_time < self.next_allowed_time[index]:
                # reject event
                continue

            callback_name = get_func_name(callback)

            name_text     = color_text(name,               Fore.BLUE)
            pattern_text  = color_text(pattern,            Fore.BLUE)
            endpoint_text = color_text(endpoint,           Fore.BLUE)
            callback_text = color_text(callback_name[:50], Fore.BLUE)

            if self.filter(event, pattern, **listen_kwargs):
                # note that delayed callbacks are added
                matches.append(self.get_delayed_callback(callback, delay, index))
                timeouts.append(ctimeout)
                conditions.append(condition)

                # set next debounce 
                self.next_allowed_time[index] = event_time + debounce

                match_text = color_text('matched', Style.BRIGHT, Fore.GREEN)
                logger.info(
                    f'Event [{name_text}] {match_text} [{pattern_text}] under [{endpoint_text}] for [{callback_text}]'
                )
            else:
                match_text = color_text('rejected', Style.BRIGHT, Fore.RED)
                logger.debug(
                    f'Event [{name_text}] {match_text} against [{pattern_text}] under [{endpoint_text}] for [{callback_text}]'
                )

        return matches, timeouts, conditions

    def get_delayed_callback(self, callback: Callable, delay: int|float, index):
        '''
        Parameters:
            callback: function to wrap  
            delay: delay in ms
        '''
        if index not in self.callback_registry:
            async def async_wrap(callback, *args, **kwargs):
                await asyncio.sleep(delay/1000)
                return await callback(*args, **kwargs) 

            def sync_wrap(callback, *args, **kwargs):
                time.sleep(delay/1000)
                return callback(*args, **kwargs) 

            wrapper = None
            if inspect.iscoroutinefunction(callback): wrapper = async_wrap
            else:                                     wrapper = sync_wrap

            self.callback_registry[index] = partial(wrapper, callback)

        return self.callback_registry[index]

    def wait_on_event_callbacks(
        self,
        event      : E,
        callbacks  : list[Callable],
        timeouts   : list[int | float | None] | None    = None,
        conditions : list[FrameDirective | None] | None = None,
    ): #, *args, **kwargs):
        '''
        Waits for event-associated callbacks to complete.

        Submits each callback in ``callbacks`` to the thread pool (via
        ``submit_callback``), passing ``event`` as the only parameter to each. Once
        started, the future for callback ``callbacks[i]`` will have ``timeouts[i]``
        seconds to complete. If it has not completed in this time, the future's result is
        set to the Timeout exception

        Overridable by inheriting classes based on callback structure
        '''
        if timeouts is None:
            timeouts = [None]*len(callbacks)

        if conditions is None:
            conditions = [FrameDirective.CONTINUE_WITHOUT]*len(callbacks)

        future_map = {}
        timed_futures = set()
        for callback, timeout, on_err in zip(callbacks, timeouts, conditions):
            # "raw" callback here is the reference that will be indexed in `.callback_start_times`
            future, submitted_time = self.submit_event_callback(callback, event) #*args, **kwargs)
            future_map[future] = (
                callback,
                get_func_name(callback),
                future,
                submitted_time,
                timeout,
                on_err
            )

            if timeout is not None:
                timed_futures.add(future)

        completed_futures = []
        # iterate while there are some futures not completed (cancelled, or finished w/
        # exception or return value). When completed, a future is removed from `futures`.
        while future_map:
            min_timeout = float('inf')
            expired_futures = [] # actively running but expired

            if timed_futures:
                time_now = time.time()
                for future in list(timed_futures):
                    callback, cback_name, future, submitted_time, timeout, on_err = future_map[future]
                    future_tuple = (future, cback_name, on_err)
                    callback_id = (callback, submitted_time)

                    if future.done():
                        timed_futures.remove(future)
                        continue

                    if future.running() and callback_id in self.callback_start_times:
                        time_running = time_now - self.callback_start_times[callback_id]
                        time_left = timeout - time_running

                        # track running futures that have timed out
                        if time_left < 0:
                            expired_futures.append(future_tuple)
                            continue
                    else:
                        # running w/o a start time, or queued. Set time-left to the timeout
                        time_left = timeout

                    # track running future w/ smallest non-zero time left before timeout
                    min_timeout = min(min_timeout, time_left)

            done_futures = [] 
            for future in list(future_map.keys()):
                _, cback_name, future, _, _, on_err = future_map[future]
                future_tuple = (future, cback_name, on_err)

                if future.done():
                    # done futures have a set result or exception and now immutable
                    done_futures.append(future_tuple) # local
                    completed_futures.append(future) # global
                    future_map.pop(future)

            done_with_exception = [
                ftuple
                for ftuple in done_futures
                if ftuple[0].exception() is not None
            ]
            frame_cancellable = expired_futures + done_with_exception

            cancel_frame = False
            # set a timeout exception for all expired futures, and determine
            # if any of these timeouts should result in a frame cancellation
            for _, _, on_err in frame_cancellable:
                if on_err == FrameDirective.CANCEL_FRAME:
                    cancel_frame = True
                    break

            # set exceptions on running expired futures
            for expired_future, cback_name, _ in expired_futures:
                # results are immutable once set; double check doneness
                if not expired_future.done():
                    expired_future.set_exception(
                        CallbackTimeoutError(
                            f'Event callback {cback_name} timed out with condition {on_err}'
                        )
                    )

            if cancel_frame:
                # cancel the active frame. Expired futures have already been handled, and
                # we now need to handle running futures (not cancellable) and pending
                # futures (cancellable)
 
                # explicitly cancel queued futures before looping through active futures
                for future in list(future_map.keys()):
                    # cancellable futures not yet started
                    # can't really be more specific here, sets a CancelledError 
                    future.cancel()

                for future in list(future_map.keys()):
                    _, cback_name, future, _, _, _ = future_map[future]

                    if future.done():
                        # includes the expired futures whose exceptions were just set &
                        # those cancelled, although managing `future_map` isn't needed at
                        # this stage
                        future_map.pop(future)
                    elif future.running():
                        # possible memory leak
                        future.set_exception(
                            CancelledFrameError(
                                f'Indirect frame cancellation for event callback {cback_name}'
                            )
                        )
                        # attempt to communicate with threaded processes for graceful shutdown
                        # -> set shutdown flags
                        ...

                # finally, raise an exception indicating the frame was cancelled
                raise CancelledFrameError

            # if no expired futures (or they can be ignored with CONTINUE_WITHOUT), carry
            # on. Wait for the future with the least time remaining; we shouldn't need to
            # do any of the above future parsing until at least min-timeout more seconds.
            timeout = None
            if min_timeout < float('inf'):
                timeout = min_timeout

            # if any remaining futures can produce CANCELLED FRAMEs, wait only on them.
            # This lets us be maximally dynamic: 
            # -> We respond before the timeout only if the newly completed might CANCEL
            #    the FRAME, which we want to do as soon as possible.
            # -> If not responding early, we wait at most `timeout` seconds to be back and
            #    checking if the *earliest possible timeout violation* has occurred. If
            #    so, we want to handle it immediately no matter what so I can set the
            #    desired Timeout exception as *early as possible*. If I don't, the future
            #    might finish and set its own result, shutting me out when I would've
            #    liked to capture the timeout violation.
            # -> When CANCEL FRAME futures are available, I don't wait on all pending
            #    futures as I *don't mind* being late to them. I only need to check
            #    immediately on CANCEL FRAMEs, and otherwise be back no later than
            #    `timeout`. If there's no `timeout` I'll wait for the first CANCEL FRAME,
            #    if no CANCEL FRAME I'll wait up to timeout on all futures, and if neither
            #    I'll simply wait until all futures complete.
            cancel_frame_futures = [
                future
                for future, ftuple in future_map.items()
                if ftuple[-1] == FrameDirective.CANCEL_FRAME
            ]

            return_when = concurrent.futures.ALL_COMPLETED
            wait_futures = future_map.keys()
            if cancel_frame_futures:
                return_when = concurrent.futures.FIRST_COMPLETED
                wait_futures = cancel_frame_futures

            # if no active future has a timeout
            done, _ = wait(
                wait_futures,
                timeout=timeout,
                return_when=return_when
            )

        # add any trailing completed futures
        completed_futures.extend(done)

        return completed_futures

    def queue_callbacks(self, event_idx, callbacks: list[Callable]):
        '''
        Overridable by inheriting classes based on callback structure
        '''
        self.running_events[event_idx].update(callbacks)

    def wrap_timed_callback(self, callback: Callable, submitted_time):
        '''
        Check for shutdown flag and exit before running the callbacks. 

        Applies primarily to jobs enqueued by the ThreadPoolExecutor but not started when
        an interrupt is received.
        '''
        def safe_callback(callback, *args, **kwargs):
            # track when this task actually begins
            self.callback_start_times[(callback, submitted_time)] = time.time()

            return callback(*args, **kwargs) 

        return partial(safe_callback, callback)

    def filter(self, event: E, pattern, **listen_kwargs) -> bool:
        '''
        Determine if a given event matches the provided pattern

        Parameters:
            event:
            pattern:
            listen_kwargs: 
        '''
        raise NotImplementedError

    def add_post_callback(self, callback: Callable):
        self.post_callbacks.append(callback)

    def get_listener(self, listener_cls=None):
        '''
        Create a new Listener to manage watched routes and their callbacks.
        '''
        if listener_cls is None:
            listener_cls = self.listener_cls

        if listener_cls is None:
            raise ValueError('No Listener class provided')

        listener = listener_cls(self)
        return self.extend_listener(listener)

    def extend_listener(self, listener):
        '''
        Extend a provided Listener object with the Router instance's ``listener_kwargs``.
        '''
        for endpoint, route_tuples in self.routemap.items():
            for route_tuple in route_tuples:
                listen_kwargs = route_tuple[-1]
                listener.listen(endpoint, **listen_kwargs)
        return listener

    def stop_event(self, event):
        '''
        Pop event out of the running events tracker and return it.
        '''
        event_idx = self.event_index(event)
        return self.running_events.pop(event_idx, None)

    def clear_event(self, event: E, future):
        '''
        Clear an event. Pops the passed event out of ``running_events``, and if the
        request counter is >0, the event is re-submitted.

        This method is attached as a "done" callback to the main event wrapping job
        ``submit_event``. The ``future`` given to this method is one to which it was
        attached as this "done" callback. This method should only be called when that
        ``future`` is finished running (or failed). If any jobs were submitted in the
        wrapper task, the future results here should be non-empty (even if the methods
        don't return anything; we'll at least have ``[None,...]`` if we scheduled at least
        one callback). We use this fact to filter out non-work threads that call this
        method. Because even the ``matching_routes`` check is threaded, we can't wait to
        see an event has no work to schedule, and thus can't prevent this method being
        attached as a "done" callback. The check for results from the passed future allows
        us to know when in fact a valid frame has finished, and a resubmission may be on
        the table.

        .. admonition:: Why we don't need to worry about resubmission w/ empty results

            Note that, even though we can't check for whether there will be any matching
            routes prior to calling ``submit_event`` (we have to wait until we're in that
            method, at which point this method will already be attached as a callback),
            the event will never be marked as "running" and added to ``running_events``.
            This means that we won't queue up any callbacks for the same event while
            waiting on it, and then exit early here, never to re-submit. The worry was
            that a event response might match 0 callbacks (e.g., due to debouncing) and
            return ``[]`` from ``submit_event``, but *while waiting for this to complete*,
            the same event is submitted and matches (e.g., debouncing timers now allowing
            the event through). This could mean that the repeat event gets queued behind
            the event in ``running_events`` and should be resubmitted here as a "queued
            callback," but *won't do so* because we exit early if no results are obtained.
            This is **not an issue** because the original event (that didn't match any
            callbacks) will never be marked as running, and thus never prevent the second
            event from itself running if in fact in matches a non-empty callback set. This
            means that an empty future result set seen here indicates both 1) that no work
            took place, and 2) no conflicting events were prevented from running, and we
            can exit early here.
        '''
        self._active_futures.remove(future)

        # result should be *something* if work was scheduled, since `submit_event` wraps
        # up futures in a list. If no result, event was never marked active, and don't
        # want to resubmit as any duplicates were allowed to start. Attempt to get result,
        # returning if it's (None or []) or raised an Exception (possibly a
        # FrameCancelledError if there's a frame issue, for a CancelledError if the tier-I
        # task was successfully cancelled following a `.shutdown()` call)
        try:
            if not future.result(): return
        except concurrent.futures.CancelledError as e:
            #logger.error(f'Tier-I future cancelled')
            # do not re-raise; outer context can handle cancellations
            return
        except CancelledFrameError as e:
            # do not re-raise; outer context will manage frame cancellations
            return
        except Exception as e:
            # print traceback for unexpected exception
            logger.error(f'Unexpected exception in tier-I future: "{e}"')
            traceback.print_exc()
            return

        self.event_log.append((event, future))
        queued_callbacks = self.stop_event(event)

        # resubmit event if some queued work remains
        if queued_callbacks and len(queued_callbacks) > 0:
            logger.debug(
                f'Event [{event.name}] resubmitted with [{len(queued_callbacks)}] queued callbacks'
            )
            self.submit(event, callbacks=queued_callbacks)

    def event_index(self, event):
        return event[:2]

    def general_callback(self, future):
        try:
            future.result()
        except concurrent.futures.CancelledError as e:
            logger.error(f'Tier-II future cancelled; "{e}"')
        except CancelledFrameError as e:
            logger.error(f'Tier-II frame cancelled; "{e}"')
        except Exception as e:
            logger.warning(f'Tier-II job failed with unknown exception "{e}"')

    def shutdown(self):
        logger.info(color_text('Router shutdown received', Fore.BLACK, Back.RED))
        self.should_exit = True

        # manually track and cancel pending futures b/c `.shutdown(cancel_futures=True)`
        # is misleading, and will cause an outer `as_completed` loop to hang
        for future in tqdm(
            list(self._active_futures),
            desc=color_text(
                f'Cancelling {len(self._active_futures)} pending futures...',
                Fore.BLACK, Back.RED),
            colour='red',
        ):
            future.cancel()

        if self._thread_pool_2 is not None:
            # cancel pending futures (i.e., those not started)
            self.secondary_thread_pool.shutdown(wait=False)

        if self._thread_pool_1 is not None:
            # cancel pending futures (i.e., those not started)
            self.primary_thread_pool.shutdown(wait=False)


class ChainRouter[E: Event](Router[E]):
    '''
    Routes events to registered callbacks
    '''
    def __init__(self, ordered_routers):
        super().__init__()

        self.ordered_routers = []
        for router in ordered_routers:
            self.add_router(router)

        self.running_events = defaultdict(lambda: defaultdict(set))

    def add_router(self, router):
        '''
        TODO: allow positional insertion in ordered list
        
        .. note::

            the ``routemap`` extensions here shouldn't be necessary, since 1) route maps
            show up only in ``matching_routes``, and 2) ``matching_routes`` is only
            invoked in ``submit_event``, which is totally overwritten for the ChainRouter
            type. All events are routed through to individual Routers, and which point
            their route maps are used.
        '''
        self.ordered_routers.append(router)
        for endpoint, routelist in router.routemap.items():
            self.routemap[endpoint].extend(routelist)

    def matching_routes(
        self,
        event: E,
        event_time=None
    ):
        '''
        Colloquial ``callbacks`` now used as a dict of lists of callbacks, indexed by
        router, and only having keys for routers with non-empty callback lists.
        '''
        if event_time is None:
            event_time = time.time()*1000

        callback_map  = {}
        timeout_map   = {}
        condition_map = {}
        for router in self.ordered_routers:
            matches, timeouts, conditions = router.matching_routes(event, event_time)
            if matches:
                callback_map[router]  = matches
                timeout_map[router]   = timeouts
                condition_map[router] = conditions

        return callback_map, timeout_map, condition_map

    def wait_on_event_callbacks(
        self,
        event      : E,
        callbacks  : dict[Router, list[Callable]],
        timeouts   : dict[Router, list[int | float | None]] | None    = None,
        conditions : dict[Router, list[FrameDirective | None]] | None = None,
    ): #, *args, **kwargs):
        '''
        Returns a dictionary mapping from 

        Note: relies on order of callback-associated dicts matching that of
        ``ordered_routers``, which should happen in ``matching_routes``.

        This method blurs the OOP lines a bit, as we're actually passing dicts rather than
        the lists expected by the super class. The parent ``submit_event`` is carefully
        designed to be indifferent; use caution when making changes.
        '''
        if timeouts is not None:
            timeouts = {}

        if conditions is not None:
            conditions = {}

        futures = {}
        for router, callback_list in callbacks.items():
            futures[router] = router.submit_event(
                event,
                callback_list,
                timeouts=timeouts.get(router),
                conditions=conditions.get(router),
            )

        return futures

    def queue_callbacks(self, event_idx, callbacks):
        for router, callback_list in callbacks.items():
            self.running_events[event_idx][router].update(callback_list)

    def stop_event(self, event):
        '''
        Sub-routers do not get a "done" callback for their ``submit_event`` jobs, as they
        would if they handled their own event submissions. They will, however, set the
        submitted event as "running." We can't rely on sub-routers' "done" callbacks to
        "unset" the running event, because the disconnect between the thread completing
        and execution of that callback may take too long. 

        Instead, we explicitly unset the running event for each of the constituent
        sub-routers at the *same time* we handle the ChainRouter's notion of event's
        ending.
        '''
        event_idx = self.event_index(event)
        for router in self.ordered_routers:
            rq_callbacks = router.running_events.pop(event_idx, [])
            assert len(rq_callbacks) == 0

        return self.running_events.pop(event_idx, None)

    def get_listener(self, listener_cls=None):
        if listener_cls is None:
            for router in self.ordered_routers:
                if router.listener_cls is not None:
                    listener_cls = router.listener_cls
                    break

        listener = super().get_listener(listener_cls)
        for router in self.ordered_routers:
            router.extend_listener(listener)
        return listener

    def shutdown(self):
        super().shutdown()

        # for router in self.ordered_routers:
        #     router.shutdown()


# RouterBuilder
def route(router, route_group, **route_kwargs):
    def decorator(f):
        f._route_data = (router, route_group, route_kwargs)
        return f

    return decorator

class RouteRegistryMeta(type):
    '''
    Metaclass handling route registry at the class level.
    '''
    def __new__(cls, name, bases, attrs):
        route_registry = defaultdict(lambda: defaultdict(list))

        def register_route(method):
            nonlocal route_registry

            if hasattr(method, '_route_data'):
                router, route_group, route_kwargs = method._route_data
                route_registry[router][route_group].append((method, route_kwargs))

        # add registered superclass methods; iterate over bases (usually just one), then
        # that base's chain down (reversed), then methods from each subclass
        for base in bases:
            for _class in reversed(base.mro()):
                methods = inspect.getmembers(_class, predicate=inspect.isfunction)
                for _, method in methods:
                    register_route(method)

        # add final registered formats for the current class, overwriting any found in
        # superclass chain
        for attr_name, attr_value in attrs.items():
            register_route(attr_value)

        attrs['route_registry'] = route_registry

        return super().__new__(cls, name, bases, attrs)

class RouterBuilder(ChainRouter, metaclass=RouteRegistryMeta):
    '''
    Builds a (Chain)Router using attached methods and passed options.

    This class can be subtyped and desired router methods attached using the provided
    ``route`` decorator. This facilitates two separate grouping mechanisms:

    1. Group methods by frame (i.e., attach to the same router in a chain router)
    2. Group by registry equivalence (i.e, within a frame, registered with the same
       parameters)

    These groups are indicated by the following collation syntax:

    .. code-block:: python

        @route('<router>/<frame>', '<route-group>', **route_kwargs)
        def method(...):
            ...

    and the following is a specific example:

    .. code-block:: python

        @route(router='convert', route_group='file', debounce=500)
        def file_convert_1(self, event):
            ...

    which will attach the method to the "convert" router (or "frame" in a chain router
    context) using parameters (endpoint, pattern, and other keyword args) associated with
    the "file" route group (as indexed by the ``register_map`` provided on instantiation)
    with the ``debounce`` route keyword (which will override the same keyword values if
    set in the route group). Note that the exact same ``@route`` signature can be used for
    an arbitrary number of methods to be handled in parallel by the associated Router.
    
    Note that there is one reserved route group keyword: "post," for post callbacks.
    Multiple post-callbacks for a particular router can be specified with the same ID
    syntax above.

    .. admonition:: Map structures

        The following is a more intuitive breakdown of the maps involved, provided and
        computed on instantiation:

        .. code-block:: python
            
            # provided
            register_map[<router-name>] -> ( Router, { <type>: ( ( endpoint, pattern ), **kwargs ) } )

            # computed
            routers[<router-name>][<type>] -> [... <methods> ...]

    .. admonition:: TODO
        
        Consider "flattening" the ``register_map`` to be indexed only by ``<type>``,
        effectively forcing the 2nd grouping mechanism to be provided here (while the 1st
        is handled by the method registration within the body of the class). This properly
        separates the group mechanisms and is a bit more elegant, but reduces the
        flexibility a bit (possibly in a good way, though).
    '''
    def __init__(
        self,
        register_map: dict[str, tuple[Router, dict[str, tuple[tuple[str, str], dict[str, Any]]]]],
    ):
        self.register_map = register_map
        routers = []

        # register
        for router_name, (router, router_options) in self.register_map.items():
            routers.append(router)
            for route_group, method_arg_list in self.route_registry[router_name].items():
                # get post-callbacks for reserved key "post"
                # assumed no kwargs for passthrough
                if route_group == 'post':
                    for method, _ in method_arg_list:
                        router.add_post_callback(
                            update_wrapper(partial(method, self), method),
                        )
                    continue

                group_options = router_options.get(route_group)
                if group_options is None:
                    continue

                # "group_route_kwargs" are route kwargs provided @ group level
                # "method_route_kwargs" are route kwargs provided @ method level 
                # |-> considered more specific and will override group kwargs
                (endpoint, pattern), group_route_kwargs = group_options
                for method, method_route_kwargs in method_arg_list:
                    router.register(
                        endpoint,
                        update_wrapper(partial(method, self), method),
                        pattern,
                        **{
                            **group_route_kwargs,
                            **method_route_kwargs
                        }
                    )

        super().__init__(routers)

    # -- disabling for now to inherit from ChainRouter directly. Require the order to
    # -- simply be specified by the order of the router keys in the register_map
    # def get_router(self, router_key_list: list[str]):
    #     return ChainRouter([self.register_map[k][0] for k in router_key_list])