olog/execlib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add Event typing, clean up docstrings, add initial tests

Browse Source
This commit is contained in:
Sam G.
2024-04-27 16:33:08 -07:00
parent 109fbefd41
commit 6c335c2a9a
21 changed files with 424 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
notebooks/listener.ipynb Normal file
View File

@@ -0,0 +1,109 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "db82df24-b51b-4315-b104-bd6337f44acc",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/execlog/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from pathlib import Path\n",
"\n",
"from router_env import chain_router, events"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "07edc761-cdd6-4df3-a4d0-f1e256431621",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"defaultdict(<function PathListener.__init__.<locals>.<lambda> at 0x7fee0e5f9800>, {1: defaultdict(<class 'int'>, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.listeners.path:Starting listener for 1 paths\n",
"INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [<flags.MODIFY: 2>, <flags.MOVED_FROM: 64>, <flags.MOVED_TO: 128>, <flags.CREATE: 256>, <flags.DELETE: 512>, <flags.DELETE_SELF: 1024>]\n"
]
}
],
"source": [
"listener = chain_router.get_listener()\n",
"listener.start()\n",
"\n",
"print(listener.watchmap)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e049fd73-227e-4574-bcad-3dbeff99804f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.CREATE: 256>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1-1 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.MODIFY: 2>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2-1 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.DELETE: 512>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2-2 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3-1 ::')]\n"
]
}
],
"source": [
"file_a = Path('endpoint_proxy/fileA')\n",
"file_a.write_text('test text')\n",
"file_a.unlink()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "891a8bfd-465a-4c5d-a5d8-ab71f61cc624",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "execlog",
"language": "python",
"name": "execlog"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

146
notebooks/router.ipynb
View File

@@ -2,21 +2,12 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "cd0a1da5-da7a-4181-bea3-07d02991398f",
"execution_count": null,
"id": "718618b7-132f-44e0-8cad-6a912a623c82",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/execlog/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"outputs": [],
"source": [
"import logging\n",
"from pathlib import Path\n",
@@ -29,6 +20,17 @@
"logging.basicConfig(level=logging.DEBUG)"
]
},
{
"cell_type": "markdown",
"id": "f3fd536f-75d6-408d-88b2-1e4a1b62a51e",
"metadata": {},
"source": [
"# Router setup\n",
"Create individual \"frame\" routers, and attach them in a chain.\n",
"\n",
"A matching event will first be processed by matching callbacks in `router1` in parallel, blocking until all are completed, and then pass on to the next router (`router2`) to repeat the same process. This trajectory can occur in parallel for several events."
]
},
{
"cell_type": "code",
"execution_count": 2,
@@ -43,6 +45,22 @@
"chain_router = ChainRouter([router1, router2, router3])"
]
},
{
"cell_type": "markdown",
"id": "5a4a1e7a-ee8b-4eec-97dd-ed9abac73989",
"metadata": {},
"source": [
"Register callbacks to each of the routers. The `Router` objects are of type `PathRouter`, so `.register` takes a path endpoint and a function that accepts `Event`s.\n",
"\n",
"Events are created with the registered endpoint path, and a `name` parameter with the filename at that path target. Here one callback is attached to Router 1, two to Router 2, and three to Router 3. A given matching event should have a trajectory that looks like:\n",
"\n",
"```\n",
"Event -> Router-1 -> C1-1 -- blocking --> Router-2 --> C2-1 -- blocking --> Router-3 --> C3-1 -->\n",
" \\-> C2-2 -/ \\-> C3-2 -/\n",
" \\-> C3-3 -/\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 3,
@@ -63,6 +81,14 @@
"]"
]
},
{
"cell_type": "markdown",
"id": "b3f98138-e381-4a98-af33-0e5310f305ce",
"metadata": {},
"source": [
"Submit the event list to an individual router. Each event will be handled in its own thread, until the thread limit is reached, at which point the events remain in a queue until they can be processed."
]
},
{
"cell_type": "code",
"execution_count": 4,
@@ -83,9 +109,9 @@
{
"data": {
"text/plain": [
"[<Future at 0x78be3b5c2cf0 state=running>,\n",
" <Future at 0x78be3b5c2ab0 state=running>,\n",
" <Future at 0x78be3b5c34a0 state=running>]"
"[<Future at 0x7802dbfc62d0 state=running>,\n",
" <Future at 0x7802dbfc6900 state=running>,\n",
" <Future at 0x7802e03660c0 state=running>]"
]
},
"execution_count": 4,
@@ -99,9 +125,6 @@
"R1 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n",
"R1 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n",
"R1 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R1 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R1 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n",
"R1 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.MODIFY: 2>])\n",
"R1 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n"
]
}
@@ -111,6 +134,14 @@
"router1.submit(events)"
]
},
{
"cell_type": "markdown",
"id": "e4de6182-aae6-43e0-9d14-04f1e291fe41",
"metadata": {},
"source": [
"Submit the event list to the chain router. Each event will be processed independently and in parallel, so long as there are threads available. Each event will make its way through the router chain, blocking until all matching callbacks for a given router are completed."
]
},
{
"cell_type": "code",
"execution_count": 5,
@@ -123,7 +154,6 @@
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [file1] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [file1] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [file1] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n"
]
@@ -131,9 +161,9 @@
{
"data": {
"text/plain": [
"[<Future at 0x78be40f24e60 state=running>,\n",
" <Future at 0x78be3b5c3740 state=running>,\n",
" <Future at 0x78be3b5c1430 state=running>]"
"[<Future at 0x7802dbfec500 state=running>,\n",
" <Future at 0x7802dbfecb90 state=running>,\n",
" <Future at 0x7802dbfec800 state=running>]"
]
},
"execution_count": 5,
@@ -144,10 +174,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [file2] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [file2] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [file2] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n",
"INFO:execlog.router:Event [file3] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [file3] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [file3] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n"
]
@@ -158,59 +186,10 @@
"text": [
"R2 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R3 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n",
"R3 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R3 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"R2 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n",
"R3 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"R2 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.MODIFY: 2>])\n",
"R3 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.MODIFY: 2>])\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"R3 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n",
"R3 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n"
]
@@ -232,15 +211,21 @@
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.listeners.path:Starting listener for 1 paths\n",
"INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [<flags.MODIFY: 2>, <flags.MOVED_FROM: 64>, <flags.MOVED_TO: 128>, <flags.CREATE: 256>, <flags.DELETE: 512>, <flags.DELETE_SELF: 1024>]\n"
"INFO:execlog.listeners.path:Starting listener for 1 paths\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"defaultdict(<function PathListener.__init__.<locals>.<lambda> at 0x78be3b5db100>, {1: defaultdict(<class 'int'>, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n"
"defaultdict(<function PathListener.__init__.<locals>.<lambda> at 0x7802dbfdfba0>, {1: defaultdict(<class 'int'>, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [<flags.MODIFY: 2>, <flags.MOVED_FROM: 64>, <flags.MOVED_TO: 128>, <flags.CREATE: 256>, <flags.DELETE: 512>, <flags.DELETE_SELF: 1024>]\n"
]
}
],
@@ -253,7 +238,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 7,
"id": "00bb2889-f266-4fb1-9a89-7d7539aba9cf",
"metadata": {},
"outputs": [
@@ -262,7 +247,10 @@
"output_type": "stream",
"text": [
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.CREATE: 256>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.MODIFY: 2>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.DELETE: 512>]\n"
]
}

49
notebooks/router_env.py Normal file
View File

@@ -0,0 +1,49 @@
import logging
from pathlib import Path
from functools import partial
from execlog import util
from execlog import ChainRouter, Event
from execlog.routers import PathRouter
from execlog.listeners import PathListener
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger.addHandler(util.generic.TqdmLoggingHandler())
# router setup
router1 = PathRouter()
router2 = PathRouter()
router3 = PathRouter()
chain_router = ChainRouter([router1, router2, router3])
# router-1
router1.register('endpoint_proxy', partial(print, 'R1-1 ::'))
# router-2
router2.register('endpoint_proxy', partial(print, 'R2-1 ::'))
router2.register('endpoint_proxy', partial(print, 'R2-2 ::'))
# router-3
router3.register('endpoint_proxy', partial(print, 'R3-1 ::'))
router3.register('endpoint_proxy', partial(print, 'R3-2 ::'))
router3.register('endpoint_proxy', partial(print, 'R3-3 ::'))
events = [
Event(endpoint='endpoint_proxy', name='file1'),
Event(endpoint='endpoint_proxy', name='file2'),
Event(endpoint='endpoint_proxy', name='file3'),
]
if __name__ == '__main__':
futures = chain_router.submit(events)
chain_router.wait_on_futures(futures)
#listener = chain_router.get_listener()
#listener.start()
#file_a = Path('endpoint_proxy/fileA')
#file_a.write_text('test text')
#file_a.unlink()