diff --git a/.gitignore b/.gitignore index 78c4f94..4ceb80c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *.egg-info/ .ipynb_checkpoints/ .python-version +.pytest_cache # vendor and build files dist/ @@ -11,3 +12,7 @@ build/ docs/_autoref/ docs/_autosummary/ docs/_build/ + +# local +notebooks/ +/Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index b63f792..0000000 --- a/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -PYTHON=/home/smgr/.pyenv/versions/execlog/bin/python -BASH=/usr/bin/bash - - -## ------------------ docs ------------------ ## -docs-build: - sphinx-apidoc --module-first --separate -o docs/_autoref/ execlog - make -C docs/ html - -docs-serve: - cd docs/_build/html && python -m http.server 9090 - -docs-clean: - make -C docs/ clean - rm -rf docs/_autoref -## ------------------------------------------ ## - -## ----------------- tests ------------------ ## -test: - pytest --pyargs tests -v diff --git a/README.md b/README.md index 96ccac4..2dbbc9e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,19 @@ # Overview -`execlog` is a lightweight multi-threaded job framework +`execlog` is a lightweight multi-threaded job framework written in Python. It implements a +simple event-based model over core Python utilities like `ThreadPoolExecutor` to +facilitate reactivity and manage concurrent responses. -- **Handler**: live-reload handshake manager for connecting pages -- **Listener**: -- **Router** -- **Server**: +There are a few top-level classes exposed by the package: + +- **Router**: Central event routing object. Routers facilitate route registration, + allowing for _pattern_-based matching of _events_ to arbitrary _callback_ functions. For + example, you could have a function that converts a PDF file to a collection images + (_callback_), and want this function to be called for a new files (_event_) that match + the glob `*.pdf` (_pattern_). +- **Listener**: Connective event listening object, often created directly by router + instances. Listeners pay attention to events arising along registered routes of an + affiliated router, passing them through (after optional delays, debouncing, filtering, + etc). In the above example, the associated `Listener` instance might wrap a tool like + iNotify to dynamically respond to file events. +- **Server**: Long-running process manager for listeners and optional live-reloading via + HTTP. Interfaces with listener `start()` and `shutdown()` for graceful interruption. diff --git a/execlog.egg-info/PKG-INFO b/execlog.egg-info/PKG-INFO deleted file mode 100644 index 4678c71..0000000 --- a/execlog.egg-info/PKG-INFO +++ /dev/null @@ -1,19 +0,0 @@ -Metadata-Version: 2.1 -Name: execlog -Version: 0.1.1 -Summary: Lightweight multi-threaded job framework -Author-email: Sam Griesemer -Classifier: Programming Language :: Python :: 3 -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Requires-Python: >=3.12 -Description-Content-Type: text/markdown -Requires-Dist: tqdm - -# Overview -`execlog` is a lightweight multi-threaded job framework - -- **Handler**: live-reload handshake manager for connecting pages -- **Listener**: -- **Router** -- **Server**: diff --git a/execlog.egg-info/SOURCES.txt b/execlog.egg-info/SOURCES.txt deleted file mode 100644 index 5f74ab4..0000000 --- a/execlog.egg-info/SOURCES.txt +++ /dev/null @@ -1,27 +0,0 @@ -MANIFEST.in -README.md -pyproject.toml -execlog/__init__.py -execlog/event.py -execlog/handler.py -execlog/listener.py -execlog/router.py -execlog/server.py -execlog.egg-info/PKG-INFO -execlog.egg-info/SOURCES.txt -execlog.egg-info/dependency_links.txt -execlog.egg-info/requires.txt -execlog.egg-info/top_level.txt -execlog/listeners/__init__.py -execlog/listeners/path.py -execlog/routers/__init__.py -execlog/routers/path.py -execlog/syncers/__init__.py -execlog/syncers/router.py -execlog/util/__init__.py -execlog/util/generic.py -execlog/util/path.py -tests/test_handler.py -tests/test_listener.py -tests/test_router.py -tests/test_server.py \ No newline at end of file diff --git a/execlog.egg-info/dependency_links.txt b/execlog.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/execlog.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/execlog.egg-info/requires.txt b/execlog.egg-info/requires.txt deleted file mode 100644 index 78620c4..0000000 --- a/execlog.egg-info/requires.txt +++ /dev/null @@ -1 +0,0 @@ -tqdm diff --git a/execlog.egg-info/top_level.txt b/execlog.egg-info/top_level.txt deleted file mode 100644 index 9968ff8..0000000 --- a/execlog.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -execlog diff --git a/execlog/listener.py b/execlog/listener.py index 5aff47d..78807f2 100644 --- a/execlog/listener.py +++ b/execlog/listener.py @@ -1,5 +1,4 @@ ''' - See also: - https://inotify-simple.readthedocs.io/en/latest/#gracefully-exit-a-blocking-read diff --git a/execlog/listeners/__init__.py b/execlog/listeners/__init__.py index 2ef9507..0d01181 100644 --- a/execlog/listeners/__init__.py +++ b/execlog/listeners/__init__.py @@ -1,5 +1 @@ -''' -Thing -''' - from execlog.listeners.path import PathListener diff --git a/notebooks/listener.ipynb b/notebooks/listener.ipynb deleted file mode 100644 index 87c30d5..0000000 --- a/notebooks/listener.ipynb +++ /dev/null @@ -1,109 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "db82df24-b51b-4315-b104-bd6337f44acc", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/smgr/.pyenv/versions/execlog/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from pathlib import Path\n", - "\n", - "from router_env import chain_router, events" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "07edc761-cdd6-4df3-a4d0-f1e256431621", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "defaultdict(. at 0x7fee0e5f9800>, {1: defaultdict(, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:execlog.listeners.path:Starting listener for 1 paths\n", - "INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [, , , , , ]\n" - ] - } - ], - "source": [ - "listener = chain_router.get_listener()\n", - "listener.start()\n", - "\n", - "print(listener.watchmap)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "e049fd73-227e-4574-bcad-3dbeff99804f", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:execlog.listeners.path:Watcher fired for [fileA]: []\n", - "INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R1-1 ::')]\n", - "DEBUG:execlog.listeners.path:Watcher fired for [fileA]: []\n", - "INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R2-1 ::')]\n", - "DEBUG:execlog.listeners.path:Watcher fired for [fileA]: []\n", - "INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R2-2 ::')]\n", - "INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R3-1 ::')]\n" - ] - } - ], - "source": [ - "file_a = Path('endpoint_proxy/fileA')\n", - "file_a.write_text('test text')\n", - "file_a.unlink()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "891a8bfd-465a-4c5d-a5d8-ab71f61cc624", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "execlog", - "language": "python", - "name": "execlog" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/router.ipynb b/notebooks/router.ipynb deleted file mode 100644 index f7875cb..0000000 --- a/notebooks/router.ipynb +++ /dev/null @@ -1,325 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "718618b7-132f-44e0-8cad-6a912a623c82", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/smgr/.pyenv/versions/execlog/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import logging\n", - "from pathlib import Path\n", - "from functools import partial\n", - "\n", - "from execlog import ChainRouter, Event\n", - "from execlog.routers import PathRouter\n", - "from execlog.listeners import PathListener\n", - "\n", - "logging.basicConfig(level=logging.DEBUG)" - ] - }, - { - "cell_type": "markdown", - "id": "f3fd536f-75d6-408d-88b2-1e4a1b62a51e", - "metadata": {}, - "source": [ - "# Router setup\n", - "Create individual \"frame\" routers, and attach them in a chain.\n", - "\n", - "A matching event will first be processed by matching callbacks in `router1` in parallel, blocking until all are completed, and then pass on to the next router (`router2`) to repeat the same process. This trajectory can occur in parallel for several events." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "f8b834c2-808e-4bd0-87eb-932dc42ba390", - "metadata": {}, - "outputs": [], - "source": [ - "router1 = PathRouter()\n", - "router2 = PathRouter()\n", - "router3 = PathRouter()\n", - "\n", - "chain_router = ChainRouter([router1, router2, router3])" - ] - }, - { - "cell_type": "markdown", - "id": "5a4a1e7a-ee8b-4eec-97dd-ed9abac73989", - "metadata": {}, - "source": [ - "Register callbacks to each of the routers. The `Router` objects are of type `PathRouter`, so `.register` takes a path endpoint and a function that accepts `Event`s.\n", - "\n", - "Events are created with the registered endpoint path, and a `name` parameter with the filename at that path target. Here one callback is attached to Router 1, two to Router 2, and three to Router 3. A given matching event should have a trajectory that looks like:\n", - "\n", - "```\n", - "Event -> Router-1 -> C1-1 -- blocking --> Router-2 --> C2-1 -- blocking --> Router-3 --> C3-1 -->\n", - " \\-> C2-2 -/ \\-> C3-2 -/\n", - " \\-> C3-3 -/\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "c5bd34bb-ce1d-4719-a77a-ee13a95c3c78", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "router1.register('endpoint_proxy', partial(print, 'R1 ::'))\n", - "router2.register('endpoint_proxy', partial(print, 'R2 ::'))\n", - "router3.register('endpoint_proxy', partial(print, 'R3 ::'))\n", - "\n", - "events = [\n", - " Event(endpoint='endpoint_proxy', name='file1'),\n", - " Event(endpoint='endpoint_proxy', name='file2'),\n", - " Event(endpoint='endpoint_proxy', name='file3'),\n", - "]" - ] - }, - { - "cell_type": "markdown", - "id": "b3f98138-e381-4a98-af33-0e5310f305ce", - "metadata": {}, - "source": [ - "Submit the event list to an individual router. Each event will be handled in its own thread, until the thread limit is reached, at which point the events remain in a queue until they can be processed." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "d7354e95-b7ce-4b8a-bcc2-cb8b161b8bae", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:execlog.router:Event [file1] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R1 ::')]\n" - ] - }, - { - "data": { - "text/plain": [ - "[,\n", - " ,\n", - " ]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:execlog.router:Event [file2] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R1 ::')]\n", - "INFO:execlog.router:Event [file3] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R1 ::')]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "R1 ::R1 :: Event(endpoint='endpoint_proxy', name='file1', action=None)Event(endpoint='endpoint_proxy', name='file2', action=None)\n", - "\n", - "R1 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n", - "R1 :: Event(endpoint='endpoint_proxy', name='fileA', action=[])\n" - ] - } - ], - "source": [ - "# multi-event single router\n", - "router1.submit(events)" - ] - }, - { - "cell_type": "markdown", - "id": "e4de6182-aae6-43e0-9d14-04f1e291fe41", - "metadata": {}, - "source": [ - "Submit the event list to the chain router. Each event will be processed independently and in parallel, so long as there are threads available. Each event will make its way through the router chain, blocking until all matching callbacks for a given router are completed." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "092a21a7-8052-4826-911c-6e4423b075ce", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:execlog.router:Event [file1] " - ] - }, - { - "data": { - "text/plain": [ - "[,\n", - " ,\n", - " ]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R2 ::')]\n", - "INFO:execlog.router:Event [file1] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R3 ::')]\n", - "INFO:execlog.router:Event [file2] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R2 ::')]\n", - "INFO:execlog.router:Event [file2] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R3 ::')]\n", - "INFO:execlog.router:Event [file3] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R2 ::')]\n", - "INFO:execlog.router:Event [file3] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R3 ::')]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "R2 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n", - "R2 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n", - "R2 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n", - "R3 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n", - "R3 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n", - "R3 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n", - "R2 :: Event(endpoint='endpoint_proxy', name='fileA', action=[])\n", - "R3 :: Event(endpoint='endpoint_proxy', name='fileA', action=[])\n" - ] - } - ], - "source": [ - "chain_router.submit(events)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "52f83d9a-7b12-4891-8b90-986a9ed399d7", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "defaultdict(. at 0x74da282f4540>, {1: defaultdict(, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:execlog.listeners.path:Starting listener for 1 paths\n", - "INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [, , , , , ]\n" - ] - } - ], - "source": [ - "listener = chain_router.get_listener()\n", - "listener.start()\n", - "\n", - "print(listener.watchmap)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "00bb2889-f266-4fb1-9a89-7d7539aba9cf", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:execlog.listeners.path:Watcher fired for [fileA]: []\n", - "INFO:execlog.router:Event [fileA] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(, 'R1 ::')]\n", - "DEBUG:execlog.listeners.path:Watcher fired for [fileA]: []\n" - ] - } - ], - "source": [ - "file_a = Path('endpoint_proxy/fileA')\n", - "file_a.write_text('test text')\n", - "file_a.unlink()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "4e993450-bdb7-4860-ba23-dbc2e5676ace", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "defaultdict(.()>,\n", - " {1: defaultdict(int,\n", - " {(PosixPath('endpoint_proxy'),\n", - " PosixPath('.')): 1986})})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "listener.watchmap" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "baa40300-fa71-404e-9dc3-90a5361c0e98", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "execlog", - "language": "python", - "name": "execlog" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/router_env.py b/notebooks/router_env.py deleted file mode 100644 index fded96b..0000000 --- a/notebooks/router_env.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging -from pathlib import Path -from functools import partial - -from execlog import util -from execlog import ChainRouter, Event -from execlog.routers import PathRouter -from execlog.listeners import PathListener - - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) -logger.addHandler(util.generic.TqdmLoggingHandler()) - -# router setup -router1 = PathRouter() -router2 = PathRouter() -router3 = PathRouter() - -chain_router = ChainRouter([router1, router2, router3]) - -# router-1 -router1.register('endpoint_proxy', partial(print, 'R1-1 ::')) - -# router-2 -router2.register('endpoint_proxy', partial(print, 'R2-1 ::')) -router2.register('endpoint_proxy', partial(print, 'R2-2 ::')) - -# router-3 -router3.register('endpoint_proxy', partial(print, 'R3-1 ::')) -router3.register('endpoint_proxy', partial(print, 'R3-2 ::')) -router3.register('endpoint_proxy', partial(print, 'R3-3 ::')) - -events = [ - Event(endpoint='endpoint_proxy', name='file1'), - Event(endpoint='endpoint_proxy', name='file2'), - Event(endpoint='endpoint_proxy', name='file3'), -] - -if __name__ == '__main__': - futures = chain_router.submit(events) - chain_router.wait_on_futures(futures) - - #listener = chain_router.get_listener() - #listener.start() - - #file_a = Path('endpoint_proxy/fileA') - #file_a.write_text('test text') - #file_a.unlink() diff --git a/pyproject.toml b/pyproject.toml index e605e99..472be41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "execlog" -version = "0.1.1" +version = "0.4.1" authors = [ { name="Sam Griesemer", email="samgriesemer@gmail.com" }, ]