olog/execlib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

clean up repo files, README, auxiliary files (pre-BFG)

Browse Source
This commit is contained in:
Sam G. 2024-05-14 15:49:23 -07:00
parent 5510062148
commit 1a19219d86
14 changed files with 23 additions and 563 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -4,6 +4,7 @@ __pycache__/
*.egg-info/
.ipynb_checkpoints/
.python-version
.pytest_cache
# vendor and build files
dist/
@ -11,3 +12,7 @@ build/
docs/_autoref/
docs/_autosummary/
docs/_build/
# local
notebooks/
/Makefile

20
Makefile
View File

@ -1,20 +0,0 @@
PYTHON=/home/smgr/.pyenv/versions/execlog/bin/python
BASH=/usr/bin/bash
## ------------------ docs ------------------ ##
docs-build:
sphinx-apidoc --module-first --separate -o docs/_autoref/ execlog
make -C docs/ html
docs-serve:
cd docs/_build/html && python -m http.server 9090
docs-clean:
make -C docs/ clean
rm -rf docs/_autoref
## ------------------------------------------ ##
## ----------------- tests ------------------ ##
test:
pytest --pyargs tests -v

22
README.md
View File

@ -1,7 +1,19 @@
# Overview
`execlog` is a lightweight multi-threaded job framework
`execlog` is a lightweight multi-threaded job framework written in Python. It implements a
simple event-based model over core Python utilities like `ThreadPoolExecutor` to
facilitate reactivity and manage concurrent responses.
- **Handler**: live-reload handshake manager for connecting pages
- **Listener**:
- **Router**
- **Server**:
There are a few top-level classes exposed by the package:
- **Router**: Central event routing object. Routers facilitate route registration,
allowing for _pattern_-based matching of _events_ to arbitrary _callback_ functions. For
example, you could have a function that converts a PDF file to a collection images
(_callback_), and want this function to be called for a new files (_event_) that match
the glob `*.pdf` (_pattern_).
- **Listener**: Connective event listening object, often created directly by router
instances. Listeners pay attention to events arising along registered routes of an
affiliated router, passing them through (after optional delays, debouncing, filtering,
etc). In the above example, the associated `Listener` instance might wrap a tool like
iNotify to dynamically respond to file events.
- **Server**: Long-running process manager for listeners and optional live-reloading via
HTTP. Interfaces with listener `start()` and `shutdown()` for graceful interruption.

19
execlog.egg-info/PKG-INFO
View File

@ -1,19 +0,0 @@
Metadata-Version: 2.1
Name: execlog
Version: 0.1.1
Summary: Lightweight multi-threaded job framework
Author-email: Sam Griesemer <samgriesemer@gmail.com>
Classifier: Programming Language :: Python :: 3
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Requires-Python: >=3.12
Description-Content-Type: text/markdown
Requires-Dist: tqdm
# Overview
`execlog` is a lightweight multi-threaded job framework
- **Handler**: live-reload handshake manager for connecting pages
- **Listener**:
- **Router**
- **Server**:

27
execlog.egg-info/SOURCES.txt
View File

@ -1,27 +0,0 @@
MANIFEST.in
README.md
pyproject.toml
execlog/__init__.py
execlog/event.py
execlog/handler.py
execlog/listener.py
execlog/router.py
execlog/server.py
execlog.egg-info/PKG-INFO
execlog.egg-info/SOURCES.txt
execlog.egg-info/dependency_links.txt
execlog.egg-info/requires.txt
execlog.egg-info/top_level.txt
execlog/listeners/__init__.py
execlog/listeners/path.py
execlog/routers/__init__.py
execlog/routers/path.py
execlog/syncers/__init__.py
execlog/syncers/router.py
execlog/util/__init__.py
execlog/util/generic.py
execlog/util/path.py
tests/test_handler.py
tests/test_listener.py
tests/test_router.py
tests/test_server.py

1
execlog.egg-info/dependency_links.txt
View File

@ -1 +0,0 @@

1
execlog.egg-info/requires.txt
View File

@ -1 +0,0 @@
tqdm

1
execlog.egg-info/top_level.txt
View File

@ -1 +0,0 @@
execlog

1
execlog/listener.py
View File

@ -1,5 +1,4 @@
'''
See also:
- https://inotify-simple.readthedocs.io/en/latest/#gracefully-exit-a-blocking-read

4
execlog/listeners/__init__.py
View File

@ -1,5 +1 @@
'''
Thing
'''
from execlog.listeners.path import PathListener

109
notebooks/listener.ipynb
View File

@ -1,109 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "db82df24-b51b-4315-b104-bd6337f44acc",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/execlog/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from pathlib import Path\n",
"\n",
"from router_env import chain_router, events"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "07edc761-cdd6-4df3-a4d0-f1e256431621",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"defaultdict(<function PathListener.__init__.<locals>.<lambda> at 0x7fee0e5f9800>, {1: defaultdict(<class 'int'>, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.listeners.path:Starting listener for 1 paths\n",
"INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [<flags.MODIFY: 2>, <flags.MOVED_FROM: 64>, <flags.MOVED_TO: 128>, <flags.CREATE: 256>, <flags.DELETE: 512>, <flags.DELETE_SELF: 1024>]\n"
]
}
],
"source": [
"listener = chain_router.get_listener()\n",
"listener.start()\n",
"\n",
"print(listener.watchmap)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e049fd73-227e-4574-bcad-3dbeff99804f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.CREATE: 256>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1-1 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.MODIFY: 2>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2-1 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.DELETE: 512>]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2-2 ::')]\n",
"INFO:execlog.router:Event [fileA] \u001b[1m\u001b[32mmatched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3-1 ::')]\n"
]
}
],
"source": [
"file_a = Path('endpoint_proxy/fileA')\n",
"file_a.write_text('test text')\n",
"file_a.unlink()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "891a8bfd-465a-4c5d-a5d8-ab71f61cc624",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "execlog",
"language": "python",
"name": "execlog"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

325
notebooks/router.ipynb
View File

@ -1,325 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "718618b7-132f-44e0-8cad-6a912a623c82",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/execlog/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"import logging\n",
"from pathlib import Path\n",
"from functools import partial\n",
"\n",
"from execlog import ChainRouter, Event\n",
"from execlog.routers import PathRouter\n",
"from execlog.listeners import PathListener\n",
"\n",
"logging.basicConfig(level=logging.DEBUG)"
]
},
{
"cell_type": "markdown",
"id": "f3fd536f-75d6-408d-88b2-1e4a1b62a51e",
"metadata": {},
"source": [
"# Router setup\n",
"Create individual \"frame\" routers, and attach them in a chain.\n",
"\n",
"A matching event will first be processed by matching callbacks in `router1` in parallel, blocking until all are completed, and then pass on to the next router (`router2`) to repeat the same process. This trajectory can occur in parallel for several events."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "f8b834c2-808e-4bd0-87eb-932dc42ba390",
"metadata": {},
"outputs": [],
"source": [
"router1 = PathRouter()\n",
"router2 = PathRouter()\n",
"router3 = PathRouter()\n",
"\n",
"chain_router = ChainRouter([router1, router2, router3])"
]
},
{
"cell_type": "markdown",
"id": "5a4a1e7a-ee8b-4eec-97dd-ed9abac73989",
"metadata": {},
"source": [
"Register callbacks to each of the routers. The `Router` objects are of type `PathRouter`, so `.register` takes a path endpoint and a function that accepts `Event`s.\n",
"\n",
"Events are created with the registered endpoint path, and a `name` parameter with the filename at that path target. Here one callback is attached to Router 1, two to Router 2, and three to Router 3. A given matching event should have a trajectory that looks like:\n",
"\n",
"```\n",
"Event -> Router-1 -> C1-1 -- blocking --> Router-2 --> C2-1 -- blocking --> Router-3 --> C3-1 -->\n",
" \\-> C2-2 -/ \\-> C3-2 -/\n",
" \\-> C3-3 -/\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "c5bd34bb-ce1d-4719-a77a-ee13a95c3c78",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"router1.register('endpoint_proxy', partial(print, 'R1 ::'))\n",
"router2.register('endpoint_proxy', partial(print, 'R2 ::'))\n",
"router3.register('endpoint_proxy', partial(print, 'R3 ::'))\n",
"\n",
"events = [\n",
" Event(endpoint='endpoint_proxy', name='file1'),\n",
" Event(endpoint='endpoint_proxy', name='file2'),\n",
" Event(endpoint='endpoint_proxy', name='file3'),\n",
"]"
]
},
{
"cell_type": "markdown",
"id": "b3f98138-e381-4a98-af33-0e5310f305ce",
"metadata": {},
"source": [
"Submit the event list to an individual router. Each event will be handled in its own thread, until the thread limit is reached, at which point the events remain in a queue until they can be processed."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d7354e95-b7ce-4b8a-bcc2-cb8b161b8bae",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [file1] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n"
]
},
{
"data": {
"text/plain": [
"[<Future at 0x74da282c7740 state=running>,\n",
" <Future at 0x74da282c7b00 state=running>,\n",
" <Future at 0x74da282ec260 state=running>]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [file2] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"INFO:execlog.router:Event [file3] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"R1 ::R1 :: Event(endpoint='endpoint_proxy', name='file1', action=None)Event(endpoint='endpoint_proxy', name='file2', action=None)\n",
"\n",
"R1 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R1 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n"
]
}
],
"source": [
"# multi-event single router\n",
"router1.submit(events)"
]
},
{
"cell_type": "markdown",
"id": "e4de6182-aae6-43e0-9d14-04f1e291fe41",
"metadata": {},
"source": [
"Submit the event list to the chain router. Each event will be processed independently and in parallel, so long as there are threads available. Each event will make its way through the router chain, blocking until all matching callbacks for a given router are completed."
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "092a21a7-8052-4826-911c-6e4423b075ce",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.router:Event [file1] "
]
},
{
"data": {
"text/plain": [
"[<Future at 0x74da282ed880 state=running>,\n",
" <Future at 0x74da282ed040 state=running>,\n",
" <Future at 0x74da282ee060 state=running>]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [file1] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n",
"INFO:execlog.router:Event [file2] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [file2] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n",
"INFO:execlog.router:Event [file3] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R2 ::')]\n",
"INFO:execlog.router:Event [file3] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R3 ::')]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"R2 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R3 :: Event(endpoint='endpoint_proxy', name='file1', action=None)\n",
"R3 :: Event(endpoint='endpoint_proxy', name='file2', action=None)\n",
"R3 :: Event(endpoint='endpoint_proxy', name='file3', action=None)\n",
"R2 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n",
"R3 :: Event(endpoint='endpoint_proxy', name='fileA', action=[<flags.CREATE: 256>])\n"
]
}
],
"source": [
"chain_router.submit(events)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "52f83d9a-7b12-4891-8b90-986a9ed399d7",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"defaultdict(<function PathListener.__init__.<locals>.<lambda> at 0x74da282f4540>, {1: defaultdict(<class 'int'>, {(PosixPath('endpoint_proxy'), PosixPath('.')): 1986})})\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:execlog.listeners.path:Starting listener for 1 paths\n",
"INFO:execlog.listeners.path:> Listening on path endpoint_proxy for flags [<flags.MODIFY: 2>, <flags.MOVED_FROM: 64>, <flags.MOVED_TO: 128>, <flags.CREATE: 256>, <flags.DELETE: 512>, <flags.DELETE_SELF: 1024>]\n"
]
}
],
"source": [
"listener = chain_router.get_listener()\n",
"listener.start()\n",
"\n",
"print(listener.watchmap)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "00bb2889-f266-4fb1-9a89-7d7539aba9cf",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.CREATE: 256>]\n",
"INFO:execlog.router:Event [fileA] matched [**/!(.*|*.tmp|*~)] under [endpoint_proxy] for [functools.partial(<built-in function print>, 'R1 ::')]\n",
"DEBUG:execlog.listeners.path:Watcher fired for [fileA]: [<flags.MODIFY: 2>]\n"
]
}
],
"source": [
"file_a = Path('endpoint_proxy/fileA')\n",
"file_a.write_text('test text')\n",
"file_a.unlink()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "4e993450-bdb7-4860-ba23-dbc2e5676ace",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"defaultdict(<function execlog.listeners.path.PathListener.__init__.<locals>.<lambda>()>,\n",
" {1: defaultdict(int,\n",
" {(PosixPath('endpoint_proxy'),\n",
" PosixPath('.')): 1986})})"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"listener.watchmap"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "baa40300-fa71-404e-9dc3-90a5361c0e98",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "execlog",
"language": "python",
"name": "execlog"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

49
notebooks/router_env.py
View File

@ -1,49 +0,0 @@
import logging
from pathlib import Path
from functools import partial
from execlog import util
from execlog import ChainRouter, Event
from execlog.routers import PathRouter
from execlog.listeners import PathListener
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger.addHandler(util.generic.TqdmLoggingHandler())
# router setup
router1 = PathRouter()
router2 = PathRouter()
router3 = PathRouter()
chain_router = ChainRouter([router1, router2, router3])
# router-1
router1.register('endpoint_proxy', partial(print, 'R1-1 ::'))
# router-2
router2.register('endpoint_proxy', partial(print, 'R2-1 ::'))
router2.register('endpoint_proxy', partial(print, 'R2-2 ::'))
# router-3
router3.register('endpoint_proxy', partial(print, 'R3-1 ::'))
router3.register('endpoint_proxy', partial(print, 'R3-2 ::'))
router3.register('endpoint_proxy', partial(print, 'R3-3 ::'))
events = [
Event(endpoint='endpoint_proxy', name='file1'),
Event(endpoint='endpoint_proxy', name='file2'),
Event(endpoint='endpoint_proxy', name='file3'),
]
if __name__ == '__main__':
futures = chain_router.submit(events)
chain_router.wait_on_futures(futures)
#listener = chain_router.get_listener()
#listener.start()
#file_a = Path('endpoint_proxy/fileA')
#file_a.write_text('test text')
#file_a.unlink()

2
pyproject.toml
View File

@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "execlog"
version = "0.1.1"
version = "0.4.1"
authors = [
{ name="Sam Griesemer", email="samgriesemer@gmail.com" },
]