olog/co3
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

implement general composition in ComposableMapper subtype

Browse Source
This commit is contained in:
Sam G. 2024-04-13 14:59:43 -07:00
parent 0f9582c391
commit 9badda5446
11 changed files with 274 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
co3/composer.py → _deprecated/composer.py
View File

@ -87,3 +87,4 @@ class Composer[C: Component]:
Retrieve the named table composition, if defined.
'''
return self.table_map.get(table_name)

2
co3.egg-info/SOURCES.txt
View File

@ -8,6 +8,7 @@ co3/collector.py
co3/component.py
co3/composer.py
co3/database.py
co3/engine.py
co3/indexer.py
co3/manager.py
co3/mapper.py
@ -26,6 +27,7 @@ co3/databases/__init__.py
co3/databases/fts.py
co3/databases/sql.py
co3/databases/vss.py
co3/engines/__init__.py
co3/managers/__init__.py
co3/managers/fts.py
co3/managers/sql.py

2
co3/__init__.py
View File

@ -102,10 +102,12 @@ from co3.manager import Manager
from co3.mapper import Mapper
from co3.component import Component
from co3.schema import Schema
from co3.engine import Engine
from co3 import accessors
from co3 import databases
from co3 import managers
from co3 import components
from co3 import schemas
from co3 import engines
from co3 import util

2
co3/accessors/sql.py
View File

@ -118,7 +118,7 @@ class SQLAccessor(RelationalAccessor[SQLTable]):
limit = 0,
mappings = False,
include_cols = False,
) -> list[dict|sa.Mapping]:
): # -> list[dict|sa.Mapping]: (double check the Mapping types)
'''
Perform a SELECT query against the provided table-like object (see
`check_table()`).

13
co3/components/__init__.py
View File

@ -1,3 +1,10 @@
'''
Dev note:
Any reason to have ComposeableComponents and Relations as separate types? The thought
is that there may be some possible Component types we want to be able to Compose that
wouldn't logically be Relations. But the gap here might be quite small
'''
from typing import Self
from abc import ABCMeta, abstractmethod
@ -12,7 +19,7 @@ class ComposableComponent[T](Component[T], metaclass=ABCMeta):
Components that can be composed with others of the same type.
'''
@abstractmethod
def compose(self, component: Self, on) -> Self:
def compose(self, component: Self, on, outer=False) -> Self:
'''
Abstract composition.
'''
@ -67,7 +74,6 @@ class SQLTable(Relation[SQLTableLike]):
Provide column:default pairs for a provided SQLAlchemy table.
Parameters:
table: SQLAlchemy table
include_all: whether to include all columns, even those without explicit defaults
'''
default_values = {}
@ -95,6 +101,9 @@ class SQLTable(Relation[SQLTableLike]):
return insert_dict
def compose(self, _with: Self, on, outer=False):
return self.obj.join(_with, on, isouter=outer)
# key-value stores
class Dictionary(Relation[dict]):
def get_attributes(self):

2
co3/engines/__init__.py
View File

@ -1,3 +1,5 @@
from contextlib import contextmanager
import sqlalchemy as sa
from co3.engine import Engine

4
co3/managers/sql.py
View File

@ -58,11 +58,11 @@ from co3.components import Relation, SQLTable
logger = logging.getLogger(__name__)
class RelationalManager[R: Relation, D: 'RelationalDatabase[R]'](Manager[R, D]):
class RelationalManager[R: Relation](Manager[R]):
pass
class SQLManager(RelationalManager[SQLTable, 'SQLDatabase[SQLTable]']):
class SQLManager(RelationalManager[SQLTable]):
'''
Core schema table manager. Exposes common operations and facilitates joint operations
needed for highly connected schemas.

174
co3/mapper.py
View File

@ -33,10 +33,10 @@ from typing import Callable
from collections import defaultdict
from co3.co3 import CO3
from co3.collector import Collector
from co3.composer import Composer
from co3.component import Component
from co3.schema import Schema
from co3.collector import Collector
from co3.component import Component
from co3.components import ComposableComponent
class Mapper[C: Component]:
@ -59,8 +59,9 @@ class Mapper[C: Component]:
this class. It may be more appropriate to have at the Schema level, or even just
dissolved altogether if arbitrary named Components can be attached to schemas.
'''
type comp_spec = str | C
_collector_cls: type[Collector[C]] = Collector[C]
_composer_cls: type[Composer[C]] = Composer[C]
def __init__(self, schema: Schema[C]):
'''
@ -71,12 +72,11 @@ class Mapper[C: Component]:
self.schema = schema
self.collector = self._collector_cls(schema)
self.composer = self._composer_cls()
self.attribute_comps: dict[type[CO3], C] = {}
self.collation_groups: dict[type[CO3], dict[str|None, C]] = defaultdict(dict)
def _check_component(self, comp: C | str):
def _check_component(self, comp: self.comp_spec):
if type(comp) is str:
comp_key = comp
comp = self.schema.get_component(comp_key)
@ -95,9 +95,9 @@ class Mapper[C: Component]:
def attach(
self,
type_ref : type[CO3],
attr_comp : C | str,
coll_comp : C | str | None = None,
coll_groups : dict[str | None, C | str] = None
attr_comp : self.comp_spec,
coll_comp : self.comp_spec | None = None,
coll_groups : dict[str | None, self.comp_spec] | None = None,
) -> None:
'''
Parameters:
@ -127,8 +127,8 @@ class Mapper[C: Component]:
def attach_many(
self,
type_list: list[type[CO3]],
attr_name_map: Callable[[type[CO3]], str],
coll_name_map: Callable[[type[CO3], str|None], str] | None = None,
attr_name_map: Callable[[type[CO3]], self.comp_spec],
coll_name_map: Callable[[type[CO3], str], self.comp_spec] | None = None,
):
'''
Auto-register a set of types to the Mapper's attached Schema. Associations are
@ -148,6 +148,8 @@ class Mapper[C: Component]:
for _type in type_list:
attr_comp = attr_name_map(_type)
coll_groups = {}
if coll_name_map:
for action_group in _type.group_registry:
coll_groups[action_group] = coll_name_map(_type, action_group)
@ -162,7 +164,7 @@ class Mapper[C: Component]:
def get_collation_comp(
self,
type_ref: type[CO3],
group=str | None
group=str | None,
) -> C | None:
return self.collation_groups.get(type_ref, {}).get(group, None)
@ -183,6 +185,11 @@ class Mapper[C: Component]:
we don't do a whole lot more here: we just call `collect` over those
components, adding them to the collector session all the same.
Parameters:
obj: CO3 instance to collect from
action_keys: keys for actions to collect from
action_group: action group names to run all actions for
Returns: dict with keys and values relevant for associated SQLite tables
'''
if action_keys is None:
@ -211,7 +218,7 @@ class Mapper[C: Component]:
_, action_groups = obj.action_registry[action_key]
for action_group in action_groups:
collation_component = self.get_collation_comp(_cls, group=action_group)
collation_component, _ = self.get_collation_comp(_cls, group=action_group)
if collation_component is None:
continue
@ -234,3 +241,144 @@ class Mapper[C: Component]:
return receipts
class ComposableMapper[C: ComposableComponent](Mapper[C]):
'''
Dev note: class design
Heavily debating between multiple possible design approaches here. The main
purpose of this subtype is make clear the need for additional compositional
mapping details, namely functions that can produce pairwise join conditions for
both the attribute tree (vertical traversal) and the collation components
(horizontal traversal). Here's a few remarks:
- I want the necessary maps to provided/stored _outside_ of `compose` calls to
reduce overhead for downstream callers. It's awkward to have think about the
exact attr-to-attr associations each time you want a type's associated
composition, especially when they don't change under the same Mapper (i.e.,
if you have the Mapper reference, the compositional associations should be
implicit).
- The barebones spec here appears to be two pairwise "composer" maps: one for
attribute comps, and one for collation comps. For now I think this makes sense
as additional init params, but there may later be reason to wrap this up a bit
more.
- Considering the full deprecation for the Composer type, or whether this could be
the place where it serves some purpose. Aesthetically, there's symmetry with the
`collect` and Collector method-type pairing, but that isn't a good enough reason
to justify a separate type here. The difference is that Collector instances
actually store type references, whereas the considered Composer type would
effectively just be a convenient collection of utility functions. Still possibly
useful, but not as clearly justifiable.
- If a separate Composer type were to be justified here, it would serve as a
"reusable connective tissue" for possibly many Mappers with the same kinds of
edge-wise relationships. Can think of it like this:
* Schemas collect up "nodes" (Components). These are explicit storage structures
in a DB, and can include some explicit attribute connections (foreign keys),
although those are connections made on the DB side.
* Mappers provide an exoskeleton for a Schema's nodes. It structures Components into
attributes and collation types, and additionally ties them to external CO3
types. The handy analogy here has been that attribute comps connect
_vertically_ (in a tree like fashion; point up for parents and down for
children), and collation comps point _horiztonally_ (or perhaps more aptly,
_outward_; at each node in the attribute tree, you have a "circle" of
collation comps that can point to it, and are not involved as formal tree
nodes. Can maybe think of these like "ornaments" or bulbs or orbitals).
* While the Mappers may provide the "bones," there's no way to communicate
_across_ them. While I might know that one attribute is the "parent" of
another, I don't know _why_ that relationship is there. A Composer, or the
composer details to be provided to this class, serve as the "nerves" to be
paired with the bone, actually establishing a line of communication. More
specifically, the nerves here are attribute-based mappings between pairs of
Components, i.e., (generalized) join conditions.
- Note that, by the above logic, we should then want/need a type to manage the
functions provided to `attach_many`. These functions help automatically
characterize the shape of the type skeleton in the same way the proposed
Composer wrapper would. In fact, the barebones presentation here is really just
the same two function signatures as are expected by that method. The above
analogy simply made me ask why the "bones" wouldn't be reusable if the "nerves"
were going to be. So we should perhaps coordinate a decision on this front; if
one goes, the other must as well. This may also help me keep it simpler for the
time being.
- One other aspect of a dedicated Composer type (and by the above point, a
hypothetical type to aid in `attach_many` specification) could have some sort of
"auto" feature about it. With a clear enough "discovery system," we could
encourage certain kinds of Schemas and components are named and structured. Such
an auto-composer could "scan" all components in a provided Schema and attempt to
find common attributes across tables that are unlinked (i.e., the reused
column names implicit across types in the attribute hierarchy; e.g., File.name
-> Note.name), as well as explicit connections which may suggest collation
attachment (e.g., `note_conversions.name` --FK-> Note.name). This, of course,
could always be overridden with manual specification, but being aware of some
automatic discovery structures could help constrain schema definitions to be
more in-line with the CO3 operational model. That all being said, this is a
large amount of complexity and should likely be avoided until necessary.
'''
def __init__(
self,
schema : Schema[C],
attr_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None
coll_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None
):
super().__init__(schema)
self.attr_compose_map = attr_compose_map
self.coll_compose_map = coll_compose_map
def compose(
self,
obj: CO3,
action_groups: list[str] = None,
*compose_args,
**compose_kwargs,
):
'''
Compose tables up the type hierarchy, and across through action groups to
collation components.
Note:
Comparing to ORM, this method would likely also still be needed, since it may
not be explicitly clear how some JOINs should be handled up the inheritance
chain (for components / sa.Relationships, it's a little easier).
Parameters:
outer: whether to use outer joins down the chain
conversion: whether to return conversion joins or base primitives
full: whether to return fully connected primitive and conversion table
'''
attr_comp_agg = None
for _cls in reversed(obj.__class__.__mro__[:-2]):
attr_comp = self.get_attribute_comp(_cls)
# require an attribute component for type consideration
if attr_comp is None:
continue
# compose horizontally with components from provided action groups
coll_comp_agg = attr_comp
for action_group in action_groups:
coll_comp = self.get_collation_comp(_cls, group=action_group)
if coll_comp is None:
continue
compose_condition = self.coll_compose_map(coll_comp_agg, coll_comp)
coll_comp_agg = coll_comp_agg.compose(
component=coll_comp,
on=compose_condition,
*compose_args,
**compose_kwargs,
)
# note the reduced attr_comp ref passed to compose map, rather than
# coll_comp_agg produced above; this is provided as the compose comp, though
compose_condition = self.attr_compose_map(attr_comp_agg, attr_comp)
attr_comp_agg = attr_comp_agg.compose(
component=coll_comp_agg,
on=compose_condition,
*compose_args,
**compose_kwargs,
)
return attr_comp_agg

6
examples/.ipynb_checkpoints/database-checkpoint.ipynb Normal file
View File

@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}

57
examples/database.ipynb Normal file
View File

@ -0,0 +1,57 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "6f6fbc7e-4fb9-4353-b2ee-9ea819a3c896",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"import vegetables"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88fd0ea8-9c94-4569-a51b-823a04f32f55",
"metadata": {},
"outputs": [],
"source": [
"tomato = vegetables.Tomato('t1', 5)\n",
"\n",
"# test a register collation action\n",
"tomato.collate('ripe')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "co3",
"language": "python",
"name": "co3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

22
examples/vegetables.py
View File

@ -103,3 +103,25 @@ tomato_cooking_table = sa.Table(
vegetable_schema = SQLSchema.from_metadata(metadata)
vegetable_mapper = Mapper(vegetable_schema)
def attr_name_map(cls):
return f'{cls.__name__.lower()}'
def coll_name_map(cls, action_group):
return f'{cls.__name__.lower()}_{action_group}_states'
vegetable_mapper.attach_many(
type_list,
attr_name_map,
coll_name_map,
)
'''
new mapping type for Mapper attachment:
Callable[ [type[CO3], str|None], tuple[str, tuple[str], tuple[str]]]
tail tuples to associate column names from central table to collation
this should complete the auto-compose horizontally
'''