clean up Database/Accessor/Manager interaction, refine Mapper group maps
This commit is contained in:
parent
9badda5446
commit
157ff69b9e
1
.gitignore
vendored
1
.gitignore
vendored
@ -2,6 +2,7 @@
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
localsys.egg-info/
|
||||
.ipynb_checkpoints/
|
||||
|
||||
# vendor and build files
|
||||
dist/
|
||||
|
||||
@ -6,7 +6,6 @@ co3/accessor.py
|
||||
co3/co3.py
|
||||
co3/collector.py
|
||||
co3/component.py
|
||||
co3/composer.py
|
||||
co3/database.py
|
||||
co3/engine.py
|
||||
co3/indexer.py
|
||||
|
||||
@ -95,11 +95,11 @@ Note: Organization for inheritance over composition
|
||||
from co3.accessor import Accessor
|
||||
from co3.co3 import CO3, collate
|
||||
from co3.collector import Collector
|
||||
from co3.composer import Composer
|
||||
#from co3.composer import Composer
|
||||
from co3.database import Database
|
||||
from co3.indexer import Indexer
|
||||
from co3.manager import Manager
|
||||
from co3.mapper import Mapper
|
||||
from co3.mapper import Mapper, ComposableMapper
|
||||
from co3.component import Component
|
||||
from co3.schema import Schema
|
||||
from co3.engine import Engine
|
||||
|
||||
@ -83,7 +83,8 @@ class RelationalAccessor[R: Relation](Accessor[R]):
|
||||
include_cols : bool = False,
|
||||
):
|
||||
res = self.select(
|
||||
relation, attributes, where, mappings, include_cols, limit=1)
|
||||
relation, attributes, where, mappings, include_cols, limit=1
|
||||
)
|
||||
|
||||
if include_cols and len(res[0]) > 0:
|
||||
return res[0][0], res[1]
|
||||
@ -97,6 +98,7 @@ class RelationalAccessor[R: Relation](Accessor[R]):
|
||||
class SQLAccessor(RelationalAccessor[SQLTable]):
|
||||
def raw_select(
|
||||
self,
|
||||
connection,
|
||||
sql,
|
||||
bind_params=None,
|
||||
mappings=False,
|
||||
@ -110,7 +112,8 @@ class SQLAccessor(RelationalAccessor[SQLTable]):
|
||||
|
||||
def select(
|
||||
self,
|
||||
table: SQLTable,
|
||||
connection,
|
||||
component: SQLTable,
|
||||
columns = None,
|
||||
where = None,
|
||||
distinct_on = None,
|
||||
@ -141,20 +144,21 @@ class SQLAccessor(RelationalAccessor[SQLTable]):
|
||||
if where is None:
|
||||
where = sa.true()
|
||||
|
||||
stmt = sa.select(table).where(where)
|
||||
if cols is not None:
|
||||
stmt = sa.select(*cols).select_from(table).where(where)
|
||||
table = component.obj
|
||||
statement = sa.select(table).where(where)
|
||||
if columns is not None:
|
||||
statement = sa.select(*columns).select_from(table).where(where)
|
||||
|
||||
if distinct_on is not None:
|
||||
stmt = stmt.group_by(distinct_on)
|
||||
statement = statement.group_by(distinct_on)
|
||||
|
||||
if order_by is not None:
|
||||
stmt = stmt.order_by(order_by)
|
||||
statement = statement.order_by(order_by)
|
||||
|
||||
if limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
statement = statement.limit(limit)
|
||||
|
||||
res = SQLEngine._execute(connection, statement, include_cols=include_cols)
|
||||
res = SQLEngine.execute(connection, statement, include_cols=include_cols)
|
||||
|
||||
if mappings:
|
||||
return res.mappings().all()
|
||||
|
||||
@ -7,13 +7,10 @@ abstractions within particular storage protocols.
|
||||
'''
|
||||
|
||||
class Component[T]:
|
||||
def __init__(self, name, obj: T, schema: 'Schema'):
|
||||
def __init__(self, name, obj: T):
|
||||
self.name = name
|
||||
self.obj = obj
|
||||
|
||||
self.schema = schema
|
||||
schema.add_component(self)
|
||||
|
||||
def __str__(self):
|
||||
return f'<Component ({self.__class__.__name__})> {self.name}'
|
||||
|
||||
|
||||
@ -63,8 +63,12 @@ class Relation[T](ComposableComponent[T]):
|
||||
|
||||
class SQLTable(Relation[SQLTableLike]):
|
||||
@classmethod
|
||||
def from_table(cls, table: sa.Table, schema: 'SQLSchema'):
|
||||
return cls(table.name, table, schema)
|
||||
def from_table(cls, table: sa.Table):
|
||||
'''
|
||||
Note that the sa.Table type is intentional here; not all matching types for
|
||||
SQLTableLike have a defined `name` property
|
||||
'''
|
||||
return cls(table.name, table)
|
||||
|
||||
def get_attributes(self) -> tuple:
|
||||
return tuple(self.obj.columns)
|
||||
@ -102,7 +106,10 @@ class SQLTable(Relation[SQLTableLike]):
|
||||
return insert_dict
|
||||
|
||||
def compose(self, _with: Self, on, outer=False):
|
||||
return self.obj.join(_with, on, isouter=outer)
|
||||
return self.__class__(
|
||||
f'{self.name}+{_with.name}',
|
||||
self.obj.join(_with.obj, on, isouter=outer)
|
||||
)
|
||||
|
||||
# key-value stores
|
||||
class Dictionary(Relation[dict]):
|
||||
|
||||
@ -55,10 +55,10 @@ Dev note: on explicit connection contexts
|
||||
import logging
|
||||
|
||||
from co3.accessor import Accessor
|
||||
from co3.composer import Composer
|
||||
from co3.manager import Manager
|
||||
from co3.indexer import Indexer
|
||||
from co3.engine import Engine
|
||||
from co3.schema import Schema
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -118,6 +118,12 @@ class Database[C: Component]:
|
||||
self._reset_cache = False
|
||||
|
||||
def select(self, component: C, *args, **kwargs):
|
||||
'''
|
||||
Dev note: args and kwargs have to be general/unspecified here due to the possible
|
||||
passthrough method adopting arbitrary parameters in subtypes. I could simply
|
||||
overload this method in the relevant inheriting DBs (i.e., by matching the
|
||||
expected Accessor's .select signature).
|
||||
'''
|
||||
with self.engine.connect() as connection:
|
||||
return self.accessor.select(
|
||||
connection,
|
||||
@ -128,13 +134,16 @@ class Database[C: Component]:
|
||||
|
||||
def insert(self, component: C, *args, **kwargs):
|
||||
with self.engine.connect() as connection:
|
||||
return self.accessor.insert(
|
||||
return self.manager.insert(
|
||||
connection,
|
||||
component,
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
def recreate(self, schema: Schema[C]):
|
||||
self.manager.recreate(schema, self.engine)
|
||||
|
||||
@property
|
||||
def index(self):
|
||||
if self.reset_cache:
|
||||
|
||||
@ -10,14 +10,13 @@ class SQLEngine(Engine):
|
||||
super().__init__(url, **kwargs)
|
||||
|
||||
def _create_manager(self):
|
||||
return sa.create_engine(*self.manager_args, self.manager_kwargs)
|
||||
return sa.create_engine(*self._manager_args, **self._manager_kwargs)
|
||||
|
||||
@contextmanager
|
||||
def connect(self, timeout=None):
|
||||
return self.manager.connect()
|
||||
|
||||
@staticmethod
|
||||
def _execute(
|
||||
def execute(
|
||||
connection,
|
||||
statement,
|
||||
bind_params=None,
|
||||
@ -42,7 +41,7 @@ class SQLEngine(Engine):
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def _exec_explicit(connection, statement, bind_params=None):
|
||||
def exec_explicit(connection, statement, bind_params=None):
|
||||
trans = connection.begin() # start a new transaction explicitly
|
||||
try:
|
||||
result = connection.execute(statement, bind_params)
|
||||
|
||||
@ -8,6 +8,7 @@ from pathlib import Path
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
from co3.schema import Schema
|
||||
from co3.engine import Engine
|
||||
|
||||
|
||||
class Manager[C: Component](metaclass=ABCMeta):
|
||||
@ -19,7 +20,7 @@ class Manager[C: Component](metaclass=ABCMeta):
|
||||
wrapped up in this class to then also be mirrored for the FTS counterparts.
|
||||
'''
|
||||
@abstractmethod
|
||||
def recreate(self, schema: Schema[C]):
|
||||
def recreate(self, schema: Schema[C], engine: Engine):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@ -49,6 +49,7 @@ from tqdm.auto import tqdm
|
||||
|
||||
from co3 import util
|
||||
from co3.schema import Schema
|
||||
from co3.engines import SQLEngine
|
||||
from co3.manager import Manager
|
||||
from co3.components import Relation, SQLTable
|
||||
|
||||
@ -76,12 +77,17 @@ class SQLManager(RelationalManager[SQLTable]):
|
||||
from an attached collector.
|
||||
'''
|
||||
def __init__(self, *args, **kwargs):
|
||||
'''
|
||||
The insert lock is a _reentrant lock_, meaning the same thread can acquire the
|
||||
lock again with out deadlocking (simplifying across methods of this class that
|
||||
need it).
|
||||
'''
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.routers = []
|
||||
|
||||
self._router = None
|
||||
self._insert_lock = threading.Lock()
|
||||
self._insert_lock = threading.RLock()
|
||||
|
||||
@property
|
||||
def router(self):
|
||||
@ -92,18 +98,42 @@ class SQLManager(RelationalManager[SQLTable]):
|
||||
def add_router(self, router):
|
||||
self.routers.append(router)
|
||||
|
||||
def recreate(self, schema: Schema[SQLTable]):
|
||||
schema.metadata.drop_all(self.engine)
|
||||
schema.metadata.create_all(self.engine, checkfirst=True)
|
||||
def recreate(self, schema: Schema[SQLTable], engine: SQLEngine):
|
||||
'''
|
||||
Ideally this remains open, as we can't necessarily rely on a SQLAlchemy metadata
|
||||
object for all kinds of SQLDatabases (would depend on the backend, for instance).
|
||||
|
||||
Haven't quite nailed down how backend instances should be determined; something
|
||||
like SQLAlchemySQLManager doesn't seem great. Nevertheless, this method likely
|
||||
cannot be generalized at the "SQL" (general) level.
|
||||
'''
|
||||
metadata = next(iter(schema._component_set)).obj.metadata
|
||||
metadata.drop_all(engine.manager)
|
||||
metadata.create_all(engine.manager, checkfirst=True)
|
||||
|
||||
def update(self): pass
|
||||
|
||||
def insert(self, inserts: dict):
|
||||
def insert(
|
||||
self,
|
||||
connection,
|
||||
component,
|
||||
inserts: list[dict],
|
||||
):
|
||||
'''
|
||||
Parameters:
|
||||
'''
|
||||
with self._insert_lock:
|
||||
connection.execute(
|
||||
sa.insert(component.obj),
|
||||
inserts
|
||||
)
|
||||
|
||||
def insert_many(self, connection, inserts: dict):
|
||||
'''
|
||||
Perform provided table inserts.
|
||||
|
||||
Parameters:
|
||||
inserts: table-indexed dictionary of insert lists
|
||||
inserts: component-indexed dictionary of insert lists
|
||||
'''
|
||||
total_inserts = sum([len(ilist) for ilist in inserts.values()])
|
||||
if total_inserts < 1: return
|
||||
@ -112,22 +142,16 @@ class SQLManager(RelationalManager[SQLTable]):
|
||||
|
||||
# TODO: add some exception handling? may be fine w default propagation
|
||||
start = time.time()
|
||||
with self.engine.connect() as connection:
|
||||
with self._insert_lock:
|
||||
for table_str in inserts:
|
||||
table_inserts = inserts[table_str]
|
||||
if len(table_inserts) == 0: continue
|
||||
|
||||
table = tables.table_map[table_str]
|
||||
for component in inserts:
|
||||
comp_inserts = inserts[component]
|
||||
if len(comp_inserts) == 0: continue
|
||||
|
||||
logger.info(
|
||||
f'Inserting {len(table_inserts)} out-of-date entries into table "{table_str}"'
|
||||
f'Inserting {len(comp_inserts)} out-of-date entries into component "{component}"'
|
||||
)
|
||||
|
||||
connection.execute(
|
||||
sa.insert(table),
|
||||
table_inserts
|
||||
)
|
||||
self.insert(connection, component, comp_inserts)
|
||||
connection.commit()
|
||||
logger.info(f'Insert transaction completed successfully in {time.time()-start:.2f}s')
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ Development log:
|
||||
hierarchy). As such, to fully collect from a type, the Mapper needs to leave
|
||||
registration open to various types, not just those part of the same hierarchy.
|
||||
'''
|
||||
from typing import Callable
|
||||
from typing import Callable, Any
|
||||
from collections import defaultdict
|
||||
|
||||
from co3.co3 import CO3
|
||||
@ -55,12 +55,14 @@ class Mapper[C: Component]:
|
||||
"dropped off" at an appropriate Database's Manager to actually perform the requested
|
||||
inserts (hence why we tie Mappers to Schemas one-to-one).
|
||||
|
||||
Dev note: the Composer needs reconsideration, or at least its positioning directly in
|
||||
this class. It may be more appropriate to have at the Schema level, or even just
|
||||
Dev note:
|
||||
the Composer needs reconsideration, or at least its positioning directly in this
|
||||
class. It may be more appropriate to have at the Schema level, or even just
|
||||
dissolved altogether if arbitrary named Components can be attached to schemas.
|
||||
'''
|
||||
type comp_spec = str | C
|
||||
|
||||
- Consider pushing this into a Mapper factory; on init, could check if provided
|
||||
Schema wraps up composable Components or not
|
||||
'''
|
||||
_collector_cls: type[Collector[C]] = Collector[C]
|
||||
|
||||
def __init__(self, schema: Schema[C]):
|
||||
@ -76,7 +78,7 @@ class Mapper[C: Component]:
|
||||
self.attribute_comps: dict[type[CO3], C] = {}
|
||||
self.collation_groups: dict[type[CO3], dict[str|None, C]] = defaultdict(dict)
|
||||
|
||||
def _check_component(self, comp: self.comp_spec):
|
||||
def _check_component(self, comp: str | C):
|
||||
if type(comp) is str:
|
||||
comp_key = comp
|
||||
comp = self.schema.get_component(comp_key)
|
||||
@ -95,9 +97,9 @@ class Mapper[C: Component]:
|
||||
def attach(
|
||||
self,
|
||||
type_ref : type[CO3],
|
||||
attr_comp : self.comp_spec,
|
||||
coll_comp : self.comp_spec | None = None,
|
||||
coll_groups : dict[str | None, self.comp_spec] | None = None,
|
||||
attr_comp : str | C,
|
||||
coll_comp : str | C | None = None,
|
||||
coll_groups : dict[str | None, str | C] | None = None,
|
||||
) -> None:
|
||||
'''
|
||||
Parameters:
|
||||
@ -127,8 +129,8 @@ class Mapper[C: Component]:
|
||||
def attach_many(
|
||||
self,
|
||||
type_list: list[type[CO3]],
|
||||
attr_name_map: Callable[[type[CO3]], self.comp_spec],
|
||||
coll_name_map: Callable[[type[CO3], str], self.comp_spec] | None = None,
|
||||
attr_name_map: Callable[[type[CO3]], str | C],
|
||||
coll_name_map: Callable[[type[CO3], str], str | C] | None = None,
|
||||
):
|
||||
'''
|
||||
Auto-register a set of types to the Mapper's attached Schema. Associations are
|
||||
@ -218,7 +220,7 @@ class Mapper[C: Component]:
|
||||
|
||||
_, action_groups = obj.action_registry[action_key]
|
||||
for action_group in action_groups:
|
||||
collation_component, _ = self.get_collation_comp(_cls, group=action_group)
|
||||
collation_component = self.get_collation_comp(_cls, group=action_group)
|
||||
|
||||
if collation_component is None:
|
||||
continue
|
||||
@ -317,8 +319,8 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]):
|
||||
def __init__(
|
||||
self,
|
||||
schema : Schema[C],
|
||||
attr_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None
|
||||
coll_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None
|
||||
attr_compose_map : Callable[[str | C, str | C], Any] | None = None,
|
||||
coll_compose_map : Callable[[str | C, str | C], Any] | None = None,
|
||||
):
|
||||
super().__init__(schema)
|
||||
|
||||
@ -328,7 +330,7 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]):
|
||||
def compose(
|
||||
self,
|
||||
obj: CO3,
|
||||
action_groups: list[str] = None,
|
||||
action_groups: list[str] | None = None,
|
||||
*compose_args,
|
||||
**compose_kwargs,
|
||||
):
|
||||
@ -356,27 +358,34 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]):
|
||||
|
||||
# compose horizontally with components from provided action groups
|
||||
coll_comp_agg = attr_comp
|
||||
if action_groups is not None:
|
||||
for action_group in action_groups:
|
||||
coll_comp = self.get_collation_comp(_cls, group=action_group)
|
||||
|
||||
if coll_comp is None:
|
||||
continue
|
||||
|
||||
compose_condition = self.coll_compose_map(coll_comp_agg, coll_comp)
|
||||
# note how the join condition is specified using the non-composite
|
||||
# `attr_comp` and new `coll_comp`; the composite doesn't typically
|
||||
# have the same attribute access and needs a ref to a specific comp
|
||||
compose_condition = self.coll_compose_map(attr_comp, coll_comp)
|
||||
|
||||
coll_comp_agg = coll_comp_agg.compose(
|
||||
component=coll_comp,
|
||||
on=compose_condition,
|
||||
coll_comp,
|
||||
compose_condition,
|
||||
*compose_args,
|
||||
**compose_kwargs,
|
||||
)
|
||||
|
||||
if attr_comp_agg is None:
|
||||
attr_comp_agg = coll_comp_agg
|
||||
else:
|
||||
# note the reduced attr_comp ref passed to compose map, rather than
|
||||
# coll_comp_agg produced above; this is provided as the compose comp, though
|
||||
compose_condition = self.attr_compose_map(attr_comp_agg, attr_comp)
|
||||
attr_comp_agg = attr_comp_agg.compose(
|
||||
component=coll_comp_agg,
|
||||
on=compose_condition,
|
||||
coll_comp_agg,
|
||||
compose_condition,
|
||||
*compose_args,
|
||||
**compose_kwargs,
|
||||
)
|
||||
|
||||
@ -15,7 +15,8 @@ class SQLSchema(RelationalSchema[SQLTable]):
|
||||
instance = cls()
|
||||
|
||||
for table in metadata.tables.values():
|
||||
SQLTable.from_table(table, instance)
|
||||
comp = SQLTable.from_table(table)
|
||||
instance.add_component(comp)
|
||||
|
||||
return instance
|
||||
|
||||
|
||||
@ -1,6 +1,204 @@
|
||||
{
|
||||
"cells": [],
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "6f6fbc7e-4fb9-4353-b2ee-9ea819a3c896",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import vegetables"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "88fd0ea8-9c94-4569-a51b-823a04f32f55",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'age': 3}"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tomato = vegetables.Tomato('t1', 5)\n",
|
||||
"\n",
|
||||
"# test a register collation action\n",
|
||||
"tomato.collate('ripe')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "348926d9-7137-4eff-a919-508788553dd2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['afff61ec-e0b0-44bb-9f0d-06008a82f6a5',\n",
|
||||
" '5edfa13c-0eb1-4bbc-b55e-1550ff7df3d2',\n",
|
||||
" '4568a2d4-eb41-4b15-9a29-e3e22906c661']"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vegetables.vegetable_mapper.collect(tomato, ['ripe'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "4e5e7319-11bf-4051-951b-08c84e9f3874",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<Component (SQLTable)> vegetable+tomato+tomato_aging_states+tomato_cooking_states"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "aa290686-8074-4038-a3cc-ce6817844653",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[Column('id', Integer(), table=<vegetable>, primary_key=True, nullable=False),\n",
|
||||
" Column('name', String(), table=<vegetable>),\n",
|
||||
" Column('color', String(), table=<vegetable>),\n",
|
||||
" Column('id', Integer(), table=<tomato>, primary_key=True, nullable=False),\n",
|
||||
" Column('name', String(), ForeignKey('vegetable.name'), table=<tomato>),\n",
|
||||
" Column('radius', Integer(), table=<tomato>),\n",
|
||||
" Column('id', Integer(), table=<tomato_aging_states>, primary_key=True, nullable=False),\n",
|
||||
" Column('name', String(), ForeignKey('tomato.name'), table=<tomato_aging_states>),\n",
|
||||
" Column('state', String(), table=<tomato_aging_states>),\n",
|
||||
" Column('age', Integer(), table=<tomato_aging_states>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tomato.__class__.__mro__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "c21d2c54-39e2-4de3-93bc-763896ed348e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from co3.databases import SQLDatabase\n",
|
||||
"\n",
|
||||
"db = SQLDatabase('sqlite://', echo=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "a785d202-99d3-4ae7-859e-ee22b481f8df",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib._GeneratorContextManager at 0x7dd5c619be60>"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"db.recreate(vegetable_schema) "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "co3",
|
||||
"language": "python",
|
||||
"name": "co3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@ import random
|
||||
import sqlalchemy as sa
|
||||
|
||||
from co3.schemas import SQLSchema
|
||||
from co3 import CO3, collate, Mapper
|
||||
from co3 import CO3, collate, Mapper, ComposableMapper
|
||||
from co3 import util
|
||||
|
||||
|
||||
@ -101,5 +101,35 @@ tomato_cooking_table = sa.Table(
|
||||
sa.Column('pieces', sa.Integer),
|
||||
)
|
||||
vegetable_schema = SQLSchema.from_metadata(metadata)
|
||||
vegetable_mapper = Mapper(vegetable_schema)
|
||||
|
||||
def general_compose_map(c1, c2):
|
||||
return c1.obj.c.name == c2.obj.c.name
|
||||
|
||||
vegetable_mapper = ComposableMapper(
|
||||
vegetable_schema,
|
||||
attr_compose_map=general_compose_map,
|
||||
coll_compose_map=general_compose_map,
|
||||
)
|
||||
|
||||
def attr_name_map(cls):
|
||||
return f'{cls.__name__.lower()}'
|
||||
|
||||
def coll_name_map(cls, action_group):
|
||||
return f'{cls.__name__.lower()}_{action_group}_states'
|
||||
|
||||
vegetable_mapper.attach_many(
|
||||
type_list,
|
||||
attr_name_map,
|
||||
coll_name_map,
|
||||
)
|
||||
|
||||
'''
|
||||
new mapping type for Mapper attachment:
|
||||
|
||||
Callable[ [type[CO3], str|None], tuple[str, tuple[str], tuple[str]]]
|
||||
|
||||
tail tuples to associate column names from central table to collation
|
||||
|
||||
this should complete the auto-compose horizontally
|
||||
'''
|
||||
|
||||
|
||||
@ -21,16 +21,388 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"id": "88fd0ea8-9c94-4569-a51b-823a04f32f55",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'age': 5}"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tomato = vegetables.Tomato('t1', 5)\n",
|
||||
"\n",
|
||||
"# test a register collation action\n",
|
||||
"tomato.collate('ripe')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "348926d9-7137-4eff-a919-508788553dd2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['9ca6772e-6621-4511-a4a6-ad451a1da91f',\n",
|
||||
" '2a91b423-4e08-491c-b1d2-5ec25259191e',\n",
|
||||
" '4a9edb2b-4ac5-467e-82ef-b254829ac2a2']"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vegetables.vegetable_mapper.collect(tomato, ['ripe'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "4e5e7319-11bf-4051-951b-08c84e9f3874",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<Component (SQLTable)> vegetable+tomato+tomato_aging_states+tomato_cooking_states"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "aa290686-8074-4038-a3cc-ce6817844653",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[Column('id', Integer(), table=<vegetable>, primary_key=True, nullable=False),\n",
|
||||
" Column('name', String(), table=<vegetable>),\n",
|
||||
" Column('color', String(), table=<vegetable>),\n",
|
||||
" Column('id', Integer(), table=<tomato>, primary_key=True, nullable=False),\n",
|
||||
" Column('name', String(), ForeignKey('vegetable.name'), table=<tomato>),\n",
|
||||
" Column('radius', Integer(), table=<tomato>),\n",
|
||||
" Column('id', Integer(), table=<tomato_aging_states>, primary_key=True, nullable=False),\n",
|
||||
" Column('name', String(), ForeignKey('tomato.name'), table=<tomato_aging_states>),\n",
|
||||
" Column('state', String(), table=<tomato_aging_states>),\n",
|
||||
" Column('age', Integer(), table=<tomato_aging_states>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tomato.__class__.__mro__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "c21d2c54-39e2-4de3-93bc-763896ed348e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from co3.databases import SQLDatabase\n",
|
||||
"\n",
|
||||
"db = SQLDatabase('sqlite://') #, echo=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "a785d202-99d3-4ae7-859e-ee22b481f8df",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"db.recreate(vegetables.vegetable_schema)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{<Component (SQLTable)> vegetable: [{'name': 't1', 'color': 'red'}],\n",
|
||||
" <Component (SQLTable)> tomato: [{'name': 't1', 'radius': 5}],\n",
|
||||
" <Component (SQLTable)> tomato_aging_states: [{'name': 't1',\n",
|
||||
" 'state': 'ripe',\n",
|
||||
" 'age': 2}]}"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vegetables.vegetable_mapper.collector.inserts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "af7124ed-3031-4f28-89a6-553eb5b3cc7a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with db.engine.connect() as connection:\n",
|
||||
" db.manager.insert_many(\n",
|
||||
" connection, \n",
|
||||
" vegetables.vegetable_mapper.collector.inserts,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "0149e14e-5d07-42af-847d-af5c190f8946",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[{'id': 1, 'name': 't1', 'radius': 5}]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"with db.engine.connect() as connection:\n",
|
||||
" print(db.accessor.select(\n",
|
||||
" connection, \n",
|
||||
" vegetables.vegetable_schema.get_component('tomato')\n",
|
||||
" ))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"id": "668d1b8c-b47f-4a58-914d-e43402443fe6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5}]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"agg_table = vegetables.vegetable_mapper.compose(tomato)\n",
|
||||
"\n",
|
||||
"with db.engine.connect() as connection:\n",
|
||||
" agg_res = db.accessor.select(\n",
|
||||
" connection, \n",
|
||||
" agg_table,\n",
|
||||
" mappings=True,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"print(agg_res)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"id": "a051d72d-a867-46dc-bb5e-69341f39a056",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5, 'id_2': 1, 'name_2': 't1', 'state': 'ripe', 'age': 2}]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"agg_table = vegetables.vegetable_mapper.compose(tomato, action_groups=['aging'])#, outer=True)\n",
|
||||
"\n",
|
||||
"with db.engine.connect() as connection:\n",
|
||||
" agg_res = db.accessor.select(\n",
|
||||
" connection, \n",
|
||||
" agg_table,\n",
|
||||
" mappings=True,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"print(agg_res)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 31,
|
||||
"id": "6a80cfd7-3175-4526-96e0-374765d64a27",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"sqlalchemy.engine.row.RowMapping"
|
||||
]
|
||||
},
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"type(agg_res[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "7cf05ddd-2328-4051-9cf8-4ac01352405e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sqlalchemy as sa\n",
|
||||
"from co3.engines import SQLEngine\n",
|
||||
"\n",
|
||||
"a = SQLEngine.execute(db.engine.connect(), sa.select(agg_table.obj))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "c1edf68e-1fde-4a1f-8ec3-084713a8da45",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[]"
|
||||
]
|
||||
},
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"a.mappings().all()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"id": "8b8a9e47-7f5f-4828-a99e-5d9a12697f46",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tomato2 = vegetables.Tomato('t2', 8)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 38,
|
||||
"id": "062aa4de-7aea-4fd3-b5db-82af147d023e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'Tomato' object has no attribute 'action_map'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mvegetables\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvegetable_mapper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtomato2\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[0;32m~/Documents/projects/ontolog/co3/build/__editable__.co3-0.1.1-py3-none-any/co3/mapper.py:198\u001b[0m, in \u001b[0;36mMapper.collect\u001b[0;34m(self, obj, action_keys, action_groups)\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 180\u001b[0m \u001b[38;5;124;03mStages inserts up the inheritance chain, and down through components.\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[38;5;124;03mReturns: dict with keys and values relevant for associated SQLite tables\u001b[39;00m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m action_keys \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 198\u001b[0m action_keys \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[43mobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maction_map\u001b[49m\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[1;32m 200\u001b[0m receipts \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _cls \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(obj\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__mro__\u001b[39m[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m]):\n",
|
||||
"\u001b[0;31mAttributeError\u001b[0m: 'Tomato' object has no attribute 'action_map'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vegetables.vegetable_mapper.collect(tomato2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4673ddc8-3f76-4d8c-8186-bbed4a682e0d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"db.insert(vegetables.vegetable_schema.get_component('tomato'), "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 36,
|
||||
"id": "9314be4e-c1d5-4af8-ad23-0b208d24b3eb",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[{'id': 1, 'name': 't1', 'radius': 5}]"
|
||||
]
|
||||
},
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"db.select(vegetables.vegetable_schema.get_component('tomato'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a2efd060-f298-4ca6-8a58-7ed5acf1dd15",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@ -6,7 +6,7 @@ import random
|
||||
import sqlalchemy as sa
|
||||
|
||||
from co3.schemas import SQLSchema
|
||||
from co3 import CO3, collate, Mapper
|
||||
from co3 import CO3, collate, Mapper, ComposableMapper
|
||||
from co3 import util
|
||||
|
||||
|
||||
@ -101,7 +101,15 @@ tomato_cooking_table = sa.Table(
|
||||
sa.Column('pieces', sa.Integer),
|
||||
)
|
||||
vegetable_schema = SQLSchema.from_metadata(metadata)
|
||||
vegetable_mapper = Mapper(vegetable_schema)
|
||||
|
||||
def general_compose_map(c1, c2):
|
||||
return c1.obj.c.name == c2.obj.c.name
|
||||
|
||||
vegetable_mapper = ComposableMapper(
|
||||
vegetable_schema,
|
||||
attr_compose_map=general_compose_map,
|
||||
coll_compose_map=general_compose_map,
|
||||
)
|
||||
|
||||
def attr_name_map(cls):
|
||||
return f'{cls.__name__.lower()}'
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
from co3 import Accessor
|
||||
from co3 import CO3
|
||||
from co3 import Collector
|
||||
from co3 import Composer
|
||||
from co3 import Database
|
||||
from co3 import Indexer
|
||||
from co3 import Manager
|
||||
|
||||
Loading…
Reference in New Issue
Block a user