clean up Database/Accessor/Manager interaction, refine Mapper group maps

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 .pytest_cache/
 localsys.egg-info/
+.ipynb_checkpoints/
 
 # vendor and build files
 dist/

co3.egg-info/SOURCES.txt

@@ -6,7 +6,6 @@ co3/accessor.py
 co3/co3.py
 co3/collector.py
 co3/component.py
-co3/composer.py
 co3/database.py
 co3/engine.py
 co3/indexer.py

co3/__init__.py

@@ -95,11 +95,11 @@ Note: Organization for inheritance over composition
 from co3.accessor  import Accessor
 from co3.co3       import CO3, collate
 from co3.collector import Collector
-from co3.composer  import Composer
+#from co3.composer  import Composer
 from co3.database  import Database
 from co3.indexer   import Indexer
 from co3.manager   import Manager
-from co3.mapper    import Mapper
+from co3.mapper    import Mapper, ComposableMapper
 from co3.component import Component
 from co3.schema    import Schema
 from co3.engine    import Engine

co3/accessors/sql.py

@@ -83,7 +83,8 @@ class RelationalAccessor[R: Relation](Accessor[R]):
         include_cols : bool = False,
     ):
         res = self.select(
-            relation, attributes, where, mappings, include_cols, limit=1)
+            relation, attributes, where, mappings, include_cols, limit=1
+        )
 
         if include_cols and len(res[0]) > 0:
             return res[0][0], res[1]
@@ -97,6 +98,7 @@ class RelationalAccessor[R: Relation](Accessor[R]):
 class SQLAccessor(RelationalAccessor[SQLTable]):
     def raw_select(
         self,
+        connection,
         sql,
         bind_params=None,
         mappings=False,
@@ -110,7 +112,8 @@ class SQLAccessor(RelationalAccessor[SQLTable]):
 
     def select(
         self,
-        table:       SQLTable,
+        connection,
+        component:   SQLTable,
         columns      = None,
         where        = None,
         distinct_on  = None,
@@ -141,20 +144,21 @@ class SQLAccessor(RelationalAccessor[SQLTable]):
         if where is None:
             where = sa.true()
 
-        stmt = sa.select(table).where(where)
+        table = component.obj
-        if cols is not None:
+        statement = sa.select(table).where(where)
-            stmt = sa.select(*cols).select_from(table).where(where)
+        if columns is not None:
+            statement = sa.select(*columns).select_from(table).where(where)
 
         if distinct_on is not None:
-            stmt = stmt.group_by(distinct_on)
+            statement = statement.group_by(distinct_on)
 
         if order_by is not None:
-            stmt = stmt.order_by(order_by)
+            statement = statement.order_by(order_by)
 
         if limit > 0:
-            stmt = stmt.limit(limit)
+            statement = statement.limit(limit)
 
-        res = SQLEngine._execute(connection, statement, include_cols=include_cols)
+        res = SQLEngine.execute(connection, statement, include_cols=include_cols)
 
         if mappings:
             return res.mappings().all()

co3/component.py

@@ -7,13 +7,10 @@ abstractions within particular storage protocols.
 '''
 
 class Component[T]:
-    def __init__(self, name, obj: T, schema: 'Schema'):
+    def __init__(self, name, obj: T):
         self.name = name
         self.obj  = obj
 
-        self.schema = schema
-        schema.add_component(self)
-
     def __str__(self):
         return f'<Component ({self.__class__.__name__})> {self.name}'
 

co3/components/__init__.py

@@ -63,8 +63,12 @@ class Relation[T](ComposableComponent[T]):
 
 class SQLTable(Relation[SQLTableLike]):
     @classmethod
-    def from_table(cls, table: sa.Table, schema: 'SQLSchema'):
+    def from_table(cls, table: sa.Table):
-        return cls(table.name, table, schema)
+        '''
+        Note that the sa.Table type is intentional here; not all matching types for
+        SQLTableLike have a defined `name` property
+        '''
+        return cls(table.name, table)
 
     def get_attributes(self) -> tuple:
         return tuple(self.obj.columns)
@@ -102,7 +106,10 @@ class SQLTable(Relation[SQLTableLike]):
         return insert_dict
 
     def compose(self, _with: Self, on, outer=False):
-        return self.obj.join(_with, on, isouter=outer)
+        return self.__class__(
+            f'{self.name}+{_with.name}',
+            self.obj.join(_with.obj, on, isouter=outer)
+        )
 
 # key-value stores
 class Dictionary(Relation[dict]):

co3/database.py

@@ -55,10 +55,10 @@ Dev note: on explicit connection contexts
 import logging
 
 from co3.accessor import Accessor
-from co3.composer import Composer
 from co3.manager  import Manager
 from co3.indexer  import Indexer
 from co3.engine   import Engine
+from co3.schema   import Schema
 
 logger = logging.getLogger(__name__)
 
@@ -118,6 +118,12 @@ class Database[C: Component]:
         self._reset_cache = False
 
     def select(self, component: C, *args, **kwargs):
+        '''
+        Dev note: args and kwargs have to be general/unspecified here due to the possible
+        passthrough method adopting arbitrary parameters in subtypes. I could simply
+        overload this method in the relevant inheriting DBs (i.e., by matching the
+        expected Accessor's .select signature).
+        '''
         with self.engine.connect() as connection:
             return self.accessor.select(
                 connection,
@@ -128,13 +134,16 @@ class Database[C: Component]:
 
     def insert(self, component: C, *args, **kwargs):
         with self.engine.connect() as connection:
-            return self.accessor.insert(
+            return self.manager.insert(
                 connection,
                 component,
                 *args,
                 **kwargs
             )
 
+    def recreate(self, schema: Schema[C]):
+        self.manager.recreate(schema, self.engine)
+
     @property
     def index(self):
         if self.reset_cache:

co3/engines/__init__.py

@@ -10,14 +10,13 @@ class SQLEngine(Engine):
         super().__init__(url, **kwargs)
 
     def _create_manager(self):
-        return sa.create_engine(*self.manager_args, self.manager_kwargs)
+        return sa.create_engine(*self._manager_args, **self._manager_kwargs)
         
-    @contextmanager
     def connect(self, timeout=None):
         return self.manager.connect()
     
     @staticmethod
-    def _execute(
+    def execute(
         connection,
         statement,
         bind_params=None,
@@ -42,7 +41,7 @@ class SQLEngine(Engine):
         return res
 
     @staticmethod
-    def _exec_explicit(connection, statement, bind_params=None):
+    def exec_explicit(connection, statement, bind_params=None):
         trans = connection.begin()  # start a new transaction explicitly
         try:
             result = connection.execute(statement, bind_params)

co3/manager.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from abc import ABCMeta, abstractmethod
 
 from co3.schema import Schema
+from co3.engine import Engine
 
 
 class Manager[C: Component](metaclass=ABCMeta):
@@ -19,7 +20,7 @@ class Manager[C: Component](metaclass=ABCMeta):
     wrapped up in this class to then also be mirrored for the FTS counterparts.
     '''
     @abstractmethod
-    def recreate(self, schema: Schema[C]):
+    def recreate(self, schema: Schema[C], engine: Engine):
         raise NotImplementedError
 
     @abstractmethod

co3/managers/sql.py

@@ -49,6 +49,7 @@ from tqdm.auto import tqdm
 
 from co3 import util
 from co3.schema import Schema
+from co3.engines import SQLEngine
 from co3.manager import Manager
 from co3.components import Relation, SQLTable
 
@@ -76,12 +77,17 @@ class SQLManager(RelationalManager[SQLTable]):
     from an attached collector.
     '''
     def __init__(self, *args, **kwargs):
+        '''
+        The insert lock is a _reentrant lock_, meaning the same thread can acquire the
+        lock again with out deadlocking (simplifying across methods of this class that
+        need it).
+        '''
         super().__init__(*args, **kwargs)
 
         self.routers  = []
 
         self._router    = None
-        self._insert_lock = threading.Lock()
+        self._insert_lock = threading.RLock()
 
     @property
     def router(self):
@@ -92,18 +98,42 @@ class SQLManager(RelationalManager[SQLTable]):
     def add_router(self, router):
         self.routers.append(router)
 
-    def recreate(self, schema: Schema[SQLTable]): 
+    def recreate(self, schema: Schema[SQLTable], engine: SQLEngine): 
-        schema.metadata.drop_all(self.engine)
+        '''
-        schema.metadata.create_all(self.engine, checkfirst=True)
+        Ideally this remains open, as we can't necessarily rely on a SQLAlchemy metadata
+        object for all kinds of SQLDatabases (would depend on the backend, for instance). 
+
+        Haven't quite nailed down how backend instances should be determined; something
+        like SQLAlchemySQLManager doesn't seem great. Nevertheless, this method likely
+        cannot be generalized at the "SQL" (general) level.
+        '''
+        metadata = next(iter(schema._component_set)).obj.metadata
+        metadata.drop_all(engine.manager)
+        metadata.create_all(engine.manager, checkfirst=True)
 
     def update(self): pass
 
-    def insert(self, inserts: dict):
+    def insert(
+        self,
+        connection,
+        component,
+        inserts: list[dict],
+    ):
+        '''
+        Parameters:
+        '''
+        with self._insert_lock:
+            connection.execute(
+                sa.insert(component.obj),
+                inserts
+            )
+
+    def insert_many(self, connection, inserts: dict):
         '''
         Perform provided table inserts.
 
         Parameters:
-            inserts: table-indexed dictionary of insert lists
+            inserts: component-indexed dictionary of insert lists
         '''
         total_inserts = sum([len(ilist) for ilist in inserts.values()])
         if total_inserts < 1: return
@@ -112,24 +142,18 @@ class SQLManager(RelationalManager[SQLTable]):
 
         # TODO: add some exception handling? may be fine w default propagation
         start = time.time()
-        with self.engine.connect() as connection:
+        with self._insert_lock:
-            with self._insert_lock:
+            for component in inserts:
-                for table_str in inserts:
+                comp_inserts = inserts[component]
-                    table_inserts = inserts[table_str]
+                if len(comp_inserts) == 0: continue
-                    if len(table_inserts) == 0: continue
 
-                    table = tables.table_map[table_str]
+                logger.info(
+                    f'Inserting {len(comp_inserts)} out-of-date entries into component "{component}"'
+                )
 
-                    logger.info(
+                self.insert(connection, component, comp_inserts)
-                        f'Inserting {len(table_inserts)} out-of-date entries into table "{table_str}"'
+            connection.commit()
-                    )
+            logger.info(f'Insert transaction completed successfully in {time.time()-start:.2f}s')
-
-                    connection.execute(
-                        sa.insert(table),
-                        table_inserts
-                    )
-                connection.commit()
-                logger.info(f'Insert transaction completed successfully in {time.time()-start:.2f}s')
 
     def _file_sync_bools(self):
         synced_bools = []

co3/mapper.py

@@ -29,7 +29,7 @@ Development log:
       hierarchy). As such, to fully collect from a type, the Mapper needs to leave
       registration open to various types, not just those part of the same hierarchy.
 '''
-from typing import Callable
+from typing import Callable, Any
 from collections import defaultdict
 
 from co3.co3        import CO3
@@ -55,12 +55,14 @@ class Mapper[C: Component]:
     "dropped off" at an appropriate Database's Manager to actually perform the requested
     inserts (hence why we tie Mappers to Schemas one-to-one). 
 
-    Dev note: the Composer needs reconsideration, or at least its positioning directly in
+    Dev note:
-    this class. It may be more appropriate to have at the Schema level, or even just
+        the Composer needs reconsideration, or at least its positioning directly in this
-    dissolved altogether if arbitrary named Components can be attached to schemas.
+        class. It may be more appropriate to have at the Schema level, or even just
-    '''
+        dissolved altogether if arbitrary named Components can be attached to schemas.
-    type comp_spec = str | C
 
+        - Consider pushing this into a Mapper factory; on init, could check if provided
+          Schema wraps up composable Components or not
+    '''
     _collector_cls: type[Collector[C]] = Collector[C]
 
     def __init__(self, schema: Schema[C]):
@@ -76,7 +78,7 @@ class Mapper[C: Component]:
         self.attribute_comps:  dict[type[CO3], C] = {}
         self.collation_groups: dict[type[CO3], dict[str|None, C]] = defaultdict(dict)
 
-    def _check_component(self, comp: self.comp_spec):
+    def _check_component(self, comp: str | C):
         if type(comp) is str:
             comp_key = comp
             comp = self.schema.get_component(comp_key)
@@ -95,9 +97,9 @@ class Mapper[C: Component]:
     def attach(
         self,
         type_ref    : type[CO3],
-        attr_comp   : self.comp_spec,
+        attr_comp   : str | C,
-        coll_comp   : self.comp_spec | None                   = None,
+        coll_comp   : str | C | None                   = None,
-        coll_groups : dict[str | None, self.comp_spec] | None = None,
+        coll_groups : dict[str | None, str | C] | None = None,
     ) -> None:
         '''
         Parameters:
@@ -127,8 +129,8 @@ class Mapper[C: Component]:
     def attach_many(
         self,
         type_list: list[type[CO3]],
-        attr_name_map: Callable[[type[CO3]], self.comp_spec],
+        attr_name_map: Callable[[type[CO3]], str | C],
-        coll_name_map: Callable[[type[CO3], str], self.comp_spec] | None = None,
+        coll_name_map: Callable[[type[CO3], str], str | C] | None = None,
     ):
         '''
         Auto-register a set of types to the Mapper's attached Schema. Associations are
@@ -218,7 +220,7 @@ class Mapper[C: Component]:
 
                 _, action_groups = obj.action_registry[action_key]
                 for action_group in action_groups:
-                    collation_component, _ = self.get_collation_comp(_cls, group=action_group)
+                    collation_component = self.get_collation_comp(_cls, group=action_group)
 
                     if collation_component is None:
                         continue
@@ -317,8 +319,8 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]):
     def __init__(
         self,
         schema           : Schema[C],
-        attr_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None
+        attr_compose_map : Callable[[str | C, str | C], Any] | None = None,
-        coll_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None
+        coll_compose_map : Callable[[str | C, str | C], Any] | None = None,
     ):
         super().__init__(schema)
 
@@ -328,7 +330,7 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]):
     def compose(
         self,
         obj:           CO3,
-        action_groups: list[str] = None,
+        action_groups: list[str] | None = None,
         *compose_args,
         **compose_kwargs,
     ):
@@ -356,29 +358,36 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]):
 
             # compose horizontally with components from provided action groups
             coll_comp_agg = attr_comp
-            for action_group in action_groups:
+            if action_groups is not None:
-                coll_comp = self.get_collation_comp(_cls, group=action_group)
+                for action_group in action_groups:
+                    coll_comp = self.get_collation_comp(_cls, group=action_group)
 
-                if coll_comp is None:
+                    if coll_comp is None:
-                    continue
+                        continue
 
-                compose_condition = self.coll_compose_map(coll_comp_agg, coll_comp)
+                    # note how the join condition is specified using the non-composite
+                    # `attr_comp` and new `coll_comp`; the composite doesn't typically
+                    # have the same attribute access and needs a ref to a specific comp
+                    compose_condition = self.coll_compose_map(attr_comp, coll_comp)
 
-                coll_comp_agg = coll_comp_agg.compose(
+                    coll_comp_agg = coll_comp_agg.compose(
-                    component=coll_comp,
+                        coll_comp,
-                    on=compose_condition,
+                        compose_condition,
+                        *compose_args,
+                        **compose_kwargs,
+                    )
+
+            if attr_comp_agg is None:
+                attr_comp_agg = coll_comp_agg
+            else:
+                # note the reduced attr_comp ref passed to compose map, rather than
+                # coll_comp_agg produced above; this is provided as the compose comp, though
+                compose_condition = self.attr_compose_map(attr_comp_agg, attr_comp)
+                attr_comp_agg = attr_comp_agg.compose(
+                    coll_comp_agg,
+                    compose_condition,
                     *compose_args,
                     **compose_kwargs,
                 )
 
-            # note the reduced attr_comp ref passed to compose map, rather than
-            # coll_comp_agg produced above; this is provided as the compose comp, though
-            compose_condition = self.attr_compose_map(attr_comp_agg, attr_comp)
-            attr_comp_agg = attr_comp_agg.compose(
-                component=coll_comp_agg,
-                on=compose_condition,
-                *compose_args,
-                **compose_kwargs,
-            )
-
         return attr_comp_agg

co3/schemas/__init__.py

@@ -15,7 +15,8 @@ class SQLSchema(RelationalSchema[SQLTable]):
         instance = cls()
 
         for table in metadata.tables.values():
-            SQLTable.from_table(table, instance)
+            comp = SQLTable.from_table(table)
+            instance.add_component(comp)
 
         return instance
         

examples/.ipynb_checkpoints/database-checkpoint.ipynb

@@ -1,6 +1,204 @@
 {
- "cells": [],
+ "cells": [
- "metadata": {},
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6f6fbc7e-4fb9-4353-b2ee-9ea819a3c896",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import vegetables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "88fd0ea8-9c94-4569-a51b-823a04f32f55",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'age': 3}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tomato = vegetables.Tomato('t1', 5)\n",
+    "\n",
+    "# test a register collation action\n",
+    "tomato.collate('ripe')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "348926d9-7137-4eff-a919-508788553dd2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['afff61ec-e0b0-44bb-9f0d-06008a82f6a5',\n",
+       " '5edfa13c-0eb1-4bbc-b55e-1550ff7df3d2',\n",
+       " '4568a2d4-eb41-4b15-9a29-e3e22906c661']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vegetables.vegetable_mapper.collect(tomato, ['ripe'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4e5e7319-11bf-4051-951b-08c84e9f3874",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Component (SQLTable)> vegetable+tomato+tomato_aging_states+tomato_cooking_states"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "aa290686-8074-4038-a3cc-ce6817844653",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Column('id', Integer(), table=<vegetable>, primary_key=True, nullable=False),\n",
+       " Column('name', String(), table=<vegetable>),\n",
+       " Column('color', String(), table=<vegetable>),\n",
+       " Column('id', Integer(), table=<tomato>, primary_key=True, nullable=False),\n",
+       " Column('name', String(), ForeignKey('vegetable.name'), table=<tomato>),\n",
+       " Column('radius', Integer(), table=<tomato>),\n",
+       " Column('id', Integer(), table=<tomato_aging_states>, primary_key=True, nullable=False),\n",
+       " Column('name', String(), ForeignKey('tomato.name'), table=<tomato_aging_states>),\n",
+       " Column('state', String(), table=<tomato_aging_states>),\n",
+       " Column('age', Integer(), table=<tomato_aging_states>)]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tomato.__class__.__mro__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c21d2c54-39e2-4de3-93bc-763896ed348e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from co3.databases import SQLDatabase\n",
+    "\n",
+    "db = SQLDatabase('sqlite://', echo=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "a785d202-99d3-4ae7-859e-ee22b481f8df",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<contextlib._GeneratorContextManager at 0x7dd5c619be60>"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "db.recreate(vegetable_schema) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "co3",
+   "language": "python",
+   "name": "co3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
  "nbformat": 4,
  "nbformat_minor": 5
 }

examples/.ipynb_checkpoints/vegetables-checkpoint.py

@@ -6,7 +6,7 @@ import random
 import sqlalchemy as sa
 
 from co3.schemas import SQLSchema
-from co3 import CO3, collate, Mapper
+from co3 import CO3, collate, Mapper, ComposableMapper
 from co3 import util
 
 
@@ -101,5 +101,35 @@ tomato_cooking_table = sa.Table(
     sa.Column('pieces', sa.Integer),
 )
 vegetable_schema = SQLSchema.from_metadata(metadata)
-vegetable_mapper = Mapper(vegetable_schema)
+
+def general_compose_map(c1, c2):
+    return c1.obj.c.name == c2.obj.c.name
+    
+vegetable_mapper = ComposableMapper(
+    vegetable_schema,
+    attr_compose_map=general_compose_map,
+    coll_compose_map=general_compose_map,
+)
+
+def attr_name_map(cls):
+    return f'{cls.__name__.lower()}'
+
+def coll_name_map(cls, action_group):
+    return f'{cls.__name__.lower()}_{action_group}_states'
+
+vegetable_mapper.attach_many(
+    type_list,
+    attr_name_map,
+    coll_name_map,
+)
+
+'''
+new mapping type for Mapper attachment:
+
+Callable[ [type[CO3], str|None], tuple[str, tuple[str], tuple[str]]]
+
+tail tuples to associate column names from central table to collation
+
+this should complete the auto-compose horizontally
+'''
 

examples/database.ipynb

@@ -21,16 +21,388 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "88fd0ea8-9c94-4569-a51b-823a04f32f55",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'age': 5}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tomato = vegetables.Tomato('t1', 5)\n",
     "\n",
     "# test a register collation action\n",
     "tomato.collate('ripe')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "348926d9-7137-4eff-a919-508788553dd2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['9ca6772e-6621-4511-a4a6-ad451a1da91f',\n",
+       " '2a91b423-4e08-491c-b1d2-5ec25259191e',\n",
+       " '4a9edb2b-4ac5-467e-82ef-b254829ac2a2']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vegetables.vegetable_mapper.collect(tomato, ['ripe'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4e5e7319-11bf-4051-951b-08c84e9f3874",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Component (SQLTable)> vegetable+tomato+tomato_aging_states+tomato_cooking_states"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "aa290686-8074-4038-a3cc-ce6817844653",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Column('id', Integer(), table=<vegetable>, primary_key=True, nullable=False),\n",
+       " Column('name', String(), table=<vegetable>),\n",
+       " Column('color', String(), table=<vegetable>),\n",
+       " Column('id', Integer(), table=<tomato>, primary_key=True, nullable=False),\n",
+       " Column('name', String(), ForeignKey('vegetable.name'), table=<tomato>),\n",
+       " Column('radius', Integer(), table=<tomato>),\n",
+       " Column('id', Integer(), table=<tomato_aging_states>, primary_key=True, nullable=False),\n",
+       " Column('name', String(), ForeignKey('tomato.name'), table=<tomato_aging_states>),\n",
+       " Column('state', String(), table=<tomato_aging_states>),\n",
+       " Column('age', Integer(), table=<tomato_aging_states>)]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tomato.__class__.__mro__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c21d2c54-39e2-4de3-93bc-763896ed348e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from co3.databases import SQLDatabase\n",
+    "\n",
+    "db = SQLDatabase('sqlite://') #, echo=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "a785d202-99d3-4ae7-859e-ee22b481f8df",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "db.recreate(vegetables.vegetable_schema)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{<Component (SQLTable)> vegetable: [{'name': 't1', 'color': 'red'}],\n",
+       " <Component (SQLTable)> tomato: [{'name': 't1', 'radius': 5}],\n",
+       " <Component (SQLTable)> tomato_aging_states: [{'name': 't1',\n",
+       "   'state': 'ripe',\n",
+       "   'age': 2}]}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vegetables.vegetable_mapper.collector.inserts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "af7124ed-3031-4f28-89a6-553eb5b3cc7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with db.engine.connect() as connection:\n",
+    "    db.manager.insert_many(\n",
+    "        connection, \n",
+    "        vegetables.vegetable_mapper.collector.inserts,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "0149e14e-5d07-42af-847d-af5c190f8946",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'id': 1, 'name': 't1', 'radius': 5}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "with db.engine.connect() as connection:\n",
+    "    print(db.accessor.select(\n",
+    "        connection, \n",
+    "        vegetables.vegetable_schema.get_component('tomato')\n",
+    "    ))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "668d1b8c-b47f-4a58-914d-e43402443fe6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "agg_table = vegetables.vegetable_mapper.compose(tomato)\n",
+    "\n",
+    "with db.engine.connect() as connection:\n",
+    "    agg_res = db.accessor.select(\n",
+    "        connection, \n",
+    "        agg_table,\n",
+    "        mappings=True,\n",
+    "    )\n",
+    "\n",
+    "print(agg_res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "a051d72d-a867-46dc-bb5e-69341f39a056",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5, 'id_2': 1, 'name_2': 't1', 'state': 'ripe', 'age': 2}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "agg_table = vegetables.vegetable_mapper.compose(tomato, action_groups=['aging'])#, outer=True)\n",
+    "\n",
+    "with db.engine.connect() as connection:\n",
+    "    agg_res = db.accessor.select(\n",
+    "        connection, \n",
+    "        agg_table,\n",
+    "        mappings=True,\n",
+    "    )\n",
+    "\n",
+    "print(agg_res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "6a80cfd7-3175-4526-96e0-374765d64a27",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "sqlalchemy.engine.row.RowMapping"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(agg_res[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "7cf05ddd-2328-4051-9cf8-4ac01352405e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sqlalchemy as sa\n",
+    "from co3.engines import SQLEngine\n",
+    "\n",
+    "a = SQLEngine.execute(db.engine.connect(), sa.select(agg_table.obj))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "c1edf68e-1fde-4a1f-8ec3-084713a8da45",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[]"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "a.mappings().all()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "8b8a9e47-7f5f-4828-a99e-5d9a12697f46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tomato2 = vegetables.Tomato('t2', 8)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "062aa4de-7aea-4fd3-b5db-82af147d023e",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'Tomato' object has no attribute 'action_map'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mvegetables\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvegetable_mapper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtomato2\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Documents/projects/ontolog/co3/build/__editable__.co3-0.1.1-py3-none-any/co3/mapper.py:198\u001b[0m, in \u001b[0;36mMapper.collect\u001b[0;34m(self, obj, action_keys, action_groups)\u001b[0m\n\u001b[1;32m    179\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m    180\u001b[0m \u001b[38;5;124;03mStages inserts up the inheritance chain, and down through components.\u001b[39;00m\n\u001b[1;32m    181\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[38;5;124;03mReturns: dict with keys and values relevant for associated SQLite tables\u001b[39;00m\n\u001b[1;32m    196\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m    197\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m action_keys \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 198\u001b[0m     action_keys \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[43mobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maction_map\u001b[49m\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[1;32m    200\u001b[0m receipts \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    201\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _cls \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(obj\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__mro__\u001b[39m[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m]):\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'Tomato' object has no attribute 'action_map'"
+     ]
+    }
+   ],
+   "source": [
+    "vegetables.vegetable_mapper.collect(tomato2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4673ddc8-3f76-4d8c-8186-bbed4a682e0d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db.insert(vegetables.vegetable_schema.get_component('tomato'), "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "9314be4e-c1d5-4af8-ad23-0b208d24b3eb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'id': 1, 'name': 't1', 'radius': 5}]"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "db.select(vegetables.vegetable_schema.get_component('tomato'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2efd060-f298-4ca6-8a58-7ed5acf1dd15",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

examples/vegetables.py

@@ -6,7 +6,7 @@ import random
 import sqlalchemy as sa
 
 from co3.schemas import SQLSchema
-from co3 import CO3, collate, Mapper
+from co3 import CO3, collate, Mapper, ComposableMapper
 from co3 import util
 
 
@@ -101,7 +101,15 @@ tomato_cooking_table = sa.Table(
     sa.Column('pieces', sa.Integer),
 )
 vegetable_schema = SQLSchema.from_metadata(metadata)
-vegetable_mapper = Mapper(vegetable_schema)
+
+def general_compose_map(c1, c2):
+    return c1.obj.c.name == c2.obj.c.name
+    
+vegetable_mapper = ComposableMapper(
+    vegetable_schema,
+    attr_compose_map=general_compose_map,
+    coll_compose_map=general_compose_map,
+)
 
 def attr_name_map(cls):
     return f'{cls.__name__.lower()}'

tests/imports.py

@@ -1,7 +1,6 @@
 from co3 import Accessor
 from co3 import CO3
 from co3 import Collector
-from co3 import Composer
 from co3 import Database
 from co3 import Indexer
 from co3 import Manager