add and integrate Component, Schema objects

README.md

@@ -15,7 +15,21 @@ a known schema.
 - **Collector** to collect data for updating storage state
 - **Database** to collect data for updating storage state
 - **Mapper** to collect data for updating storage state
-- **Relation** to collect data for updating storage state
+- **Component** to collect data for updating storage state
 
 **CO3** is an abstract base class that makes it easy to integrate this model with object
 hierarchies that mirror a storage schema.
+
+# Detailed structural breakdown
+There are a few pillars of the CO3 model that meaningfully group up functionality:
+
+- Database: generic to a Component type, provides basic connection to a database at a
+  specific address/location. The explicit Component type makes it easy to hook into
+  appropriately typed functional objects:
+  * Manager: generic to a Component and Database type, provides a supported set of
+    state-modifying operations to a constituent database
+  * Accessor: generic to a Component and Database type, provides a supported set of
+    state inspection operations on a constituent database
+  * Indexer: 
+- Mapper: generic to a Component, serves as the fundamental connective component between
+  types in the data representation hierarchy (CO3 subclasses) and database Components.

build/__editable__.co3-0.1.1-py3-none-any/co3/component.py

@@ -0,0 +1 @@
+/home/smgr/Documents/projects/ontolog/co3/co3/component.py

build/__editable__.co3-0.1.1-py3-none-any/co3/components/__init__.py

@@ -0,0 +1 @@
+/home/smgr/Documents/projects/ontolog/co3/co3/components/__init__.py

build/__editable__.co3-0.1.1-py3-none-any/co3/relation.py

@@ -1 +0,0 @@
-/home/smgr/Documents/projects/ontolog/co3/co3/relation.py

build/__editable__.co3-0.1.1-py3-none-any/co3/relations/__init__.py

@@ -1 +0,0 @@
-/home/smgr/Documents/projects/ontolog/co3/co3/relations/__init__.py

build/__editable__.co3-0.1.1-py3-none-any/co3/schema.py

@@ -0,0 +1 @@
+/home/smgr/Documents/projects/ontolog/co3/co3/schema.py

build/__editable__.co3-0.1.1-py3-none-any/co3/schemas/__init__.py

@@ -0,0 +1 @@
+/home/smgr/Documents/projects/ontolog/co3/co3/schemas/__init__.py

build/__editable__.co3-0.1.1-py3-none-any/co3/util/types.py

@@ -0,0 +1 @@
+/home/smgr/Documents/projects/ontolog/co3/co3/util/types.py

co3/__init__.py

@@ -93,17 +93,19 @@ Note: Organization for inheritance over composition
 '''
 
 from co3.accessor  import Accessor
-from co3.co3       import CO3
+from co3.co3       import CO3, collate
 from co3.collector import Collector
 from co3.composer  import Composer
 from co3.database  import Database
 from co3.indexer   import Indexer
 from co3.manager   import Manager
 from co3.mapper    import Mapper
-from co3.relation  import Relation
+from co3.component import Component
+from co3.schema    import Schema
 
 from co3 import accessors
 from co3 import databases
 from co3 import managers
-from co3 import relations
+from co3 import components
+from co3 import schemas
 from co3 import util

co3/accessor.py

@@ -8,13 +8,14 @@ schema-specific queries.
 import inspect
 from pathlib import Path
 from collections import defaultdict
+from abc import ABCMeta, abstractmethod
 
 import sqlalchemy as sa
 
-#from co3.database import Database
+from co3.component import Component
 
 
-class Accessor[D: 'Database']:
+class Accessor[C: Component, D: 'Database[C]'](metaclass=ABCMeta):
     '''
     Access wrapper class for complex queries and easy integration with Composer tables.
     Implements high-level access to things like common constrained SELECT queries.

co3/accessors/sql.py

@@ -1,3 +1,42 @@
+'''
+Design proposal: variable backends
+
+One particular feature not supported by the current type hierarchy is the possible use of
+different backends to implement a general interface like SQLAccessor. One could imagine,
+for instance, using `sqlalchemy` or `sqlite` to define the same methods laid out in a
+parent class blueprint. It's not too difficult to imagine cases where both of these may be
+useful, but for now it is outside the development scope. Should it ever enter the scope,
+however, we might consider a simple `backend` argument on instantiation, keeping just the
+SQLAccessor exposed rather than a whole set of backend-specific types:
+
+```py
+class SQLAlchemyAccessor(RelationalAccessor): # may also inherit from a dedicated interface parent
+    def select(...):
+        ...
+
+class SQLiteAccessor(RelationalAccessor): 
+    def select(...):
+        ...
+
+class SQLAccessor(RelationalAccessor): 
+    backends = {
+        'sqlalchemy': SQLAlchemyAccessor,
+        'sqlite':     SQLteAccessor,
+    }
+
+    def __init__(self, backend: str):
+        self.backend = self.backends.get(backend)
+
+    def select(...):
+        return self.backend.select(...)
+
+```
+
+For now, we can look at SQLAccessor (and equivalents in other type hierarchies, like
+SQLManagers) as being SQLAlchemyAccessors and not supporting any backend swapping. But in
+theory, to make the above change, we'd just need to rename it and wrap it up.
+'''
+
 from pathlib import Path
 from collections.abc import Iterable
 import inspect
@@ -7,21 +46,44 @@ import sqlalchemy as sa
 
 from co3 import util
 from co3.accessor import Accessor
-from co3.relation import Relation
+from co3.components import Relation, SQLTable
-
-#from co3.databases.sql import RelationalDatabase, TabularDatabase, SQLDatabase
-from co3.relations import TabularRelation, SQLTable
 
 
-class RelationalAccessor[D: 'RelationalDatabase', R: Relation](Accessor[D]):
+class RelationalAccessor[R: Relation, D: 'RelationalDatabase[R]'](Accessor[R, D]):
-    pass
+    def raw_select(self, sql: str):
+        raise NotImplementedError
+
+    def select(
+        self,
+        relation: R,
+        cols         = None,
+        where        = None,
+        distinct_on  = None,
+        order_by     = None,
+        limit        = 0,
+    ):
+        raise NotImplementedError
+
+    def select_one(
+        self,
+        relation     : R,
+        cols                = None,
+        where               = None,
+        mappings     : bool = False,
+        include_cols : bool = False,
+    ):
+        res = self.select(relation, cols, where, mappings, include_cols, limit=1)
+
+        if include_cols and len(res[0]) > 0:
+            return res[0][0], res[1]
+
+        if len(res) > 0:
+            return res[0]
+
+        return None
 
 
-class TabularAccessor[D: 'TabularDatabase', R: TabularRelation](RelationalAccessor[D, R]):
+class SQLAccessor(RelationalAccessor[SQLTable, 'SQLDatabase[SQLTable]']):
-    pass
-
-
-class SQLAccessor(TabularAccessor['SQLDatabase', SQLTable]):
     def raw_select(
         self,
         sql,
@@ -37,7 +99,7 @@ class SQLAccessor(TabularAccessor['SQLDatabase', SQLTable]):
 
     def select(
         self,
-        table: sa.Table | sa.Subquery | sa.Join,
+        table: SQLTable,
         cols         = None,
         where        = None,
         distinct_on  = None,
@@ -82,15 +144,3 @@ class SQLAccessor(TabularAccessor['SQLDatabase', SQLTable]):
             stmt = stmt.limit(limit)
 
         return res_method(self.engine, stmt, include_cols=include_cols)
-
-    def select_one(self, table, cols=None, where=None, mappings=False, include_cols=False):
-        res = self.select(table, cols, where, mappings, include_cols, limit=1)
-
-        if include_cols and len(res[0]) > 0:
-            return res[0][0], res[1]
-
-        if len(res) > 0:
-            return res[0]
-
-        return None
-

co3/co3.py

@@ -29,6 +29,8 @@ logger = logging.getLogger(__name__)
 
 def collate(action_key, action_groups=None):
     def decorator(func):
+        nonlocal action_groups
+        
         if action_groups is None:
             action_groups = [None]
         func._action_data = (action_key, action_groups)

co3/collector.py

@@ -29,12 +29,13 @@ from uuid import uuid4
 import sqlalchemy as sa
 
 from co3 import util
+from co3.component import Component
 #from localsys.db.schema import tables
 
 
 logger = logging.getLogger(__name__)
 
-class Collector:
+class Collector[C: Component, M: 'Mapper[C]']:
     def __init__(self):
         self._inserts = defaultdict(lambda: defaultdict(list))
 

co3/component.py

@@ -0,0 +1,19 @@
+'''
+Component
+
+General wrapper for storage components to be used in various database contexts. Relations
+can be thought of generally as named data containers/entities serving as a fundamental
+abstractions within particular storage protocols.
+'''
+
+class Component[T]:
+    def __init__(self, name, obj: T, schema: 'Schema'):
+        self.name = name
+        self.obj  = obj
+
+        self.schema = schema
+        schema.add_component(self)
+
+    def get_attributes(self):
+        raise NotImplementedError
+

co3/components/__init__.py

@@ -0,0 +1,79 @@
+from typing import Self
+from abc import ABCMeta, abstractmethod
+
+import sqlalchemy as sa
+
+from co3.util.types import TableLike
+from co3.component import Component
+
+
+class ComposableComponent[T](Component[T], metaclass=ABCMeta):
+    '''
+    Components that can be composed with others of the same type.
+    '''
+    @abstractmethod
+    def compose(self, component: Self, on) -> Self:
+        '''
+        Abstract composition.
+        '''
+        raise NotImplementedError
+
+
+# relational databases
+class Relation[T](ComposableComponent[T]):
+    '''
+    Relation base for tabular components to be used in relation DB settings. Attempts to
+    adhere to the set-theoretic base outlined in the relational model [1]. Some
+    terminology:
+
+    Relation: table-like container
+    | -> Heading: set of attributes
+    |    | -> Attribute: column name
+    | -> Body: set of tuples with domain matching the heading
+    |    | -> Tuple: collection of values
+
+
+    [1]: https://en.wikipedia.org/wiki/Relational_model#Set-theoretic_formulation
+
+    Note: development tasks
+        As it stands, the Relation skeleton is incredibly lax compared to the properties and
+        operations that should be formally available, according its pure relational algebra
+        analog. 
+
+        Relations are also generic up to a type T, which ultimately serves as the base object
+        for Relation instances. We aren't attempting to implement some generally useful
+        table-like class here; instead we're just exposing a lightweight interface that's
+        needed for a few CO3 contexts, and commonly off-loading most of the heavy-lifting to
+        true relation objects like SQLAlchemy tables.
+    '''
+    def compose(
+        self,
+        _with: Self,
+        on,
+        outer=False
+    ):
+        return self
+
+class SQLTable(Relation[TableLike]):
+    @classmethod
+    def from_table(cls, table: sa.Table, schema: 'SQLSchema'):
+        return cls(table.name, table, schema)
+
+    def get_attributes(self):
+        return tuple(self.obj.columns)
+
+
+# key-value stores
+class Dictionary(Relation[dict]):
+    def get_attributes(self):
+        return tuple(self.obj.keys())
+
+
+# document databases
+class Document[T](Component[T]):
+    pass
+
+
+# graph databases
+class Node[T](Component[T]):
+    pass

co3/composer.py

@@ -26,7 +26,7 @@ class ExampleComposer(Composer):
 '''
 from pathlib import Path
 
-from co3.mapper import Mapper
+from co3.component import Component
 
 
 def register_table(table_name=None):
@@ -43,7 +43,7 @@ def register_table(table_name=None):
         return func
     return decorator
 
-class Composer[M: Mapper]:
+class Composer[C: Component, M: 'Mapper[C]']:
     '''
     Base composer wrapper for table groupings.
 

co3/database.py

@@ -21,9 +21,9 @@ from co3.indexer  import Indexer
 logger = logging.getLogger(__name__)
 
 
-class Database:
+class Database[C: Component]:
-    accessor: type[Accessor[Self]] = Accessor
+    accessor: type[Accessor[C, Self]] = Accessor[C, Self]
-    manager:  type[Manager[Self]]  = Manager
+    manager:  type[Manager[C, Self]]  = Manager[C, Self]
 
     def __init__(self, resource):
         '''

co3/databases/sql.py

@@ -2,30 +2,24 @@ from typing import Self
 
 from co3.database import Database
 
-from co3.accessors.sql import RelationalAccessor, TabularAccessor, SQLAccessor
+from co3.accessors.sql import RelationalAccessor, SQLAccessor
-from co3.managers.sql  import RelationalManager,  TabularManager,  SQLManager
+from co3.managers.sql  import RelationalManager,  SQLManager
 
-from co3.relation import Relation
+from co3.components import Relation, SQLTable
-from co3.relations import TabularRelation, SQLTable
 
 
 class RelationalDatabase[R: Relation](Database):
-    accessor: type[RelationalAccessor[Self, R]] = RelationalAccessor[Self, R]
-    manager:  type[RelationalManager[Self, R]]  = RelationalManager[Self, R]
-
-
-class TabularDatabase[R: TabularRelation](RelationalDatabase[R]):
     '''
     accessor/manager assignments satisfy supertype's type settings;
     `TabluarAccessor[Self, R]` is of type `type[RelationalAccessor[Self, R]]`
     (and yes, `type[]` specifies that the variable is itself being set to a type or a
     class, rather than a satisfying _instance_)
     '''
-    accessor: type[TabularAccessor[Self, R]] = TabularAccessor[Self, R]
+    accessor: type[RelationalAccessor[Self, R]] = RelationalAccessor[Self, R]
-    manager:  type[TabularManager[Self, R]]  = TabularManager[Self, R]
+    manager:  type[RelationalManager[Self, R]]  = RelationalManager[Self, R]
 
 
-class SQLDatabase[R: SQLTable](TabularDatabase[R]):
+class SQLDatabase[R: SQLTable](RelationalDatabase[R]):
     accessor = SQLAccessor
     manager  = SQLManager
 

co3/manager.py

@@ -7,10 +7,11 @@ interacting with an underlying database, like inserts and schema recreation.
 from pathlib import Path
 from abc import ABCMeta, abstractmethod
 
+from co3.schema import Schema
 #from co3.database import Database
 
 
-class Manager[D: 'Database'](metaclass=ABCMeta):
+class Manager[C: Component, D: 'Database[C]'](metaclass=ABCMeta):
     '''
     Management wrapper class for table groupings.
 
@@ -22,7 +23,7 @@ class Manager[D: 'Database'](metaclass=ABCMeta):
         self.database = database
 
     @abstractmethod
-    def recreate(self):
+    def recreate(self, schema: Schema[C]):
         raise NotImplementedError
 
     @abstractmethod

co3/managers/sql.py

@@ -48,26 +48,21 @@ import sqlalchemy as sa
 from tqdm.auto import tqdm
 
 from co3 import util
-
+from co3.schema import Schema
 from co3.manager import Manager
-from co3.relation import Relation
+from co3.components import Relation, SQLTable
-#from co3.databases import SQLDatabase
+
 #from localsys.reloader.router._base import ChainRouter, Event
-from co3.relations import TabularRelation, SQLTable
 
 
 logger = logging.getLogger(__name__)
 
 
-class RelationalManager[D: 'RelationalDatabase', R: Relation](Manager[D]):
+class RelationalManager[R: Relation, D: 'RelationalDatabase[R]'](Manager[R, D]):
     pass
 
 
-class TabularManager[D: 'TabularDatabase', R: TabularRelation](RelationalManager[D, R]):
+class SQLManager(RelationalManager[SQLTable, 'SQLDatabase[SQLTable]']):
-    pass
-
-
-class SQLManager(TabularManager['SQLDatabase', SQLTable]):
     '''
     Core schema table manager. Exposes common operations and facilitates joint operations
     needed for highly connected schemas.
@@ -80,8 +75,6 @@ class SQLManager(TabularManager['SQLDatabase', SQLTable]):
     saturates a router with events (dynamically) and sweeps up inserts on session basis
     from an attached collector.
     '''
-    conversion_formats = ['src', 'html5']
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -99,9 +92,9 @@ class SQLManager(TabularManager['SQLDatabase', SQLTable]):
     def add_router(self, router):
         self.routers.append(router)
 
-    def recreate(self): 
+    def recreate(self, schema: Schema[SQLTable]): 
-        tables.metadata.drop_all(self.engine)
+        schema.metadata.drop_all(self.engine)
-        tables.metadata.create_all(self.engine, checkfirst=True)
+        schema.metadata.create_all(self.engine, checkfirst=True)
 
     def update(self): pass
 

co3/mapper.py

@@ -18,27 +18,56 @@ mapper.attach(
     }
 )
 '''
+from typing import Callable, Self
 from collections import defaultdict
 
-from co3.co3 import CO3
+from co3.co3       import CO3
-from co3.relation import Relation
+from co3.collector import Collector
+from co3.composer  import Composer
+from co3.component import Component
+from co3.schema    import Schema
 
 
-class Mapper[R: Relation]:
+class Mapper[C: Component]:
     '''
     Mapper base class for housing schema components and managing relationships between CO3
-    types and storage targets (of type R).
+    types and storage components (of type C).
     '''
-    def __init__(self):
+    _collector_cls: type[Collector[C, Self]] = Collector[C, Self]
-        self.attribute_comps:  dict[CO3, R] = {}
+    _composer_cls:  type[Composer[C, Self]]  = Composer[C, Self]
-        self.collation_groups: dict[CO3, dict[str|None, R]] = defaultdict(dict)
+
+    def __init__(self, co3_root: type[CO3], schema: Schema):
+        self.co3_root = co3_root
+        self.schema   = schema
+
+        self.collector = self._collector_cls()
+        self.composer  = self._composer_cls()
+
+        self.attribute_comps:  dict[type[CO3], C] = {}
+        self.collation_groups: dict[type[CO3], dict[str|None, C]] = defaultdict(dict)
+
+    def _check_component(self, comp: C | str):
+        if type(comp) is str:
+            comp_key = comp
+            comp = self.schema.get_component(comp_key)
+            if comp is None:
+                raise ValueError(
+                    f'Component key {comp_key} not available in attached schema'
+                )
+        else:
+            if comp not in self.schema:
+                raise TypeError(
+                    f'Component {comp} not registered to Mapper schema {self.schema}'
+                )
+
+        return comp
 
     def attach(
         self,
-        type_ref    : CO3,
+        type_ref    : type[CO3],
-        attr_comp   : R,
+        attr_comp   : C | str,
-        coll_comp   : R | None            = None,
+        coll_comp   : C | str | None      = None,
-        coll_groups : dict[str | None, R] = None
+        coll_groups : dict[str | None, C | str] = None
     ) -> None:
         '''
         Parameters:
@@ -49,24 +78,33 @@ class Mapper[R: Relation]:
             coll_groups: storage components for named collation groups; dict mapping group
                          names to components
         '''
+        # check for type compatibility with CO3 root
+        if not issubclass(type_ref, self.co3_root):
+            raise TypeError(
+                f'Type ref {type_ref} not a subclass of Mapper CO3 root {self.co3_root}'
+            )
+
+        # check attribute component in registered schema
+        attr_comp = self._check_component(attr_comp)
         self.attribute_comps[type_ref] = attr_comp
 
+        # check default component in registered schema
         if coll_comp is not None:
-            self.collation_groups[type_ref][None] = attr_comp
+            coll_comp = self._check_component(coll_comp)
+            self.collation_groups[type_ref][None] = coll_comp
 
+        # check if any component in group dict not in registered schema
         if coll_groups is not None:
-            self.collation_groups[type_ref].update(attr_comp)
+            for coll_key in coll_groups:
+                coll_groups[coll_key] = self._check_component(coll_groups[coll_key])
 
-    def join_attribute_relations(self, r1: R, r2: R) -> R:
+            self.collation_groups[type_ref].update(coll_groups)
-        '''
-        Specific mechanism for joining attribute-based relations.
-        '''
-        pass
 
-    def join_collation_relations(self, r1: R, r2: R) -> R:
+    def attach_hierarchy(
-        '''
+        self,
-        Specific mechanism for joining collation-based relations.
+        type_ref: type[CO3],
-        '''
+        obj_name_map: Callable[[type[CO3]], str],
+    ):
         pass
 
     def get_connective_data(
@@ -84,13 +122,13 @@ class Mapper[R: Relation]:
         '''
         return {}
 
-    def get_attribute_comp(self, type_ref: CO3) -> R | None:
+    def get_attribute_comp(self, type_ref: CO3) -> C | None:
         return self.attribute_comps.get(type_ref, None)
 
-    def get_collation_comp(self, type_ref: CO3, group=str | None) -> R | None:
+    def get_collation_comp(self, type_ref: CO3, group=str | None) -> C | None:
         return self.collation_group.get(type_ref, {}).get(group, None)
 
-    def collect(self, collector, mapper, action_keys=None) -> dict:
+    def collect(self, obj, action_keys=None) -> dict:
         '''
         Stages inserts up the inheritance chain, and down through components.
 
@@ -105,38 +143,38 @@ class Mapper[R: Relation]:
         Returns: dict with keys and values relevant for associated SQLite tables
         '''
         if action_keys is None:
-            action_keys = list(self.action_map.keys())
+            action_keys = list(obj.action_map.keys())
 
         receipts = []
-        for _cls in reversed(self.__class__.__mro__[:-2]):
+        for _cls in reversed(obj.__class__.__mro__[:-2]):
-            attribute_component = mapper.get_attribute_comp(_cls)
+            attribute_component = self.get_attribute_comp(_cls)
 
             # require an attribute component for type consideration
             if attribute_component is None:
                 continue
 
-            collector.add_insert(
+            self.collector.add_insert(
                 attribute_component,
-                self.attributes,
+                obj.attributes,
                 receipts=receipts,
             )
 
             for action_key in action_keys:
-                collation_data = self.collate(action_key)
+                collation_data = obj.collate(action_key)
 
                 # if method either returned no data or isn't registered, ignore
                 if collation_data is None:
                     continue
 
-                _, action_groups = self.action_map[action_key]
+                _, action_groups = obj.action_map[action_key]
                 for action_group in action_groups:
-                    collation_component = mapper.get_collation_comp(_cls, group=action_group)
+                    collation_component = self.get_collation_comp(_cls, group=action_group)
 
                     if collation_component is None:
                         continue
 
                     # gather connective data for collation components
-                    connective_data = mapper.get_connective_data(self, action_key, action_group)
+                    connective_data = self.get_connective_data(_cls, action_key, action_group)
 
                     collector.add_insert(
                         collation_component,
@@ -153,56 +191,3 @@ class Mapper[R: Relation]:
 
         return receipts
 
-    @classmethod
-    def compose(cls, outer=False, conversion=False, full=False):
-        '''
-        Note:
-            Comparing to ORM, this method would likely also still be needed, since it may
-            not be explicitly clear how some JOINs should be handled up the inheritance
-            chain (for components / sa.Relationships, it's a little easier).
-
-        Parameters:
-            outer: whether to use outer joins down the chain
-            conversion: whether to return conversion joins or base primitives
-            full: whether to return fully connected primitive and conversion table
-        '''
-        def join_builder(outer=False, conversion=False):
-            head_table = None
-            last_table = None
-            join_table = None
-
-            for _cls in reversed(cls.__mro__[:-2]):
-                table_str    = None
-                table_prefix = _cls.table_prefix
-
-                if conversion: table_str = f'{table_prefix}_conversions'
-                else:          table_str = f'{table_prefix}s'
-
-                if table_str not in tables.table_map:
-                    continue
-
-                table = tables.table_map[table_str]
-
-                if join_table is None:
-                    head_table = table
-                    join_table = table
-                else:
-                    if conversion:
-                        join_condition = last_table.c.name_fmt == table.c.name_fmt
-                    else:
-                        join_condition = last_table.c.name == table.c.name
-
-                    join_table = join_table.join(table, join_condition, isouter=outer)
-
-                last_table = table
-
-            return join_table, head_table
-
-        if full:
-            # note how the join isn't an OUTER join b/w the two
-            core, core_h = join_builder(outer=outer, conversion=False)
-            conv, conv_h = join_builder(outer=outer, conversion=True)
-            return core.join(conv, core_h.c.name == conv_h.c.name)
-
-        join_table, _ = join_builder(outer=outer, conversion=conversion)
-        return join_table

co3/mappers/__init__.py

@@ -0,0 +1,74 @@
+from typing import Self
+
+import sqlalchemy as sa
+
+from co3.mapper import Mapper
+from co3.component import ComposableComponent
+
+
+class ComposableMapper[C: ComposableComponent](Mapper[C]):
+    def join_attribute_relations(self, r1: C, r2: C) -> C:
+        '''
+        Specific mechanism for joining attribute-based relations.
+        '''
+        pass
+
+    def join_collation_relations(self, r1: C, r2: C) -> C:
+        '''
+        Specific mechanism for joining collation-based relations.
+        '''
+        pass
+
+    @classmethod
+    def compose(cls, outer=False, conversion=False, full=False):
+        '''
+        Note:
+            Comparing to ORM, this method would likely also still be needed, since it may
+            not be explicitly clear how some JOINs should be handled up the inheritance
+            chain (for components / sa.Relationships, it's a little easier).
+
+        Parameters:
+            outer: whether to use outer joins down the chain
+            conversion: whether to return conversion joins or base primitives
+            full: whether to return fully connected primitive and conversion table
+        '''
+        def join_builder(outer=False, conversion=False):
+            head_table = None
+            last_table = None
+            join_table = None
+
+            for _cls in reversed(cls.__mro__[:-2]):
+                table_str    = None
+                table_prefix = _cls.table_prefix
+
+                if conversion: table_str = f'{table_prefix}_conversions'
+                else:          table_str = f'{table_prefix}s'
+
+                if table_str not in tables.table_map:
+                    continue
+
+                table = tables.table_map[table_str]
+
+                if join_table is None:
+                    head_table = table
+                    join_table = table
+                else:
+                    if conversion:
+                        join_condition = last_table.c.name_fmt == table.c.name_fmt
+                    else:
+                        join_condition = last_table.c.name == table.c.name
+
+                    join_table = join_table.join(table, join_condition, isouter=outer)
+
+                last_table = table
+
+            return join_table, head_table
+
+        if full:
+            # note how the join isn't an OUTER join b/w the two
+            core, core_h = join_builder(outer=outer, conversion=False)
+            conv, conv_h = join_builder(outer=outer, conversion=True)
+            return core.join(conv, core_h.c.name == conv_h.c.name)
+
+        join_table, _ = join_builder(outer=outer, conversion=conversion)
+        return join_table

co3/relation.py

@@ -1,37 +0,0 @@
-'''
-Relation
-
-Loose wrapper for table-like objects to be used in various database contexts. Relations
-can be thought of generally as named data containers that contain tuples of attributes
-(adhering to relational algebra terms). Relation subtypes are referred to commonly across
-CO3 generics, serving as a fundamental abstraction within particular storage protocols.
-
-Note: development tasks
-    As it stands, the Relation skeleton is incredibly lax compared to the properties and
-    operations that should be formally available, according its pure relational algebra
-    analog. 
-
-    Relations are also generic up to a type T, which ultimately serves as the base object
-    for Relation instances. We aren't attempting to implement some generally useful
-    table-like class here; instead we're just exposing a lightweight interface that's
-    needed for a few CO3 contexts, and commonly off-loading most of the heavy-lifting to
-    true relation objects like SQLAlchemy tables.
-'''
-from typing import Self
-
-
-class Relation[T]:
-    def __init__(self, name, obj: T):
-        self.name = name
-        self.obj  = obj
-
-    def get_attributes(self):
-        raise NotImplementedError
-
-    def join(
-        self,
-        corelation: Self,
-        on,
-        outer=False
-    ):
-        raise NotImplementedError

co3/relations/__init__.py

@@ -1,16 +0,0 @@
-import sqlalchemy as sa
-
-from co3.relation import Relation
-
-
-class DictRelation(Relation[dict]):
-    def get_attributes(self):
-        return tuple(self.obj.keys())
-
-class TabularRelation(Relation[sa.Table]):
-    def get_attributes(self):
-        return tuple(self.obj.columns)
-
-
-class SQLTable(Relation[sa.Table]):
-    pass

co3/schema.py

@@ -0,0 +1,32 @@
+'''
+Schema
+
+Collection of related storage components, often representing the data structure of an
+entire database. Some databases support multiple schemas, however. In general, a Schema
+can wrap up a relevant subset of tables within a single database, so long as
+`Manager.recreate()` supports creating components in separate calls.
+
+Schema objects are used to:
+
+- Semantically group related storage components
+- Tell databases what components to create/remove together
+- Provide target contexts for connected CO3 type systems within Mappers
+'''
+
+from co3.component import Component
+
+
+class Schema[C: Component]:
+    def __init__(self):
+        self._component_set = set()
+        self._component_map = {}
+
+    def __contains__(self, component: C):
+        return component in self._component_set
+
+    def add_component(self, component: C):
+        self._component_set.add(component)
+        self._component_map[component.name] = component
+
+    def get_component(self, name: str):
+        return self._component_map.get(name)

co3/schemas/__init__.py

@@ -0,0 +1,21 @@
+from typing import Self
+
+import sqlalchemy as sa
+
+from co3.schema import Schema
+from co3.components import Relation, SQLTable
+
+
+class RelationalSchema[R: Relation](Schema[R]):
+    pass
+
+class SQLSchema(RelationalSchema[SQLTable]):
+    @classmethod
+    def from_metadata(cls, metadata: sa.MetaData):
+        instance = cls()
+
+        for table in metadata.tables.values():
+            SQLTable.from_table(table, instance)
+
+        return instance
+        

co3/util/types.py

@@ -0,0 +1,6 @@
+from typing import TypeVar
+
+import sqlalchemy as sa
+
+
+TableLike = TypeVar('TableLike', bound=sa.Table | sa.Subquery | sa.Join)

examples/.ipynb_checkpoints/mapper-checkpoint.ipynb

@@ -0,0 +1,145 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e02ccafe-e04d-4312-acba-e41cf7b1c021",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from co3 import Mapper\n",
+    "\n",
+    "import vegetables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7d80f7b9-7458-4ad4-8c1a-3ea56e796b4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vegetable_mapper = Mapper(\n",
+    "    vegetables.Vegetable,\n",
+    "    vegetables.vegetable_schema\n",
+    ")\n",
+    "\n",
+    "vegetable_mapper.attach(\n",
+    "    vegetables.Vegetable,\n",
+    "    vegetables.vegetable_table,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f9408562-bf50-4522-909c-318557f85948",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# manually attach component\n",
+    "vegetable_mapper.attach(\n",
+    "    vegetables.Tomato,\n",
+    "    vegetables.tomato_table,\n",
+    "    coll_groups={\n",
+    "        'aging':   vegetables.vegetable_schema.get_component('tomato_aging_states'),\n",
+    "        'cooking': vegetables.vegetable_schema.get_component('tomato_cooking_states'),\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05fdd404-87ee-4187-832f-2305272758ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# attach by name in schema\n",
+    "vegetable_mapper.attach(\n",
+    "    vegetables.Tomato,\n",
+    "    vegetables.tomato_table,\n",
+    "    coll_groups={\n",
+    "        'aging':   'tomato_aging_states',\n",
+    "        'cooking': 'tomato_cooking_states',\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9b6af49-a69d-41cc-beae-1b6f171cd2f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# attach entire type hierarchy w/ type->name map\n",
+    "vegetable_mapper.attach_hierarchy(\n",
+    "# this might make more sense during init\n",
+    "    lambda x:x.__name__.lower())\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "2e4336ab-5b5f-484d-815d-164d4b6f40a0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "defaultdict(dict,\n",
+       "            {vegetables.Tomato: {'aging': <co3.components.SQLTable at 0x7ece94358aa0>,\n",
+       "              'cooking': <co3.components.SQLTable at 0x7ece94358ad0>}})"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vegetable_mapper.collation_groups"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d416f9cd-2cb6-4a6e-bab7-86ac21216b8c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "co3",
+   "language": "python",
+   "name": "co3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/.ipynb_checkpoints/vegetables-checkpoint.py

@@ -0,0 +1,80 @@
+import random
+
+import sqlalchemy as sa
+
+from co3.schemas import SQLSchema
+from co3 import CO3, collate
+from co3 import util
+
+
+class Vegetable(CO3):
+    def __init__(self, name, color):
+        self.name = name
+        self.color = color
+
+class Tomato(Vegetable):
+    def __init__(self, name, radius):
+        super().__init__(name, 'red')
+        self.radius = radius
+
+    @property
+    def attributes(self):
+        return vars(self)
+
+    @collate('ripe', action_groups=['aging'])
+    def ripen(self):
+        return {
+            'age': random.randint(1, 6)
+        }
+
+    @collate('rotten', action_groups=['aging'])
+    def rot(self):
+        return {
+            'age': random.randint(4, 9)
+        }
+        
+    @collate('diced', action_groups=['cooking'])
+    def dice(self):
+        return {
+            'pieces': random.randint(2, 12)
+        }
+
+'''
+VEGETABLE
+|
+TOMATO -- AGING
+       |
+       -- COOKING
+'''
+metadata = sa.MetaData()
+vegetable_table = sa.Table(
+    'vegetable',
+    metadata,
+    sa.Column('id',    sa.Integer, primary_key=True),
+    sa.Column('name',  sa.String),
+    sa.Column('color', sa.String),
+)
+tomato_table = sa.Table(
+    'tomato',
+    metadata,
+    sa.Column('id',   sa.Integer, primary_key=True),
+    sa.Column('vegetable_id',   sa.Integer, util.db.deferred_cd_fkey('vegetables.id')),
+    sa.Column('radius', sa.Integer),
+)
+tomato_aging_table = sa.Table(
+    'tomato_aging_states',
+    metadata,
+    sa.Column('id',   sa.Integer, primary_key=True),
+    sa.Column('vegetable_id',   sa.Integer, util.db.deferred_cd_fkey('vegetables.id')),
+    sa.Column('state', sa.String),
+    sa.Column('age', sa.Integer),
+)
+tomato_cooking_table = sa.Table(
+    'tomato_cooking_states',
+    metadata,
+    sa.Column('id',   sa.Integer, primary_key=True),
+    sa.Column('vegetable_id',   sa.Integer, util.db.deferred_cd_fkey('vegetables.id')),
+    sa.Column('state', sa.String),
+    sa.Column('pieces', sa.Integer),
+)
+vegetable_schema = SQLSchema.from_metadata(metadata)

examples/mapper.ipynb

@@ -0,0 +1,200 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e02ccafe-e04d-4312-acba-e41cf7b1c021",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from co3 import Mapper\n",
+    "\n",
+    "import vegetables"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c0914069-7f3c-4213-8d34-f7566033e054",
+   "metadata": {},
+   "source": [
+    "## Development notes\n",
+    "- No registry actually needs to take place if there's a default type2component map or one supplied on creation. Can just collect right out of the gate\n",
+    "- Need connective function (type to collation) and attribute map. Do we need to this with a subclass? If a func is passed in on init, I can type it appropriately I guess `Callable[[type[CO3],str,str|None],dict]`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "7d80f7b9-7458-4ad4-8c1a-3ea56e796b4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vegetable_mapper = Mapper(\n",
+    "    vegetables.Vegetable,\n",
+    "    vegetables.vegetable_schema\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d24d31b4-c4a6-4a1e-8bea-c44378aadfdd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# not valid; tables need to be wrapped in CO3 Components\n",
+    "vegetable_mapper.attach(\n",
+    "    vegetables.Vegetable,\n",
+    "    vegetables.vegetable_table,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f9408562-bf50-4522-909c-318557f85948",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# manually attach component\n",
+    "vegetable_mapper.attach(\n",
+    "    vegetables.Tomato,\n",
+    "    vegetables.vegetable_schema.get_component('tomato'),\n",
+    "    coll_groups={\n",
+    "        'aging':   vegetables.vegetable_schema.get_component('tomato_aging_states'),\n",
+    "        'cooking': vegetables.vegetable_schema.get_component('tomato_cooking_states'),\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "05fdd404-87ee-4187-832f-2305272758ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# attach by name in schema\n",
+    "vegetable_mapper.attach(\n",
+    "    vegetables.Tomato,\n",
+    "    'tomato',\n",
+    "    coll_groups={\n",
+    "        'aging':   'tomato_aging_states',\n",
+    "        'cooking': 'tomato_cooking_states',\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9b6af49-a69d-41cc-beae-1b6f171cd2f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# attach entire type hierarchy w/ type->name map\n",
+    "vegetable_mapper.attach_hierarchy(\n",
+    "# this might make more sense during init\n",
+    "    lambda x:x.__name__.lower())\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "2e4336ab-5b5f-484d-815d-164d4b6f40a0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'co3_root': vegetables.Vegetable,\n",
+       " 'schema': <co3.schemas.SQLSchema at 0x74ac03f5c8c0>,\n",
+       " 'collector': <co3.collector.Collector at 0x74ac0357ae70>,\n",
+       " 'composer': <co3.composer.Composer at 0x74ac0357a4b0>,\n",
+       " 'attribute_comps': {vegetables.Tomato: <co3.components.SQLTable at 0x74ac09d4a720>},\n",
+       " 'collation_groups': defaultdict(dict,\n",
+       "             {vegetables.Tomato: {'aging': <co3.components.SQLTable at 0x74ac03f5cad0>,\n",
+       "               'cooking': <co3.components.SQLTable at 0x74ac03f5cb00>}})}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vars(vegetable_mapper)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c16786d4-0b71-42d9-97f7-7893c542104e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tomato = vegetables.Tomato('t1', 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "884d6753-c763-4e71-824a-711436e203e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<vegetables.Tomato at 0x74ac082bacc0>"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tomato"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "137d0bf1-940d-448c-91e9-01e7fc4f31b4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "co3",
+   "language": "python",
+   "name": "co3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/vegetables.py

@@ -0,0 +1,80 @@
+import random
+
+import sqlalchemy as sa
+
+from co3.schemas import SQLSchema
+from co3 import CO3, collate
+from co3 import util
+
+
+class Vegetable(CO3):
+    def __init__(self, name, color):
+        self.name = name
+        self.color = color
+
+class Tomato(Vegetable):
+    def __init__(self, name, radius):
+        super().__init__(name, 'red')
+        self.radius = radius
+
+    @property
+    def attributes(self):
+        return vars(self)
+
+    @collate('ripe', action_groups=['aging'])
+    def ripen(self):
+        return {
+            'age': random.randint(1, 6)
+        }
+
+    @collate('rotten', action_groups=['aging'])
+    def rot(self):
+        return {
+            'age': random.randint(4, 9)
+        }
+        
+    @collate('diced', action_groups=['cooking'])
+    def dice(self):
+        return {
+            'pieces': random.randint(2, 12)
+        }
+
+'''
+VEGETABLE
+|
+TOMATO -- AGING
+       |
+       -- COOKING
+'''
+metadata = sa.MetaData()
+vegetable_table = sa.Table(
+    'vegetable',
+    metadata,
+    sa.Column('id',    sa.Integer, primary_key=True),
+    sa.Column('name',  sa.String),
+    sa.Column('color', sa.String),
+)
+tomato_table = sa.Table(
+    'tomato',
+    metadata,
+    sa.Column('id',   sa.Integer, primary_key=True),
+    sa.Column('vegetable_id',   sa.Integer, util.db.deferred_cd_fkey('vegetables.id')),
+    sa.Column('radius', sa.Integer),
+)
+tomato_aging_table = sa.Table(
+    'tomato_aging_states',
+    metadata,
+    sa.Column('id',   sa.Integer, primary_key=True),
+    sa.Column('vegetable_id',   sa.Integer, util.db.deferred_cd_fkey('vegetables.id')),
+    sa.Column('state', sa.String),
+    sa.Column('age', sa.Integer),
+)
+tomato_cooking_table = sa.Table(
+    'tomato_cooking_states',
+    metadata,
+    sa.Column('id',   sa.Integer, primary_key=True),
+    sa.Column('vegetable_id',   sa.Integer, util.db.deferred_cd_fkey('vegetables.id')),
+    sa.Column('state', sa.String),
+    sa.Column('pieces', sa.Integer),
+)
+vegetable_schema = SQLSchema.from_metadata(metadata)

tests/co4_example.py

@@ -26,7 +26,13 @@ class Tomato(CO3):
         return self.size / 2
 
 
-tomato_table = sa.Table()
+metadata = sa.MetaData()
+tomato_table = sa.Table(
+    'tomato',
+    metadata,
+    sa.Column('id', sa.Integer, primary_key=True),
+)
+tomato_schema = Schema.from_metadata(metadata)
 
 mapper = Mapper()
 mapper.attach(
@@ -39,6 +45,15 @@ tomato = Tomato(5, False)
 mapper.collect(tomato, for='diced')
 
 db = SQLiteDatabse('resource.sqlite')
+db.recreate(tomato_schema)
+
+# for non-raw DB ops, consider requiring a verify step first. Keeps up integrity between
+# runs
+db.verify(tomato_schema)
+# then
+# not too straightforward, but can also verify mapper compositions if they use a schema
+# that's been verified by the database
+
 db.sync(mapper)
 
 dict_results = db.select(