From 157ff69b9e13a83787f69bc4c6f9293c766057f8 Mon Sep 17 00:00:00 2001 From: "Sam G." Date: Wed, 17 Apr 2024 18:42:26 -0700 Subject: [PATCH] clean up Database/Accessor/Manager interaction, refine Mapper group maps --- .gitignore | 1 + co3.egg-info/SOURCES.txt | 1 - co3/__init__.py | 4 +- co3/accessors/sql.py | 22 +- co3/component.py | 5 +- co3/components/__init__.py | 13 +- co3/database.py | 13 +- co3/engines/__init__.py | 7 +- co3/manager.py | 3 +- co3/managers/sql.py | 68 +++- co3/mapper.py | 77 ++-- co3/schemas/__init__.py | 3 +- .../database-checkpoint.ipynb | 202 +++++++++- .../vegetables-checkpoint.py | 34 +- examples/database.ipynb | 376 +++++++++++++++++- examples/vegetables.py | 12 +- tests/imports.py | 1 - 17 files changed, 750 insertions(+), 92 deletions(-) diff --git a/.gitignore b/.gitignore index 88e6867..81c1349 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ .pytest_cache/ localsys.egg-info/ +.ipynb_checkpoints/ # vendor and build files dist/ diff --git a/co3.egg-info/SOURCES.txt b/co3.egg-info/SOURCES.txt index dcda5a5..4df4b94 100644 --- a/co3.egg-info/SOURCES.txt +++ b/co3.egg-info/SOURCES.txt @@ -6,7 +6,6 @@ co3/accessor.py co3/co3.py co3/collector.py co3/component.py -co3/composer.py co3/database.py co3/engine.py co3/indexer.py diff --git a/co3/__init__.py b/co3/__init__.py index 2c9aecd..d6118a8 100644 --- a/co3/__init__.py +++ b/co3/__init__.py @@ -95,11 +95,11 @@ Note: Organization for inheritance over composition from co3.accessor import Accessor from co3.co3 import CO3, collate from co3.collector import Collector -from co3.composer import Composer +#from co3.composer import Composer from co3.database import Database from co3.indexer import Indexer from co3.manager import Manager -from co3.mapper import Mapper +from co3.mapper import Mapper, ComposableMapper from co3.component import Component from co3.schema import Schema from co3.engine import Engine diff --git a/co3/accessors/sql.py b/co3/accessors/sql.py index 310612b..9047ac5 100644 --- a/co3/accessors/sql.py +++ b/co3/accessors/sql.py @@ -83,7 +83,8 @@ class RelationalAccessor[R: Relation](Accessor[R]): include_cols : bool = False, ): res = self.select( - relation, attributes, where, mappings, include_cols, limit=1) + relation, attributes, where, mappings, include_cols, limit=1 + ) if include_cols and len(res[0]) > 0: return res[0][0], res[1] @@ -97,6 +98,7 @@ class RelationalAccessor[R: Relation](Accessor[R]): class SQLAccessor(RelationalAccessor[SQLTable]): def raw_select( self, + connection, sql, bind_params=None, mappings=False, @@ -110,7 +112,8 @@ class SQLAccessor(RelationalAccessor[SQLTable]): def select( self, - table: SQLTable, + connection, + component: SQLTable, columns = None, where = None, distinct_on = None, @@ -141,20 +144,21 @@ class SQLAccessor(RelationalAccessor[SQLTable]): if where is None: where = sa.true() - stmt = sa.select(table).where(where) - if cols is not None: - stmt = sa.select(*cols).select_from(table).where(where) + table = component.obj + statement = sa.select(table).where(where) + if columns is not None: + statement = sa.select(*columns).select_from(table).where(where) if distinct_on is not None: - stmt = stmt.group_by(distinct_on) + statement = statement.group_by(distinct_on) if order_by is not None: - stmt = stmt.order_by(order_by) + statement = statement.order_by(order_by) if limit > 0: - stmt = stmt.limit(limit) + statement = statement.limit(limit) - res = SQLEngine._execute(connection, statement, include_cols=include_cols) + res = SQLEngine.execute(connection, statement, include_cols=include_cols) if mappings: return res.mappings().all() diff --git a/co3/component.py b/co3/component.py index e08ebe9..a75aba0 100644 --- a/co3/component.py +++ b/co3/component.py @@ -7,13 +7,10 @@ abstractions within particular storage protocols. ''' class Component[T]: - def __init__(self, name, obj: T, schema: 'Schema'): + def __init__(self, name, obj: T): self.name = name self.obj = obj - self.schema = schema - schema.add_component(self) - def __str__(self): return f' {self.name}' diff --git a/co3/components/__init__.py b/co3/components/__init__.py index 51ab355..3f3dd29 100644 --- a/co3/components/__init__.py +++ b/co3/components/__init__.py @@ -63,8 +63,12 @@ class Relation[T](ComposableComponent[T]): class SQLTable(Relation[SQLTableLike]): @classmethod - def from_table(cls, table: sa.Table, schema: 'SQLSchema'): - return cls(table.name, table, schema) + def from_table(cls, table: sa.Table): + ''' + Note that the sa.Table type is intentional here; not all matching types for + SQLTableLike have a defined `name` property + ''' + return cls(table.name, table) def get_attributes(self) -> tuple: return tuple(self.obj.columns) @@ -102,7 +106,10 @@ class SQLTable(Relation[SQLTableLike]): return insert_dict def compose(self, _with: Self, on, outer=False): - return self.obj.join(_with, on, isouter=outer) + return self.__class__( + f'{self.name}+{_with.name}', + self.obj.join(_with.obj, on, isouter=outer) + ) # key-value stores class Dictionary(Relation[dict]): diff --git a/co3/database.py b/co3/database.py index 2026d6f..920de59 100644 --- a/co3/database.py +++ b/co3/database.py @@ -55,10 +55,10 @@ Dev note: on explicit connection contexts import logging from co3.accessor import Accessor -from co3.composer import Composer from co3.manager import Manager from co3.indexer import Indexer from co3.engine import Engine +from co3.schema import Schema logger = logging.getLogger(__name__) @@ -118,6 +118,12 @@ class Database[C: Component]: self._reset_cache = False def select(self, component: C, *args, **kwargs): + ''' + Dev note: args and kwargs have to be general/unspecified here due to the possible + passthrough method adopting arbitrary parameters in subtypes. I could simply + overload this method in the relevant inheriting DBs (i.e., by matching the + expected Accessor's .select signature). + ''' with self.engine.connect() as connection: return self.accessor.select( connection, @@ -128,13 +134,16 @@ class Database[C: Component]: def insert(self, component: C, *args, **kwargs): with self.engine.connect() as connection: - return self.accessor.insert( + return self.manager.insert( connection, component, *args, **kwargs ) + def recreate(self, schema: Schema[C]): + self.manager.recreate(schema, self.engine) + @property def index(self): if self.reset_cache: diff --git a/co3/engines/__init__.py b/co3/engines/__init__.py index c13f127..44f225a 100644 --- a/co3/engines/__init__.py +++ b/co3/engines/__init__.py @@ -10,14 +10,13 @@ class SQLEngine(Engine): super().__init__(url, **kwargs) def _create_manager(self): - return sa.create_engine(*self.manager_args, self.manager_kwargs) + return sa.create_engine(*self._manager_args, **self._manager_kwargs) - @contextmanager def connect(self, timeout=None): return self.manager.connect() @staticmethod - def _execute( + def execute( connection, statement, bind_params=None, @@ -42,7 +41,7 @@ class SQLEngine(Engine): return res @staticmethod - def _exec_explicit(connection, statement, bind_params=None): + def exec_explicit(connection, statement, bind_params=None): trans = connection.begin() # start a new transaction explicitly try: result = connection.execute(statement, bind_params) diff --git a/co3/manager.py b/co3/manager.py index 04a6ec0..19eeda7 100644 --- a/co3/manager.py +++ b/co3/manager.py @@ -8,6 +8,7 @@ from pathlib import Path from abc import ABCMeta, abstractmethod from co3.schema import Schema +from co3.engine import Engine class Manager[C: Component](metaclass=ABCMeta): @@ -19,7 +20,7 @@ class Manager[C: Component](metaclass=ABCMeta): wrapped up in this class to then also be mirrored for the FTS counterparts. ''' @abstractmethod - def recreate(self, schema: Schema[C]): + def recreate(self, schema: Schema[C], engine: Engine): raise NotImplementedError @abstractmethod diff --git a/co3/managers/sql.py b/co3/managers/sql.py index 228255f..cc12fed 100644 --- a/co3/managers/sql.py +++ b/co3/managers/sql.py @@ -49,6 +49,7 @@ from tqdm.auto import tqdm from co3 import util from co3.schema import Schema +from co3.engines import SQLEngine from co3.manager import Manager from co3.components import Relation, SQLTable @@ -76,12 +77,17 @@ class SQLManager(RelationalManager[SQLTable]): from an attached collector. ''' def __init__(self, *args, **kwargs): + ''' + The insert lock is a _reentrant lock_, meaning the same thread can acquire the + lock again with out deadlocking (simplifying across methods of this class that + need it). + ''' super().__init__(*args, **kwargs) self.routers = [] self._router = None - self._insert_lock = threading.Lock() + self._insert_lock = threading.RLock() @property def router(self): @@ -92,18 +98,42 @@ class SQLManager(RelationalManager[SQLTable]): def add_router(self, router): self.routers.append(router) - def recreate(self, schema: Schema[SQLTable]): - schema.metadata.drop_all(self.engine) - schema.metadata.create_all(self.engine, checkfirst=True) + def recreate(self, schema: Schema[SQLTable], engine: SQLEngine): + ''' + Ideally this remains open, as we can't necessarily rely on a SQLAlchemy metadata + object for all kinds of SQLDatabases (would depend on the backend, for instance). + + Haven't quite nailed down how backend instances should be determined; something + like SQLAlchemySQLManager doesn't seem great. Nevertheless, this method likely + cannot be generalized at the "SQL" (general) level. + ''' + metadata = next(iter(schema._component_set)).obj.metadata + metadata.drop_all(engine.manager) + metadata.create_all(engine.manager, checkfirst=True) def update(self): pass - def insert(self, inserts: dict): + def insert( + self, + connection, + component, + inserts: list[dict], + ): + ''' + Parameters: + ''' + with self._insert_lock: + connection.execute( + sa.insert(component.obj), + inserts + ) + + def insert_many(self, connection, inserts: dict): ''' Perform provided table inserts. Parameters: - inserts: table-indexed dictionary of insert lists + inserts: component-indexed dictionary of insert lists ''' total_inserts = sum([len(ilist) for ilist in inserts.values()]) if total_inserts < 1: return @@ -112,24 +142,18 @@ class SQLManager(RelationalManager[SQLTable]): # TODO: add some exception handling? may be fine w default propagation start = time.time() - with self.engine.connect() as connection: - with self._insert_lock: - for table_str in inserts: - table_inserts = inserts[table_str] - if len(table_inserts) == 0: continue + with self._insert_lock: + for component in inserts: + comp_inserts = inserts[component] + if len(comp_inserts) == 0: continue - table = tables.table_map[table_str] + logger.info( + f'Inserting {len(comp_inserts)} out-of-date entries into component "{component}"' + ) - logger.info( - f'Inserting {len(table_inserts)} out-of-date entries into table "{table_str}"' - ) - - connection.execute( - sa.insert(table), - table_inserts - ) - connection.commit() - logger.info(f'Insert transaction completed successfully in {time.time()-start:.2f}s') + self.insert(connection, component, comp_inserts) + connection.commit() + logger.info(f'Insert transaction completed successfully in {time.time()-start:.2f}s') def _file_sync_bools(self): synced_bools = [] diff --git a/co3/mapper.py b/co3/mapper.py index da0d104..83bfc34 100644 --- a/co3/mapper.py +++ b/co3/mapper.py @@ -29,7 +29,7 @@ Development log: hierarchy). As such, to fully collect from a type, the Mapper needs to leave registration open to various types, not just those part of the same hierarchy. ''' -from typing import Callable +from typing import Callable, Any from collections import defaultdict from co3.co3 import CO3 @@ -55,12 +55,14 @@ class Mapper[C: Component]: "dropped off" at an appropriate Database's Manager to actually perform the requested inserts (hence why we tie Mappers to Schemas one-to-one). - Dev note: the Composer needs reconsideration, or at least its positioning directly in - this class. It may be more appropriate to have at the Schema level, or even just - dissolved altogether if arbitrary named Components can be attached to schemas. - ''' - type comp_spec = str | C + Dev note: + the Composer needs reconsideration, or at least its positioning directly in this + class. It may be more appropriate to have at the Schema level, or even just + dissolved altogether if arbitrary named Components can be attached to schemas. + - Consider pushing this into a Mapper factory; on init, could check if provided + Schema wraps up composable Components or not + ''' _collector_cls: type[Collector[C]] = Collector[C] def __init__(self, schema: Schema[C]): @@ -76,7 +78,7 @@ class Mapper[C: Component]: self.attribute_comps: dict[type[CO3], C] = {} self.collation_groups: dict[type[CO3], dict[str|None, C]] = defaultdict(dict) - def _check_component(self, comp: self.comp_spec): + def _check_component(self, comp: str | C): if type(comp) is str: comp_key = comp comp = self.schema.get_component(comp_key) @@ -95,9 +97,9 @@ class Mapper[C: Component]: def attach( self, type_ref : type[CO3], - attr_comp : self.comp_spec, - coll_comp : self.comp_spec | None = None, - coll_groups : dict[str | None, self.comp_spec] | None = None, + attr_comp : str | C, + coll_comp : str | C | None = None, + coll_groups : dict[str | None, str | C] | None = None, ) -> None: ''' Parameters: @@ -127,8 +129,8 @@ class Mapper[C: Component]: def attach_many( self, type_list: list[type[CO3]], - attr_name_map: Callable[[type[CO3]], self.comp_spec], - coll_name_map: Callable[[type[CO3], str], self.comp_spec] | None = None, + attr_name_map: Callable[[type[CO3]], str | C], + coll_name_map: Callable[[type[CO3], str], str | C] | None = None, ): ''' Auto-register a set of types to the Mapper's attached Schema. Associations are @@ -218,7 +220,7 @@ class Mapper[C: Component]: _, action_groups = obj.action_registry[action_key] for action_group in action_groups: - collation_component, _ = self.get_collation_comp(_cls, group=action_group) + collation_component = self.get_collation_comp(_cls, group=action_group) if collation_component is None: continue @@ -317,8 +319,8 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]): def __init__( self, schema : Schema[C], - attr_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None - coll_compose_map : Callable[[self.comp_spec, self.comp_spec], Any] | None = None + attr_compose_map : Callable[[str | C, str | C], Any] | None = None, + coll_compose_map : Callable[[str | C, str | C], Any] | None = None, ): super().__init__(schema) @@ -328,7 +330,7 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]): def compose( self, obj: CO3, - action_groups: list[str] = None, + action_groups: list[str] | None = None, *compose_args, **compose_kwargs, ): @@ -356,29 +358,36 @@ class ComposableMapper[C: ComposableComponent](Mapper[C]): # compose horizontally with components from provided action groups coll_comp_agg = attr_comp - for action_group in action_groups: - coll_comp = self.get_collation_comp(_cls, group=action_group) + if action_groups is not None: + for action_group in action_groups: + coll_comp = self.get_collation_comp(_cls, group=action_group) - if coll_comp is None: - continue + if coll_comp is None: + continue - compose_condition = self.coll_compose_map(coll_comp_agg, coll_comp) + # note how the join condition is specified using the non-composite + # `attr_comp` and new `coll_comp`; the composite doesn't typically + # have the same attribute access and needs a ref to a specific comp + compose_condition = self.coll_compose_map(attr_comp, coll_comp) - coll_comp_agg = coll_comp_agg.compose( - component=coll_comp, - on=compose_condition, + coll_comp_agg = coll_comp_agg.compose( + coll_comp, + compose_condition, + *compose_args, + **compose_kwargs, + ) + + if attr_comp_agg is None: + attr_comp_agg = coll_comp_agg + else: + # note the reduced attr_comp ref passed to compose map, rather than + # coll_comp_agg produced above; this is provided as the compose comp, though + compose_condition = self.attr_compose_map(attr_comp_agg, attr_comp) + attr_comp_agg = attr_comp_agg.compose( + coll_comp_agg, + compose_condition, *compose_args, **compose_kwargs, ) - # note the reduced attr_comp ref passed to compose map, rather than - # coll_comp_agg produced above; this is provided as the compose comp, though - compose_condition = self.attr_compose_map(attr_comp_agg, attr_comp) - attr_comp_agg = attr_comp_agg.compose( - component=coll_comp_agg, - on=compose_condition, - *compose_args, - **compose_kwargs, - ) - return attr_comp_agg diff --git a/co3/schemas/__init__.py b/co3/schemas/__init__.py index be666a5..ab72828 100644 --- a/co3/schemas/__init__.py +++ b/co3/schemas/__init__.py @@ -15,7 +15,8 @@ class SQLSchema(RelationalSchema[SQLTable]): instance = cls() for table in metadata.tables.values(): - SQLTable.from_table(table, instance) + comp = SQLTable.from_table(table) + instance.add_component(comp) return instance diff --git a/examples/.ipynb_checkpoints/database-checkpoint.ipynb b/examples/.ipynb_checkpoints/database-checkpoint.ipynb index 363fcab..025b7ed 100644 --- a/examples/.ipynb_checkpoints/database-checkpoint.ipynb +++ b/examples/.ipynb_checkpoints/database-checkpoint.ipynb @@ -1,6 +1,204 @@ { - "cells": [], - "metadata": {}, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6f6fbc7e-4fb9-4353-b2ee-9ea819a3c896", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import vegetables" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "88fd0ea8-9c94-4569-a51b-823a04f32f55", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'age': 3}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tomato = vegetables.Tomato('t1', 5)\n", + "\n", + "# test a register collation action\n", + "tomato.collate('ripe')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "348926d9-7137-4eff-a919-508788553dd2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['afff61ec-e0b0-44bb-9f0d-06008a82f6a5',\n", + " '5edfa13c-0eb1-4bbc-b55e-1550ff7df3d2',\n", + " '4568a2d4-eb41-4b15-9a29-e3e22906c661']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vegetables.vegetable_mapper.collect(tomato, ['ripe'])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4e5e7319-11bf-4051-951b-08c84e9f3874", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + " vegetable+tomato+tomato_aging_states+tomato_cooking_states" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "aa290686-8074-4038-a3cc-ce6817844653", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Column('id', Integer(), table=, primary_key=True, nullable=False),\n", + " Column('name', String(), table=),\n", + " Column('color', String(), table=),\n", + " Column('id', Integer(), table=, primary_key=True, nullable=False),\n", + " Column('name', String(), ForeignKey('vegetable.name'), table=),\n", + " Column('radius', Integer(), table=),\n", + " Column('id', Integer(), table=, primary_key=True, nullable=False),\n", + " Column('name', String(), ForeignKey('tomato.name'), table=),\n", + " Column('state', String(), table=),\n", + " Column('age', Integer(), table=)]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tomato.__class__.__mro__" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c21d2c54-39e2-4de3-93bc-763896ed348e", + "metadata": {}, + "outputs": [], + "source": [ + "from co3.databases import SQLDatabase\n", + "\n", + "db = SQLDatabase('sqlite://', echo=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a785d202-99d3-4ae7-859e-ee22b481f8df", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.recreate(vegetable_schema) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "co3", + "language": "python", + "name": "co3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/examples/.ipynb_checkpoints/vegetables-checkpoint.py b/examples/.ipynb_checkpoints/vegetables-checkpoint.py index a8cf1ea..ed09698 100644 --- a/examples/.ipynb_checkpoints/vegetables-checkpoint.py +++ b/examples/.ipynb_checkpoints/vegetables-checkpoint.py @@ -6,7 +6,7 @@ import random import sqlalchemy as sa from co3.schemas import SQLSchema -from co3 import CO3, collate, Mapper +from co3 import CO3, collate, Mapper, ComposableMapper from co3 import util @@ -101,5 +101,35 @@ tomato_cooking_table = sa.Table( sa.Column('pieces', sa.Integer), ) vegetable_schema = SQLSchema.from_metadata(metadata) -vegetable_mapper = Mapper(vegetable_schema) + +def general_compose_map(c1, c2): + return c1.obj.c.name == c2.obj.c.name + +vegetable_mapper = ComposableMapper( + vegetable_schema, + attr_compose_map=general_compose_map, + coll_compose_map=general_compose_map, +) + +def attr_name_map(cls): + return f'{cls.__name__.lower()}' + +def coll_name_map(cls, action_group): + return f'{cls.__name__.lower()}_{action_group}_states' + +vegetable_mapper.attach_many( + type_list, + attr_name_map, + coll_name_map, +) + +''' +new mapping type for Mapper attachment: + +Callable[ [type[CO3], str|None], tuple[str, tuple[str], tuple[str]]] + +tail tuples to associate column names from central table to collation + +this should complete the auto-compose horizontally +''' diff --git a/examples/database.ipynb b/examples/database.ipynb index 44e6fd5..1bc7e69 100644 --- a/examples/database.ipynb +++ b/examples/database.ipynb @@ -21,16 +21,388 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "88fd0ea8-9c94-4569-a51b-823a04f32f55", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'age': 5}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tomato = vegetables.Tomato('t1', 5)\n", "\n", "# test a register collation action\n", "tomato.collate('ripe')" ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "348926d9-7137-4eff-a919-508788553dd2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['9ca6772e-6621-4511-a4a6-ad451a1da91f',\n", + " '2a91b423-4e08-491c-b1d2-5ec25259191e',\n", + " '4a9edb2b-4ac5-467e-82ef-b254829ac2a2']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vegetables.vegetable_mapper.collect(tomato, ['ripe'])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4e5e7319-11bf-4051-951b-08c84e9f3874", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + " vegetable+tomato+tomato_aging_states+tomato_cooking_states" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "aa290686-8074-4038-a3cc-ce6817844653", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Column('id', Integer(), table=, primary_key=True, nullable=False),\n", + " Column('name', String(), table=),\n", + " Column('color', String(), table=),\n", + " Column('id', Integer(), table=, primary_key=True, nullable=False),\n", + " Column('name', String(), ForeignKey('vegetable.name'), table=),\n", + " Column('radius', Integer(), table=),\n", + " Column('id', Integer(), table=, primary_key=True, nullable=False),\n", + " Column('name', String(), ForeignKey('tomato.name'), table=),\n", + " Column('state', String(), table=),\n", + " Column('age', Integer(), table=)]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tomato.__class__.__mro__" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c21d2c54-39e2-4de3-93bc-763896ed348e", + "metadata": {}, + "outputs": [], + "source": [ + "from co3.databases import SQLDatabase\n", + "\n", + "db = SQLDatabase('sqlite://') #, echo=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a785d202-99d3-4ae7-859e-ee22b481f8df", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "db.recreate(vegetables.vegetable_schema)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{ vegetable: [{'name': 't1', 'color': 'red'}],\n", + " tomato: [{'name': 't1', 'radius': 5}],\n", + " tomato_aging_states: [{'name': 't1',\n", + " 'state': 'ripe',\n", + " 'age': 2}]}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vegetables.vegetable_mapper.collector.inserts" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "af7124ed-3031-4f28-89a6-553eb5b3cc7a", + "metadata": {}, + "outputs": [], + "source": [ + "with db.engine.connect() as connection:\n", + " db.manager.insert_many(\n", + " connection, \n", + " vegetables.vegetable_mapper.collector.inserts,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0149e14e-5d07-42af-847d-af5c190f8946", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'id': 1, 'name': 't1', 'radius': 5}]\n" + ] + } + ], + "source": [ + "with db.engine.connect() as connection:\n", + " print(db.accessor.select(\n", + " connection, \n", + " vegetables.vegetable_schema.get_component('tomato')\n", + " ))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "668d1b8c-b47f-4a58-914d-e43402443fe6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5}]\n" + ] + } + ], + "source": [ + "agg_table = vegetables.vegetable_mapper.compose(tomato)\n", + "\n", + "with db.engine.connect() as connection:\n", + " agg_res = db.accessor.select(\n", + " connection, \n", + " agg_table,\n", + " mappings=True,\n", + " )\n", + "\n", + "print(agg_res)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "a051d72d-a867-46dc-bb5e-69341f39a056", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5, 'id_2': 1, 'name_2': 't1', 'state': 'ripe', 'age': 2}]\n" + ] + } + ], + "source": [ + "agg_table = vegetables.vegetable_mapper.compose(tomato, action_groups=['aging'])#, outer=True)\n", + "\n", + "with db.engine.connect() as connection:\n", + " agg_res = db.accessor.select(\n", + " connection, \n", + " agg_table,\n", + " mappings=True,\n", + " )\n", + "\n", + "print(agg_res)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "6a80cfd7-3175-4526-96e0-374765d64a27", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "sqlalchemy.engine.row.RowMapping" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(agg_res[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "7cf05ddd-2328-4051-9cf8-4ac01352405e", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlalchemy as sa\n", + "from co3.engines import SQLEngine\n", + "\n", + "a = SQLEngine.execute(db.engine.connect(), sa.select(agg_table.obj))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "c1edf68e-1fde-4a1f-8ec3-084713a8da45", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a.mappings().all()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "8b8a9e47-7f5f-4828-a99e-5d9a12697f46", + "metadata": {}, + "outputs": [], + "source": [ + "tomato2 = vegetables.Tomato('t2', 8)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "062aa4de-7aea-4fd3-b5db-82af147d023e", + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'Tomato' object has no attribute 'action_map'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mvegetables\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvegetable_mapper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtomato2\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/projects/ontolog/co3/build/__editable__.co3-0.1.1-py3-none-any/co3/mapper.py:198\u001b[0m, in \u001b[0;36mMapper.collect\u001b[0;34m(self, obj, action_keys, action_groups)\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 180\u001b[0m \u001b[38;5;124;03mStages inserts up the inheritance chain, and down through components.\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[38;5;124;03mReturns: dict with keys and values relevant for associated SQLite tables\u001b[39;00m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m action_keys \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 198\u001b[0m action_keys \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[43mobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maction_map\u001b[49m\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[1;32m 200\u001b[0m receipts \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _cls \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(obj\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__mro__\u001b[39m[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m]):\n", + "\u001b[0;31mAttributeError\u001b[0m: 'Tomato' object has no attribute 'action_map'" + ] + } + ], + "source": [ + "vegetables.vegetable_mapper.collect(tomato2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4673ddc8-3f76-4d8c-8186-bbed4a682e0d", + "metadata": {}, + "outputs": [], + "source": [ + "db.insert(vegetables.vegetable_schema.get_component('tomato'), " + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "9314be4e-c1d5-4af8-ad23-0b208d24b3eb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 't1', 'radius': 5}]" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.select(vegetables.vegetable_schema.get_component('tomato'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2efd060-f298-4ca6-8a58-7ed5acf1dd15", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/vegetables.py b/examples/vegetables.py index d17297c..ed09698 100644 --- a/examples/vegetables.py +++ b/examples/vegetables.py @@ -6,7 +6,7 @@ import random import sqlalchemy as sa from co3.schemas import SQLSchema -from co3 import CO3, collate, Mapper +from co3 import CO3, collate, Mapper, ComposableMapper from co3 import util @@ -101,7 +101,15 @@ tomato_cooking_table = sa.Table( sa.Column('pieces', sa.Integer), ) vegetable_schema = SQLSchema.from_metadata(metadata) -vegetable_mapper = Mapper(vegetable_schema) + +def general_compose_map(c1, c2): + return c1.obj.c.name == c2.obj.c.name + +vegetable_mapper = ComposableMapper( + vegetable_schema, + attr_compose_map=general_compose_map, + coll_compose_map=general_compose_map, +) def attr_name_map(cls): return f'{cls.__name__.lower()}' diff --git a/tests/imports.py b/tests/imports.py index d8a7813..db8a53c 100644 --- a/tests/imports.py +++ b/tests/imports.py @@ -1,7 +1,6 @@ from co3 import Accessor from co3 import CO3 from co3 import Collector -from co3 import Composer from co3 import Database from co3 import Indexer from co3 import Manager