clean up repo files, README, auxiliary files (pre-BFG)

This commit is contained in:
Sam G. 2024-05-15 19:56:01 -07:00
parent a302560c72
commit 6ece9c1721
13 changed files with 96 additions and 1006 deletions

6
.gitignore vendored
View File

@ -1,7 +1,7 @@
# generic py # generic py
__pycache__/ __pycache__/
.pytest_cache/ .pytest_cache/
localsys.egg-info/ *.egg-info/
.ipynb_checkpoints/ .ipynb_checkpoints/
.pytest_cache/ .pytest_cache/
.python-version .python-version
@ -12,3 +12,7 @@ build/
docs/_autoref/ docs/_autoref/
docs/_autosummary/ docs/_autosummary/
docs/_build/ docs/_build/
# local
notebooks/
/Makefile

22
LICENSE Normal file
View File

@ -0,0 +1,22 @@
MIT License
Copyright (c) 2024 Sam Griesemer
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -1,21 +0,0 @@
PYTHON=/home/smgr/.pyenv/versions/co4/bin/python
BASH=/usr/bin/bash
## ------------------ docs ------------------ ##
docs-build:
sphinx-apidoc --module-first --separate -o docs/_autoref/ co3
make -C docs/ html
docs-serve:
cd docs/_build/html && python -m http.server 9090
docs-clean:
make -C docs/ clean
rm -rf docs/_autoref
rm -rf docs/_autosummary
## ------------------------------------------ ##
## ----------------- tests ------------------ ##
test:
pytest --pyargs tests -v

View File

@ -1,35 +1,35 @@
# Overview # Overview
`co3` is a package for file conversion and associated database operations. The `CO3` base class `co3` is a lightweight Python ORM for hierarchical storage management. It implements a
provides a standard interface for performing conversions, preparing inserts, and general type system for defining database components like relations, schemas, engines,
interacting with database schemas that mirror the class hierarchy. etc. Objects inheriting from the `CO3` base class can then define data transformations
that connect to database components, and can be automatically collected for coordinated
database insertion.
Simplified description of the operational model: `co3` attempts to provide a general interface for interacting with a storage media (e.g.,
database, pickled objects, VSS framework, in-memory key-value stores, etc). The following
top-level classes capture the bulk of the operational model:
**Goal**: interact with a storage medium (database, pickled structure, VSS framework) with - **Database**: reference to a storage medium, with an `Accessor` for accessing data,
a known schema. `Manager` for managing database state, and an `Engine` for managing connections and
external operations.
- **Accessor**: provides access to stored items in a `Database`, typically via a supported
`select` operation over known `Component` types
- **Manager**: manages database storage state (e.g., supported inserts or database sync
operations)
- **Mapper**: associates `CO3` types with `Schema` components, and provides automatic
collection and composition operations for supported items
- **Collector**: collects data from defined `CO3` type transformations and prepares for
`Database` insert operations
- **Component**: atomic storage groups for databases (i.e., generalized notion of a
"relation" in relational algebra).
- **Indexer**: automatic caching of supported access queries to a `Database`
- **Schema**: general schema analog for grouping related `Component` sets
- **Differ**: facilitates set operations on results from selectable resources (e.g.,
automatic comparison between file data on disk and file rows in a SQL database)
- **Syncer**: generalized syncing procedure for items between data resources (e.g.,
syncing new, modified, and deleted files from disk to a SQL database that stores file
metadata).
- **Accessor** to provide access to stored items The **CO3** an abstract base class then makes it easy to integrate this model with regular
- **Composer** to compose common access points (e.g., JOINed tables) Python object hierarchies that can be mapped to a storage schema.
- **Indexer** to index/cache access queries
- **Manager** to manage storage state (e.g., supported inserts, database syncs)
- **Collector** to collect data for updating storage state
- **Database** to collect data for updating storage state
- **Mapper** to collect data for updating storage state
- **Component** to collect data for updating storage state
**CO3** is an abstract base class that makes it easy to integrate this model with object
hierarchies that mirror a storage schema.
# Detailed structural breakdown
There are a few pillars of the CO3 model that meaningfully group up functionality:
- Database: generic to a Component type, provides basic connection to a database at a
specific address/location. The explicit Component type makes it easy to hook into
appropriately typed functional objects:
* Manager: generic to a Component and Database type, provides a supported set of
state-modifying operations to a constituent database
* Accessor: generic to a Component and Database type, provides a supported set of
state inspection operations on a constituent database
* Indexer:
- Mapper: generic to a Component, serves as the fundamental connective component between
types in the data representation hierarchy (CO3 subclasses) and database Components.

View File

@ -1,47 +0,0 @@
Metadata-Version: 2.1
Name: co3
Version: 0.1.1
Summary: Lightweight ORM
Author-email: Sam Griesemer <samgriesemer@gmail.com>
Classifier: Programming Language :: Python :: 3
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Requires-Python: >=3.11
Description-Content-Type: text/markdown
Requires-Dist: tqdm
# Overview
`co3` is a package for file conversion and associated database operations. The `CO3` base class
provides a standard interface for performing conversions, preparing inserts, and
interacting with database schemas that mirror the class hierarchy.
Simplified description of the operational model:
**Goal**: interact with a storage medium (database, pickled structure, VSS framework) with
a known schema.
- **Accessor** to provide access to stored items
- **Composer** to compose common access points (e.g., JOINed tables)
- **Indexer** to index/cache access queries
- **Manager** to manage storage state (e.g., supported inserts, database syncs)
- **Collector** to collect data for updating storage state
- **Database** to collect data for updating storage state
- **Mapper** to collect data for updating storage state
- **Component** to collect data for updating storage state
**CO3** is an abstract base class that makes it easy to integrate this model with object
hierarchies that mirror a storage schema.
# Detailed structural breakdown
There are a few pillars of the CO3 model that meaningfully group up functionality:
- Database: generic to a Component type, provides basic connection to a database at a
specific address/location. The explicit Component type makes it easy to hook into
appropriately typed functional objects:
* Manager: generic to a Component and Database type, provides a supported set of
state-modifying operations to a constituent database
* Accessor: generic to a Component and Database type, provides a supported set of
state inspection operations on a constituent database
* Indexer:
- Mapper: generic to a Component, serves as the fundamental connective component between
types in the data representation hierarchy (CO3 subclasses) and database Components.

View File

@ -1,52 +0,0 @@
MANIFEST.in
README.md
pyproject.toml
co3/__init__.py
co3/accessor.py
co3/co3.py
co3/collector.py
co3/component.py
co3/database.py
co3/differ.py
co3/engine.py
co3/indexer.py
co3/manager.py
co3/mapper.py
co3/resource.py
co3/schema.py
co3/syncer.py
co3.egg-info/PKG-INFO
co3.egg-info/SOURCES.txt
co3.egg-info/dependency_links.txt
co3.egg-info/requires.txt
co3.egg-info/top_level.txt
co3/accessors/__init__.py
co3/accessors/fts.py
co3/accessors/sql.py
co3/accessors/vss.py
co3/components/__init__.py
co3/databases/__init__.py
co3/databases/fts.py
co3/databases/sql.py
co3/databases/vss.py
co3/engines/__init__.py
co3/indexers/relational.py
co3/managers/__init__.py
co3/managers/fts.py
co3/managers/sql.py
co3/managers/vss.py
co3/mappers/__init__.py
co3/resources/__init__.py
co3/resources/disk.py
co3/schemas/__init__.py
co3/util/__init__.py
co3/util/db.py
co3/util/generic.py
co3/util/paths.py
co3/util/regex.py
co3/util/types.py
tests/test_co3.py
tests/test_database.py
tests/test_imports.py
tests/test_mapper.py
tests/test_schema.py

View File

@ -1 +0,0 @@

View File

@ -1 +0,0 @@
tqdm

View File

@ -1 +0,0 @@
co3

View File

@ -11,6 +11,8 @@ from co3.accessor import Accessor
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class VSSAccessor(Accessor): class VSSAccessor(Accessor):
_model_cls = None
def __init__(self, cache_path): def __init__(self, cache_path):
super().__init__() super().__init__()
@ -35,8 +37,7 @@ class VSSAccessor(Accessor):
@property @property
def model(self): def model(self):
if self._model is None: if self._model is None:
# model trained with 128 token seqs self._model = self._model_cls()
self._model = SentenceTransformer('sentence-transformers/all-MiniLM-L12-v2')
return self._model return self._model
@property @property
@ -60,12 +61,8 @@ class VSSAccessor(Accessor):
index_name : str, index_name : str,
query : str, query : str,
limit : int = 10, limit : int = 10,
score_threshold = 0.5, score_threshold = 0.5,
): ):
'''
Parameters:
index_name: one of ['chunks','blocks','notes']
'''
if not query: if not query:
return None return None

View File

@ -1,429 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "6f6fbc7e-4fb9-4353-b2ee-9ea819a3c896",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"import vegetables"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "88fd0ea8-9c94-4569-a51b-823a04f32f55",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'age': 5}"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tomato = vegetables.Tomato('t1', 5)\n",
"\n",
"# test a register collation action\n",
"tomato.collate('ripe')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "348926d9-7137-4eff-a919-508788553dd2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['9ca6772e-6621-4511-a4a6-ad451a1da91f',\n",
" '2a91b423-4e08-491c-b1d2-5ec25259191e',\n",
" '4a9edb2b-4ac5-467e-82ef-b254829ac2a2']"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.collect(tomato, ['ripe'])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "4e5e7319-11bf-4051-951b-08c84e9f3874",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<Component (SQLTable)> vegetable+tomato+tomato_aging_states+tomato_cooking_states"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.compose(tomato, action_groups=['aging', 'cooking'])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "aa290686-8074-4038-a3cc-ce6817844653",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Column('id', Integer(), table=<vegetable>, primary_key=True, nullable=False),\n",
" Column('name', String(), table=<vegetable>),\n",
" Column('color', String(), table=<vegetable>),\n",
" Column('id', Integer(), table=<tomato>, primary_key=True, nullable=False),\n",
" Column('name', String(), ForeignKey('vegetable.name'), table=<tomato>),\n",
" Column('radius', Integer(), table=<tomato>),\n",
" Column('id', Integer(), table=<tomato_aging_states>, primary_key=True, nullable=False),\n",
" Column('name', String(), ForeignKey('tomato.name'), table=<tomato_aging_states>),\n",
" Column('state', String(), table=<tomato_aging_states>),\n",
" Column('age', Integer(), table=<tomato_aging_states>)]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(vegetables.vegetable_mapper.compose(tomato, action_groups=['aging']).obj.columns)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f3c7e37d-ba9e-4bae-ae44-adc922bf5f4c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(vegetables.Tomato, vegetables.Vegetable, co3.co3.CO3, object)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tomato.__class__.__mro__"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "c21d2c54-39e2-4de3-93bc-763896ed348e",
"metadata": {},
"outputs": [],
"source": [
"from co3.databases import SQLDatabase\n",
"\n",
"db = SQLDatabase('sqlite://') #, echo=True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "a785d202-99d3-4ae7-859e-ee22b481f8df",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"db.recreate(vegetables.vegetable_schema)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "cda01cb0-1666-4cb1-aa64-bcdca871aff5",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{<Component (SQLTable)> vegetable: [{'name': 't1', 'color': 'red'}],\n",
" <Component (SQLTable)> tomato: [{'name': 't1', 'radius': 5}],\n",
" <Component (SQLTable)> tomato_aging_states: [{'name': 't1',\n",
" 'state': 'ripe',\n",
" 'age': 2}]}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.collector.inserts"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "af7124ed-3031-4f28-89a6-553eb5b3cc7a",
"metadata": {},
"outputs": [],
"source": [
"with db.engine.connect() as connection:\n",
" db.manager.insert_many(\n",
" connection,\n",
" vegetables.vegetable_mapper.collector.inserts,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "0149e14e-5d07-42af-847d-af5c190f8946",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'id': 1, 'name': 't1', 'radius': 5}]\n"
]
}
],
"source": [
"with db.engine.connect() as connection:\n",
" print(db.accessor.select(\n",
" connection, \n",
" vegetables.vegetable_schema.get_component('tomato')\n",
" ))"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "668d1b8c-b47f-4a58-914d-e43402443fe6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5}]\n"
]
}
],
"source": [
"agg_table = vegetables.vegetable_mapper.compose(tomato)\n",
"\n",
"with db.engine.connect() as connection:\n",
" agg_res = db.accessor.select(\n",
" connection, \n",
" agg_table,\n",
" mappings=True,\n",
" )\n",
"\n",
"print(agg_res)"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "a051d72d-a867-46dc-bb5e-69341f39a056",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'id': 1, 'name': 't1', 'color': 'red', 'id_1': 1, 'name_1': 't1', 'radius': 5, 'id_2': 1, 'name_2': 't1', 'state': 'ripe', 'age': 2}]\n"
]
}
],
"source": [
"agg_table = vegetables.vegetable_mapper.compose(tomato, action_groups=['aging'])#, outer=True)\n",
"\n",
"with db.engine.connect() as connection:\n",
" agg_res = db.accessor.select(\n",
" connection, \n",
" agg_table,\n",
" mappings=True,\n",
" )\n",
"\n",
"print(agg_res)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "6a80cfd7-3175-4526-96e0-374765d64a27",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"sqlalchemy.engine.row.RowMapping"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"type(agg_res[0])"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "7cf05ddd-2328-4051-9cf8-4ac01352405e",
"metadata": {},
"outputs": [],
"source": [
"import sqlalchemy as sa\n",
"from co3.engines import SQLEngine\n",
"\n",
"a = SQLEngine.execute(db.engine.connect(), sa.select(agg_table.obj))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "c1edf68e-1fde-4a1f-8ec3-084713a8da45",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[]"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"a.mappings().all()\n"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "8b8a9e47-7f5f-4828-a99e-5d9a12697f46",
"metadata": {},
"outputs": [],
"source": [
"tomato2 = vegetables.Tomato('t2', 8)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "062aa4de-7aea-4fd3-b5db-82af147d023e",
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'Tomato' object has no attribute 'action_map'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mvegetables\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvegetable_mapper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtomato2\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/Documents/projects/ontolog/co3/build/__editable__.co3-0.1.1-py3-none-any/co3/mapper.py:198\u001b[0m, in \u001b[0;36mMapper.collect\u001b[0;34m(self, obj, action_keys, action_groups)\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 180\u001b[0m \u001b[38;5;124;03mStages inserts up the inheritance chain, and down through components.\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[38;5;124;03mReturns: dict with keys and values relevant for associated SQLite tables\u001b[39;00m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m action_keys \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 198\u001b[0m action_keys \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[43mobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maction_map\u001b[49m\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[1;32m 200\u001b[0m receipts \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _cls \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(obj\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__mro__\u001b[39m[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m]):\n",
"\u001b[0;31mAttributeError\u001b[0m: 'Tomato' object has no attribute 'action_map'"
]
}
],
"source": [
"vegetables.vegetable_mapper.collect(tomato2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4673ddc8-3f76-4d8c-8186-bbed4a682e0d",
"metadata": {},
"outputs": [],
"source": [
"db.insert(vegetables.vegetable_schema.get_component('tomato'), "
]
},
{
"cell_type": "code",
"execution_count": 36,
"id": "9314be4e-c1d5-4af8-ad23-0b208d24b3eb",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'id': 1, 'name': 't1', 'radius': 5}]"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db.select(vegetables.vegetable_schema.get_component('tomato'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2efd060-f298-4ca6-8a58-7ed5acf1dd15",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "co3",
"language": "python",
"name": "co3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -1,403 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "e02ccafe-e04d-4312-acba-e41cf7b1c021",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/smgr/.pyenv/versions/co4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"import vegetables"
]
},
{
"cell_type": "markdown",
"id": "c0914069-7f3c-4213-8d34-f7566033e054",
"metadata": {},
"source": [
"## Development notes\n",
"- No registry actually needs to take place if there's a default type2component map or one supplied on creation. Can just collect right out of the gate\n",
"- Need connective function (type to collation) and attribute map. Do we need to this with a subclass? If a func is passed in on init, I can type it appropriately I guess `Callable[[type[CO3],str,str|None],dict]`"
]
},
{
"cell_type": "markdown",
"id": "ef733715-bb75-4263-b216-45e778a06b21",
"metadata": {},
"source": [
"## Usage\n",
"The Mapper's primary job is to associate class hierarchies with database components. This can be done in a few ways:\n",
"\n",
"1. Manually attaching a type reference to a Component\n",
"2. Attaching a type reference to a Component's name as registered in a schema\n",
"3. Automatically register the CO3 heirarchy to matching schema component names (through transformation)"
]
},
{
"cell_type": "markdown",
"id": "d2672422-3596-4eab-ac44-5da617f74b80",
"metadata": {},
"source": [
"## Explicit example steps"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "7d80f7b9-7458-4ad4-8c1a-3ea56e796b4e",
"metadata": {},
"outputs": [],
"source": [
"from co3 import Mapper\n",
"\n",
"vegetable_mapper = Mapper(\n",
" vegetables.Vegetable,\n",
" vegetables.vegetable_schema\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "d24d31b4-c4a6-4a1e-8bea-c44378aadfdd",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'\\nvegetable_mapper.attach(\\n vegetables.Vegetable,\\n vegetables.vegetable_table,\\n)\\n'"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# not valid; tables need to be wrapped in CO3 Components\n",
"'''\n",
"vegetable_mapper.attach(\n",
" vegetables.Vegetable,\n",
" vegetables.vegetable_table,\n",
")\n",
"'''"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "f9408562-bf50-4522-909c-318557f85948",
"metadata": {},
"outputs": [],
"source": [
"# manually attach component\n",
"vegetable_mapper.attach(\n",
" vegetables.Tomato,\n",
" vegetables.vegetable_schema.get_component('tomato'),\n",
" coll_groups={\n",
" 'aging': vegetables.vegetable_schema.get_component('tomato_aging_states'),\n",
" 'cooking': vegetables.vegetable_schema.get_component('tomato_cooking_states'),\n",
" },\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "05fdd404-87ee-4187-832f-2305272758ae",
"metadata": {},
"outputs": [],
"source": [
"# attach by name in schema\n",
"vegetable_mapper.attach(\n",
" vegetables.Tomato,\n",
" 'tomato',\n",
" coll_groups={\n",
" 'aging': 'tomato_aging_states',\n",
" 'cooking': 'tomato_cooking_states',\n",
" },\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "e9b6af49-a69d-41cc-beae-1b6f171cd2f5",
"metadata": {},
"outputs": [],
"source": [
"# attach entire type hierarchy w/ type->name map\n",
"vegetable_mapper.attach_hierarchy(\n",
" # this might make more sense during init\n",
" vegetables.Vegetable,\n",
" lambda x:x.__name__.lower()\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "0fb45a86-5c9b-41b1-a3ab-5691444f175e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<co3.components.SQLTable at 0x7f2012b23f80>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetable_mapper.get_collation_comp(vegetables.Tomato, 'cooking')"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "2e4336ab-5b5f-484d-815d-164d4b6f40a0",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'co3_root': vegetables.Vegetable,\n",
" 'schema': <co3.schemas.SQLSchema at 0x732074224aa0>,\n",
" 'collector': <co3.collector.Collector at 0x7320757da120>,\n",
" 'composer': <co3.composer.Composer at 0x7320757da9c0>,\n",
" 'attribute_comps': {vegetables.Tomato: <co3.components.SQLTable at 0x732074224cb0>},\n",
" 'collation_groups': defaultdict(dict,\n",
" {vegetables.Tomato: {'aging': <co3.components.SQLTable at 0x732074224ce0>,\n",
" 'cooking': <co3.components.SQLTable at 0x732074224d10>}})}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vars(vegetable_mapper)"
]
},
{
"cell_type": "markdown",
"id": "47859e25-b803-4459-a581-f10bbcfac716",
"metadata": {},
"source": [
"## Holistic attachment"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "70c9baed-b870-4021-8949-9b713d863de6",
"metadata": {},
"outputs": [],
"source": [
"def attr_name_map(cls):\n",
" return f'{cls.__name__.lower()}'\n",
"\n",
"def coll_name_map(cls, action_group):\n",
" return f'{cls.__name__.lower()}_{action_group}_states'\n",
"\n",
"vegetables.vegetable_mapper.attach_many(\n",
" vegetables.type_list,\n",
" attr_name_map,\n",
" coll_name_map,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "c16786d4-0b71-42d9-97f7-7893c542104e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'age': 4}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# create new CO3 descendant\n",
"tomato = vegetables.Tomato('t1', 5)\n",
"\n",
"# test a register collation action\n",
"tomato.collate('ripe')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d7fa94ca-3ecd-4ee3-b0dc-f3b2b65ee47c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<Component (SQLTable)> tomato"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.get_attribute_comp(vegetables.Tomato)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "1adc3bc5-957f-4b5a-bc2c-2d172675826d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'schema': <co3.schemas.SQLSchema at 0x7ab568224e60>,\n",
" 'collector': <co3.collector.Collector at 0x7ab568225190>,\n",
" 'composer': <co3.composer.Composer at 0x7ab5682251c0>,\n",
" 'attribute_comps': {vegetables.Vegetable: <Component (SQLTable)> vegetable,\n",
" vegetables.Tomato: <Component (SQLTable)> tomato},\n",
" 'collation_groups': defaultdict(dict,\n",
" {vegetables.Vegetable: {},\n",
" vegetables.Tomato: {'aging': <Component (SQLTable)> tomato_aging_states,\n",
" 'cooking': <Component (SQLTable)> tomato_cooking_states}})}"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vars(vegetables.vegetable_mapper)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f32d1f65-9b1d-4600-b396-8551fbd1fcf7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['3bf42abc-8a12-452f-baf6-38a05fc5d420',\n",
" '271b7b84-846e-4d1d-87f6-bcabc90a7b55',\n",
" 'f9fc5d16-c5cb-47a7-9eca-7df8a3ba5d10']"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.collect(tomato, ['ripe'])"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "380dfbea-90cc-49fc-aef1-ebb342872632",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"defaultdict(<function co3.collector.Collector.__init__.<locals>.<lambda>()>,\n",
" {'3bf42abc-8a12-452f-baf6-38a05fc5d420': (<Component (SQLTable)> vegetable,\n",
" {'name': 't1', 'color': 'red'}),\n",
" '271b7b84-846e-4d1d-87f6-bcabc90a7b55': (<Component (SQLTable)> tomato,\n",
" {'name': 't1', 'radius': 5}),\n",
" 'f9fc5d16-c5cb-47a7-9eca-7df8a3ba5d10': (<Component (SQLTable)> tomato_aging_states,\n",
" {'name': 't1', 'state': 'ripe', 'age': 1})})"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.collector._inserts"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "905bb2a9-9c22-4187-be15-3dd32d206e26",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{<Component (SQLTable)> vegetable: [{'name': 't1', 'color': 'red'}],\n",
" <Component (SQLTable)> tomato: [{'name': 't1', 'radius': 5}],\n",
" <Component (SQLTable)> tomato_aging_states: [{'name': 't1',\n",
" 'state': 'ripe',\n",
" 'age': 1}]}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vegetables.vegetable_mapper.collector.inserts"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d166b9af-e3ba-4750-9dcb-d8d4e08fe4d3",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "co3",
"language": "python",
"name": "co3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -1,27 +1,49 @@
[build-system] [build-system]
requires = ["setuptools"] requires = ["setuptools", "wheel", "setuptools-git-versioning>=2.0,<3"]
build-backend = "setuptools.build_meta" build-backend = "setuptools.build_meta"
[project] [project]
name = "co3" name = "co3"
version = "0.1.1" description = "Lightweight Python ORM for hierarchical storage management"
authors = [
{ name="Sam Griesemer", email="samgriesemer@gmail.com" },
]
description = "Lightweight ORM"
readme = "README.md" readme = "README.md"
requires-python = ">=3.11" requires-python = ">=3.12"
dynamic = ["version"]
#license = {file = "LICENSE"}
authors = [
{ name="Sam Griesemer", email="samgriesemer+git@gmail.com" },
]
keywords = ["database", "orm"]
classifiers = [ classifiers = [
"Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12",
"License :: OSI Approved :: MIT License", "License :: OSI Approved :: MIT License",
"Operating System :: OS Independent", "Operating System :: OS Independent",
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
] ]
dependencies = [ dependencies = [
"tqdm" "tqdm",
"wcmatch",
"numpy",
"sqlalchemy",
"colorama",
] ]
[tool.setuptools.packages.find] [project.optional-dependencies]
#where = ["localsys"] # this is too deeply nested, need to remain at root level & use tests = ["pytest"]
#include; by default, `where` is `.` docs = [
include = ["co3*"] # pattern to match package names "sphinx",
"sphinx-togglebutton",
"sphinx-autodoc-typehints",
"furo",
"myst-parser",
]
[project.urls]
Homepage = "https://doc.olog.io/co3"
Documentation = "https://doc.olog.io/co3"
Repository = "https://git.olog.io/olog/co3"
Issues = "https://git.olog.io/olog/co3/issues"
[tool.setuptools.packages.find]
include = ["co3*"] # pattern to match package names