diff --git a/datalite/__init__.py b/datalite/__init__.py index c6ab2f5..a45fd71 100644 --- a/datalite/__init__.py +++ b/datalite/__init__.py @@ -1,3 +1,3 @@ -__all__ = ['commons', 'datalite_decorator', 'fetch', 'datalite'] +__all__ = ['commons', 'datalite_decorator', 'fetch', 'migrations', 'datalite'] from .datalite_decorator import datalite diff --git a/datalite/commons.py b/datalite/commons.py index bc04e43..2876db5 100644 --- a/datalite/commons.py +++ b/datalite/commons.py @@ -1,4 +1,6 @@ -from typing import Any, Optional, Dict +from dataclasses import Field +from typing import Any, Optional, Dict, List +import sqlite3 as sql def _convert_type(type_: Optional[type], type_overload: Dict[Optional[type], str]) -> str: @@ -33,4 +35,52 @@ def _convert_sql_format(value: Any) -> str: elif isinstance(value, bytes): return '"' + str(value).replace("b'", "")[:-1] + '"' else: - return str(value) \ No newline at end of file + return str(value) + + +def _get_table_cols(cur: sql.Cursor, table_name: str) -> List[str]: + """ + Get the column data of a table. + + :param cur: Cursor in database. + :param table_name: Name of the table. + :return: the information about columns. + """ + cur.execute(f"PRAGMA table_info({table_name});") + return [row_info[1] for row_info in cur.fetchall()][1:] + + +def _get_default(default_object: object, type_overload: Dict[Optional[type], str]) -> str: + """ + Check if the field's default object is filled, + if filled return the string to be put in the, + database. + :param default_object: The default field of the field. + :param type_overload: Type overload table. + :return: The string to be put on the table statement, + empty string if no string is necessary. + """ + if type(default_object) in type_overload: + return f' DEFAULT {_convert_sql_format(default_object)}' + return "" + + +def _create_table(class_: type, cursor: sql.Cursor, type_overload: Dict[Optional[type], str]) -> None: + """ + Create the table for a specific dataclass given + :param class_: A dataclass. + :param cursor: Current cursor instance. + :param type_overload: Overload the Python -> SQLDatatype table + with a custom table, this is that custom table. + :return: None. + """ + fields: List[Field] = [class_.__dataclass_fields__[key] for + key in class_.__dataclass_fields__.keys()] + fields.sort(key=lambda field: field.name) # Since dictionaries *may* be unsorted. + sql_fields = ', '.join(f"{field.name} {_convert_type(field.type, type_overload)}" + f"{_get_default(field.default, type_overload)}" for field in fields) + sql_fields = "obj_id INTEGER PRIMARY KEY AUTOINCREMENT, " + sql_fields + cursor.execute(f"CREATE TABLE IF NOT EXISTS {class_.__name__.lower()} ({sql_fields});") + +type_table: Dict[Optional[type], str] = {None: "NULL", int: "INTEGER", float: "REAL", + str: "TEXT", bytes: "BLOB"} diff --git a/datalite/datalite_decorator.py b/datalite/datalite_decorator.py index b3d5bad..b9c69c1 100644 --- a/datalite/datalite_decorator.py +++ b/datalite/datalite_decorator.py @@ -6,40 +6,7 @@ a class bound to a sqlite3 database. from typing import Dict, Optional, List, Callable from dataclasses import Field, asdict import sqlite3 as sql -from .commons import _convert_sql_format, _convert_type - - -def _get_default(default_object: object, type_overload: Dict[Optional[type], str]) -> str: - """ - Check if the field's default object is filled, - if filled return the string to be put in the, - database. - :param default_object: The default field of the field. - :param type_overload: Type overload table. - :return: The string to be put on the table statement, - empty string if no string is necessary. - """ - if type(default_object) in type_overload: - return f' DEFAULT {_convert_sql_format(default_object)}' - return "" - - -def _create_table(class_: type, cursor: sql.Cursor, type_overload: Dict[Optional[type], str]) -> None: - """ - Create the table for a specific dataclass given - :param class_: A dataclass. - :param cursor: Current cursor instance. - :param type_overload: Overload the Python -> SQLDatatype table - with a custom table, this is that custom table. - :return: None. - """ - fields: List[Field] = [class_.__dataclass_fields__[key] for - key in class_.__dataclass_fields__.keys()] - fields.sort(key=lambda field: field.name) # Since dictionaries *may* be unsorted. - sql_fields = ', '.join(f"{field.name} {_convert_type(field.type, type_overload)}" - f"{_get_default(field.default, type_overload)}" for field in fields) - sql_fields = "obj_id INTEGER PRIMARY KEY AUTOINCREMENT, " + sql_fields - cursor.execute(f"CREATE TABLE IF NOT EXISTS {class_.__name__.lower()} ({sql_fields});") +from .commons import _convert_sql_format, _convert_type, _create_table, type_table def _create_entry(self) -> None: @@ -105,14 +72,14 @@ def datalite(db_path: str, type_overload: Optional[Dict[Optional[type], str]] = :return: The new dataclass. """ def decorator(dataclass_: type, *args_i, **kwargs_i): - type_table: Dict[Optional[type], str] = {None: "NULL", int: "INTEGER", float: "REAL", - str: "TEXT", bytes: "BLOB"} + types_table = type_table.copy() if type_overload is not None: - type_table.update(type_overload) + types_table.update(type_overload) with sql.connect(db_path) as con: cur: sql.Cursor = con.cursor() - _create_table(dataclass_, cur, type_table) + _create_table(dataclass_, cur, types_table) setattr(dataclass_, 'db_path', db_path) # We add the path of the database to class itself. + setattr(dataclass_, 'types_table', types_table) # We add the type table for migration. dataclass_.create_entry = _create_entry dataclass_.remove_entry = _remove_entry dataclass_.update_entry = _update_entry diff --git a/datalite/fetch.py b/datalite/fetch.py index 1b9d164..c39f100 100644 --- a/datalite/fetch.py +++ b/datalite/fetch.py @@ -1,6 +1,6 @@ import sqlite3 as sql from typing import List, Tuple, Any -from .commons import _convert_sql_format +from .commons import _convert_sql_format, _get_table_cols def _insert_pagination(query: str, page: int, element_count: int) -> str: @@ -35,18 +35,6 @@ def is_fetchable(class_: type, obj_id: int) -> bool: return bool(cur.fetchall()) -def _get_table_cols(cur: sql.Cursor, table_name: str) -> List[str]: - """ - Get the column data of a table. - - :param cur: Cursor in database. - :param table_name: Name of the table. - :return: the information about columns. - """ - cur.execute(f"PRAGMA table_info({table_name});") - return [row_info[1] for row_info in cur.fetchall()][1:] - - def fetch_equals(class_: type, field: str, value: Any, ) -> Any: """ Fetch a class_ type variable from its bound db. @@ -112,7 +100,7 @@ def fetch_if(class_: type, condition: str, page: int = 0, element_count: int = 1 :param page: Which page to retrieve, default all. (0 means closed). :param element_count: Element count in each page. :return: A tuple of records that fit the given condition - of given type class_. + of given type class_. """ table_name = class_.__name__.lower() with sql.connect(getattr(class_, 'db_path')) as con: @@ -146,7 +134,7 @@ def fetch_range(class_: type, range_: range) -> tuple: :param class_: Class of the records. :param range_: Range of the object ids. :return: A tuple of class_ type objects whose values - come from the class_' bound database. + come from the class_' bound database. """ return tuple(fetch_from(class_, obj_id) for obj_id in range_ if is_fetchable(class_, obj_id)) @@ -159,7 +147,7 @@ def fetch_all(class_: type, page: int = 0, element_count: int = 10) -> tuple: :param page: Which page to retrieve, default all. (0 means closed). :param element_count: Element count in each page. :return: All the records of type class_ in - the bound database as a tuple. + the bound database as a tuple. """ try: db_path = getattr(class_, 'db_path') diff --git a/datalite/migrations.py b/datalite/migrations.py new file mode 100644 index 0000000..c766985 --- /dev/null +++ b/datalite/migrations.py @@ -0,0 +1,160 @@ +""" +Migrations module deals with migrating data when the object +definitions change. This functions deal with Schema Migrations. +""" +from dataclasses import Field +from os.path import exists +from typing import Dict, Tuple, List +import sqlite3 as sql + +from .commons import _create_table, _get_table_cols + + +def _get_db_table(class_: type) -> Tuple[str, str]: + """ + Check if the class is a datalite class, the database exists + and the table exists. Return database and table names. + + :param class_: A datalite class. + :return: A tuple of database and table names. + """ + database_name: str = getattr(class_, 'db_path', None) + if not database_name: + raise TypeError(f"{class_.__name__} is not a datalite class.") + table_name: str = class_.__name__.lower() + if not exists(database_name): + raise FileNotFoundError(f"{database_name} does not exist") + with sql.connect(database_name) as con: + cur: sql.Cursor = con.cursor() + cur.execute(f"SELECT count(*) FROM sqlite_master " + f"WHERE type='table' AND name='{table_name}';") + count: int = int(cur.fetchone()[0]) + if not count: + raise FileExistsError(f"Table, {table_name}, already exists.") + return database_name, table_name + + +def _get_table_column_names(database_name: str, table_name: str) -> Tuple[str]: + """ + Get the column names of table. + + :param database_name: Name of the database the table + resides in. + :param table_name: Name of the table. + :return: A tuple holding the column names of the table. + """ + with sql.connect(database_name) as con: + cur: sql.Cursor = con.cursor() + cols: List[str] = _get_table_cols(cur, table_name) + return tuple(cols) + + +def _copy_records(database_name: str, table_name: str): + """ + Copy all records from a table. + + :param database_name: Name of the database. + :param table_name: Name of the table. + :return: A generator holding dataclass asdict representations. + """ + with sql.connect(database_name) as con: + cur: sql.Cursor = con.cursor() + cur.execute(f'SELECT * FROM {table_name};') + values = cur.fetchall() + keys = _get_table_cols(cur, table_name) + keys.insert(0, 'obj_id') + records = (dict(zip(keys, value)) for value in values) + return records + + +def _drop_table(database_name: str, table_name: str) -> None: + """ + Drop a table. + + :param database_name: Name of the database. + :param table_name: Name of the table to be dropped. + :return: None. + """ + with sql.connect(database_name) as con: + cur: sql.Cursor = con.cursor() + cur.execute(f'DROP TABLE {table_name};') + con.commit() + + +def _modify_records(data, col_to_del: Tuple[str], col_to_add: Tuple[str], + flow: Dict[str, str]) -> Tuple[Dict[str, str]]: + """ + Modify the asdict records in accordance + with schema migration rules provided. + + :param data: Data kept as asdict in tuple. + :param col_to_del: Column names to delete. + :param col_to_add: Column names to add. + :param flow: A dictionary that explain + if the data from a deleted column + will be transferred to a column + to be added. + :return: The modified data records. + """ + records = [] + for record in data: + record_mod = {} + for key in record.keys(): + if key in col_to_del and key in flow: + record_mod[flow[key]] = record[key] + else: + record_mod[key] = record[key] + for key_to_add in col_to_add: + if key_to_add not in record_mod: + record_mod[key_to_add] = None + records.append(record_mod) + return records + + +def _migrate_records(class_: type, database_name: str, data, + col_to_del: Tuple[str], col_to_add: Tuple[str], flow: Dict[str, str]) -> None: + """ + Migrate the records into the modified table. + + :param class_: Class of the entries. + :param database_name: Name of the database. + :param data: Data, asdict tuple. + :param col_to_del: Columns to be deleted. + :param col_to_add: Columns to be added. + :param flow: Flow dictionary stating where + column data will be transferred. + :return: None. + """ + with sql.connect(database_name) as con: + cur: sql.Cursor = con.cursor() + _create_table(class_, cur, getattr(class_, 'types_table')) + con.commit() + new_records = _modify_records(data, col_to_del, col_to_add, flow) + for record in new_records: + del record['obj_id'] + class_(**record).create_entry() + + +def basic_migrate(class_: type, column_transfer: dict = None) -> None: + """ + Given a class, compare its previous table, + delete the fields that no longer exist, + create new columns for new fields. If the + column_flow parameter is given, migrate elements + from previous column to the new ones. + + :param class_: Datalite class to migrate. + :param column_transfer: A dictionary showing which + columns will be copied to new ones. + :return: None. + """ + database_name, table_name = _get_db_table(class_) + table_column_names: Tuple[str] = _get_table_column_names(database_name, table_name) + values = class_.__dataclass_fields__.values() + data_fields: Tuple[Field] = tuple(field for field in values) + data_field_names: Tuple[str] = tuple(field.name for field in data_fields) + columns_to_be_deleted: Tuple[str] = tuple(column for column in table_column_names if column not in data_field_names) + columns_to_be_added: Tuple[str] = tuple(column for column in data_field_names if column not in table_column_names) + records = _copy_records(database_name, table_name) + _drop_table(database_name, table_name) + _migrate_records(class_, database_name, records, columns_to_be_deleted, columns_to_be_added, column_transfer) diff --git a/docs/_build/doctrees/datalite.doctree b/docs/_build/doctrees/datalite.doctree index e83131f..c5e0eea 100644 Binary files a/docs/_build/doctrees/datalite.doctree and b/docs/_build/doctrees/datalite.doctree differ diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index e02ee48..6ed89e5 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/html/_sources/datalite.rst.txt b/docs/_build/html/_sources/datalite.rst.txt index 4670c1f..5c6c468 100644 --- a/docs/_build/html/_sources/datalite.rst.txt +++ b/docs/_build/html/_sources/datalite.rst.txt @@ -14,4 +14,10 @@ datalite.fetch module :undoc-members: :show-inheritance: +datalite.migrations module +---------------------------- +.. automodule:: datalite.migrations + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/_build/html/datalite.html b/docs/_build/html/datalite.html index 439eca7..66e5b6d 100644 --- a/docs/_build/html/datalite.html +++ b/docs/_build/html/datalite.html @@ -36,7 +36,7 @@ - +
@@ -83,12 +83,10 @@Contents:
All the records of type class_ in
+All the records of type class_ in +the bound database as a tuple.
the bound database as a tuple.
A tuple of records that fit the given condition
+A tuple of records that fit the given condition +of given type class_.
of given type class_.
A tuple of class_ type objects whose values
+A tuple of class_ type objects whose values +come from the class_’ bound database.
come from the class_’ bound database.
Migrations module deals with migrating data when the object +definitions change. This functions deal with Schema Migrations.
+datalite.migrations.basic_migrate(class_: type, column_transfer: dict = None) → None¶Given a class, compare its previous table, +delete the fields that no longer exist, +create new columns for new fields. If the +column_flow parameter is given, migrate elements +from previous column to the new ones.
+class – Datalite class to migrate.
column_transfer – A dictionary showing which +columns will be copied to new ones.
None.
+| + |
| - |
|
+
|
@@ -209,6 +225,8 @@
diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html
index 078c510..ecc3a79 100644
--- a/docs/_build/html/index.html
+++ b/docs/_build/html/index.html
@@ -162,6 +162,7 @@ sqlite3 database.
|
| + |
+ datalite.migrations | + |